Upload folder using huggingface_hub (part 3)

a/r033/v.jsonl

@@ -0,0 +1,40 @@
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.045540650612370046, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1823844611644745, "grad_var_mean_grad_norm_sq": 0.02404226433230939, "grad_var_noise_scale": 0.9538041723153284, "grad_var_ot_time_s_mean": 0.00330320093780756, "grad_var_time_s": 2.8934333324432373, "grad_var_trace": 0.0229316120320647, "grad_var_trace_biased": 0.021498386280060656, "step": 5000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.03956879209043941, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1825156807899475, "grad_var_mean_grad_norm_sq": 0.02623479413188424, "grad_var_noise_scale": 0.5421399948592112, "grad_var_ot_time_s_mean": 0.004373612813651562, "grad_var_time_s": 2.3080947399139404, "grad_var_trace": 0.014222931155792184, "grad_var_trace_biased": 0.013333997958555173, "step": 10000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.04461596353607551, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.18009281158447266, "grad_var_mean_grad_norm_sq": 0.029397127974118906, "grad_var_noise_scale": 0.5522112436872136, "grad_var_ot_time_s_mean": 0.004235608037561178, "grad_var_time_s": 2.318991184234619, "grad_var_trace": 0.01623342459942038, "grad_var_trace_biased": 0.015218835561956606, "step": 15000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.033664815830143795, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17558541893959045, "grad_var_mean_grad_norm_sq": 0.01956569455408029, "grad_var_noise_scale": 0.7686444584371499, "grad_var_ot_time_s_mean": 0.004584266804158688, "grad_var_time_s": 2.294569969177246, "grad_var_trace": 0.015039062694467737, "grad_var_trace_biased": 0.014099121276063505, "step": 20000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.08746609787130953, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17711594700813293, "grad_var_mean_grad_norm_sq": 0.07602124742012728, "grad_var_noise_scale": 0.16058458517255184, "grad_var_ot_time_s_mean": 0.00471621286123991, "grad_var_time_s": 2.376974582672119, "grad_var_trace": 0.012207840481261067, "grad_var_trace_biased": 0.01144485045118225, "step": 25000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.025222290464819323, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1759609580039978, "grad_var_mean_grad_norm_sq": 0.018197334800424562, "grad_var_noise_scale": 0.4117793140700061, "grad_var_ot_time_s_mean": 0.004303409717977047, "grad_var_time_s": 2.30934476852417, "grad_var_trace": 0.0074932860420210785, "grad_var_trace_biased": 0.007024955664394761, "step": 30000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.026167290433959232, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.176254004240036, "grad_var_mean_grad_norm_sq": 0.01913752171396991, "grad_var_noise_scale": 0.39181771182602787, "grad_var_ot_time_s_mean": 0.004444379359483719, "grad_var_time_s": 2.3054251670837402, "grad_var_trace": 0.007498419967988612, "grad_var_trace_biased": 0.007029768719989324, "step": 35000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.011945297084525857, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17746774852275848, "grad_var_mean_grad_norm_sq": 0.003315219624059289, "grad_var_noise_scale": 2.776713763101921, "grad_var_ot_time_s_mean": 0.0042929197661578655, "grad_var_time_s": 2.320284128189087, "grad_var_trace": 0.009205415957831005, "grad_var_trace_biased": 0.008630077460466568, "step": 40000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.032184382104506276, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1792590469121933, "grad_var_mean_grad_norm_sq": 0.023704768839646482, "grad_var_noise_scale": 0.3815654511139008, "grad_var_ot_time_s_mean": 0.004815762862563133, "grad_var_time_s": 2.295628786087036, "grad_var_trace": 0.009044920815850448, "grad_var_trace_biased": 0.008479613264859794, "step": 45000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.029106684934258886, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17452719807624817, "grad_var_mean_grad_norm_sq": 0.024288712277701862, "grad_var_noise_scale": 0.21158679698218572, "grad_var_ot_time_s_mean": 0.004426221363246441, "grad_var_time_s": 2.3113162517547607, "grad_var_trace": 0.0051391708336608255, "grad_var_trace_biased": 0.004817972656557024, "step": 50000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.019821843770103986, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17467959225177765, "grad_var_mean_grad_norm_sq": 0.013549050987013173, "grad_var_noise_scale": 0.49383377293683045, "grad_var_ot_time_s_mean": 0.004295232705771923, "grad_var_time_s": 2.3255631923675537, "grad_var_trace": 0.006690978968630201, "grad_var_trace_biased": 0.006272792783090814, "step": 55000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.014633351277497224, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17673026025295258, "grad_var_mean_grad_norm_sq": 0.0070672343853252235, "grad_var_noise_scale": 1.1419636373942157, "grad_var_ot_time_s_mean": 0.0035106828436255455, "grad_var_time_s": 2.1067147254943848, "grad_var_trace": 0.008070524684983467, "grad_var_trace_biased": 0.007566116892172001, "step": 60000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.012369373259698151, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17535439133644104, "grad_var_mean_grad_norm_sq": 0.00887441074177351, "grad_var_noise_scale": 0.42007972445668, "grad_var_ot_time_s_mean": 0.004320792853832245, "grad_var_time_s": 2.4419009685516357, "grad_var_trace": 0.0037279600191196176, "grad_var_trace_biased": 0.0034949625179246414, "step": 65000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.009236356906462219, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17246809601783752, "grad_var_mean_grad_norm_sq": 0.0036275391613028074, "grad_var_noise_scale": 1.6492555041146344, "grad_var_ot_time_s_mean": 0.004370593465864658, "grad_var_time_s": 2.305939197540283, "grad_var_trace": 0.0059827389281700396, "grad_var_trace_biased": 0.005608817745159412, "step": 70000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.008969140181645, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17271515727043152, "grad_var_mean_grad_norm_sq": 0.005001232504423232, "grad_var_noise_scale": 0.8462783627755626, "grad_var_ot_time_s_mean": 0.004592684097588062, "grad_var_time_s": 2.3039560317993164, "grad_var_trace": 0.004232434855703219, "grad_var_trace_biased": 0.003967907677221768, "step": 75000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.007286797769772785, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1724115014076233, "grad_var_mean_grad_norm_sq": 0.003233739950333694, "grad_var_noise_scale": 1.3369231108463873, "grad_var_ot_time_s_mean": 0.004290437325835228, "grad_var_time_s": 2.303687572479248, "grad_var_trace": 0.004323261674068364, "grad_var_trace_biased": 0.004053057819439091, "step": 80000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0072775539758803565, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17429295182228088, "grad_var_mean_grad_norm_sq": 0.0031108012643800767, "grad_var_noise_scale": 1.4287432233270645, "grad_var_ot_time_s_mean": 0.004304477944970131, "grad_var_time_s": 2.309382915496826, "grad_var_trace": 0.004444536225600299, "grad_var_trace_biased": 0.00416675271150028, "step": 85000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.01110500624110023, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17376592755317688, "grad_var_mean_grad_norm_sq": 0.006772175759599312, "grad_var_noise_scale": 0.682452141083793, "grad_var_ot_time_s_mean": 0.0045967018231749535, "grad_var_time_s": 2.285616159439087, "grad_var_trace": 0.004621685846934313, "grad_var_trace_biased": 0.004332830481500918, "step": 90000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.003664818062237018, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17138010263442993, "grad_var_mean_grad_norm_sq": 0.0014076666741821783, "grad_var_noise_scale": 1.7103680803250327, "grad_var_ot_time_s_mean": 0.004526999779045582, "grad_var_time_s": 2.3100178241729736, "grad_var_trace": 0.0024076281472584955, "grad_var_trace_biased": 0.0022571513880548398, "step": 95000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.012225854652794462, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1694040298461914, "grad_var_mean_grad_norm_sq": 0.009017404854126313, "grad_var_noise_scale": 0.37952676044556727, "grad_var_ot_time_s_mean": 0.004446091130375862, "grad_var_time_s": 2.3008484840393066, "grad_var_trace": 0.0034223464519126928, "grad_var_trace_biased": 0.0032084497986681493, "step": 100000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004025729434471248, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17194005846977234, "grad_var_mean_grad_norm_sq": 0.0015580003321473412, "grad_var_noise_scale": 1.6895018065780705, "grad_var_ot_time_s_mean": 0.004589271731674671, "grad_var_time_s": 2.3198704719543457, "grad_var_trace": 0.0026322443758121667, "grad_var_trace_biased": 0.0024677291023239064, "step": 105000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0044297654971129545, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17618496716022491, "grad_var_mean_grad_norm_sq": 0.0016813152370890121, "grad_var_noise_scale": 1.7436826911975836, "grad_var_ot_time_s_mean": 0.004318242892622948, "grad_var_time_s": 2.319131851196289, "grad_var_trace": 0.002931680277358872, "grad_var_trace_biased": 0.0027484502600239424, "step": 110000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0035725402841362036, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17318463325500488, "grad_var_mean_grad_norm_sq": 0.0011888363337001983, "grad_var_noise_scale": 2.138744817142288, "grad_var_ot_time_s_mean": 0.00390950683504343, "grad_var_time_s": 2.3337981700897217, "grad_var_trace": 0.002542617547131739, "grad_var_trace_biased": 0.002383703950436005, "step": 115000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0031851219065165285, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1690099835395813, "grad_var_mean_grad_norm_sq": 0.000948412300983966, "grad_var_noise_scale": 2.5155974640559498, "grad_var_ot_time_s_mean": 0.004593398422002792, "grad_var_time_s": 2.2942917346954346, "grad_var_trace": 0.002385823579234733, "grad_var_trace_biased": 0.0022367096055325624, "step": 120000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.003149444412775918, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17065352201461792, "grad_var_mean_grad_norm_sq": 0.0010344538425638048, "grad_var_noise_scale": 2.180851236405397, "grad_var_ot_time_s_mean": 0.0048763807862997055, "grad_var_time_s": 2.480076551437378, "grad_var_trace": 0.0022559899415595876, "grad_var_trace_biased": 0.0021149905702121134, "step": 125000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.003352003111885865, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.173875629901886, "grad_var_mean_grad_norm_sq": 0.0013940227475489947, "grad_var_noise_scale": 1.498191039061643, "grad_var_ot_time_s_mean": 0.00024858000688254833, "grad_var_time_s": 1.2360552549362183, "grad_var_trace": 0.002088512388625995, "grad_var_trace_biased": 0.0019579803643368704, "step": 130000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0034765872668149622, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1752699911594391, "grad_var_mean_grad_norm_sq": 0.0012269564657469397, "grad_var_noise_scale": 1.9557386547900635, "grad_var_ot_time_s_mean": 0.00027765892446041107, "grad_var_time_s": 1.254072904586792, "grad_var_trace": 0.0023996061878058905, "grad_var_trace_biased": 0.0022496308010680225, "step": 135000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004323404279600145, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17221316695213318, "grad_var_mean_grad_norm_sq": 0.0021959260461651914, "grad_var_noise_scale": 1.0334182791023003, "grad_var_ot_time_s_mean": 0.000268435338512063, "grad_var_time_s": 1.2751078605651855, "grad_var_trace": 0.0022693101156639507, "grad_var_trace_biased": 0.002127478233434954, "step": 140000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004465502480316247, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17143282294273376, "grad_var_mean_grad_norm_sq": 0.00270313522165909, "grad_var_noise_scale": 0.6954363193419933, "grad_var_ot_time_s_mean": 0.00024870410561561584, "grad_var_time_s": 1.2388920783996582, "grad_var_trace": 0.001879858409234301, "grad_var_trace_biased": 0.0017623672586571573, "step": 145000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.002574031558286635, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1694883406162262, "grad_var_mean_grad_norm_sq": 0.0006831234763855925, "grad_var_noise_scale": 2.9525681526365184, "grad_var_ot_time_s_mean": 0.0002716090530157089, "grad_var_time_s": 1.2697691917419434, "grad_var_trace": 0.002016968620694445, "grad_var_trace_biased": 0.0018909080819010423, "step": 150000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004020444400270311, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17418694496154785, "grad_var_mean_grad_norm_sq": 0.0016722804931819939, "grad_var_noise_scale": 1.4977799345102354, "grad_var_ot_time_s_mean": 0.00024989363737404346, "grad_var_time_s": 1.2381705045700073, "grad_var_trace": 0.002504708167560871, "grad_var_trace_biased": 0.0023481639070883167, "step": 155000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0022232465381953437, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1707119643688202, "grad_var_mean_grad_norm_sq": 0.00063728098160099, "grad_var_noise_scale": 2.654553709496759, "grad_var_ot_time_s_mean": 0.00025799707509577274, "grad_var_time_s": 1.238373875617981, "grad_var_trace": 0.001691696593700644, "grad_var_trace_biased": 0.0015859655565943537, "step": 160000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0030040882470424744, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17022031545639038, "grad_var_mean_grad_norm_sq": 0.0011871759943098432, "grad_var_noise_scale": 1.6324788789001805, "grad_var_ot_time_s_mean": 0.0002481087576597929, "grad_var_time_s": 1.2364979982376099, "grad_var_trace": 0.0019380397362481398, "grad_var_trace_biased": 0.0018169122527326312, "step": 165000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0031038162825805173, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17294050753116608, "grad_var_mean_grad_norm_sq": 0.0013349104798145378, "grad_var_noise_scale": 1.413452725718247, "grad_var_ot_time_s_mean": 0.000280893174931407, "grad_var_time_s": 1.2913070917129517, "grad_var_trace": 0.0018868328562837114, "grad_var_trace_biased": 0.0017689058027659794, "step": 170000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0030000064631597358, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17016297578811646, "grad_var_mean_grad_norm_sq": 0.0013237012721759266, "grad_var_noise_scale": 1.3508024113654307, "grad_var_ot_time_s_mean": 0.0002561798319220543, "grad_var_time_s": 1.2397804260253906, "grad_var_trace": 0.0017880588703827298, "grad_var_trace_biased": 0.0016763051909838092, "step": 175000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.003162212771623573, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1700807809829712, "grad_var_mean_grad_norm_sq": 0.0014527883487287924, "grad_var_noise_scale": 1.255094076630812, "grad_var_ot_time_s_mean": 0.00026213284581899643, "grad_var_time_s": 1.2402234077453613, "grad_var_trace": 0.001823386051087766, "grad_var_trace_biased": 0.0017094244228947806, "step": 180000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004620431961932685, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17073087394237518, "grad_var_mean_grad_norm_sq": 0.002649169397585028, "grad_var_noise_scale": 0.7937129541637825, "grad_var_ot_time_s_mean": 0.00030241813510656357, "grad_var_time_s": 1.4259934425354004, "grad_var_trace": 0.0021026800686375006, "grad_var_trace_biased": 0.001971262564347657, "step": 185000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.003719618303664467, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17470653355121613, "grad_var_mean_grad_norm_sq": 0.002130535857343268, "grad_var_noise_scale": 0.7955844865195559, "grad_var_ot_time_s_mean": 0.00025584734976291656, "grad_var_time_s": 1.2374252080917358, "grad_var_trace": 0.0016950212760759456, "grad_var_trace_biased": 0.001589082446321199, "step": 190000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0020285916332315585, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16928191483020782, "grad_var_mean_grad_norm_sq": 0.00075626283743394, "grad_var_noise_scale": 1.7945489958522622, "grad_var_ot_time_s_mean": 0.00025819940492510796, "grad_var_time_s": 1.244761347770691, "grad_var_trace": 0.0013571507155174596, "grad_var_trace_biased": 0.0012723287957976184, "step": 195000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.002313576102463404, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17086412012577057, "grad_var_mean_grad_norm_sq": 0.0010248937890508449, "grad_var_noise_scale": 1.3412067497385034, "grad_var_ot_time_s_mean": 0.00025441404432058334, "grad_var_time_s": 1.2376923561096191, "grad_var_trace": 0.001374594467640063, "grad_var_trace_biased": 0.0012886823134125591, "step": 200000}

a/r034/c.json

@@ -0,0 +1,55 @@
+{
+  "archive_status": "complete",
+  "artifact_id": "r034",
+  "available_steps": [
+    25000,
+    50000,
+    75000,
+    100000,
+    125000,
+    150000,
+    175000,
+    200000
+  ],
+  "config": {
+    "amp": false,
+    "attention_resolutions": "16",
+    "batch_size": 256,
+    "channel_mult": "1,2,2,2",
+    "context_size": 128,
+    "cost_feature_dim": 0,
+    "coupling_mode": "official_otcfm_exact",
+    "data_dir": "<node-path-redacted>",
+    "dataset": "cifar100",
+    "dropout": 0.1,
+    "ema_decay": 0.9999,
+    "eps": 0.05,
+    "grad_clip": 1.0,
+    "grad_var_batches": 16,
+    "grad_var_every": 5000,
+    "integration_steps": 100,
+    "log_step": 20,
+    "lr": 0.0002,
+    "lr_schedule": "constant",
+    "min_lr_ratio": 0.0,
+    "no_flash_autotune": false,
+    "no_flash_tf32": false,
+    "num_channel": 128,
+    "num_head_channels": 64,
+    "num_heads": 4,
+    "num_res_blocks": 2,
+    "num_workers": 4,
+    "output_dir": "<node-path-redacted>",
+    "pot_max_context": 2048,
+    "pot_num_threads": 1,
+    "sample_batch": 64,
+    "sample_every": 0,
+    "save_step": 25000,
+    "seed": 0,
+    "sinkhorn_iters": 20,
+    "total_steps": 200000,
+    "val_batches": 8,
+    "val_every": 0,
+    "warmup": 0
+  }
+}

a/r034/p00025000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42bdcd7132fbbff85565d52f943361daacef1de4b4e1219b6b1a7891e5209ddf
+size 572463003

a/r034/p00050000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b07f2d6003da5d4010c331e3ed118e7fec3aee18945306e412f4e2cbb82000b5
+size 572463003

a/r034/p00075000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96e5442045ab45e73d470259fce5c40723cc20d88fbb93f2ce2eaba2c143d36c
+size 572463003

a/r034/p00100000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f7725d406fa7dc072346d0ab0758b0bd0a38009769b869b21273c249342c6a7
+size 572463003

a/r034/p00125000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a00bb2163ad727cc4ec4d8a9889ac17f508e25bb8c276a36c0c70805d9fca453
+size 572463003

a/r034/p00150000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bbf646f3ef41c761f907df64e753b8115bccbeb93c94ff52a42b832ae56d4a8
+size 572463003

a/r034/p00175000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f57382655081d5d42a382c0af63a6a4251b1e4bcc228fc61b3cde05bdcdc100
+size 572463003

a/r034/p00200000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee36febbc86bc3a8d3d8d44d72e3e72c9dd58dfe44fd33445e6500ea53f7d141
+size 572463003

a/r034/v.jsonl

@@ -0,0 +1,40 @@
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.11705985422101325, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.15813396871089935, "grad_var_mean_grad_norm_sq": 0.10068703632512445, "grad_var_noise_scale": 0.17345171460360265, "grad_var_ot_time_s_mean": 0.010218793526291847, "grad_var_time_s": 2.6324667930603027, "grad_var_trace": 0.01746433908894806, "grad_var_trace_biased": 0.016372817895888805, "step": 5000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.13088909975154306, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.15723425149917603, "grad_var_mean_grad_norm_sq": 0.11632474680737742, "grad_var_noise_scale": 0.13355120241813198, "grad_var_ot_time_s_mean": 0.010769672691822052, "grad_var_time_s": 2.487294912338257, "grad_var_trace": 0.015535309807110013, "grad_var_trace_biased": 0.014564352944165637, "step": 10000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.035546315285303456, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.15368637442588806, "grad_var_mean_grad_norm_sq": 0.02322651947555586, "grad_var_noise_scale": 0.5657806605172789, "grad_var_ot_time_s_mean": 0.010826794430613518, "grad_var_time_s": 2.515678882598877, "grad_var_trace": 0.013141115530397436, "grad_var_trace_biased": 0.012319795809747597, "step": 15000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.047210331309026, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.15295401215553284, "grad_var_mean_grad_norm_sq": 0.036791085262617255, "grad_var_noise_scale": 0.30208031022110815, "grad_var_ot_time_s_mean": 0.010581748560070992, "grad_var_time_s": 2.5210070610046387, "grad_var_trace": 0.011113862449502661, "grad_var_trace_biased": 0.010419246046408745, "step": 20000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.022787420095832865, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.15033267438411713, "grad_var_mean_grad_norm_sq": 0.01348209267385797, "grad_var_noise_scale": 0.7362123094351645, "grad_var_ot_time_s_mean": 0.0100928395986557, "grad_var_time_s": 2.4941763877868652, "grad_var_trace": 0.009925682583439888, "grad_var_trace_biased": 0.009305327421974895, "step": 25000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.012519792011867094, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1520509421825409, "grad_var_mean_grad_norm_sq": 0.004527967081221859, "grad_var_noise_scale": 1.8826579580730025, "grad_var_ot_time_s_mean": 0.010813782922923565, "grad_var_time_s": 2.4699292182922363, "grad_var_trace": 0.008524613259354918, "grad_var_trace_biased": 0.007991824930645236, "step": 30000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.021822304607500367, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14672864973545074, "grad_var_mean_grad_norm_sq": 0.015600522782296157, "grad_var_noise_scale": 0.42540672340475394, "grad_var_ot_time_s_mean": 0.008760718628764153, "grad_var_time_s": 2.23512864112854, "grad_var_trace": 0.006636567280217823, "grad_var_trace_biased": 0.00622178182520421, "step": 35000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.014442475179243366, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1509070247411728, "grad_var_mean_grad_norm_sq": 0.00815717404483355, "grad_var_noise_scale": 0.8218926276659998, "grad_var_ot_time_s_mean": 0.010113196447491646, "grad_var_time_s": 2.5035550594329834, "grad_var_trace": 0.006704321210037138, "grad_var_trace_biased": 0.006285301134409817, "step": 40000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0527757966024024, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14920055866241455, "grad_var_mean_grad_norm_sq": 0.04693020850506732, "grad_var_noise_scale": 0.13286312098565442, "grad_var_ot_time_s_mean": 0.008886098861694336, "grad_var_time_s": 2.2439541816711426, "grad_var_trace": 0.006235293970490748, "grad_var_trace_biased": 0.005845588097335076, "step": 45000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.024102637865436666, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1495736539363861, "grad_var_mean_grad_norm_sq": 0.019377981569532618, "grad_var_noise_scale": 0.2600700885287922, "grad_var_ot_time_s_mean": 0.010094910860061646, "grad_var_time_s": 2.4813220500946045, "grad_var_trace": 0.005039633382297652, "grad_var_trace_biased": 0.0047246562959040485, "step": 50000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.008704011532344933, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.147516131401062, "grad_var_mean_grad_norm_sq": 0.00400855347590706, "grad_var_noise_scale": 1.2494503624902757, "grad_var_ot_time_s_mean": 0.008636457845568657, "grad_var_time_s": 2.343820571899414, "grad_var_trace": 0.005008488593533731, "grad_var_trace_biased": 0.004695458056437873, "step": 55000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.013885466328950689, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14713290333747864, "grad_var_mean_grad_norm_sq": 0.007903956651528307, "grad_var_noise_scale": 0.807225706129961, "grad_var_ot_time_s_mean": 0.010444123297929764, "grad_var_time_s": 2.4893298149108887, "grad_var_trace": 0.00638027698925054, "grad_var_trace_biased": 0.0059815096774223815, "step": 60000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.007250881876452795, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14887765049934387, "grad_var_mean_grad_norm_sq": 0.004432265439402209, "grad_var_noise_scale": 0.6783267474896815, "grad_var_ot_time_s_mean": 0.011044296436011791, "grad_var_time_s": 2.6661391258239746, "grad_var_trace": 0.0030065241995206244, "grad_var_trace_biased": 0.0028186164370505853, "step": 65000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.013220267351744393, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1477774679660797, "grad_var_mean_grad_norm_sq": 0.008209914589407754, "grad_var_noise_scale": 0.6509661241446947, "grad_var_ot_time_s_mean": 0.010235151275992393, "grad_var_time_s": 2.510610342025757, "grad_var_trace": 0.005344376279825748, "grad_var_trace_biased": 0.005010352762336639, "step": 70000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.00818883230315403, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14785930514335632, "grad_var_mean_grad_norm_sq": 0.0037910380828570173, "grad_var_noise_scale": 1.237386805176832, "grad_var_ot_time_s_mean": 0.010291654616594315, "grad_var_time_s": 2.51298189163208, "grad_var_trace": 0.004690980501650147, "grad_var_trace_biased": 0.004397794220297012, "step": 75000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.014987335078836605, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14440587162971497, "grad_var_mean_grad_norm_sq": 0.01233009685548434, "grad_var_noise_scale": 0.22987552096816663, "grad_var_ot_time_s_mean": 0.010203458368778229, "grad_var_time_s": 2.5165436267852783, "grad_var_trace": 0.0028343874382424157, "grad_var_trace_biased": 0.0026572382233522648, "step": 80000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004276895442502587, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14857754111289978, "grad_var_mean_grad_norm_sq": 0.0012037242687495452, "grad_var_noise_scale": 2.7232559292075695, "grad_var_ot_time_s_mean": 0.009383506141602993, "grad_var_time_s": 2.462214231491089, "grad_var_trace": 0.0032780492520032446, "grad_var_trace_biased": 0.003073171173753042, "step": 85000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.007041800073916256, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1480880081653595, "grad_var_mean_grad_norm_sq": 0.004273622659479134, "grad_var_noise_scale": 0.6909179426148663, "grad_var_ot_time_s_mean": 0.009722718968987465, "grad_var_time_s": 2.4759583473205566, "grad_var_trace": 0.0029527225753995965, "grad_var_trace_biased": 0.002768177414437122, "step": 90000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.005854927047773739, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14817504584789276, "grad_var_mean_grad_norm_sq": 0.0026044540352641483, "grad_var_noise_scale": 1.33124684344522, "grad_var_ot_time_s_mean": 0.010369465686380863, "grad_var_time_s": 2.479964256286621, "grad_var_trace": 0.0034671712133435635, "grad_var_trace_biased": 0.003250473012509591, "step": 95000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.006672190926377713, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14564859867095947, "grad_var_mean_grad_norm_sq": 0.0038889254144039414, "grad_var_noise_scale": 0.7634027989092687, "grad_var_ot_time_s_mean": 0.010058568790555, "grad_var_time_s": 2.479537010192871, "grad_var_trace": 0.0029688165461053562, "grad_var_trace_biased": 0.0027832655119737716, "step": 100000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004609624604050093, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14784632623195648, "grad_var_mean_grad_norm_sq": 0.0017453339520906162, "grad_var_noise_scale": 1.7505207862543675, "grad_var_ot_time_s_mean": 0.010809745639562607, "grad_var_time_s": 2.5163729190826416, "grad_var_trace": 0.003055243362090108, "grad_var_trace_biased": 0.0028642906519594762, "step": 105000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.005483466176143129, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14486941695213318, "grad_var_mean_grad_norm_sq": 0.0031099099329421815, "grad_var_noise_scale": 0.8141050321948602, "grad_var_ot_time_s_mean": 0.01087122317403555, "grad_var_time_s": 2.48417592048645, "grad_var_trace": 0.0025317933260810103, "grad_var_trace_biased": 0.0023735562432009474, "step": 110000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.003659797246325293, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1513141393661499, "grad_var_mean_grad_norm_sq": 0.0015178707145065196, "grad_var_noise_scale": 1.5052149119846605, "grad_var_ot_time_s_mean": 0.01062791794538498, "grad_var_time_s": 2.4887454509735107, "grad_var_trace": 0.0022847216339400246, "grad_var_trace_biased": 0.002141926531818773, "step": 115000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.009297054778411569, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14639034867286682, "grad_var_mean_grad_norm_sq": 0.006753928896012909, "grad_var_noise_scale": 0.40164290291733606, "grad_var_ot_time_s_mean": 0.010321324691176414, "grad_var_time_s": 2.4590461254119873, "grad_var_trace": 0.0027126676078919037, "grad_var_trace_biased": 0.0025431258823986596, "step": 120000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.005050928692695316, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14515486359596252, "grad_var_mean_grad_norm_sq": 0.003065549990937992, "grad_var_noise_scale": 0.690818054879117, "grad_var_ot_time_s_mean": 0.010956180281937122, "grad_var_time_s": 2.6548829078674316, "grad_var_trace": 0.0021177372818744784, "grad_var_trace_biased": 0.0019853787017573236, "step": 125000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004418847994860128, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14559254050254822, "grad_var_mean_grad_norm_sq": 0.002190904838923133, "grad_var_noise_scale": 1.0846991879547838, "grad_var_ot_time_s_mean": 0.010296262800693512, "grad_var_time_s": 2.4753689765930176, "grad_var_trace": 0.0023764726996661286, "grad_var_trace_biased": 0.0022279431559369955, "step": 130000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.003196446327069868, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14851996302604675, "grad_var_mean_grad_norm_sq": 0.0010034713515001685, "grad_var_noise_scale": 2.3310813046902945, "grad_var_ot_time_s_mean": 0.00996012706309557, "grad_var_time_s": 2.498211622238159, "grad_var_trace": 0.002339173307274346, "grad_var_trace_biased": 0.0021929749755696996, "step": 135000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.003300303243448862, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1462240219116211, "grad_var_mean_grad_norm_sq": 0.0010274422554562327, "grad_var_noise_scale": 2.3596314449635556, "grad_var_ot_time_s_mean": 0.010915648192167282, "grad_var_time_s": 2.4963786602020264, "grad_var_trace": 0.002424385053858805, "grad_var_trace_biased": 0.0022728609879926294, "step": 140000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0054512922391341595, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14519979059696198, "grad_var_mean_grad_norm_sq": 0.003717662590307821, "grad_var_noise_scale": 0.4974106481769184, "grad_var_ot_time_s_mean": 0.010575643740594387, "grad_var_time_s": 2.4950528144836426, "grad_var_trace": 0.0018492049587480945, "grad_var_trace_biased": 0.0017336296488263386, "step": 145000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.006283210046309933, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14376729726791382, "grad_var_mean_grad_norm_sq": 0.004347467995407368, "grad_var_noise_scale": 0.4749411664775832, "grad_var_ot_time_s_mean": 0.009203031659126282, "grad_var_time_s": 2.310805320739746, "grad_var_trace": 0.002064791520962736, "grad_var_trace_biased": 0.001935742050902565, "step": 150000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004736984675565464, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14720582962036133, "grad_var_mean_grad_norm_sq": 0.003281953223779821, "grad_var_noise_scale": 0.4728993507055162, "grad_var_ot_time_s_mean": 0.01004410907626152, "grad_var_time_s": 2.453282356262207, "grad_var_trace": 0.001552033548571353, "grad_var_trace_biased": 0.0014550314517856435, "step": 155000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.003241053666055432, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1465429663658142, "grad_var_mean_grad_norm_sq": 0.0009476045728401275, "grad_var_noise_scale": 2.5816102724130543, "grad_var_ot_time_s_mean": 0.009870784357190132, "grad_var_time_s": 2.493056058883667, "grad_var_trace": 0.002446345699429658, "grad_var_trace_biased": 0.0022934490932153043, "step": 160000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.002173818084370049, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14298605918884277, "grad_var_mean_grad_norm_sq": 0.0006993828444183765, "grad_var_noise_scale": 2.248741065307425, "grad_var_ot_time_s_mean": 0.002396529773250222, "grad_var_time_s": 1.2916779518127441, "grad_var_trace": 0.0015727309226151172, "grad_var_trace_biased": 0.0014744352399516723, "step": 165000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.002567546603866629, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14645740389823914, "grad_var_mean_grad_norm_sq": 0.0009108556466925241, "grad_var_noise_scale": 1.940084608798875, "grad_var_ot_time_s_mean": 0.002468178980052471, "grad_var_time_s": 1.3228745460510254, "grad_var_trace": 0.0017671370209857119, "grad_var_trace_biased": 0.001656690957174105, "step": 170000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0025461541286867814, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14883287250995636, "grad_var_mean_grad_norm_sq": 0.0011020654411535748, "grad_var_noise_scale": 1.3977039921420011, "grad_var_ot_time_s_mean": 0.002315242774784565, "grad_var_time_s": 1.2879979610443115, "grad_var_trace": 0.0015403612667020871, "grad_var_trace_biased": 0.0014440886875332066, "step": 175000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.002942856112624935, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14319342374801636, "grad_var_mean_grad_norm_sq": 0.001556508115158714, "grad_var_noise_scale": 0.9500568502635243, "grad_var_ot_time_s_mean": 0.0023723056074231863, "grad_var_time_s": 1.285091519355774, "grad_var_trace": 0.0014787711972973027, "grad_var_trace_biased": 0.0013863479974662212, "step": 180000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0033957293298395672, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1441018283367157, "grad_var_mean_grad_norm_sq": 0.0015317902886932686, "grad_var_noise_scale": 1.2979594260161222, "grad_var_ot_time_s_mean": 0.002488615456968546, "grad_var_time_s": 1.4972141981124878, "grad_var_trace": 0.001988201643889385, "grad_var_trace_biased": 0.0018639390411462986, "step": 185000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0027853192236332214, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14680635929107666, "grad_var_mean_grad_norm_sq": 0.0008240767370614659, "grad_var_noise_scale": 2.5385888129014162, "grad_var_ot_time_s_mean": 0.002335695782676339, "grad_var_time_s": 1.2799072265625, "grad_var_trace": 0.002091991985676539, "grad_var_trace_biased": 0.0019612424865717553, "step": 190000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0022446583835466797, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.144911527633667, "grad_var_mean_grad_norm_sq": 0.0009675136938160644, "grad_var_noise_scale": 1.4080293413448848, "grad_var_ot_time_s_mean": 0.002363398903980851, "grad_var_time_s": 1.280186414718628, "grad_var_trace": 0.0013622876690459896, "grad_var_trace_biased": 0.0012771446897306153, "step": 195000}
+{"grad_var_batches": 16, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.002299719935983759, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.14275899529457092, "grad_var_mean_grad_norm_sq": 0.0005785007008509753, "grad_var_noise_scale": 3.173664580597613, "grad_var_ot_time_s_mean": 0.0023260065354406834, "grad_var_time_s": 1.279867172241211, "grad_var_trace": 0.0018359671841416357, "grad_var_trace_biased": 0.0017212192351327834, "step": 200000}

a/r035/c.json

@@ -0,0 +1,55 @@
+{
+  "archive_status": "complete",
+  "artifact_id": "r035",
+  "available_steps": [
+    25000,
+    50000,
+    75000,
+    100000,
+    125000,
+    150000,
+    175000,
+    200000
+  ],
+  "config": {
+    "amp": false,
+    "attention_resolutions": "16",
+    "batch_size": 256,
+    "channel_mult": "1,2,2,2",
+    "context_size": 2048,
+    "cost_feature_dim": 0,
+    "coupling_mode": "flash_global_entropic",
+    "data_dir": "<node-path-redacted>",
+    "dataset": "cifar100",
+    "dropout": 0.1,
+    "ema_decay": 0.9999,
+    "eps": 0.02,
+    "grad_clip": 1.0,
+    "grad_var_batches": 128,
+    "grad_var_every": 10000,
+    "integration_steps": 100,
+    "log_step": 20,
+    "lr": 0.0002,
+    "lr_schedule": "constant",
+    "min_lr_ratio": 0.0,
+    "no_flash_autotune": false,
+    "no_flash_tf32": false,
+    "num_channel": 128,
+    "num_head_channels": 64,
+    "num_heads": 4,
+    "num_res_blocks": 2,
+    "num_workers": 4,
+    "output_dir": "<node-path-redacted>",
+    "pot_max_context": 2048,
+    "pot_num_threads": 1,
+    "sample_batch": 64,
+    "sample_every": 0,
+    "save_step": 25000,
+    "seed": 0,
+    "sinkhorn_iters": 30,
+    "total_steps": 200000,
+    "val_batches": 8,
+    "val_every": 0,
+    "warmup": 0
+  }
+}

a/r035/p00025000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb9c835de5d9eedf406cd2e4d27f532bcb5dbd4e203aae3e69f3a6eda51b6864
+size 572463067

a/r035/p00050000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb17c364abd276aead8119bfc4b0fae66220d33e373eb174152cadfc8521b67e
+size 572463067

a/r035/p00075000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c028a7f425bd8f295821266f0734c6293449f4c1ae866b805b23e76a8e1cae6
+size 572463067

a/r035/p00100000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aed5499bd79230c9d9eecdd57179daa98aed15acfdf1f9e137711c210ff92349
+size 572463067

a/r035/p00125000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3586fc98d2d00774efe649694666ad4b99da867f5d85c93f47518da2bf4e40a6
+size 572463067

a/r035/p00150000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8cb51c3c27022b36a1490226e1dfcf6e7bd36783f6155fcfe3ade2567109b06
+size 572463067

a/r035/p00175000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e39219d6ac44c1d48aef8e34af217b8265a2c7617d4eaa0e284661dbc12dee31
+size 572463067

a/r035/p00200000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09389bbaa8eb815048426fa6cc2b1434e626c00ecf0bd8255c377b32fe6264f0
+size 572463067

a/r035/v.jsonl

@@ -0,0 +1,20 @@
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.26691284754808076, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17726220190525055, "grad_var_mean_grad_norm_sq": 0.2524021815429067, "grad_var_noise_scale": 0.05794293507454118, "grad_var_ot_time_s_mean": 0.020911701023578644, "grad_var_time_s": 12.73222541809082, "grad_var_trace": 0.0146249232178132, "grad_var_trace_biased": 0.014510666005174033, "step": 10000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.036140507823682166, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17234188318252563, "grad_var_mean_grad_norm_sq": 0.025179931935938475, "grad_var_noise_scale": 0.4387176130180256, "grad_var_ot_time_s_mean": 0.019934814423322678, "grad_var_time_s": 12.619917869567871, "grad_var_trace": 0.01104687963489128, "grad_var_trace_biased": 0.010960575887743691, "step": 20000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.02459179030552215, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17198723554611206, "grad_var_mean_grad_norm_sq": 0.01567138410391643, "grad_var_noise_scale": 0.5736982490442023, "grad_var_ot_time_s_mean": 0.02003622055053711, "grad_var_time_s": 12.424349784851074, "grad_var_trace": 0.008990645620516002, "grad_var_trace_biased": 0.00892040620160572, "step": 30000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.012863268559155174, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17130538821220398, "grad_var_mean_grad_norm_sq": 0.0064080770207685924, "grad_var_noise_scale": 1.0152842728216862, "grad_var_ot_time_s_mean": 0.020172007381916046, "grad_var_time_s": 12.681075096130371, "grad_var_trace": 0.0065060198182163975, "grad_var_trace_biased": 0.006455191538386582, "step": 40000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.023454509769585973, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17234016954898834, "grad_var_mean_grad_norm_sq": 0.01773451693620627, "grad_var_noise_scale": 0.32507410084898564, "grad_var_ot_time_s_mean": 0.019897010177373886, "grad_var_time_s": 12.609047889709473, "grad_var_trace": 0.0057650321470283615, "grad_var_trace_biased": 0.005719992833379702, "step": 50000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.04556175478319542, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17146876454353333, "grad_var_mean_grad_norm_sq": 0.039728667538976206, "grad_var_noise_scale": 0.14797921574571232, "grad_var_ot_time_s_mean": 0.01998908817768097, "grad_var_time_s": 12.425143241882324, "grad_var_trace": 0.005879017065039838, "grad_var_trace_biased": 0.005833087244219215, "step": 60000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.01545355050002629, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16985267400741577, "grad_var_mean_grad_norm_sq": 0.010967805406672146, "grad_var_noise_scale": 0.4122124484548684, "grad_var_ot_time_s_mean": 0.020291034132242203, "grad_var_time_s": 12.712332725524902, "grad_var_trace": 0.004521065920860869, "grad_var_trace_biased": 0.004485745093354143, "step": 70000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0061524862812476, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16846907138824463, "grad_var_mean_grad_norm_sq": 0.0026677452711165264, "grad_var_noise_scale": 1.3165349607208605, "grad_var_ot_time_s_mean": 0.01992710679769516, "grad_var_time_s": 12.621126174926758, "grad_var_trace": 0.0035121799157226572, "grad_var_trace_biased": 0.003484741010131074, "step": 80000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0061781209291608095, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16883134841918945, "grad_var_mean_grad_norm_sq": 0.0025555978389916207, "grad_var_noise_scale": 1.428646885798513, "grad_var_ot_time_s_mean": 0.020064229145646095, "grad_var_time_s": 12.444055557250977, "grad_var_trace": 0.0036510468940287887, "grad_var_trace_biased": 0.003622523090169189, "step": 90000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.011027462377410577, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1678621470928192, "grad_var_mean_grad_norm_sq": 0.008051558899463775, "grad_var_noise_scale": 0.37251615820590095, "grad_var_ot_time_s_mean": 0.02028479427099228, "grad_var_time_s": 12.684062004089355, "grad_var_trace": 0.002999335788796777, "grad_var_trace_biased": 0.002975903477946802, "step": 100000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.008203957887293502, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16873715817928314, "grad_var_mean_grad_norm_sq": 0.005478599062471015, "grad_var_noise_scale": 0.5013723968129321, "grad_var_ot_time_s_mean": 0.019906945526599884, "grad_var_time_s": 12.91117000579834, "grad_var_trace": 0.002746818343128176, "grad_var_trace_biased": 0.002725358824822487, "step": 110000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004846236146147274, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16873851418495178, "grad_var_mean_grad_norm_sq": 0.0024587705427688822, "grad_var_noise_scale": 0.9786454259483371, "grad_var_ot_time_s_mean": 0.0200272798538208, "grad_var_time_s": 12.443248748779297, "grad_var_trace": 0.002406264545137277, "grad_var_trace_biased": 0.002387465603378392, "step": 120000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0031932472484188642, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16673392057418823, "grad_var_mean_grad_norm_sq": 0.0007825149141630015, "grad_var_noise_scale": 3.10500724605226, "grad_var_ot_time_s_mean": 0.020324118435382843, "grad_var_time_s": 12.678802490234375, "grad_var_trace": 0.002429714478620082, "grad_var_trace_biased": 0.0024107323342558626, "step": 130000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.005898292995241835, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1671137660741806, "grad_var_mean_grad_norm_sq": 0.003737201830007, "grad_var_noise_scale": 0.5828177685278363, "grad_var_ot_time_s_mean": 0.019983576610684395, "grad_var_time_s": 12.623482704162598, "grad_var_trace": 0.002178107631102826, "grad_var_trace_biased": 0.0021610911652348353, "step": 140000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004731988524648098, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16642668843269348, "grad_var_mean_grad_norm_sq": 0.002560300397392841, "grad_var_noise_scale": 0.8548950099753627, "grad_var_ot_time_s_mean": 0.020040161907672882, "grad_var_time_s": 12.435620307922363, "grad_var_trace": 0.002188788033769078, "grad_var_trace_biased": 0.002171688127255257, "step": 150000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0034783783132874826, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1666087508201599, "grad_var_mean_grad_norm_sq": 0.0014404283899824158, "grad_var_noise_scale": 1.4259624340783403, "grad_var_ot_time_s_mean": 0.020220525562763214, "grad_var_time_s": 12.65867805480957, "grad_var_trace": 0.0020539967730948704, "grad_var_trace_biased": 0.002037949923305067, "step": 160000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.00351201022740438, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16602054238319397, "grad_var_mean_grad_norm_sq": 0.0017002471877683551, "grad_var_noise_scale": 1.0739784799988752, "grad_var_ot_time_s_mean": 0.020004242658615112, "grad_var_time_s": 12.673449516296387, "grad_var_trace": 0.0018260288903418202, "grad_var_trace_biased": 0.0018117630396360248, "step": 170000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.002905508661056089, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1666477620601654, "grad_var_mean_grad_norm_sq": 0.0009765728032127012, "grad_var_noise_scale": 1.990762309547496, "grad_var_ot_time_s_mean": 0.020004257559776306, "grad_var_time_s": 12.427947044372559, "grad_var_trace": 0.0019441243291649893, "grad_var_trace_biased": 0.0019289358578433878, "step": 180000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0021516634717102393, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16669709980487823, "grad_var_mean_grad_norm_sq": 0.00047332231906997875, "grad_var_noise_scale": 3.573793943692374, "grad_var_ot_time_s_mean": 0.020431865006685257, "grad_var_time_s": 12.724798202514648, "grad_var_trace": 0.0016915564373067194, "grad_var_trace_biased": 0.0016783411526402605, "step": 190000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0028937055662004983, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16681362688541412, "grad_var_mean_grad_norm_sq": 0.0011664252288861154, "grad_var_noise_scale": 1.4924925548413643, "grad_var_ot_time_s_mean": 0.019992269575595856, "grad_var_time_s": 12.615896224975586, "grad_var_trace": 0.0017408809698916615, "grad_var_trace_biased": 0.001727280337314383, "step": 200000}

a/r036/c.json

@@ -0,0 +1,55 @@
+{
+  "archive_status": "complete",
+  "artifact_id": "r036",
+  "available_steps": [
+    25000,
+    50000,
+    75000,
+    100000,
+    125000,
+    150000,
+    175000,
+    200000
+  ],
+  "config": {
+    "amp": false,
+    "attention_resolutions": "16",
+    "batch_size": 256,
+    "channel_mult": "1,2,2,2",
+    "context_size": 4096,
+    "cost_feature_dim": 0,
+    "coupling_mode": "flash_global_entropic",
+    "data_dir": "<node-path-redacted>",
+    "dataset": "cifar100",
+    "dropout": 0.1,
+    "ema_decay": 0.9999,
+    "eps": 0.01,
+    "grad_clip": 1.0,
+    "grad_var_batches": 128,
+    "grad_var_every": 10000,
+    "integration_steps": 100,
+    "log_step": 20,
+    "lr": 0.0002,
+    "lr_schedule": "constant",
+    "min_lr_ratio": 0.0,
+    "no_flash_autotune": false,
+    "no_flash_tf32": false,
+    "num_channel": 128,
+    "num_head_channels": 64,
+    "num_heads": 4,
+    "num_res_blocks": 2,
+    "num_workers": 4,
+    "output_dir": "<node-path-redacted>",
+    "pot_max_context": 2048,
+    "pot_num_threads": 1,
+    "sample_batch": 64,
+    "sample_every": 0,
+    "save_step": 25000,
+    "seed": 0,
+    "sinkhorn_iters": 30,
+    "total_steps": 200000,
+    "val_batches": 8,
+    "val_every": 0,
+    "warmup": 0
+  }
+}

a/r036/p00025000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7abe93eb7d424428d3e49f5c964af4f24b96af0de38504aceb1388296118a01
+size 572463067

a/r036/p00050000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3558be4edbd1f092d1c7607dd8bc7855dd4a5ad0943a3ec316ff3574099e7e89
+size 572463067

a/r036/p00075000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fe3cf04af4d321246ccb5c02e1f2a19b1da079d055b86e1ed149461b8c44c77
+size 572463067

a/r036/p00100000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa6681e942beae81eba3e76728e70f2b2b62f157e3224c9f0e0088088c56aebd
+size 572463067

a/r036/p00125000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db8fef075eba9e1ea1579a12c2156f9ed1fd3c25ed0949c35c25f16053af8e36
+size 572463067

a/r036/p00150000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f22f0139de4a5549ead9eaba94356af01e664ea46081fe011780fa2db4ccf43
+size 572463067

a/r036/p00175000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4873a285da6646f579ea45a17d79de56233b2e94d6f7fd300c037232c7b3417
+size 572463067

a/r036/p00200000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f643d89356159e5a817246fba88d25bb8f197db416e8bd4813adba8cc4cbd29
+size 572463067

a/r036/p00200000/q/e000.json

@@ -0,0 +1,20 @@
+{
+  "artifact_id": "r036",
+  "result": {
+    "checkpoint": "<node-path-redacted>",
+    "dataset_name": "cifar100",
+    "dataset_res": 32,
+    "dataset_split": "test",
+    "elapsed_s": 180.05523680895567,
+    "fid": 13.792430159056607,
+    "fid_mode": "clean",
+    "integration_method": "euler",
+    "integration_steps": 100,
+    "num_gen": 10000,
+    "run": "flash_global_entropic_ctx4096_eps0.01_it30_bs256_seed0",
+    "seed": 1234,
+    "step": 200000
+  },
+  "schema_version": 1,
+  "step": 200000
+}

a/r036/p00200000/q/e001.json

@@ -0,0 +1,20 @@
+{
+  "artifact_id": "r036",
+  "result": {
+    "checkpoint": "<node-path-redacted>",
+    "dataset_name": "cifar100_train",
+    "dataset_res": 32,
+    "dataset_split": "custom",
+    "elapsed_s": 850.586355432868,
+    "fid": 8.881779055492245,
+    "fid_mode": "clean",
+    "integration_method": "euler",
+    "integration_steps": 100,
+    "num_gen": 50000,
+    "run": "flash_global_entropic_ctx4096_eps0.01_it30_bs256_seed0",
+    "seed": 1234,
+    "step": 200000
+  },
+  "schema_version": 1,
+  "step": 200000
+}

a/r036/v.jsonl

@@ -0,0 +1,20 @@
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.23435635644012862, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17295300960540771, "grad_var_mean_grad_norm_sq": 0.22099546010041524, "grad_var_noise_scale": 0.0609338320424384, "grad_var_ot_time_s_mean": 0.04296017438173294, "grad_var_time_s": 15.490266799926758, "grad_var_trace": 0.0134661002479001, "grad_var_trace_biased": 0.01336089633971338, "step": 10000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.039023610462815946, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16904380917549133, "grad_var_mean_grad_norm_sq": 0.02714372341532763, "grad_var_noise_scale": 0.44111227048621726, "grad_var_ot_time_s_mean": 0.04284455627202988, "grad_var_time_s": 15.484089851379395, "grad_var_trace": 0.011973429465185072, "grad_var_trace_biased": 0.011879887047488314, "step": 20000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.05394362196019182, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16694952547550201, "grad_var_mean_grad_norm_sq": 0.045729414183348686, "grad_var_noise_scale": 0.18104073113733793, "grad_var_ot_time_s_mean": 0.04279389977455139, "grad_var_time_s": 15.275511741638184, "grad_var_trace": 0.008278886578235597, "grad_var_trace_biased": 0.008214207776843133, "step": 30000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0347591430863244, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1660780906677246, "grad_var_mean_grad_norm_sq": 0.028618973177936137, "grad_var_noise_scale": 0.21623828585553695, "grad_var_ot_time_s_mean": 0.042834170162677765, "grad_var_time_s": 15.514619827270508, "grad_var_trace": 0.0061885177029424995, "grad_var_trace_biased": 0.006140169908388261, "step": 40000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.00828321793060033, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16705648601055145, "grad_var_mean_grad_norm_sq": 0.002341553882395823, "grad_var_noise_scale": 2.5574678633328487, "grad_var_ot_time_s_mean": 0.04287228733301163, "grad_var_time_s": 15.494446754455566, "grad_var_trace": 0.005988448804489582, "grad_var_trace_biased": 0.005941664048204507, "step": 50000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.014509369813514212, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1663496494293213, "grad_var_mean_grad_norm_sq": 0.01014745155001147, "grad_var_noise_scale": 0.43323824262025856, "grad_var_ot_time_s_mean": 0.04275985807180405, "grad_var_time_s": 15.27303695678711, "grad_var_trace": 0.0043962640766011885, "grad_var_trace_biased": 0.0043619182635027415, "step": 60000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.013883104409902942, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16509675979614258, "grad_var_mean_grad_norm_sq": 0.009960257727289324, "grad_var_noise_scale": 0.3969510977951005, "grad_var_ot_time_s_mean": 0.0428246334195137, "grad_var_time_s": 15.484768867492676, "grad_var_trace": 0.00395373523916963, "grad_var_trace_biased": 0.003922846682613617, "step": 70000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.006236590847323863, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1637563407421112, "grad_var_mean_grad_norm_sq": 0.0032475171038806932, "grad_var_noise_scale": 0.9276655551932531, "grad_var_ot_time_s_mean": 0.042854368686676025, "grad_var_time_s": 15.482183456420898, "grad_var_trace": 0.0030126097571710688, "grad_var_trace_biased": 0.00298907374344317, "step": 80000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0065078280857790265, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1648389995098114, "grad_var_mean_grad_norm_sq": 0.003477410127977782, "grad_var_noise_scale": 0.878320187760034, "grad_var_ot_time_s_mean": 0.042810387909412384, "grad_var_time_s": 15.285748481750488, "grad_var_trace": 0.003054279516524089, "grad_var_trace_biased": 0.0030304179578012447, "step": 90000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0054075750912947, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16196762025356293, "grad_var_mean_grad_norm_sq": 0.002592340186978947, "grad_var_noise_scale": 1.0945330873388903, "grad_var_ot_time_s_mean": 0.04287852719426155, "grad_var_time_s": 15.523757934570312, "grad_var_trace": 0.002837402108286743, "grad_var_trace_biased": 0.0028152349043157527, "step": 100000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0057889806363197365, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16471520066261292, "grad_var_mean_grad_norm_sq": 0.0035176132070755804, "grad_var_noise_scale": 0.6507970255361857, "grad_var_ot_time_s_mean": 0.04285100847482681, "grad_var_time_s": 15.495355606079102, "grad_var_trace": 0.0022892522121515904, "grad_var_trace_biased": 0.002271367429244156, "step": 110000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.00546197044742529, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16464656591415405, "grad_var_mean_grad_norm_sq": 0.0031884771255918554, "grad_var_noise_scale": 0.7186486695046495, "grad_var_ot_time_s_mean": 0.042795293033123016, "grad_var_time_s": 15.283310890197754, "grad_var_trace": 0.002291394844052596, "grad_var_trace_biased": 0.002273493321833435, "step": 120000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.003692692788142056, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16321396827697754, "grad_var_mean_grad_norm_sq": 0.0015916163221798203, "grad_var_noise_scale": 1.3304842037827458, "grad_var_ot_time_s_mean": 0.04282323271036148, "grad_var_time_s": 15.502002716064453, "grad_var_trace": 0.0021176203751430405, "grad_var_trace_biased": 0.0021010764659622357, "step": 130000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0037011386381066583, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16163179278373718, "grad_var_mean_grad_norm_sq": 0.001736649059729031, "grad_var_noise_scale": 1.140102537793991, "grad_var_ot_time_s_mean": 0.04286385327577591, "grad_var_time_s": 15.504843711853027, "grad_var_trace": 0.0019799580002546166, "grad_var_trace_biased": 0.0019644895783776273, "step": 140000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.002998403129343426, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16247470676898956, "grad_var_mean_grad_norm_sq": 0.0013168661660894873, "grad_var_noise_scale": 1.2869777168140328, "grad_var_ot_time_s_mean": 0.04278390854597092, "grad_var_time_s": 15.285304069519043, "grad_var_trace": 0.0016947774117834972, "grad_var_trace_biased": 0.0016815369632539386, "step": 150000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.002837801871144907, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16239656507968903, "grad_var_mean_grad_norm_sq": 0.00109392897465822, "grad_var_noise_scale": 1.6066894833691763, "grad_var_ot_time_s_mean": 0.04281488060951233, "grad_var_time_s": 15.488615989685059, "grad_var_trace": 0.0017576041791361884, "grad_var_trace_biased": 0.001743872896486687, "step": 160000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0028659767654860093, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.162322998046875, "grad_var_mean_grad_norm_sq": 0.001164481548179823, "grad_var_noise_scale": 1.4726663725354594, "grad_var_ot_time_s_mean": 0.04282078146934509, "grad_var_time_s": 15.491320610046387, "grad_var_trace": 0.0017148928174424555, "grad_var_trace_biased": 0.0017014952173061864, "step": 170000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0025145946819838867, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16241022944450378, "grad_var_mean_grad_norm_sq": 0.0010267087543690725, "grad_var_noise_scale": 1.4605909986241958, "grad_var_ot_time_s_mean": 0.04271450638771057, "grad_var_time_s": 15.256288528442383, "grad_var_trace": 0.0014996015648401277, "grad_var_trace_biased": 0.0014878859276148142, "step": 180000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0022826955878025865, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1615026295185089, "grad_var_mean_grad_norm_sq": 0.0008421151107533399, "grad_var_noise_scale": 1.7241391489970725, "grad_var_ot_time_s_mean": 0.04285664111375809, "grad_var_time_s": 15.491759300231934, "grad_var_trace": 0.001451923630411839, "grad_var_trace_biased": 0.0014405804770492466, "step": 190000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0025702122587324665, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16133365035057068, "grad_var_mean_grad_norm_sq": 0.0011269505743077167, "grad_var_noise_scale": 1.290762862914387, "grad_var_ot_time_s_mean": 0.04290389642119408, "grad_var_time_s": 15.494614601135254, "grad_var_trace": 0.001454625949656441, "grad_var_trace_biased": 0.0014432616844247499, "step": 200000}

a/r037/c.json

@@ -0,0 +1,55 @@
+{
+  "archive_status": "complete",
+  "artifact_id": "r037",
+  "available_steps": [
+    25000,
+    50000,
+    75000,
+    100000,
+    125000,
+    150000,
+    175000,
+    200000
+  ],
+  "config": {
+    "amp": false,
+    "attention_resolutions": "16",
+    "batch_size": 256,
+    "channel_mult": "1,2,2,2",
+    "context_size": 4096,
+    "cost_feature_dim": 0,
+    "coupling_mode": "flash_global_entropic",
+    "data_dir": "<node-path-redacted>",
+    "dataset": "cifar100",
+    "dropout": 0.1,
+    "ema_decay": 0.9999,
+    "eps": 0.02,
+    "grad_clip": 1.0,
+    "grad_var_batches": 128,
+    "grad_var_every": 10000,
+    "integration_steps": 100,
+    "log_step": 20,
+    "lr": 0.0002,
+    "lr_schedule": "constant",
+    "min_lr_ratio": 0.0,
+    "no_flash_autotune": false,
+    "no_flash_tf32": false,
+    "num_channel": 128,
+    "num_head_channels": 64,
+    "num_heads": 4,
+    "num_res_blocks": 2,
+    "num_workers": 4,
+    "output_dir": "<node-path-redacted>",
+    "pot_max_context": 2048,
+    "pot_num_threads": 1,
+    "sample_batch": 64,
+    "sample_every": 0,
+    "save_step": 25000,
+    "seed": 0,
+    "sinkhorn_iters": 30,
+    "total_steps": 200000,
+    "val_batches": 8,
+    "val_every": 0,
+    "warmup": 0
+  }
+}

a/r037/p00025000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e781df45ec78a0954e81deb92bb6a1970a84f3be4310db16ebfc88888844d3b8
+size 572463067

a/r037/p00050000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71dfc04d5a73c97c12920d783d7e19c5c3531a97ac49bd1339b6ddda5f5c61ce
+size 572463067

a/r037/p00075000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b191b976182ed09e0cad392c88a3c044eef89f1c047a825f6455d2da1f6554a
+size 572463067

a/r037/p00100000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:689d4b6dbad9f5cd7c648c9f158fe137ed909d6b29b0286d938e66f014e1a4f1
+size 572463067

a/r037/p00125000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ae8e56854f6eee90f587d55a13c77b654e93a706cafb19370c4a58de97dc31a
+size 572463067

a/r037/p00150000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9788d1bda43214a83e1063bb334d235543c86fe84b6b3fc95f0e05b3838775c6
+size 572463067

a/r037/p00175000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd6da7adfeb6a6bacc62e06f6aea58a7c8f5fc4ad77a30a4a924e1511ba19e60
+size 572463067

a/r037/p00200000/m.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f244f043fbabd83f826a7f30d21b1c60edd5cca6c2146e079d99f44a901a36e1
+size 572463067

a/r037/p00200000/q/e000.json

@@ -0,0 +1,20 @@
+{
+  "artifact_id": "r037",
+  "result": {
+    "checkpoint": "<node-path-redacted>",
+    "dataset_name": "cifar100",
+    "dataset_res": 32,
+    "dataset_split": "test",
+    "elapsed_s": 181.67174997925758,
+    "fid": 13.760396504093137,
+    "fid_mode": "clean",
+    "integration_method": "euler",
+    "integration_steps": 100,
+    "num_gen": 10000,
+    "run": "flash_global_entropic_ctx4096_eps0.02_it30_bs256_seed0",
+    "seed": 1234,
+    "step": 200000
+  },
+  "schema_version": 1,
+  "step": 200000
+}

a/r037/p00200000/q/e001.json

@@ -0,0 +1,20 @@
+{
+  "artifact_id": "r037",
+  "result": {
+    "checkpoint": "<node-path-redacted>",
+    "dataset_name": "cifar100_train",
+    "dataset_res": 32,
+    "dataset_split": "custom",
+    "elapsed_s": 848.4761484637856,
+    "fid": 8.758304760409146,
+    "fid_mode": "clean",
+    "integration_method": "euler",
+    "integration_steps": 100,
+    "num_gen": 50000,
+    "run": "flash_global_entropic_ctx4096_eps0.02_it30_bs256_seed0",
+    "seed": 1234,
+    "step": 200000
+  },
+  "schema_version": 1,
+  "step": 200000
+}

a/r037/v.jsonl

@@ -0,0 +1,20 @@
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.24582122083968008, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17783299088478088, "grad_var_mean_grad_norm_sq": 0.23204942194256345, "grad_var_noise_scale": 0.05981587087058924, "grad_var_ot_time_s_mean": 0.04580262303352356, "grad_var_time_s": 16.137977600097656, "grad_var_trace": 0.013880238258511252, "grad_var_trace_biased": 0.013771798897116633, "step": 10000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.028521623436205408, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1740533858537674, "grad_var_mean_grad_norm_sq": 0.014695189852489061, "grad_var_noise_scale": 0.9482901057677243, "grad_var_ot_time_s_mean": 0.04313850402832031, "grad_var_time_s": 15.662483215332031, "grad_var_trace": 0.01393530313949364, "grad_var_trace_biased": 0.013826433583716347, "step": 20000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0285029377701706, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17130175232887268, "grad_var_mean_grad_norm_sq": 0.019906815188946575, "grad_var_noise_scale": 0.4352182156496457, "grad_var_ot_time_s_mean": 0.04306216165423393, "grad_var_time_s": 15.424135208129883, "grad_var_trace": 0.008663808585800592, "grad_var_trace_biased": 0.008596122581224025, "step": 30000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.01627987692343192, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17098385095596313, "grad_var_mean_grad_norm_sq": 0.009816665043708166, "grad_var_noise_scale": 0.6635759988594775, "grad_var_ot_time_s_mean": 0.04316183179616928, "grad_var_time_s": 15.633390426635742, "grad_var_trace": 0.006514103311847563, "grad_var_trace_biased": 0.006463211879723754, "step": 40000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.013806527240715448, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17187076807022095, "grad_var_mean_grad_norm_sq": 0.00758759634669893, "grad_var_noise_scale": 0.8260717317334421, "grad_var_ot_time_s_mean": 0.04299076646566391, "grad_var_time_s": 15.614068031311035, "grad_var_trace": 0.006267898853811924, "grad_var_trace_biased": 0.006218930894016518, "step": 50000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0174010841237301, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.170869380235672, "grad_var_mean_grad_norm_sq": 0.01268264071494143, "grad_var_noise_scale": 0.37496895271133057, "grad_var_ot_time_s_mean": 0.04283928498625755, "grad_var_time_s": 15.383203506469727, "grad_var_trace": 0.004755596506495668, "grad_var_trace_biased": 0.0047184434087886706, "step": 60000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.01062471305998173, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.17048482596874237, "grad_var_mean_grad_norm_sq": 0.006411391258127116, "grad_var_noise_scale": 0.6623363624378809, "grad_var_ot_time_s_mean": 0.04300960153341293, "grad_var_time_s": 15.646422386169434, "grad_var_trace": 0.004246497564073942, "grad_var_trace_biased": 0.004213321801854615, "step": 70000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.005337654115228253, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16822290420532227, "grad_var_mean_grad_norm_sq": 0.0019551814280195684, "grad_var_noise_scale": 1.7436265921716458, "grad_var_ot_time_s_mean": 0.043061282485723495, "grad_var_time_s": 15.60032844543457, "grad_var_trace": 0.003409106330415052, "grad_var_trace_biased": 0.0033824726872086846, "step": 80000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.006061874289332472, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1695079803466797, "grad_var_mean_grad_norm_sq": 0.002651294182478944, "grad_var_noise_scale": 1.2965121302045912, "grad_var_ot_time_s_mean": 0.04287668317556381, "grad_var_time_s": 15.37692642211914, "grad_var_trace": 0.003437435068324816, "grad_var_trace_biased": 0.003410580106853528, "step": 90000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.006543570312895881, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16683226823806763, "grad_var_mean_grad_norm_sq": 0.0036736931829992993, "grad_var_noise_scale": 0.7873478931223132, "grad_var_ot_time_s_mean": 0.04297080636024475, "grad_var_time_s": 15.597126960754395, "grad_var_trace": 0.002892474587612303, "grad_var_trace_biased": 0.002869877129896582, "step": 100000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0059672262667834555, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1693994700908661, "grad_var_mean_grad_norm_sq": 0.0036300186730070794, "grad_var_noise_scale": 0.6489252577934593, "grad_var_ot_time_s_mean": 0.04297948628664017, "grad_var_time_s": 15.59744930267334, "grad_var_trace": 0.0023556108031761902, "grad_var_trace_biased": 0.002337207593776376, "step": 110000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.005103109568396417, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16859713196754456, "grad_var_mean_grad_norm_sq": 0.002868237292793991, "grad_var_noise_scale": 0.7853149740275485, "grad_var_ot_time_s_mean": 0.04294613003730774, "grad_var_time_s": 15.392197608947754, "grad_var_trace": 0.002252469695095359, "grad_var_trace_biased": 0.0022348722756024265, "step": 120000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0038195257227681433, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16785484552383423, "grad_var_mean_grad_norm_sq": 0.001696449237509957, "grad_var_noise_scale": 1.2613366646196658, "grad_var_ot_time_s_mean": 0.043232180178165436, "grad_var_time_s": 15.661909103393555, "grad_var_trace": 0.0021397936229373845, "grad_var_trace_biased": 0.0021230764852581863, "step": 130000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0040827247215810695, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1662738025188446, "grad_var_mean_grad_norm_sq": 0.0020591985308448984, "grad_var_noise_scale": 0.9904142010978332, "grad_var_ot_time_s_mean": 0.043134719133377075, "grad_var_time_s": 15.643660545349121, "grad_var_trace": 0.002039459467828582, "grad_var_trace_biased": 0.002023526190736171, "step": 140000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0029493208712441663, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16746816039085388, "grad_var_mean_grad_norm_sq": 0.0012013578764145019, "grad_var_noise_scale": 1.4664460254223903, "grad_var_ot_time_s_mean": 0.04301213473081589, "grad_var_time_s": 15.420177459716797, "grad_var_trace": 0.0017617264829779295, "grad_var_trace_biased": 0.0017479629948296644, "step": 150000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.003182494365854648, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16702121496200562, "grad_var_mean_grad_norm_sq": 0.0014598870566439144, "grad_var_noise_scale": 1.1892503179542935, "grad_var_ot_time_s_mean": 0.04295826330780983, "grad_var_time_s": 15.602690696716309, "grad_var_trace": 0.001736171146291133, "grad_var_trace_biased": 0.0017226073092107335, "step": 160000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.004207449937491651, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16703733801841736, "grad_var_mean_grad_norm_sq": 0.0024455338381697916, "grad_var_noise_scale": 0.7261357118507957, "grad_var_ot_time_s_mean": 0.04295350983738899, "grad_var_time_s": 15.593636512756348, "grad_var_trace": 0.0017757894544346304, "grad_var_trace_biased": 0.0017619160993218599, "step": 170000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.00228694302886974, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16710735857486725, "grad_var_mean_grad_norm_sq": 0.0006698961305160921, "grad_var_noise_scale": 2.4328839604447303, "grad_var_ot_time_s_mean": 0.043039627373218536, "grad_var_time_s": 15.41994571685791, "grad_var_trace": 0.00162977955109659, "grad_var_trace_biased": 0.001617046898353648, "step": 180000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0025464116013788206, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.16623197495937347, "grad_var_mean_grad_norm_sq": 0.0010610617297576023, "grad_var_noise_scale": 1.4108939168352608, "grad_var_ot_time_s_mean": 0.042961522936820984, "grad_var_time_s": 15.590353965759277, "grad_var_trace": 0.0014970455399017004, "grad_var_trace_biased": 0.0014853498716212183, "step": 190000}
+{"grad_var_batches": 128, "grad_var_estimator": "ghost_batch_global_ddp", "grad_var_global_batch": 256, "grad_var_grad_norm_sq_mean": 0.0025214404755681346, "grad_var_local_batch": 128, "grad_var_loss_mean": 0.1661214530467987, "grad_var_mean_grad_norm_sq": 0.0009918962968119025, "grad_var_noise_scale": 1.5541824671207602, "grad_var_ot_time_s_mean": 0.04301982372999191, "grad_var_time_s": 15.593084335327148, "grad_var_trace": 0.0015415878337070685, "grad_var_trace_biased": 0.0015295441787562321, "step": 200000}

a/r038/c.json

@@ -0,0 +1,55 @@
+{
+  "archive_status": "complete",
+  "artifact_id": "r038",
+  "available_steps": [
+    25000,
+    50000,
+    75000,
+    100000,
+    125000,
+    150000,
+    175000,
+    200000
+  ],
+  "config": {
+    "amp": false,
+    "attention_resolutions": "16",
+    "batch_size": 256,
+    "channel_mult": "1,2,2,2",
+    "context_size": 2048,
+    "cost_feature_dim": 0,
+    "coupling_mode": "flash_global_entropic",
+    "data_dir": "<node-path-redacted>",
+    "dataset": "cifar100",
+    "dropout": 0.1,
+    "ema_decay": 0.9999,
+    "eps": 0.01,
+    "grad_clip": 1.0,
+    "grad_var_batches": 128,
+    "grad_var_every": 10000,
+    "integration_steps": 100,
+    "log_step": 20,
+    "lr": 0.0002,
+    "lr_schedule": "constant",
+    "min_lr_ratio": 0.0,
+    "no_flash_autotune": false,
+    "no_flash_tf32": false,
+    "num_channel": 128,
+    "num_head_channels": 64,
+    "num_heads": 4,
+    "num_res_blocks": 2,
+    "num_workers": 4,
+    "output_dir": "<node-path-redacted>",
+    "pot_max_context": 2048,
+    "pot_num_threads": 1,
+    "sample_batch": 64,
+    "sample_every": 0,
+    "save_step": 25000,
+    "seed": 0,
+    "sinkhorn_iters": 30,
+    "total_steps": 200000,
+    "val_batches": 8,
+    "val_every": 0,
+    "warmup": 0
+  }
+}