AbstractPhil commited on
Commit
eff0f8e
·
verified ·
1 Parent(s): c99b07c

agatha_v2_coco256_agatha_s0: step 50000

Browse files
.gitattributes CHANGED
@@ -61,3 +61,7 @@ coco256/agatha_v2_coco256_plain_s0/agatha_v2_coco256_plain_s0_raw_step020000_raw
61
  coco256/agatha_v2_coco256_plain_s0/agatha_v2_coco256_plain_s0_raw_step020000_raw_g4.5.png filter=lfs diff=lfs merge=lfs -text
62
  coco256/agatha_v2_coco256_plain_s0/agatha_v2_coco256_plain_s0_raw_step020000_raw_g6.png filter=lfs diff=lfs merge=lfs -text
63
  coco256/agatha_v2_coco256_plain_s0/agatha_v2_coco256_plain_s0_raw_step020000_raw_g7.5.png filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
61
  coco256/agatha_v2_coco256_plain_s0/agatha_v2_coco256_plain_s0_raw_step020000_raw_g4.5.png filter=lfs diff=lfs merge=lfs -text
62
  coco256/agatha_v2_coco256_plain_s0/agatha_v2_coco256_plain_s0_raw_step020000_raw_g6.png filter=lfs diff=lfs merge=lfs -text
63
  coco256/agatha_v2_coco256_plain_s0/agatha_v2_coco256_plain_s0_raw_step020000_raw_g7.5.png filter=lfs diff=lfs merge=lfs -text
64
+ coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_step050000_raw_g1.png filter=lfs diff=lfs merge=lfs -text
65
+ coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_step050000_raw_g3.png filter=lfs diff=lfs merge=lfs -text
66
+ coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_step050000_ema_g1.png filter=lfs diff=lfs merge=lfs -text
67
+ coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_step050000_ema_g3.png filter=lfs diff=lfs merge=lfs -text
coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_evals.jsonl CHANGED
@@ -31,3 +31,48 @@
31
  {"run": "agatha_v2_coco256_agatha_s0", "step": 20000, "hook": "clip", "ok": true, "seconds": 93.24, "result": {"items": 500, "raw_g1": {"clip_score": {"mean": 17.267433848614147, "se": 0.16077567708888485, "n": 500}, "grounding": {"mean": 0.0351663559619662, "se": 0.0017532900736217964, "n": 500, "own_cos": 0.5818204405502635, "shuffled_cos": 0.5466540845882973}}, "raw_g3": {"clip_score": {"mean": 20.52512256257751, "se": 0.1757505617236494, "n": 500}, "grounding": {"mean": 0.08900275551476, "se": 0.0031491628507076073, "n": 500, "own_cos": 0.631160993020558, "shuffled_cos": 0.5421582375057978}}, "ema_g1": {"clip_score": {"mean": 13.837846539626902, "se": 0.1312145055467162, "n": 500}, "grounding": {"mean": 0.0069783765678626255, "se": 0.0011604861450748856, "n": 500, "own_cos": 0.5397440180583679, "shuffled_cos": 0.5327656414905053}}, "ema_g3": {"clip_score": {"mean": 15.058272319013025, "se": 0.15455398548859203, "n": 500}, "grounding": {"mean": 0.017291083208881706, "se": 0.001961039816383613, "n": 500, "own_cos": 0.5469900434870235, "shuffled_cos": 0.5296989602781419}}}}
32
  {"run": "agatha_v2_coco256_agatha_s0", "step": 20000, "hook": "checkpoint", "ok": true, "seconds": 7.89, "result": {"file": "agatha_v2_coco256_agatha_s0_state_step020000.pt", "bytes": 672811184, "pushed": true}}
33
  {"run": "agatha_v2_coco256_agatha_s0", "step": 20000, "hook": "milestone", "ok": true, "seconds": 2.51, "result": {"files": ["agatha_v2_coco256_agatha_s0_step020000_raw.pt", "agatha_v2_coco256_agatha_s0_step020000_ema.pt", "agatha_v2_coco256_agatha_s0_step020000_raw_g1.png", "agatha_v2_coco256_agatha_s0_step020000_raw_g3.png", "agatha_v2_coco256_agatha_s0_step020000_ema_g1.png", "agatha_v2_coco256_agatha_s0_step020000_ema_g3.png", "agatha_v2_coco256_agatha_s0_evals.jsonl", "agatha_v2_coco256_agatha_s0_train.log"], "pushed": true}}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
  {"run": "agatha_v2_coco256_agatha_s0", "step": 20000, "hook": "clip", "ok": true, "seconds": 93.24, "result": {"items": 500, "raw_g1": {"clip_score": {"mean": 17.267433848614147, "se": 0.16077567708888485, "n": 500}, "grounding": {"mean": 0.0351663559619662, "se": 0.0017532900736217964, "n": 500, "own_cos": 0.5818204405502635, "shuffled_cos": 0.5466540845882973}}, "raw_g3": {"clip_score": {"mean": 20.52512256257751, "se": 0.1757505617236494, "n": 500}, "grounding": {"mean": 0.08900275551476, "se": 0.0031491628507076073, "n": 500, "own_cos": 0.631160993020558, "shuffled_cos": 0.5421582375057978}}, "ema_g1": {"clip_score": {"mean": 13.837846539626902, "se": 0.1312145055467162, "n": 500}, "grounding": {"mean": 0.0069783765678626255, "se": 0.0011604861450748856, "n": 500, "own_cos": 0.5397440180583679, "shuffled_cos": 0.5327656414905053}}, "ema_g3": {"clip_score": {"mean": 15.058272319013025, "se": 0.15455398548859203, "n": 500}, "grounding": {"mean": 0.017291083208881706, "se": 0.001961039816383613, "n": 500, "own_cos": 0.5469900434870235, "shuffled_cos": 0.5296989602781419}}}}
32
  {"run": "agatha_v2_coco256_agatha_s0", "step": 20000, "hook": "checkpoint", "ok": true, "seconds": 7.89, "result": {"file": "agatha_v2_coco256_agatha_s0_state_step020000.pt", "bytes": 672811184, "pushed": true}}
33
  {"run": "agatha_v2_coco256_agatha_s0", "step": 20000, "hook": "milestone", "ok": true, "seconds": 2.51, "result": {"files": ["agatha_v2_coco256_agatha_s0_step020000_raw.pt", "agatha_v2_coco256_agatha_s0_step020000_ema.pt", "agatha_v2_coco256_agatha_s0_step020000_raw_g1.png", "agatha_v2_coco256_agatha_s0_step020000_raw_g3.png", "agatha_v2_coco256_agatha_s0_step020000_ema_g1.png", "agatha_v2_coco256_agatha_s0_step020000_ema_g3.png", "agatha_v2_coco256_agatha_s0_evals.jsonl", "agatha_v2_coco256_agatha_s0_train.log"], "pushed": true}}
34
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 22500, "hook": "heldout", "ok": true, "seconds": 2.57, "result": {"items": 2000, "raw": {"loss": 1.0075055359750986, "bands": [1.1567052381464231, 0.9319374745382109, 0.9248591465828222], "loss_shuffled": 1.0178440022468567, "loss_null": 1.0132328387647866, "gap_shuffled": 0.010338466271758079, "gap_shuffled_se": 0.0005550015631519094, "gap_null": 0.00572730278968811, "gap_null_se": 0.0002952067454100206, "gap_shuffled_bands": [0.00027084051046758774, 0.002181969124293363, 0.03896853213048963]}, "ema": {"loss": 1.122865512728691, "bands": [1.2743238010519233, 1.0608894778804518, 1.0133402351491179], "loss_shuffled": 1.1307370544970035, "loss_null": 1.1236155165135862, "gap_shuffled": 0.007871541768312455, "gap_shuffled_se": 0.0003557410943305438, "gap_null": 0.0007500037848949433, "gap_null_se": 0.00024270665068575823, "gap_shuffled_bands": [8.365416618501435e-05, 0.00356277361099041, 0.02653897638561938]}}}
35
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 22500, "hook": "sheet", "ok": true, "seconds": 4.47, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step022500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step022500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step022500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step022500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.012956196442246437, "latent_std": 0.9833062291145325, "image_mean": 0.464700847864151, "image_std": 0.25995585322380066}, "raw_g3": {"finite": true, "latent_mean": -0.022838708013296127, "latent_std": 1.0192276239395142, "image_mean": 0.41655096411705017, "image_std": 0.25658154487609863}, "ema_g1": {"finite": true, "latent_mean": -0.014242708683013916, "latent_std": 0.7155914902687073, "image_mean": 0.3727278411388397, "image_std": 0.12965896725654602}, "ema_g3": {"finite": true, "latent_mean": -0.03491150960326195, "latent_std": 0.7426594495773315, "image_mean": 0.3599265515804291, "image_std": 0.14492379128932953}}}
36
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 22500, "hook": "sphere", "ok": true, "seconds": 20.97, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.0731788916394056, "a90": -0.024099922287143374, "a180": -0.12225786099166783, "b": 0.08219076285095522, "c": 0.051273073880629044, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06625964382246814, "a90": -0.0015222602777786858, "a180": -0.13099702736715757, "b": 0.09395719696951117, "c": 0.058701118249194266, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.06738431658573132, "a90": -0.016104800750891823, "a180": -0.11866383242057083, "b": 0.051225351288091255, "c": 0.20865245156304035, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.06761536328740768, "a90": -0.014251128381211597, "a180": -0.12097959819360375, "b": 0.052679176454532746, "c": 0.12283461834164328, "c_pairs": 32640, "sphere_like": false}}}
37
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 25000, "hook": "heldout", "ok": true, "seconds": 2.57, "result": {"items": 2000, "raw": {"loss": 1.004552861958742, "bands": [1.1538778805305967, 0.9281726643992544, 0.9231390815234812], "loss_shuffled": 1.0142670361548662, "loss_null": 1.0101286323517562, "gap_shuffled": 0.009714174196124076, "gap_shuffled_se": 0.0005222589650733089, "gap_null": 0.005575770393013955, "gap_null_se": 0.00028011582841490677, "gap_shuffled_bands": [0.0002635128281734902, 0.0020793365612427235, 0.03655179639271458]}, "ema": {"loss": 1.0839227839708327, "bands": [1.2364621530382611, 1.0146353994008481, 0.9855617822632381], "loss_shuffled": 1.092250598564744, "loss_null": 1.0868294352591037, "gap_shuffled": 0.008327814593911171, "gap_shuffled_se": 0.0003913121507624097, "gap_null": 0.00290665128827095, "gap_null_se": 0.0002483642642448718, "gap_shuffled_bands": [0.00011146244134948854, 0.003119116029102625, 0.029175099072304148]}}}
38
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 25000, "hook": "sheet", "ok": true, "seconds": 4.49, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step025000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step025000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step025000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step025000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.004955981392413378, "latent_std": 0.9797589778900146, "image_mean": 0.4009890556335449, "image_std": 0.2583664655685425}, "raw_g3": {"finite": true, "latent_mean": -0.016357138752937317, "latent_std": 1.012208104133606, "image_mean": 0.40718555450439453, "image_std": 0.25081107020378113}, "ema_g1": {"finite": true, "latent_mean": -0.01353401131927967, "latent_std": 0.757308840751648, "image_mean": 0.38581812381744385, "image_std": 0.15009891986846924}, "ema_g3": {"finite": true, "latent_mean": -0.03233369067311287, "latent_std": 0.7939048409461975, "image_mean": 0.37736833095550537, "image_std": 0.1643049269914627}}}
39
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 25000, "hook": "sphere", "ok": true, "seconds": 21.05, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.08602345083685795, "a90": -0.023177835507274615, "a180": -0.14886906616644127, "b": 0.08265584769185244, "c": 0.05804308991576799, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.08019170866710744, "a90": -0.005276447553900185, "a180": -0.1551069697803147, "b": 0.095500945352775, "c": 0.059290530909033475, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.07752371243954134, "a90": -0.018872163407077563, "a180": -0.13617526147200512, "b": 0.0573859671748032, "c": 0.14431273313160506, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.07996664725359451, "a90": -0.016358023995280333, "a180": -0.1435752705119087, "b": 0.06078176536985545, "c": 0.10303236367941289, "c_pairs": 32640, "sphere_like": false}}}
40
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 27500, "hook": "heldout", "ok": true, "seconds": 2.57, "result": {"items": 2000, "raw": {"loss": 1.0017577249705791, "bands": [1.151489833152805, 0.9247537247872626, 0.9208447337014549], "loss_shuffled": 1.0119334336966277, "loss_null": 1.007362949192524, "gap_shuffled": 0.01017570872604847, "gap_shuffled_se": 0.0005456133121567798, "gap_null": 0.005605224221944809, "gap_null_se": 0.00028216210665405676, "gap_shuffled_bands": [0.0002695484855149642, 0.0021902040101384724, 0.038276728532752886]}, "ema": {"loss": 1.0571803880035877, "bands": [1.2104239784141129, 0.983702232623368, 0.9650972793529747], "loss_shuffled": 1.065899378836155, "loss_null": 1.0613148892521858, "gap_shuffled": 0.008718990832567214, "gap_shuffled_se": 0.0004226212118182306, "gap_null": 0.004134501248598098, "gap_null_se": 0.00025657546451684083, "gap_shuffled_bands": [0.00017016286810353073, 0.0028283600089679868, 0.031229274979316594]}}}
41
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 27500, "hook": "sheet", "ok": true, "seconds": 4.45, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step027500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step027500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step027500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step027500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.0014907550066709518, "latent_std": 0.9782348275184631, "image_mean": 0.4144320785999298, "image_std": 0.2521023154258728}, "raw_g3": {"finite": true, "latent_mean": -0.014177752658724785, "latent_std": 1.0055549144744873, "image_mean": 0.39244845509529114, "image_std": 0.24415431916713715}, "ema_g1": {"finite": true, "latent_mean": -0.011982282623648643, "latent_std": 0.7968525290489197, "image_mean": 0.39549490809440613, "image_std": 0.1698218286037445}, "ema_g3": {"finite": true, "latent_mean": -0.028468109667301178, "latent_std": 0.8383164405822754, "image_mean": 0.3892265856266022, "image_std": 0.17949366569519043}}}
42
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 27500, "hook": "sphere", "ok": true, "seconds": 21.05, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.07620633403512192, "a90": -0.022530329566383973, "a180": -0.12988233850385988, "b": 0.08152688236065224, "c": 0.05867853012064703, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.07083574642783205, "a90": 0.001392620800791779, "a180": -0.14306411365645583, "b": 0.0998887952029455, "c": 0.050970987737836, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08386918444727468, "a90": -0.02158232565028352, "a180": -0.14615604324426584, "b": 0.062282517690039174, "c": 0.10782790276249699, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.08676069390869567, "a90": -0.01752570611669773, "a180": -0.1559956817006936, "b": 0.06911588996495611, "c": 0.09087823073618244, "c_pairs": 32640, "sphere_like": false}}}
43
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 30000, "hook": "heldout", "ok": true, "seconds": 2.56, "result": {"items": 2000, "raw": {"loss": 0.9996160053312778, "bands": [1.1493058688381215, 0.9221301953199464, 0.9196015561633768], "loss_shuffled": 1.009975941747427, "loss_null": 1.0054592868536711, "gap_shuffled": 0.01035993641614914, "gap_shuffled_se": 0.0005547178997835947, "gap_null": 0.005843281522393226, "gap_null_se": 0.00029273774294579896, "gap_shuffled_bands": [0.00027125014926849937, 0.0021576854463736285, 0.0390997913877982]}, "ema": {"loss": 1.0385947472453116, "bands": [1.191536692016599, 0.963071284848882, 0.9505014903410713], "loss_shuffled": 1.047655609920621, "loss_null": 1.043414490222931, "gap_shuffled": 0.009060862675309181, "gap_shuffled_se": 0.00044975192973092373, "gap_null": 0.0048197429776191715, "gap_null_se": 0.00026418731512296704, "gap_shuffled_bands": [0.00021192728559185276, 0.0026643592499624785, 0.03288151417256367]}}}
44
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 30000, "hook": "sheet", "ok": true, "seconds": 4.47, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step030000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step030000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step030000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step030000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.0011366000398993492, "latent_std": 0.9832133054733276, "image_mean": 0.42694029211997986, "image_std": 0.25170499086380005}, "raw_g3": {"finite": true, "latent_mean": -0.01344543881714344, "latent_std": 0.9996625781059265, "image_mean": 0.4144284725189209, "image_std": 0.2297329604625702}, "ema_g1": {"finite": true, "latent_mean": -0.01060970313847065, "latent_std": 0.8322986960411072, "image_mean": 0.39999476075172424, "image_std": 0.18671579658985138}, "ema_g3": {"finite": true, "latent_mean": -0.0261564701795578, "latent_std": 0.873981237411499, "image_mean": 0.39565667510032654, "image_std": 0.1937992423772812}}}
45
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 30000, "hook": "sphere", "ok": true, "seconds": 21.07, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.07079328611354818, "a90": -0.020600830988065488, "a180": -0.12098574123903083, "b": 0.08222325025046327, "c": 0.06416632828547979, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06946606586208988, "a90": -0.0005268782714830844, "a180": -0.13840525345269666, "b": 0.09546428739744542, "c": 0.04999929975306608, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08642655431180128, "a90": -0.024243939355912646, "a180": -0.14860916926768997, "b": 0.06632429364488486, "c": 0.08447607279476636, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.08588948545639881, "a90": -0.01441795753673063, "a180": -0.157361013376067, "b": 0.07737903253003237, "c": 0.07609266728663777, "c_pairs": 32640, "sphere_like": false}}}
46
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 30000, "hook": "fid", "ok": true, "seconds": 174.22, "result": {"prompts": 2000, "raw_g1": {"fid": 90.30696242636509, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.060568449314432976, "kid_std": 0.0025767595289995777, "subset_size_used": 1000, "subsets": 100}, "raw_g3": {"fid": 47.0912924556431, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.00927644288925148, "kid_std": 0.0006208131746383627, "subset_size_used": 1000, "subsets": 100}, "ema_g1": {"fid": 136.59572721472585, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.06710842996259431, "kid_std": 0.0015796655427474657, "subset_size_used": 1000, "subsets": 100}, "ema_g3": {"fid": 95.6313821168215, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.026245106059880306, "kid_std": 0.0007288271286567662, "subset_size_used": 1000, "subsets": 100}}}
47
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 30000, "hook": "clip", "ok": true, "seconds": 95.65, "result": {"items": 500, "raw_g1": {"clip_score": {"mean": 17.760412688279953, "se": 0.16020246063270716, "n": 500}, "grounding": {"mean": 0.04090906849503714, "se": 0.001925396452777906, "n": 500, "own_cos": 0.5885852056733812, "shuffled_cos": 0.5476761371783441}}, "raw_g3": {"clip_score": {"mean": 21.1570587074097, "se": 0.16709688876306378, "n": 500}, "grounding": {"mean": 0.09991515301389113, "se": 0.0033494479278777752, "n": 500, "own_cos": 0.6430092262693128, "shuffled_cos": 0.5430940732554216}}, "ema_g1": {"clip_score": {"mean": 15.899601040568712, "se": 0.15882704632861847, "n": 500}, "grounding": {"mean": 0.025685857311887814, "se": 0.0017907586302350373, "n": 500, "own_cos": 0.5598469546548628, "shuffled_cos": 0.5341610973429749}}, "ema_g3": {"clip_score": {"mean": 19.028678033145493, "se": 0.18336186154670242, "n": 500}, "grounding": {"mean": 0.06544452148299923, "se": 0.0030227208111161127, "n": 500, "own_cos": 0.6023231648256788, "shuffled_cos": 0.5368786433426795}}}}
48
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 30000, "hook": "checkpoint", "ok": true, "seconds": 8.5, "result": {"file": "agatha_v2_coco256_agatha_s0_state_step030000.pt", "bytes": 672811376, "pushed": true}}
49
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 32500, "hook": "heldout", "ok": true, "seconds": 2.57, "result": {"items": 2000, "raw": {"loss": 0.9975015075951814, "bands": [1.1474800473956697, 0.9195169849844909, 0.9179401970597068], "loss_shuffled": 1.0076155796796082, "loss_null": 1.003554979994893, "gap_shuffled": 0.01011407208442688, "gap_shuffled_se": 0.0005400395155606212, "gap_null": 0.006053472399711609, "gap_null_se": 0.0003082463124078024, "gap_shuffled_bands": [0.00026943839523980084, 0.002157590238192071, 0.03807634011702275]}, "ema": {"loss": 1.025573652163148, "bands": [1.1782356464647725, 0.948682230553197, 0.9402611229120063], "loss_shuffled": 1.0349219785034656, "loss_null": 1.0308039297461509, "gap_shuffled": 0.009348326340317726, "gap_shuffled_se": 0.0004717917424637162, "gap_null": 0.0052302775830030445, "gap_null_se": 0.0002710842255850784, "gap_shuffled_bands": [0.0002434849759927279, 0.002567966434175822, 0.03420372376153851]}}}
50
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 32500, "hook": "sheet", "ok": true, "seconds": 4.45, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step032500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step032500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step032500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step032500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": 0.0023221690207719803, "latent_std": 0.991806149482727, "image_mean": 0.4053514003753662, "image_std": 0.2499089539051056}, "raw_g3": {"finite": true, "latent_mean": -0.01907120645046234, "latent_std": 1.0095555782318115, "image_mean": 0.4143296480178833, "image_std": 0.24480658769607544}, "ema_g1": {"finite": true, "latent_mean": -0.009433354251086712, "latent_std": 0.8617717027664185, "image_mean": 0.4052046239376068, "image_std": 0.20103813707828522}, "ema_g3": {"finite": true, "latent_mean": -0.024123746901750565, "latent_std": 0.9012882709503174, "image_mean": 0.40022268891334534, "image_std": 0.2034190148115158}}}
51
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 32500, "hook": "sphere", "ok": true, "seconds": 20.96, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.06580480914007873, "a90": -0.022498309787040018, "a180": -0.10911130849311744, "b": 0.08358832300092879, "c": 0.03854672490892364, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.044288626279376984, "a90": 0.004034000865685403, "a180": -0.09261125342443935, "b": 0.11169561622091974, "c": 0.036431421315312976, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08734294284974566, "a90": -0.025801201891612204, "a180": -0.1488846838078791, "b": 0.06954770972256288, "c": 0.0708917371771681, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.08427844973491218, "a90": -0.01139304001718568, "a180": -0.15716385945263867, "b": 0.08223808849833499, "c": 0.06805869014263936, "c_pairs": 32640, "sphere_like": false}}}
52
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 35000, "hook": "heldout", "ok": true, "seconds": 2.57, "result": {"items": 2000, "raw": {"loss": 0.9956515621542931, "bands": [1.1454308504192658, 0.9172393213517642, 0.9171199919894699], "loss_shuffled": 1.0060662276297807, "loss_null": 1.001427468791604, "gap_shuffled": 0.010414665475487708, "gap_shuffled_se": 0.0005577217452162206, "gap_null": 0.005775906637310982, "gap_null_se": 0.00029608500500502783, "gap_shuffled_bands": [0.0002640030081347063, 0.002162439641366138, 0.039330355623569686]}, "ema": {"loss": 1.015946264013648, "bands": [1.1681886324927278, 0.9382327910847469, 0.9326654610713062], "loss_shuffled": 1.0255437313616276, "loss_null": 1.0214043532013892, "gap_shuffled": 0.009597467347979546, "gap_shuffled_se": 0.000491754578582832, "gap_null": 0.005458089187741279, "gap_null_se": 0.000277089872753708, "gap_shuffled_bands": [0.0002555006392117438, 0.0024789176447165026, 0.035381244250932524]}}}
53
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 35000, "hook": "sheet", "ok": true, "seconds": 4.48, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step035000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step035000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step035000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step035000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": 0.003093262203037739, "latent_std": 0.9774841666221619, "image_mean": 0.4385228157043457, "image_std": 0.24403247237205505}, "raw_g3": {"finite": true, "latent_mean": -0.01297785434871912, "latent_std": 0.9947904348373413, "image_mean": 0.43908262252807617, "image_std": 0.23835723102092743}, "ema_g1": {"finite": true, "latent_mean": -0.008638239465653896, "latent_std": 0.8886299729347229, "image_mean": 0.41179218888282776, "image_std": 0.21248453855514526}, "ema_g3": {"finite": true, "latent_mean": -0.022835996001958847, "latent_std": 0.9254502654075623, "image_mean": 0.404263973236084, "image_std": 0.21025913953781128}}}
54
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 35000, "hook": "sphere", "ok": true, "seconds": 21.02, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.060476037785298434, "a90": -0.024037617580166808, "a180": -0.09691445799043007, "b": 0.08273980681789037, "c": 0.03159741401271493, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.04550947245014339, "a90": 0.0018946954234521016, "a180": -0.09291364032373889, "b": 0.10357542769968983, "c": 0.043266705050904825, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08675035688891934, "a90": -0.02659199972959452, "a180": -0.14690871404824418, "b": 0.07219013437585747, "c": 0.06474317035349494, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.08115917543887652, "a90": -0.00878795563471901, "a180": -0.15353039524303402, "b": 0.08608565506183385, "c": 0.060643172140483835, "c_pairs": 32640, "sphere_like": false}}}
55
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 37500, "hook": "heldout", "ok": true, "seconds": 2.57, "result": {"items": 2000, "raw": {"loss": 0.9942010100483895, "bands": [1.1440137335759293, 0.9153982110246731, 0.9163006992833538], "loss_shuffled": 1.004725433319807, "loss_null": 1.0001862834692001, "gap_shuffled": 0.010524423271417617, "gap_shuffled_se": 0.0005651169151879088, "gap_null": 0.0059852734208107, "gap_null_se": 0.0002984633362382758, "gap_shuffled_bands": [0.0002506544454249726, 0.0021477612055645977, 0.03983315511080236]}, "ema": {"loss": 1.009167422696948, "bands": [1.1608672620118499, 0.9312522847508846, 0.9270157528599009], "loss_shuffled": 1.018981810644269, "loss_null": 1.0147676702141761, "gap_shuffled": 0.009814387947320938, "gap_shuffled_se": 0.0005080074807084496, "gap_null": 0.005600247517228127, "gap_null_se": 0.0002816598737873118, "gap_shuffled_bands": [0.00027256285261735905, 0.0024247691991494337, 0.036356342604885815]}}}
56
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 37500, "hook": "sheet", "ok": true, "seconds": 4.43, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step037500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step037500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step037500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step037500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": 0.001031022984534502, "latent_std": 0.9829272031784058, "image_mean": 0.44295358657836914, "image_std": 0.26201891899108887}, "raw_g3": {"finite": true, "latent_mean": -0.014128895476460457, "latent_std": 1.0054771900177002, "image_mean": 0.43984511494636536, "image_std": 0.2550239861011505}, "ema_g1": {"finite": true, "latent_mean": -0.00859617069363594, "latent_std": 0.9053779244422913, "image_mean": 0.41496482491493225, "image_std": 0.220839723944664}, "ema_g3": {"finite": true, "latent_mean": -0.022941432893276215, "latent_std": 0.9399293661117554, "image_mean": 0.4053475856781006, "image_std": 0.21534040570259094}}}
57
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 37500, "hook": "sphere", "ok": true, "seconds": 21.0, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.06794602329134479, "a90": -0.018840361784659327, "a180": -0.11705168479803027, "b": 0.07979309069075083, "c": 0.040602054928910225, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06306782912251169, "a90": 0.0013693086932178582, "a180": -0.1275049669382412, "b": 0.09072673151083707, "c": 0.030337566565647563, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08576330649604855, "a90": -0.026774296626750536, "a180": -0.14475231636534658, "b": 0.0744940800269063, "c": 0.05996642728591204, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.07820768768190192, "a90": -0.007561533996279549, "a180": -0.14885384136752425, "b": 0.08936990833059402, "c": 0.05593985698468102, "c_pairs": 32640, "sphere_like": false}}}
58
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 40000, "hook": "heldout", "ok": true, "seconds": 2.58, "result": {"items": 2000, "raw": {"loss": 0.9925916863083839, "bands": [1.1424173048386788, 0.9134369175449736, 0.9152849953407941], "loss_shuffled": 1.003037084132433, "loss_null": 0.9985187782049179, "gap_shuffled": 0.010445397824048996, "gap_shuffled_se": 0.0005539167998577369, "gap_null": 0.005927091896533966, "gap_null_se": 0.0002992679605988222, "gap_shuffled_bands": [0.00027397493154700947, 0.0022200467666541796, 0.039344135920759894]}, "ema": {"loss": 1.0038266747891902, "bands": [1.1551798546049183, 0.9254693086562198, 0.922941478455642], "loss_shuffled": 1.013815280035138, "loss_null": 1.0095379827618598, "gap_shuffled": 0.009988605245947838, "gap_shuffled_se": 0.0005210518728847121, "gap_null": 0.005711307972669602, "gap_null_se": 0.00028598220841736567, "gap_shuffled_bands": [0.0002741551468818911, 0.0023812372461607854, 0.03715693360001259]}}}
59
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 40000, "hook": "sheet", "ok": true, "seconds": 4.47, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step040000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step040000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step040000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step040000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.007130899466574192, "latent_std": 0.973584771156311, "image_mean": 0.4039740264415741, "image_std": 0.2376425713300705}, "raw_g3": {"finite": true, "latent_mean": -0.022779952734708786, "latent_std": 1.000127911567688, "image_mean": 0.4018864631652832, "image_std": 0.22921015322208405}, "ema_g1": {"finite": true, "latent_mean": -0.007786852773278952, "latent_std": 0.9215919375419617, "image_mean": 0.41528812050819397, "image_std": 0.22793488204479218}, "ema_g3": {"finite": true, "latent_mean": -0.022402752190828323, "latent_std": 0.9533793330192566, "image_mean": 0.40727970004081726, "image_std": 0.22079569101333618}}}
60
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 40000, "hook": "sphere", "ok": true, "seconds": 20.98, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.0648783933289942, "a90": -0.02249444046130115, "a180": -0.10726234619668724, "b": 0.08591765323630368, "c": 0.05141972163743776, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.05455865466514041, "a90": 0.0019451205844071007, "a180": -0.11106242991468793, "b": 0.10335067652049498, "c": 0.05342073874080179, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08414000296796657, "a90": -0.026578322766527412, "a180": -0.14170168316940573, "b": 0.07612659733982136, "c": 0.0562400746707223, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.07528620824418138, "a90": -0.005271873739031475, "a180": -0.14530054274933124, "b": 0.092082733856643, "c": 0.05142303146185009, "c_pairs": 32640, "sphere_like": false}}}
61
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 40000, "hook": "fid", "ok": true, "seconds": 173.1, "result": {"prompts": 2000, "raw_g1": {"fid": 85.90093593973478, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.056421703432409306, "kid_std": 0.0025616237564751215, "subset_size_used": 1000, "subsets": 100}, "raw_g3": {"fid": 45.75301377308705, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.008849132026788586, "kid_std": 0.0006797347510201995, "subset_size_used": 1000, "subsets": 100}, "ema_g1": {"fid": 95.08436080868574, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.05139846026891051, "kid_std": 0.0019575498037017578, "subset_size_used": 1000, "subsets": 100}, "ema_g3": {"fid": 57.72401214174988, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.012115624546197799, "kid_std": 0.0005973665529957693, "subset_size_used": 1000, "subsets": 100}}}
62
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 40000, "hook": "clip", "ok": true, "seconds": 93.18, "result": {"items": 500, "raw_g1": {"clip_score": {"mean": 18.082584813004544, "se": 0.161266706530065, "n": 500}, "grounding": {"mean": 0.04462400122201957, "se": 0.002021391788343266, "n": 500, "own_cos": 0.5946540468069335, "shuffled_cos": 0.550030045584914}}, "raw_g3": {"clip_score": {"mean": 21.6563236357106, "se": 0.17475593629041294, "n": 500}, "grounding": {"mean": 0.10544687142847742, "se": 0.003358959098872691, "n": 500, "own_cos": 0.6485122166407505, "shuffled_cos": 0.5430653452122731}}, "ema_g1": {"clip_score": {"mean": 17.282634243569355, "se": 0.16650190507187712, "n": 500}, "grounding": {"mean": 0.038514586010377334, "se": 0.0019682205943743706, "n": 500, "own_cos": 0.5828570467327169, "shuffled_cos": 0.5443424607223394}}, "ema_g3": {"clip_score": {"mean": 20.66087287120425, "se": 0.1795397316369954, "n": 500}, "grounding": {"mean": 0.09267463906782984, "se": 0.003322212728663139, "n": 500, "own_cos": 0.6338464814111124, "shuffled_cos": 0.5411718423432826}}}}
63
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 40000, "hook": "checkpoint", "ok": true, "seconds": 8.03, "result": {"file": "agatha_v2_coco256_agatha_s0_state_step040000.pt", "bytes": 672811376, "pushed": true}}
64
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 42500, "hook": "heldout", "ok": true, "seconds": 2.57, "result": {"items": 2000, "raw": {"loss": 0.9911648867577314, "bands": [1.1410497310278922, 0.911815516137545, 0.9141116577696009], "loss_shuffled": 1.0017949780523776, "loss_null": 0.9970391308814287, "gap_shuffled": 0.010630091294646263, "gap_shuffled_se": 0.0005753054371152865, "gap_null": 0.0058742441236972805, "gap_null_se": 0.00030298051852427305, "gap_shuffled_bands": [0.000255623377782189, 0.002193079371186511, 0.040188260701091806]}, "ema": {"loss": 0.9996603723317384, "bands": [1.1508516768696908, 0.9209136360448174, 0.9196845973270974], "loss_shuffled": 1.0097980411350727, "loss_null": 1.0054312279373407, "gap_shuffled": 0.010137668803334237, "gap_shuffled_se": 0.0005318427858913773, "gap_null": 0.005770855605602265, "gap_null_se": 0.00028957094912768135, "gap_shuffled_bands": [0.00027778708682791225, 0.0023395886772808066, 0.037846333169715106]}}}
65
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 42500, "hook": "sheet", "ok": true, "seconds": 4.48, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step042500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step042500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step042500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step042500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.0044710938818752766, "latent_std": 0.9776895046234131, "image_mean": 0.4415592849254608, "image_std": 0.2545410692691803}, "raw_g3": {"finite": true, "latent_mean": -0.02658580057322979, "latent_std": 1.0064430236816406, "image_mean": 0.4598585367202759, "image_std": 0.2564338147640228}, "ema_g1": {"finite": true, "latent_mean": -0.007025165483355522, "latent_std": 0.9347200393676758, "image_mean": 0.4164487421512604, "image_std": 0.23230715095996857}, "ema_g3": {"finite": true, "latent_mean": -0.022257471457123756, "latent_std": 0.9642946720123291, "image_mean": 0.4077458679676056, "image_std": 0.2253562957048416}}}
66
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 42500, "hook": "sphere", "ok": true, "seconds": 20.94, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.07331778478452466, "a90": -0.021781301047365313, "a180": -0.124854268521684, "b": 0.08295564770867445, "c": 0.055614956780567916, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06029725976674423, "a90": 0.004497490069866652, "a180": -0.12509200960335512, "b": 0.09830378971700432, "c": 0.03886903658260276, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08311633560568824, "a90": -0.026590533998343477, "a180": -0.13964213721303298, "b": 0.07726003607684787, "c": 0.054598580839204894, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.07350416537270457, "a90": -0.004069097159841602, "a180": -0.14293923358556754, "b": 0.09420439029751458, "c": 0.04993662933122874, "c_pairs": 32640, "sphere_like": false}}}
67
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 45000, "hook": "heldout", "ok": true, "seconds": 2.57, "result": {"items": 2000, "raw": {"loss": 0.9900695872157812, "bands": [1.1399282413097909, 0.9104416586714078, 0.9135383840235362], "loss_shuffled": 1.000771230816841, "loss_null": 0.995885888710618, "gap_shuffled": 0.010701643601059914, "gap_shuffled_se": 0.0005764172648094155, "gap_null": 0.005816301494836807, "gap_null_se": 0.0003015290774555751, "gap_shuffled_bands": [0.00027466111795146846, 0.0022092913388780736, 0.040431404057629204]}, "ema": {"loss": 0.9963440653681755, "bands": [1.147343615149537, 0.9173767356837835, 0.91702705743346], "loss_shuffled": 1.006629373818636, "loss_null": 1.0021570444107055, "gap_shuffled": 0.010285308450460435, "gap_shuffled_se": 0.0005427194502118652, "gap_null": 0.00581297904253006, "gap_null_se": 0.0002926422192339688, "gap_shuffled_bands": [0.0002754375447810044, 0.0023157079202514716, 0.03850747104313651]}}}
68
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 45000, "hook": "sheet", "ok": true, "seconds": 4.53, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step045000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step045000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step045000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step045000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.008647803217172623, "latent_std": 0.9684465527534485, "image_mean": 0.42697834968566895, "image_std": 0.24865294992923737}, "raw_g3": {"finite": true, "latent_mean": -0.028440747410058975, "latent_std": 0.9958139061927795, "image_mean": 0.4329111576080322, "image_std": 0.24955014884471893}, "ema_g1": {"finite": true, "latent_mean": -0.006869482807815075, "latent_std": 0.9462983012199402, "image_mean": 0.4179980754852295, "image_std": 0.23600859940052032}, "ema_g3": {"finite": true, "latent_mean": -0.022474149242043495, "latent_std": 0.9739304780960083, "image_mean": 0.4090571701526642, "image_std": 0.22872039675712585}}}
69
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 45000, "hook": "sphere", "ok": true, "seconds": 20.98, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.0759486923642054, "a90": -0.02316786586263217, "a180": -0.12872951886577866, "b": 0.08243427488191366, "c": 0.05310502136096665, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06576589311812805, "a90": 0.0010510538031149909, "a180": -0.1325828400393711, "b": 0.09928025979390448, "c": 0.06275022074410612, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08168759212433618, "a90": -0.025976030446909673, "a180": -0.13739915380176268, "b": 0.07809127658649982, "c": 0.052651045918638784, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.07130358427601523, "a90": -0.0026049764135894315, "a180": -0.140002192138441, "b": 0.09555125305515008, "c": 0.04850419381480049, "c_pairs": 32640, "sphere_like": false}}}
70
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 47500, "hook": "heldout", "ok": true, "seconds": 2.57, "result": {"items": 2000, "raw": {"loss": 0.9887935122400522, "bands": [1.1385441364020927, 0.9089453825241609, 0.9128002670776602], "loss_shuffled": 0.9994015937447548, "loss_null": 0.9947267057299614, "gap_shuffled": 0.010608081504702568, "gap_shuffled_se": 0.0005690346072220111, "gap_null": 0.005933193489909172, "gap_null_se": 0.0003065871097589053, "gap_shuffled_bands": [0.0002461266916215229, 0.0021753917793172306, 0.040140780462962586]}, "ema": {"loss": 0.9936771188527346, "bands": [1.14459997374616, 0.9144557761773826, 0.9149119131445796], "loss_shuffled": 1.0040820005238056, "loss_null": 0.999532770767808, "gap_shuffled": 0.010404881671071052, "gap_shuffled_se": 0.0005510894643730225, "gap_null": 0.005855651915073395, "gap_null_se": 0.0002954543171568289, "gap_shuffled_bands": [0.0002735537311942242, 0.002295806478448423, 0.03904387332727163]}}}
71
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 47500, "hook": "sheet", "ok": true, "seconds": 4.48, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step047500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step047500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step047500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step047500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.003589596599340439, "latent_std": 0.9748448729515076, "image_mean": 0.43466517329216003, "image_std": 0.25748735666275024}, "raw_g3": {"finite": true, "latent_mean": -0.022544672712683678, "latent_std": 0.9911666512489319, "image_mean": 0.4333069324493408, "image_std": 0.24967829883098602}, "ema_g1": {"finite": true, "latent_mean": -0.006600101478397846, "latent_std": 0.9547484517097473, "image_mean": 0.42020130157470703, "image_std": 0.23924267292022705}, "ema_g3": {"finite": true, "latent_mean": -0.022751707583665848, "latent_std": 0.9810546040534973, "image_mean": 0.413918137550354, "image_std": 0.23192714154720306}}}
72
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 47500, "hook": "sphere", "ok": true, "seconds": 20.95, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.07624964919861611, "a90": -0.02357392866454802, "a180": -0.12892536973268423, "b": 0.08266127163259065, "c": 0.057493868625526225, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.059190900851421965, "a90": 0.0026234583214433185, "a180": -0.12100526002428723, "b": 0.10021933415127762, "c": 0.053495141538335955, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.0803718049956828, "a90": -0.025716413872275933, "a180": -0.13502719611908967, "b": 0.07877730393274172, "c": 0.05250446955078225, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.0698549036483242, "a90": -0.0017049302758255214, "a180": -0.13800487702082284, "b": 0.0961822413492455, "c": 0.04924032034689978, "c_pairs": 32640, "sphere_like": false}}}
73
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 50000, "hook": "heldout", "ok": true, "seconds": 2.57, "result": {"items": 2000, "raw": {"loss": 0.9876734229773283, "bands": [1.1375199083025993, 0.9074680035238283, 0.9121640113932399], "loss_shuffled": 0.998096381559968, "loss_null": 0.9936989120543003, "gap_shuffled": 0.010422958582639695, "gap_shuffled_se": 0.0005623147763035434, "gap_null": 0.006025489076972008, "gap_null_se": 0.00030951230248076766, "gap_shuffled_bands": [0.00026430255935556236, 0.0021782747403356535, 0.03933700123613604]}, "ema": {"loss": 0.9915075074881315, "bands": [1.1423221589775987, 0.9120700816664888, 0.9132733469209854], "loss_shuffled": 1.0019971336424351, "loss_null": 0.9973817363828421, "gap_shuffled": 0.010489626154303551, "gap_shuffled_se": 0.0005572637833011597, "gap_null": 0.0058742288947105405, "gap_null_se": 0.00029793071913090865, "gap_shuffled_bands": [0.00026840231035598445, 0.00228570884561255, 0.03942254169471158]}}}
74
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 50000, "hook": "sheet", "ok": true, "seconds": 4.53, "result": {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step050000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step050000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step050000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step050000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.005326401442289352, "latent_std": 0.9730930328369141, "image_mean": 0.4317779242992401, "image_std": 0.2536206841468811}, "raw_g3": {"finite": true, "latent_mean": -0.016666021198034286, "latent_std": 1.00787353515625, "image_mean": 0.4061306416988373, "image_std": 0.2535746991634369}, "ema_g1": {"finite": true, "latent_mean": -0.007217263802886009, "latent_std": 0.9567620754241943, "image_mean": 0.4227927029132843, "image_std": 0.24122270941734314}, "ema_g3": {"finite": true, "latent_mean": -0.023708097636699677, "latent_std": 0.982517659664154, "image_mean": 0.41566768288612366, "image_std": 0.2351980209350586}}}
75
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 50000, "hook": "sphere", "ok": true, "seconds": 20.94, "result": {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.07757875936154576, "a90": -0.019779807734990815, "a180": -0.13537771098810075, "b": 0.08613481194668048, "c": 0.05589262942411118, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06364668598291538, "a90": -0.003272238170361669, "a180": -0.12402113379546911, "b": 0.09817541246483291, "c": 0.05484722803662785, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08048647691735841, "a90": -0.02561708593489507, "a180": -0.1353558678998218, "b": 0.07959609583447838, "c": 0.052551416500645896, "c_pairs": 32640, "sphere_like": false}, "ema_g3": {"n": 256, "size": 256, "a": -0.06934620714135528, "a90": -0.0011100502139925437, "a180": -0.137582364068718, "b": 0.09710788470016721, "c": 0.04988464387477503, "c_pairs": 32640, "sphere_like": false}}}
76
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 50000, "hook": "fid", "ok": true, "seconds": 172.5, "result": {"prompts": 2000, "raw_g1": {"fid": 78.38395796824767, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.04673764649934381, "kid_std": 0.002182370195363323, "subset_size_used": 1000, "subsets": 100}, "raw_g3": {"fid": 43.406524504769834, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.007405634777095038, "kid_std": 0.00047614576406396393, "subset_size_used": 1000, "subsets": 100}, "ema_g1": {"fid": 82.24183137085606, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.0480243107028998, "kid_std": 0.0022158037502084975, "subset_size_used": 1000, "subsets": 100}, "ema_g3": {"fid": 46.46724147557586, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.00835478776316989, "kid_std": 0.000512931433318383, "subset_size_used": 1000, "subsets": 100}}}
77
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 50000, "hook": "clip", "ok": true, "seconds": 93.06, "result": {"items": 500, "raw_g1": {"clip_score": {"mean": 18.194331548616525, "se": 0.16202108310482533, "n": 500}, "grounding": {"mean": 0.0475769885760916, "se": 0.0019883150388673243, "n": 500, "own_cos": 0.5994609751705561, "shuffled_cos": 0.5518839865944645}}, "raw_g3": {"clip_score": {"mean": 21.81861600245198, "se": 0.17558844295985107, "n": 500}, "grounding": {"mean": 0.10984932104188458, "se": 0.0034055360238698803, "n": 500, "own_cos": 0.6572034583605368, "shuffled_cos": 0.5473541373186522}}, "ema_g1": {"clip_score": {"mean": 17.91791611400637, "se": 0.16553803799783726, "n": 500}, "grounding": {"mean": 0.047002753521252626, "se": 0.002063502015335097, "n": 500, "own_cos": 0.5954830841987963, "shuffled_cos": 0.5484803306775436}}, "ema_g3": {"clip_score": {"mean": 21.39030382640341, "se": 0.1794481884640946, "n": 500}, "grounding": {"mean": 0.10666849887925443, "se": 0.0034433669503893464, "n": 500, "own_cos": 0.6496313385515439, "shuffled_cos": 0.5429628396722894}}}}
78
+ {"run": "agatha_v2_coco256_agatha_s0", "step": 50000, "hook": "checkpoint", "ok": true, "seconds": 7.96, "result": {"file": "agatha_v2_coco256_agatha_s0_state_step050000.pt", "bytes": 672811376, "pushed": true}}
coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_step050000_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00a7f1cc6df102e7e6a64c691fe87a1d011148f82a0ba1c523a400aed8301e81
3
+ size 168199875
coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_step050000_ema_g1.png ADDED

Git LFS Details

  • SHA256: 52dbc2ae75f79b74b374b4e019570f808a7d7b7484ccca75419f0c34a8ef4854
  • Pointer size: 132 Bytes
  • Size of remote file: 4.09 MB
coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_step050000_ema_g3.png ADDED

Git LFS Details

  • SHA256: 50a558af762ec2d8cb5b63dd875823e419041147fcd6674c47281abc4785574a
  • Pointer size: 132 Bytes
  • Size of remote file: 4.2 MB
coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_step050000_raw.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4685075ee01ca7db8d2ffaa687da18b83feb24b7ca52971ee40c5d6440897e73
3
+ size 168199875
coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_step050000_raw_g1.png ADDED

Git LFS Details

  • SHA256: 85e22656926d3a2f632f72920910903172e212fe0e25ac320bb12006d06ecabc
  • Pointer size: 132 Bytes
  • Size of remote file: 3.98 MB
coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_step050000_raw_g3.png ADDED

Git LFS Details

  • SHA256: a197fd9fca7b877b8a1b9c7a605fcffbc272b36ebaab4279a122e769ba025552
  • Pointer size: 132 Bytes
  • Size of remote file: 4.17 MB
coco256/agatha_v2_coco256_agatha_s0/agatha_v2_coco256_agatha_s0_train.log CHANGED
@@ -234,3 +234,362 @@ arm agatha: {"arm": "agatha"}; 42,023,588 trained parameters; start step 0
234
  [hook clip] step 20000 ok in 93.24 s: {"items": 500, "raw_g1": {"clip_score": {"mean": 17.267433848614147, "se": 0.16077567708888485, "n": 500}, "grounding": {"mean": 0.0351663559619662, "se": 0.0017532900736217964, "n": 500, "own_cos": 0.5818204405502635, "shuffled_cos": 0.5466540845882973}}, "raw_g3": {"clip_score": {"mean": 20.52512256257751, "se": 0.1757505617236494, "n": 500}, "grounding": {"mean": 0.08900275551476, "se": 0.0031491628507076073, "n": 500, "own_cos": 0.631160993020558, "shuffled_cos": 0.5421582375057978}}, "ema_g1": {"clip_score": {"mean": 13.837846539626902, "se": 0.1312145055467162, "n": 500}, "grounding": {" ...
235
  [hook checkpoint] step 20000 ok in 7.89 s: {"file": "agatha_v2_coco256_agatha_s0_state_step020000.pt", "bytes": 672811184, "pushed": true}
236
  [hook milestone] step 20000 ok in 2.51 s: {"files": ["agatha_v2_coco256_agatha_s0_step020000_raw.pt", "agatha_v2_coco256_agatha_s0_step020000_ema.pt", "agatha_v2_coco256_agatha_s0_step020000_raw_g1.png", "agatha_v2_coco256_agatha_s0_step020000_raw_g3.png", "agatha_v2_coco256_agatha_s0_step020000_ema_g1.png", "agatha_v2_coco256_agatha_s0_step020000_ema_g3.png", "agatha_v2_coco256_agatha_s0_evals.jsonl", "agatha_v2_coco256_agatha_s0_train.log"], "pushed": true}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
234
  [hook clip] step 20000 ok in 93.24 s: {"items": 500, "raw_g1": {"clip_score": {"mean": 17.267433848614147, "se": 0.16077567708888485, "n": 500}, "grounding": {"mean": 0.0351663559619662, "se": 0.0017532900736217964, "n": 500, "own_cos": 0.5818204405502635, "shuffled_cos": 0.5466540845882973}}, "raw_g3": {"clip_score": {"mean": 20.52512256257751, "se": 0.1757505617236494, "n": 500}, "grounding": {"mean": 0.08900275551476, "se": 0.0031491628507076073, "n": 500, "own_cos": 0.631160993020558, "shuffled_cos": 0.5421582375057978}}, "ema_g1": {"clip_score": {"mean": 13.837846539626902, "se": 0.1312145055467162, "n": 500}, "grounding": {" ...
235
  [hook checkpoint] step 20000 ok in 7.89 s: {"file": "agatha_v2_coco256_agatha_s0_state_step020000.pt", "bytes": 672811184, "pushed": true}
236
  [hook milestone] step 20000 ok in 2.51 s: {"files": ["agatha_v2_coco256_agatha_s0_step020000_raw.pt", "agatha_v2_coco256_agatha_s0_step020000_ema.pt", "agatha_v2_coco256_agatha_s0_step020000_raw_g1.png", "agatha_v2_coco256_agatha_s0_step020000_raw_g3.png", "agatha_v2_coco256_agatha_s0_step020000_ema_g1.png", "agatha_v2_coco256_agatha_s0_step020000_ema_g3.png", "agatha_v2_coco256_agatha_s0_evals.jsonl", "agatha_v2_coco256_agatha_s0_train.log"], "pushed": true}
237
+ run records to the repository: pushed (3 files: ['agatha_v2_coco256_agatha_s0_summary_step020000.json', 'agatha_v2_coco256_agatha_s0_evals.jsonl', 'agatha_v2_coco256_agatha_s0_train.log'])
238
+ TRAIN DONE agatha_v2_coco256_agatha_s0: step 20000, last mean loss 1.0105029344558716, 1:13:39, hook errors 0
239
+ TRAIN agatha_v2_coco256_agatha_s0 on NVIDIA RTX PRO 6000 Blackwell Server Edition | torch 2.14.1+cu130 | steps to 100000 | 2026-11-17 14:01:16
240
+ RESUME agatha_v2_coco256_agatha_s0 from the bucket: agatha_v2_coco256_agatha_s0_state_step020000.pt (step 20000; data position {'sampler': 'step_order', 'seed': 0, 'batch_size': 256, 'num_images': 82783, 'next_step': 20000})
241
+ arm agatha: {"arm": "agatha"}; 42,023,588 trained parameters; start step 20000
242
+ [agatha_v2_coco256_agatha_s0] 42,023,588 trained parameters (agatha); steps 20000 -> 100000 at batch 256; lr 0.0001; EMA 0.9999; autocast bf16
243
+ [agatha_v2_coco256_agatha_s0] step 20100/100000 loss 1.00955 | tower grad norms 0.027 0.0237 0.0274 0.028 0.0265 0.0162 | 0.322 s/step | spent 0:00:32 left 7:08:27
244
+ [agatha_v2_coco256_agatha_s0] step 20200/100000 loss 1.01009 | tower grad norms 0.0306 0.0274 0.0302 0.0302 0.0286 0.0187 | 0.368 s/step | spent 0:01:08 left 7:38:30
245
+ [agatha_v2_coco256_agatha_s0] step 20300/100000 loss 1.00841 | tower grad norms 0.0303 0.0231 0.0267 0.0261 0.0241 0.0164 | 0.368 s/step | spent 0:01:45 left 7:48:06
246
+ [agatha_v2_coco256_agatha_s0] step 20400/100000 loss 1.01103 | tower grad norms 0.0281 0.0233 0.0257 0.0252 0.0225 0.0155 | 0.368 s/step | spent 0:02:22 left 7:52:32
247
+ TRAIN agatha_v2_coco256_agatha_s0 on NVIDIA RTX PRO 6000 Blackwell Server Edition | torch 2.14.1+cu130 | steps to 100000 | 2026-11-17 14:25:28
248
+ RESUME agatha_v2_coco256_agatha_s0 from the bucket: agatha_v2_coco256_agatha_s0_state_step020000.pt (step 20000; data position {'sampler': 'step_order', 'seed': 0, 'batch_size': 256, 'num_images': 82783, 'next_step': 20000})
249
+ arm agatha: {"arm": "agatha"}; 42,023,588 trained parameters; start step 20000
250
+ [agatha_v2_coco256_agatha_s0] 42,023,588 trained parameters (agatha); steps 20000 -> 100000 at batch 256; lr 0.0001; EMA 0.9999; autocast bf16
251
+ [agatha_v2_coco256_agatha_s0] step 20100/100000 loss 1.00955 | tower grad norms 0.027 0.0237 0.0274 0.028 0.0265 0.0162 | 0.184 s/step | spent 0:00:18 left 4:05:25
252
+ [agatha_v2_coco256_agatha_s0] step 20200/100000 loss 1.01009 | tower grad norms 0.0306 0.0274 0.0302 0.0302 0.0286 0.0187 | 0.168 s/step | spent 0:00:35 left 3:54:22
253
+ [agatha_v2_coco256_agatha_s0] step 20300/100000 loss 1.00841 | tower grad norms 0.0303 0.0231 0.0267 0.0261 0.0241 0.0164 | 0.168 s/step | spent 0:00:52 left 3:50:26
254
+ [agatha_v2_coco256_agatha_s0] step 20400/100000 loss 1.01103 | tower grad norms 0.0281 0.0233 0.0257 0.0252 0.0225 0.0155 | 0.168 s/step | spent 0:01:08 left 3:48:21
255
+ [agatha_v2_coco256_agatha_s0] step 20500/100000 loss 1.00942 | tower grad norms 0.0324 0.0296 0.0336 0.0323 0.0274 0.0205 | 0.168 s/step | spent 0:01:25 left 3:46:58
256
+ [agatha_v2_coco256_agatha_s0] step 20600/100000 loss 1.00806 | tower grad norms 0.0294 0.0253 0.0297 0.0306 0.027 0.0188 | 0.168 s/step | spent 0:01:42 left 3:45:59
257
+ [agatha_v2_coco256_agatha_s0] step 20700/100000 loss 1.00889 | tower grad norms 0.0274 0.0227 0.0288 0.0278 0.0241 0.017 | 0.168 s/step | spent 0:01:59 left 3:45:15
258
+ [agatha_v2_coco256_agatha_s0] step 20800/100000 loss 1.00793 | tower grad norms 0.0334 0.0263 0.0325 0.0333 0.0293 0.0191 | 0.169 s/step | spent 0:02:16 left 3:44:39
259
+ [agatha_v2_coco256_agatha_s0] step 20900/100000 loss 1.01032 | tower grad norms 0.0292 0.0257 0.0271 0.0266 0.0237 0.0161 | 0.168 s/step | spent 0:02:32 left 3:44:03
260
+ [agatha_v2_coco256_agatha_s0] step 21000/100000 loss 1.00888 | tower grad norms 0.0288 0.025 0.0291 0.0291 0.0258 0.0169 | 0.168 s/step | spent 0:02:49 left 3:43:30
261
+ [agatha_v2_coco256_agatha_s0] step 21100/100000 loss 1.00880 | tower grad norms 0.0307 0.0253 0.0287 0.0282 0.0247 0.0173 | 0.168 s/step | spent 0:03:06 left 3:43:00
262
+ [agatha_v2_coco256_agatha_s0] step 21200/100000 loss 1.01004 | tower grad norms 0.0281 0.0234 0.026 0.0265 0.0229 0.0163 | 0.168 s/step | spent 0:03:23 left 3:42:33
263
+ [agatha_v2_coco256_agatha_s0] step 21300/100000 loss 1.01050 | tower grad norms 0.0312 0.0282 0.0324 0.032 0.0293 0.0195 | 0.168 s/step | spent 0:03:40 left 3:42:10
264
+ [agatha_v2_coco256_agatha_s0] step 21400/100000 loss 1.00878 | tower grad norms 0.0297 0.0248 0.0306 0.03 0.0291 0.0184 | 0.168 s/step | spent 0:03:57 left 3:41:46
265
+ [agatha_v2_coco256_agatha_s0] step 21500/100000 loss 1.01084 | tower grad norms 0.0254 0.0219 0.0257 0.0258 0.0231 0.0152 | 0.168 s/step | spent 0:04:13 left 3:41:23
266
+ [agatha_v2_coco256_agatha_s0] step 21600/100000 loss 1.00898 | tower grad norms 0.0308 0.0259 0.0297 0.029 0.0257 0.0176 | 0.168 s/step | spent 0:04:30 left 3:41:02
267
+ [agatha_v2_coco256_agatha_s0] step 21700/100000 loss 1.00857 | tower grad norms 0.0261 0.0235 0.0244 0.0251 0.0218 0.015 | 0.169 s/step | spent 0:04:47 left 3:40:42
268
+ [agatha_v2_coco256_agatha_s0] step 21800/100000 loss 1.00755 | tower grad norms 0.0243 0.0204 0.0262 0.0258 0.0242 0.0154 | 0.168 s/step | spent 0:05:04 left 3:40:21
269
+ [agatha_v2_coco256_agatha_s0] step 21900/100000 loss 1.00748 | tower grad norms 0.0304 0.0248 0.0297 0.0296 0.0239 0.017 | 0.168 s/step | spent 0:05:21 left 3:39:59
270
+ [agatha_v2_coco256_agatha_s0] step 22000/100000 loss 1.00911 | tower grad norms 0.0278 0.0246 0.0295 0.0297 0.0255 0.0175 | 0.168 s/step | spent 0:05:37 left 3:39:39
271
+ [agatha_v2_coco256_agatha_s0] step 22100/100000 loss 1.00969 | tower grad norms 0.0245 0.0205 0.0248 0.0246 0.022 0.0149 | 0.168 s/step | spent 0:05:54 left 3:39:20
272
+ [agatha_v2_coco256_agatha_s0] step 22200/100000 loss 1.00746 | tower grad norms 0.0339 0.027 0.0346 0.033 0.0283 0.0195 | 0.168 s/step | spent 0:06:11 left 3:39:01
273
+ [agatha_v2_coco256_agatha_s0] step 22300/100000 loss 1.00759 | tower grad norms 0.0273 0.022 0.0257 0.0263 0.021 0.0159 | 0.168 s/step | spent 0:06:28 left 3:38:43
274
+ [agatha_v2_coco256_agatha_s0] step 22400/100000 loss 1.00903 | tower grad norms 0.0254 0.0219 0.0275 0.0267 0.0239 0.0154 | 0.168 s/step | spent 0:06:45 left 3:38:23
275
+ [agatha_v2_coco256_agatha_s0] step 22500/100000 loss 1.00807 | tower grad norms 0.0267 0.0229 0.0269 0.0264 0.0247 0.0168 | 0.168 s/step | spent 0:07:02 left 3:38:04
276
+ [hook heldout] step 22500 ok in 2.57 s: {"items": 2000, "raw": {"loss": 1.0075055359750986, "bands": [1.1567052381464231, 0.9319374745382109, 0.9248591465828222], "loss_shuffled": 1.0178440022468567, "loss_null": 1.0132328387647866, "gap_shuffled": 0.010338466271758079, "gap_shuffled_se": 0.0005550015631519094, "gap_null": 0.00572730278968811, "gap_null_se": 0.0002952067454100206, "gap_shuffled_bands": [0.00027084051046758774, 0.002181969124293363, 0.03896853213048963]}, "ema": {"loss": 1.122865512728691, "bands": [1.2743238010519233, 1.0608894778804518, 1.0133402351491179], "loss_shuffled": 1.1307370544970035, "loss_null": 1.123615 ...
277
+ [hook sheet] step 22500 ok in 4.47 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step022500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step022500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step022500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step022500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.012956196442246437, "latent_std": 0.9833062291145325, "image_mean" ...
278
+ [hook sphere] step 22500 ok in 20.97 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.0731788916394056, "a90": -0.024099922287143374, "a180": -0.12225786099166783, "b": 0.08219076285095522, "c": 0.051273073880629044, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06625964382246814, "a90": -0.0015222602777786858, "a180": -0.13099702736715757, "b": 0.09395719696951117, "c": 0.058701118249194266, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.06738431658573132, "a90": -0.016104800750891 ...
279
+ [agatha_v2_coco256_agatha_s0] step 22600/100000 loss 1.00753 | tower grad norms 0.0271 0.0219 0.0267 0.0255 0.0244 0.0162 | 0.449 s/step | spent 0:07:46 left 3:51:41
280
+ [agatha_v2_coco256_agatha_s0] step 22700/100000 loss 1.00661 | tower grad norms 0.0294 0.0237 0.0286 0.0285 0.0243 0.0159 | 0.169 s/step | spent 0:08:03 left 3:50:51
281
+ [agatha_v2_coco256_agatha_s0] step 22800/100000 loss 1.00617 | tower grad norms 0.0306 0.0246 0.0278 0.027 0.0241 0.0191 | 0.169 s/step | spent 0:08:20 left 3:50:05
282
+ [agatha_v2_coco256_agatha_s0] step 22900/100000 loss 1.00575 | tower grad norms 0.0263 0.0216 0.0266 0.0261 0.024 0.0153 | 0.168 s/step | spent 0:08:37 left 3:49:19
283
+ [agatha_v2_coco256_agatha_s0] step 23000/100000 loss 1.00717 | tower grad norms 0.0297 0.025 0.0283 0.0294 0.0255 0.0188 | 0.168 s/step | spent 0:08:54 left 3:48:36
284
+ [agatha_v2_coco256_agatha_s0] step 23100/100000 loss 1.00677 | tower grad norms 0.0284 0.0252 0.0276 0.0283 0.025 0.0176 | 0.169 s/step | spent 0:09:11 left 3:47:54
285
+ [agatha_v2_coco256_agatha_s0] step 23200/100000 loss 1.00563 | tower grad norms 0.0252 0.0211 0.0237 0.0234 0.0208 0.0148 | 0.169 s/step | spent 0:09:28 left 3:47:15
286
+ [agatha_v2_coco256_agatha_s0] step 23300/100000 loss 1.00575 | tower grad norms 0.0306 0.0253 0.0313 0.031 0.027 0.0178 | 0.169 s/step | spent 0:09:44 left 3:46:36
287
+ [agatha_v2_coco256_agatha_s0] step 23400/100000 loss 1.00686 | tower grad norms 0.0322 0.0273 0.0288 0.0289 0.0241 0.017 | 0.168 s/step | spent 0:10:01 left 3:45:57
288
+ [agatha_v2_coco256_agatha_s0] step 23500/100000 loss 1.00694 | tower grad norms 0.027 0.023 0.028 0.0285 0.0239 0.0152 | 0.168 s/step | spent 0:10:18 left 3:45:20
289
+ [agatha_v2_coco256_agatha_s0] step 23600/100000 loss 1.00539 | tower grad norms 0.0268 0.022 0.0275 0.0264 0.0221 0.0148 | 0.168 s/step | spent 0:10:35 left 3:44:44
290
+ [agatha_v2_coco256_agatha_s0] step 23700/100000 loss 1.00539 | tower grad norms 0.0295 0.0256 0.0298 0.0288 0.0275 0.018 | 0.169 s/step | spent 0:10:52 left 3:44:10
291
+ [agatha_v2_coco256_agatha_s0] step 23800/100000 loss 1.00665 | tower grad norms 0.0294 0.0236 0.0272 0.0274 0.0246 0.016 | 0.169 s/step | spent 0:11:09 left 3:43:37
292
+ [agatha_v2_coco256_agatha_s0] step 23900/100000 loss 1.00559 | tower grad norms 0.0316 0.0257 0.0316 0.0312 0.0272 0.0182 | 0.168 s/step | spent 0:11:25 left 3:43:04
293
+ [agatha_v2_coco256_agatha_s0] step 24000/100000 loss 1.00560 | tower grad norms 0.0326 0.0281 0.0298 0.0299 0.0273 0.0189 | 0.168 s/step | spent 0:11:42 left 3:42:33
294
+ [agatha_v2_coco256_agatha_s0] step 24100/100000 loss 1.00489 | tower grad norms 0.0339 0.0269 0.0298 0.0291 0.0251 0.0172 | 0.169 s/step | spent 0:11:59 left 3:42:02
295
+ [agatha_v2_coco256_agatha_s0] step 24200/100000 loss 1.00708 | tower grad norms 0.0251 0.0229 0.0257 0.0265 0.0235 0.016 | 0.169 s/step | spent 0:12:16 left 3:41:32
296
+ [agatha_v2_coco256_agatha_s0] step 24300/100000 loss 1.00636 | tower grad norms 0.0306 0.0264 0.0272 0.0263 0.0233 0.0151 | 0.168 s/step | spent 0:12:33 left 3:41:02
297
+ [agatha_v2_coco256_agatha_s0] step 24400/100000 loss 1.00503 | tower grad norms 0.03 0.0247 0.0269 0.0264 0.0221 0.0173 | 0.169 s/step | spent 0:12:50 left 3:40:34
298
+ [agatha_v2_coco256_agatha_s0] step 24500/100000 loss 1.00644 | tower grad norms 0.0296 0.0246 0.0289 0.0292 0.0256 0.017 | 0.169 s/step | spent 0:13:07 left 3:40:06
299
+ [agatha_v2_coco256_agatha_s0] step 24600/100000 loss 1.00654 | tower grad norms 0.0262 0.0217 0.0271 0.0266 0.0239 0.0151 | 0.169 s/step | spent 0:13:23 left 3:39:38
300
+ [agatha_v2_coco256_agatha_s0] step 24700/100000 loss 1.00411 | tower grad norms 0.0277 0.0241 0.0272 0.0274 0.0228 0.0171 | 0.168 s/step | spent 0:13:40 left 3:39:10
301
+ [agatha_v2_coco256_agatha_s0] step 24800/100000 loss 1.00171 | tower grad norms 0.0258 0.0232 0.0289 0.0278 0.0252 0.0162 | 0.169 s/step | spent 0:13:57 left 3:38:43
302
+ [agatha_v2_coco256_agatha_s0] step 24900/100000 loss 1.00500 | tower grad norms 0.0272 0.0237 0.0281 0.0276 0.0243 0.0161 | 0.169 s/step | spent 0:14:14 left 3:38:17
303
+ [agatha_v2_coco256_agatha_s0] step 25000/100000 loss 1.00504 | tower grad norms 0.0257 0.0214 0.0254 0.0255 0.0223 0.0149 | 0.168 s/step | spent 0:14:31 left 3:37:50
304
+ [hook heldout] step 25000 ok in 2.57 s: {"items": 2000, "raw": {"loss": 1.004552861958742, "bands": [1.1538778805305967, 0.9281726643992544, 0.9231390815234812], "loss_shuffled": 1.0142670361548662, "loss_null": 1.0101286323517562, "gap_shuffled": 0.009714174196124076, "gap_shuffled_se": 0.0005222589650733089, "gap_null": 0.005575770393013955, "gap_null_se": 0.00028011582841490677, "gap_shuffled_bands": [0.0002635128281734902, 0.0020793365612427235, 0.03655179639271458]}, "ema": {"loss": 1.0839227839708327, "bands": [1.2364621530382611, 1.0146353994008481, 0.9855617822632381], "loss_shuffled": 1.092250598564744, "loss_null": 1.08682 ...
305
+ [hook sheet] step 25000 ok in 4.49 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step025000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step025000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step025000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step025000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.004955981392413378, "latent_std": 0.9797589778900146, "image_mean" ...
306
+ [hook sphere] step 25000 ok in 21.05 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.08602345083685795, "a90": -0.023177835507274615, "a180": -0.14886906616644127, "b": 0.08265584769185244, "c": 0.05804308991576799, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.08019170866710744, "a90": -0.005276447553900185, "a180": -0.1551069697803147, "b": 0.095500945352775, "c": 0.059290530909033475, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.07752371243954134, "a90": -0.018872163407077563, ...
307
+ [agatha_v2_coco256_agatha_s0] step 25100/100000 loss 1.00555 | tower grad norms 0.025 0.0202 0.0258 0.0254 0.0212 0.015 | 0.450 s/step | spent 0:15:16 left 3:44:17
308
+ [agatha_v2_coco256_agatha_s0] step 25200/100000 loss 1.00563 | tower grad norms 0.0259 0.0215 0.0256 0.0254 0.0235 0.0145 | 0.168 s/step | spent 0:15:33 left 3:43:43
309
+ [agatha_v2_coco256_agatha_s0] step 25300/100000 loss 1.00351 | tower grad norms 0.0298 0.0248 0.0314 0.0323 0.0276 0.0178 | 0.169 s/step | spent 0:15:50 left 3:43:09
310
+ [agatha_v2_coco256_agatha_s0] step 25400/100000 loss 1.00562 | tower grad norms 0.0281 0.0241 0.0288 0.0287 0.0256 0.0176 | 0.169 s/step | spent 0:16:06 left 3:42:37
311
+ [agatha_v2_coco256_agatha_s0] step 25500/100000 loss 1.00393 | tower grad norms 0.0274 0.0245 0.0271 0.0271 0.024 0.0161 | 0.169 s/step | spent 0:16:23 left 3:42:05
312
+ [agatha_v2_coco256_agatha_s0] step 25600/100000 loss 1.00252 | tower grad norms 0.0277 0.0237 0.0295 0.0294 0.025 0.0163 | 0.168 s/step | spent 0:16:40 left 3:41:33
313
+ [agatha_v2_coco256_agatha_s0] step 25700/100000 loss 1.00466 | tower grad norms 0.0281 0.025 0.0271 0.0278 0.0242 0.0162 | 0.168 s/step | spent 0:16:57 left 3:41:01
314
+ [agatha_v2_coco256_agatha_s0] step 25800/100000 loss 1.00361 | tower grad norms 0.0264 0.0217 0.0249 0.0243 0.0216 0.0141 | 0.169 s/step | spent 0:17:14 left 3:40:31
315
+ [agatha_v2_coco256_agatha_s0] step 25900/100000 loss 1.00380 | tower grad norms 0.0257 0.0208 0.0239 0.0247 0.0216 0.0153 | 0.168 s/step | spent 0:17:31 left 3:40:01
316
+ [agatha_v2_coco256_agatha_s0] step 26000/100000 loss 1.00376 | tower grad norms 0.0261 0.0217 0.0249 0.0255 0.0213 0.0145 | 0.168 s/step | spent 0:17:47 left 3:39:31
317
+ [agatha_v2_coco256_agatha_s0] step 26100/100000 loss 1.00355 | tower grad norms 0.025 0.0224 0.026 0.0248 0.0213 0.0147 | 0.169 s/step | spent 0:18:04 left 3:39:01
318
+ [agatha_v2_coco256_agatha_s0] step 26200/100000 loss 1.00378 | tower grad norms 0.0268 0.0227 0.0275 0.0291 0.0246 0.0162 | 0.169 s/step | spent 0:18:21 left 3:38:33
319
+ [agatha_v2_coco256_agatha_s0] step 26300/100000 loss 1.00188 | tower grad norms 0.0278 0.0227 0.025 0.0252 0.0213 0.0153 | 0.169 s/step | spent 0:18:38 left 3:38:04
320
+ [agatha_v2_coco256_agatha_s0] step 26400/100000 loss 1.00380 | tower grad norms 0.0272 0.0221 0.0298 0.0299 0.0246 0.0156 | 0.169 s/step | spent 0:18:55 left 3:37:36
321
+ [agatha_v2_coco256_agatha_s0] step 26500/100000 loss 1.00239 | tower grad norms 0.0254 0.0227 0.0248 0.0255 0.0228 0.015 | 0.169 s/step | spent 0:19:12 left 3:37:09
322
+ [agatha_v2_coco256_agatha_s0] step 26600/100000 loss 1.00301 | tower grad norms 0.0261 0.0213 0.024 0.0245 0.0205 0.0145 | 0.169 s/step | spent 0:19:29 left 3:36:42
323
+ [agatha_v2_coco256_agatha_s0] step 26700/100000 loss 1.00325 | tower grad norms 0.0268 0.0226 0.0254 0.0266 0.0212 0.015 | 0.168 s/step | spent 0:19:45 left 3:36:14
324
+ [agatha_v2_coco256_agatha_s0] step 26800/100000 loss 1.00225 | tower grad norms 0.0269 0.0228 0.0276 0.0271 0.0232 0.0155 | 0.169 s/step | spent 0:20:02 left 3:35:48
325
+ [agatha_v2_coco256_agatha_s0] step 26900/100000 loss 1.00373 | tower grad norms 0.0269 0.0231 0.0281 0.028 0.0234 0.0163 | 0.169 s/step | spent 0:20:19 left 3:35:21
326
+ [agatha_v2_coco256_agatha_s0] step 27000/100000 loss 1.00283 | tower grad norms 0.027 0.0232 0.0259 0.026 0.0222 0.0153 | 0.169 s/step | spent 0:20:36 left 3:34:55
327
+ [agatha_v2_coco256_agatha_s0] step 27100/100000 loss 1.00308 | tower grad norms 0.0289 0.0229 0.0263 0.0252 0.0212 0.0156 | 0.169 s/step | spent 0:20:53 left 3:34:30
328
+ [agatha_v2_coco256_agatha_s0] step 27200/100000 loss 1.00181 | tower grad norms 0.0314 0.0243 0.0301 0.0311 0.0259 0.0183 | 0.169 s/step | spent 0:21:10 left 3:34:04
329
+ [agatha_v2_coco256_agatha_s0] step 27300/100000 loss 1.00369 | tower grad norms 0.0294 0.0223 0.0267 0.0269 0.0211 0.0155 | 0.169 s/step | spent 0:21:27 left 3:33:38
330
+ [agatha_v2_coco256_agatha_s0] step 27400/100000 loss 1.00132 | tower grad norms 0.0271 0.0219 0.0267 0.0278 0.0234 0.0153 | 0.168 s/step | spent 0:21:44 left 3:33:13
331
+ [agatha_v2_coco256_agatha_s0] step 27500/100000 loss 1.00153 | tower grad norms 0.0282 0.0245 0.0252 0.0249 0.0194 0.0148 | 0.168 s/step | spent 0:22:00 left 3:32:47
332
+ [hook heldout] step 27500 ok in 2.57 s: {"items": 2000, "raw": {"loss": 1.0017577249705791, "bands": [1.151489833152805, 0.9247537247872626, 0.9208447337014549], "loss_shuffled": 1.0119334336966277, "loss_null": 1.007362949192524, "gap_shuffled": 0.01017570872604847, "gap_shuffled_se": 0.0005456133121567798, "gap_null": 0.005605224221944809, "gap_null_se": 0.00028216210665405676, "gap_shuffled_bands": [0.0002695484855149642, 0.0021902040101384724, 0.038276728532752886]}, "ema": {"loss": 1.0571803880035877, "bands": [1.2104239784141129, 0.983702232623368, 0.9650972793529747], "loss_shuffled": 1.065899378836155, "loss_null": 1.0613148 ...
333
+ [hook sheet] step 27500 ok in 4.45 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step027500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step027500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step027500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step027500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.0014907550066709518, "latent_std": 0.9782348275184631, "image_mean ...
334
+ [hook sphere] step 27500 ok in 21.05 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.07620633403512192, "a90": -0.022530329566383973, "a180": -0.12988233850385988, "b": 0.08152688236065224, "c": 0.05867853012064703, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.07083574642783205, "a90": 0.001392620800791779, "a180": -0.14306411365645583, "b": 0.0998887952029455, "c": 0.050970987737836, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08386918444727468, "a90": -0.02158232565028352, "a ...
335
+ [agatha_v2_coco256_agatha_s0] step 27600/100000 loss 1.00277 | tower grad norms 0.0248 0.0212 0.0283 0.0283 0.0247 0.0152 | 0.449 s/step | spent 0:22:45 left 3:36:50
336
+ [agatha_v2_coco256_agatha_s0] step 27700/100000 loss 1.00330 | tower grad norms 0.0265 0.0221 0.0268 0.0274 0.0236 0.0152 | 0.169 s/step | spent 0:23:02 left 3:36:22
337
+ [agatha_v2_coco256_agatha_s0] step 27800/100000 loss 1.00250 | tower grad norms 0.0262 0.0214 0.0265 0.0281 0.0232 0.015 | 0.168 s/step | spent 0:23:19 left 3:35:53
338
+ [agatha_v2_coco256_agatha_s0] step 27900/100000 loss 1.00154 | tower grad norms 0.0297 0.026 0.0278 0.0271 0.0227 0.0163 | 0.168 s/step | spent 0:23:36 left 3:35:25
339
+ [agatha_v2_coco256_agatha_s0] step 28000/100000 loss 1.00071 | tower grad norms 0.0272 0.0226 0.027 0.0278 0.0261 0.0166 | 0.168 s/step | spent 0:23:53 left 3:34:57
340
+ [agatha_v2_coco256_agatha_s0] step 28100/100000 loss 1.00085 | tower grad norms 0.0282 0.0264 0.0255 0.0254 0.0214 0.0157 | 0.168 s/step | spent 0:24:09 left 3:34:30
341
+ [agatha_v2_coco256_agatha_s0] step 28200/100000 loss 1.00162 | tower grad norms 0.0293 0.0232 0.0286 0.029 0.0242 0.0159 | 0.169 s/step | spent 0:24:26 left 3:34:03
342
+ [agatha_v2_coco256_agatha_s0] step 28300/100000 loss 1.00283 | tower grad norms 0.0256 0.0223 0.0264 0.0261 0.0238 0.0163 | 0.169 s/step | spent 0:24:43 left 3:33:36
343
+ [agatha_v2_coco256_agatha_s0] step 28400/100000 loss 1.00064 | tower grad norms 0.0259 0.0237 0.0248 0.026 0.0207 0.0151 | 0.169 s/step | spent 0:25:00 left 3:33:09
344
+ [agatha_v2_coco256_agatha_s0] step 28500/100000 loss 1.00184 | tower grad norms 0.0289 0.0245 0.0263 0.0269 0.023 0.0159 | 0.169 s/step | spent 0:25:17 left 3:32:43
345
+ [agatha_v2_coco256_agatha_s0] step 28600/100000 loss 1.00170 | tower grad norms 0.0287 0.0236 0.0285 0.0285 0.0241 0.0165 | 0.169 s/step | spent 0:25:34 left 3:32:17
346
+ [agatha_v2_coco256_agatha_s0] step 28700/100000 loss 1.00003 | tower grad norms 0.0323 0.0256 0.031 0.0307 0.0241 0.0172 | 0.168 s/step | spent 0:25:51 left 3:31:51
347
+ [agatha_v2_coco256_agatha_s0] step 28800/100000 loss 1.00032 | tower grad norms 0.0249 0.0226 0.0249 0.0254 0.0211 0.0151 | 0.168 s/step | spent 0:26:07 left 3:31:25
348
+ [agatha_v2_coco256_agatha_s0] step 28900/100000 loss 1.00026 | tower grad norms 0.0311 0.0257 0.0336 0.0327 0.0295 0.0189 | 0.169 s/step | spent 0:26:24 left 3:30:59
349
+ [agatha_v2_coco256_agatha_s0] step 29000/100000 loss 1.00063 | tower grad norms 0.0305 0.024 0.0276 0.0281 0.0229 0.0154 | 0.169 s/step | spent 0:26:41 left 3:30:34
350
+ [agatha_v2_coco256_agatha_s0] step 29100/100000 loss 1.00138 | tower grad norms 0.0272 0.0215 0.0277 0.0275 0.0219 0.0153 | 0.169 s/step | spent 0:26:58 left 3:30:09
351
+ [agatha_v2_coco256_agatha_s0] step 29200/100000 loss 1.00029 | tower grad norms 0.0258 0.0217 0.0258 0.0244 0.0215 0.0153 | 0.168 s/step | spent 0:27:15 left 3:29:44
352
+ [agatha_v2_coco256_agatha_s0] step 29300/100000 loss 0.99934 | tower grad norms 0.0232 0.0209 0.0244 0.0236 0.0215 0.0147 | 0.169 s/step | spent 0:27:32 left 3:29:19
353
+ [agatha_v2_coco256_agatha_s0] step 29400/100000 loss 1.00024 | tower grad norms 0.0277 0.0231 0.0276 0.0281 0.0244 0.0164 | 0.169 s/step | spent 0:27:48 left 3:28:55
354
+ [agatha_v2_coco256_agatha_s0] step 29500/100000 loss 1.00033 | tower grad norms 0.0261 0.0223 0.0252 0.0262 0.0212 0.0148 | 0.169 s/step | spent 0:28:05 left 3:28:30
355
+ [agatha_v2_coco256_agatha_s0] step 29600/100000 loss 1.00077 | tower grad norms 0.0256 0.0239 0.026 0.0268 0.0225 0.0148 | 0.169 s/step | spent 0:28:22 left 3:28:06
356
+ [agatha_v2_coco256_agatha_s0] step 29700/100000 loss 1.00327 | tower grad norms 0.0285 0.0233 0.0283 0.0274 0.0229 0.0168 | 0.169 s/step | spent 0:28:39 left 3:27:42
357
+ [agatha_v2_coco256_agatha_s0] step 29800/100000 loss 0.99999 | tower grad norms 0.0266 0.0228 0.0265 0.0269 0.0212 0.0155 | 0.168 s/step | spent 0:28:56 left 3:27:18
358
+ [agatha_v2_coco256_agatha_s0] step 29900/100000 loss 0.99954 | tower grad norms 0.0275 0.0214 0.0266 0.0256 0.0218 0.0147 | 0.168 s/step | spent 0:29:13 left 3:26:54
359
+ [agatha_v2_coco256_agatha_s0] step 30000/100000 loss 0.99953 | tower grad norms 0.0257 0.0213 0.0243 0.0244 0.0213 0.0147 | 0.169 s/step | spent 0:29:30 left 3:26:30
360
+ [hook heldout] step 30000 ok in 2.56 s: {"items": 2000, "raw": {"loss": 0.9996160053312778, "bands": [1.1493058688381215, 0.9221301953199464, 0.9196015561633768], "loss_shuffled": 1.009975941747427, "loss_null": 1.0054592868536711, "gap_shuffled": 0.01035993641614914, "gap_shuffled_se": 0.0005547178997835947, "gap_null": 0.005843281522393226, "gap_null_se": 0.00029273774294579896, "gap_shuffled_bands": [0.00027125014926849937, 0.0021576854463736285, 0.0390997913877982]}, "ema": {"loss": 1.0385947472453116, "bands": [1.191536692016599, 0.963071284848882, 0.9505014903410713], "loss_shuffled": 1.047655609920621, "loss_null": 1.04341449 ...
361
+ [hook sheet] step 30000 ok in 4.47 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step030000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step030000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step030000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step030000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.0011366000398993492, "latent_std": 0.9832133054733276, "image_mean ...
362
+ [hook sphere] step 30000 ok in 21.07 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.07079328611354818, "a90": -0.020600830988065488, "a180": -0.12098574123903083, "b": 0.08222325025046327, "c": 0.06416632828547979, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06946606586208988, "a90": -0.0005268782714830844, "a180": -0.13840525345269666, "b": 0.09546428739744542, "c": 0.04999929975306608, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08642655431180128, "a90": -0.0242439393559126 ...
363
+ [hook fid] step 30000 ok in 174.22 s: {"prompts": 2000, "raw_g1": {"fid": 90.30696242636509, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.060568449314432976, "kid_std": 0.0025767595289995777, "subset_size_used": 1000, "subsets": 100}, "raw_g3": {"fid": 47.0912924556431, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.00927644288925148, "kid_std": 0.0006208131746383627, "subset_size_used": 1000, "subsets": 100}, "ema_g1": {"fid": 136.59572721472585, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.06710842996259431, "kid_std": 0.0015796655427474657, "subset_size_used": 1000, "subsets": 100}, "ema_g3": {"fid": 95.6313821168215, "n_re ...
364
+ [hook clip] step 30000 ok in 95.65 s: {"items": 500, "raw_g1": {"clip_score": {"mean": 17.760412688279953, "se": 0.16020246063270716, "n": 500}, "grounding": {"mean": 0.04090906849503714, "se": 0.001925396452777906, "n": 500, "own_cos": 0.5885852056733812, "shuffled_cos": 0.5476761371783441}}, "raw_g3": {"clip_score": {"mean": 21.1570587074097, "se": 0.16709688876306378, "n": 500}, "grounding": {"mean": 0.09991515301389113, "se": 0.0033494479278777752, "n": 500, "own_cos": 0.6430092262693128, "shuffled_cos": 0.5430940732554216}}, "ema_g1": {"clip_score": {"mean": 15.899601040568712, "se": 0.15882704632861847, "n": 500}, "grounding ...
365
+ [hook checkpoint] step 30000 ok in 8.5 s: {"file": "agatha_v2_coco256_agatha_s0_state_step030000.pt", "bytes": 672811376, "pushed": true}
366
+ [agatha_v2_coco256_agatha_s0] step 30100/100000 loss 0.99843 | tower grad norms 0.0303 0.0269 0.0259 0.0266 0.0216 0.0161 | 3.233 s/step | spent 0:34:53 left 4:01:28
367
+ [agatha_v2_coco256_agatha_s0] step 30200/100000 loss 1.00058 | tower grad norms 0.026 0.0222 0.0244 0.0253 0.0204 0.014 | 0.168 s/step | spent 0:35:10 left 4:00:40
368
+ [agatha_v2_coco256_agatha_s0] step 30300/100000 loss 0.99994 | tower grad norms 0.0273 0.0215 0.0243 0.0256 0.0209 0.0145 | 0.168 s/step | spent 0:35:27 left 3:59:54
369
+ [agatha_v2_coco256_agatha_s0] step 30400/100000 loss 1.00069 | tower grad norms 0.0248 0.0212 0.026 0.0266 0.0222 0.0143 | 0.168 s/step | spent 0:35:43 left 3:59:07
370
+ [agatha_v2_coco256_agatha_s0] step 30500/100000 loss 1.00070 | tower grad norms 0.025 0.0223 0.0258 0.0261 0.0212 0.0136 | 0.168 s/step | spent 0:36:00 left 3:58:22
371
+ [agatha_v2_coco256_agatha_s0] step 30600/100000 loss 0.99988 | tower grad norms 0.0266 0.022 0.0238 0.0262 0.0224 0.0152 | 0.168 s/step | spent 0:36:17 left 3:57:37
372
+ [agatha_v2_coco256_agatha_s0] step 30700/100000 loss 0.99941 | tower grad norms 0.0268 0.022 0.024 0.0245 0.0193 0.0143 | 0.169 s/step | spent 0:36:34 left 3:56:52
373
+ [agatha_v2_coco256_agatha_s0] step 30800/100000 loss 0.99791 | tower grad norms 0.0282 0.0241 0.0284 0.029 0.024 0.0166 | 0.168 s/step | spent 0:36:51 left 3:56:08
374
+ [agatha_v2_coco256_agatha_s0] step 30900/100000 loss 0.99905 | tower grad norms 0.0296 0.0229 0.0276 0.0277 0.022 0.0157 | 0.168 s/step | spent 0:37:08 left 3:55:24
375
+ [agatha_v2_coco256_agatha_s0] step 31000/100000 loss 0.99881 | tower grad norms 0.0286 0.0238 0.028 0.0289 0.0247 0.0162 | 0.168 s/step | spent 0:37:24 left 3:54:41
376
+ [agatha_v2_coco256_agatha_s0] step 31100/100000 loss 0.99732 | tower grad norms 0.0275 0.0243 0.0237 0.0244 0.0212 0.0145 | 0.168 s/step | spent 0:37:41 left 3:53:59
377
+ [agatha_v2_coco256_agatha_s0] step 31200/100000 loss 0.99700 | tower grad norms 0.0265 0.0221 0.026 0.0266 0.023 0.0158 | 0.169 s/step | spent 0:37:58 left 3:53:16
378
+ [agatha_v2_coco256_agatha_s0] step 31300/100000 loss 0.99950 | tower grad norms 0.0298 0.0227 0.0255 0.0249 0.0219 0.0142 | 0.168 s/step | spent 0:38:15 left 3:52:35
379
+ [agatha_v2_coco256_agatha_s0] step 31400/100000 loss 1.00045 | tower grad norms 0.0279 0.0224 0.0229 0.0239 0.0199 0.0137 | 0.169 s/step | spent 0:38:32 left 3:51:54
380
+ [agatha_v2_coco256_agatha_s0] step 31500/100000 loss 0.99987 | tower grad norms 0.0281 0.0246 0.0255 0.0264 0.0219 0.0149 | 0.168 s/step | spent 0:38:49 left 3:51:13
381
+ [agatha_v2_coco256_agatha_s0] step 31600/100000 loss 0.99771 | tower grad norms 0.0259 0.0215 0.0273 0.0269 0.0217 0.0147 | 0.168 s/step | spent 0:39:05 left 3:50:32
382
+ [agatha_v2_coco256_agatha_s0] step 31700/100000 loss 0.99752 | tower grad norms 0.0272 0.0238 0.027 0.0277 0.023 0.0158 | 0.168 s/step | spent 0:39:22 left 3:49:52
383
+ [agatha_v2_coco256_agatha_s0] step 31800/100000 loss 0.99728 | tower grad norms 0.0269 0.023 0.0283 0.0277 0.0228 0.0154 | 0.168 s/step | spent 0:39:39 left 3:49:12
384
+ [agatha_v2_coco256_agatha_s0] step 31900/100000 loss 0.99860 | tower grad norms 0.031 0.0254 0.0283 0.0292 0.0246 0.0176 | 0.168 s/step | spent 0:39:56 left 3:48:33
385
+ [agatha_v2_coco256_agatha_s0] step 32000/100000 loss 0.99888 | tower grad norms 0.0272 0.024 0.0251 0.0262 0.0212 0.0149 | 0.168 s/step | spent 0:40:13 left 3:47:54
386
+ [agatha_v2_coco256_agatha_s0] step 32100/100000 loss 0.99824 | tower grad norms 0.0288 0.0211 0.0256 0.0269 0.0225 0.0151 | 0.168 s/step | spent 0:40:29 left 3:47:16
387
+ [agatha_v2_coco256_agatha_s0] step 32200/100000 loss 0.99853 | tower grad norms 0.0287 0.0257 0.0268 0.028 0.0244 0.0157 | 0.168 s/step | spent 0:40:46 left 3:46:37
388
+ [agatha_v2_coco256_agatha_s0] step 32300/100000 loss 0.99936 | tower grad norms 0.0256 0.0228 0.0247 0.0253 0.0199 0.0148 | 0.168 s/step | spent 0:41:03 left 3:45:59
389
+ [agatha_v2_coco256_agatha_s0] step 32400/100000 loss 0.99806 | tower grad norms 0.0256 0.0203 0.0234 0.0242 0.0211 0.0134 | 0.168 s/step | spent 0:41:20 left 3:45:22
390
+ [agatha_v2_coco256_agatha_s0] step 32500/100000 loss 0.99685 | tower grad norms 0.0252 0.0197 0.0266 0.0254 0.0207 0.0134 | 0.168 s/step | spent 0:41:37 left 3:44:45
391
+ [hook heldout] step 32500 ok in 2.57 s: {"items": 2000, "raw": {"loss": 0.9975015075951814, "bands": [1.1474800473956697, 0.9195169849844909, 0.9179401970597068], "loss_shuffled": 1.0076155796796082, "loss_null": 1.003554979994893, "gap_shuffled": 0.01011407208442688, "gap_shuffled_se": 0.0005400395155606212, "gap_null": 0.006053472399711609, "gap_null_se": 0.0003082463124078024, "gap_shuffled_bands": [0.00026943839523980084, 0.002157590238192071, 0.03807634011702275]}, "ema": {"loss": 1.025573652163148, "bands": [1.1782356464647725, 0.948682230553197, 0.9402611229120063], "loss_shuffled": 1.0349219785034656, "loss_null": 1.03080392 ...
392
+ [hook sheet] step 32500 ok in 4.45 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step032500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step032500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step032500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step032500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": 0.0023221690207719803, "latent_std": 0.991806149482727, "image_mean": ...
393
+ [hook sphere] step 32500 ok in 20.96 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.06580480914007873, "a90": -0.022498309787040018, "a180": -0.10911130849311744, "b": 0.08358832300092879, "c": 0.03854672490892364, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.044288626279376984, "a90": 0.004034000865685403, "a180": -0.09261125342443935, "b": 0.11169561622091974, "c": 0.036431421315312976, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08734294284974566, "a90": -0.0258012018916122 ...
394
+ [agatha_v2_coco256_agatha_s0] step 32600/100000 loss 0.99753 | tower grad norms 0.0261 0.0213 0.0241 0.0244 0.0198 0.014 | 0.448 s/step | spent 0:42:22 left 3:46:38
395
+ [agatha_v2_coco256_agatha_s0] step 32700/100000 loss 0.99953 | tower grad norms 0.0271 0.0235 0.0262 0.0265 0.0212 0.0148 | 0.168 s/step | spent 0:42:38 left 3:46:00
396
+ [agatha_v2_coco256_agatha_s0] step 32800/100000 loss 0.99917 | tower grad norms 0.0263 0.0213 0.0247 0.025 0.0221 0.0142 | 0.168 s/step | spent 0:42:55 left 3:45:22
397
+ [agatha_v2_coco256_agatha_s0] step 32900/100000 loss 0.99810 | tower grad norms 0.0289 0.0238 0.0279 0.0284 0.0232 0.0163 | 0.168 s/step | spent 0:43:12 left 3:44:45
398
+ [agatha_v2_coco256_agatha_s0] step 33000/100000 loss 0.99531 | tower grad norms 0.0321 0.0276 0.0241 0.0253 0.02 0.0136 | 0.168 s/step | spent 0:43:29 left 3:44:08
399
+ [agatha_v2_coco256_agatha_s0] step 33100/100000 loss 0.99791 | tower grad norms 0.025 0.0215 0.0252 0.0248 0.0212 0.0142 | 0.169 s/step | spent 0:43:46 left 3:43:31
400
+ [agatha_v2_coco256_agatha_s0] step 33200/100000 loss 0.99613 | tower grad norms 0.0236 0.0206 0.0246 0.0248 0.0201 0.0138 | 0.169 s/step | spent 0:44:03 left 3:42:55
401
+ [agatha_v2_coco256_agatha_s0] step 33300/100000 loss 0.99717 | tower grad norms 0.0279 0.0234 0.0253 0.027 0.0214 0.0149 | 0.168 s/step | spent 0:44:19 left 3:42:19
402
+ [agatha_v2_coco256_agatha_s0] step 33400/100000 loss 0.99726 | tower grad norms 0.0278 0.0225 0.0259 0.0276 0.0222 0.0149 | 0.168 s/step | spent 0:44:36 left 3:41:44
403
+ [agatha_v2_coco256_agatha_s0] step 33500/100000 loss 0.99677 | tower grad norms 0.0291 0.0234 0.0262 0.0269 0.0204 0.0149 | 0.169 s/step | spent 0:44:53 left 3:41:08
404
+ [agatha_v2_coco256_agatha_s0] step 33600/100000 loss 0.99731 | tower grad norms 0.0279 0.0229 0.0271 0.0282 0.022 0.0155 | 0.169 s/step | spent 0:45:10 left 3:40:33
405
+ [agatha_v2_coco256_agatha_s0] step 33700/100000 loss 0.99737 | tower grad norms 0.0299 0.0255 0.0281 0.0304 0.0245 0.0169 | 0.168 s/step | spent 0:45:27 left 3:39:58
406
+ [agatha_v2_coco256_agatha_s0] step 33800/100000 loss 0.99629 | tower grad norms 0.0262 0.021 0.0229 0.024 0.0194 0.0131 | 0.169 s/step | spent 0:45:44 left 3:39:24
407
+ [agatha_v2_coco256_agatha_s0] step 33900/100000 loss 0.99626 | tower grad norms 0.0244 0.0202 0.0261 0.0253 0.0215 0.014 | 0.169 s/step | spent 0:46:01 left 3:38:50
408
+ [agatha_v2_coco256_agatha_s0] step 34000/100000 loss 0.99763 | tower grad norms 0.0304 0.0245 0.0269 0.0267 0.0235 0.0145 | 0.168 s/step | spent 0:46:17 left 3:38:15
409
+ [agatha_v2_coco256_agatha_s0] step 34100/100000 loss 0.99430 | tower grad norms 0.0248 0.0203 0.0246 0.0245 0.0201 0.0132 | 0.168 s/step | spent 0:46:34 left 3:37:42
410
+ [agatha_v2_coco256_agatha_s0] step 34200/100000 loss 0.99678 | tower grad norms 0.0286 0.0233 0.0263 0.027 0.0209 0.015 | 0.168 s/step | spent 0:46:51 left 3:37:08
411
+ [agatha_v2_coco256_agatha_s0] step 34300/100000 loss 0.99640 | tower grad norms 0.0277 0.0227 0.0273 0.0279 0.0228 0.0151 | 0.168 s/step | spent 0:47:08 left 3:36:35
412
+ [agatha_v2_coco256_agatha_s0] step 34400/100000 loss 0.99538 | tower grad norms 0.0275 0.0229 0.026 0.0261 0.0215 0.0152 | 0.168 s/step | spent 0:47:25 left 3:36:01
413
+ [agatha_v2_coco256_agatha_s0] step 34500/100000 loss 0.99805 | tower grad norms 0.0289 0.0236 0.0268 0.0275 0.022 0.0153 | 0.168 s/step | spent 0:47:42 left 3:35:28
414
+ [agatha_v2_coco256_agatha_s0] step 34600/100000 loss 0.99696 | tower grad norms 0.0223 0.0189 0.0236 0.0234 0.0206 0.0129 | 0.168 s/step | spent 0:47:58 left 3:34:56
415
+ [agatha_v2_coco256_agatha_s0] step 34700/100000 loss 0.99852 | tower grad norms 0.03 0.026 0.0241 0.0256 0.0212 0.0151 | 0.168 s/step | spent 0:48:15 left 3:34:23
416
+ [agatha_v2_coco256_agatha_s0] step 34800/100000 loss 0.99857 | tower grad norms 0.0248 0.0216 0.0253 0.0266 0.0232 0.0148 | 0.169 s/step | spent 0:48:32 left 3:33:51
417
+ [agatha_v2_coco256_agatha_s0] step 34900/100000 loss 0.99596 | tower grad norms 0.0274 0.0236 0.0288 0.0291 0.0239 0.0156 | 0.168 s/step | spent 0:48:49 left 3:33:19
418
+ [agatha_v2_coco256_agatha_s0] step 35000/100000 loss 0.99475 | tower grad norms 0.0294 0.0277 0.025 0.0268 0.0209 0.0152 | 0.168 s/step | spent 0:49:06 left 3:32:47
419
+ [hook heldout] step 35000 ok in 2.57 s: {"items": 2000, "raw": {"loss": 0.9956515621542931, "bands": [1.1454308504192658, 0.9172393213517642, 0.9171199919894699], "loss_shuffled": 1.0060662276297807, "loss_null": 1.001427468791604, "gap_shuffled": 0.010414665475487708, "gap_shuffled_se": 0.0005577217452162206, "gap_null": 0.005775906637310982, "gap_null_se": 0.00029608500500502783, "gap_shuffled_bands": [0.0002640030081347063, 0.002162439641366138, 0.039330355623569686]}, "ema": {"loss": 1.015946264013648, "bands": [1.1681886324927278, 0.9382327910847469, 0.9326654610713062], "loss_shuffled": 1.0255437313616276, "loss_null": 1.02140 ...
420
+ [hook sheet] step 35000 ok in 4.48 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step035000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step035000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step035000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step035000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": 0.003093262203037739, "latent_std": 0.9774841666221619, "image_mean": ...
421
+ [hook sphere] step 35000 ok in 21.02 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.060476037785298434, "a90": -0.024037617580166808, "a180": -0.09691445799043007, "b": 0.08273980681789037, "c": 0.03159741401271493, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.04550947245014339, "a90": 0.0018946954234521016, "a180": -0.09291364032373889, "b": 0.10357542769968983, "c": 0.043266705050904825, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08675035688891934, "a90": -0.026591999729594 ...
422
+ [agatha_v2_coco256_agatha_s0] step 35100/100000 loss 0.99510 | tower grad norms 0.0275 0.0227 0.0267 0.028 0.0218 0.0153 | 0.449 s/step | spent 0:49:51 left 3:34:16
423
+ [agatha_v2_coco256_agatha_s0] step 35200/100000 loss 0.99573 | tower grad norms 0.0296 0.0246 0.0279 0.0294 0.0228 0.015 | 0.169 s/step | spent 0:50:08 left 3:33:43
424
+ [agatha_v2_coco256_agatha_s0] step 35300/100000 loss 0.99659 | tower grad norms 0.0274 0.022 0.0286 0.028 0.0218 0.015 | 0.168 s/step | spent 0:50:24 left 3:33:11
425
+ [agatha_v2_coco256_agatha_s0] step 35400/100000 loss 0.99609 | tower grad norms 0.0252 0.0199 0.0253 0.0252 0.0223 0.0131 | 0.168 s/step | spent 0:50:41 left 3:32:39
426
+ [agatha_v2_coco256_agatha_s0] step 35500/100000 loss 0.99676 | tower grad norms 0.0273 0.0234 0.0262 0.0265 0.0205 0.0148 | 0.169 s/step | spent 0:50:58 left 3:32:07
427
+ [agatha_v2_coco256_agatha_s0] step 35600/100000 loss 0.99720 | tower grad norms 0.0298 0.0235 0.0256 0.026 0.0201 0.015 | 0.168 s/step | spent 0:51:15 left 3:31:35
428
+ [agatha_v2_coco256_agatha_s0] step 35700/100000 loss 0.99562 | tower grad norms 0.028 0.0238 0.0285 0.0284 0.023 0.0146 | 0.168 s/step | spent 0:51:32 left 3:31:04
429
+ [agatha_v2_coco256_agatha_s0] step 35800/100000 loss 0.99495 | tower grad norms 0.0256 0.0221 0.0266 0.0259 0.0211 0.0148 | 0.168 s/step | spent 0:51:49 left 3:30:32
430
+ [agatha_v2_coco256_agatha_s0] step 35900/100000 loss 0.99659 | tower grad norms 0.0265 0.0201 0.0262 0.0272 0.02 0.0136 | 0.168 s/step | spent 0:52:05 left 3:30:01
431
+ [agatha_v2_coco256_agatha_s0] step 36000/100000 loss 0.99500 | tower grad norms 0.0262 0.0219 0.0249 0.026 0.0201 0.0141 | 0.168 s/step | spent 0:52:22 left 3:29:30
432
+ [agatha_v2_coco256_agatha_s0] step 36100/100000 loss 0.99534 | tower grad norms 0.0253 0.0214 0.0251 0.0253 0.0209 0.0142 | 0.168 s/step | spent 0:52:39 left 3:28:59
433
+ [agatha_v2_coco256_agatha_s0] step 36200/100000 loss 0.99625 | tower grad norms 0.0259 0.0222 0.025 0.0256 0.0207 0.0141 | 0.168 s/step | spent 0:52:56 left 3:28:28
434
+ [agatha_v2_coco256_agatha_s0] step 36300/100000 loss 0.99522 | tower grad norms 0.0282 0.0214 0.0232 0.0237 0.0188 0.0131 | 0.168 s/step | spent 0:53:13 left 3:27:58
435
+ [agatha_v2_coco256_agatha_s0] step 36400/100000 loss 0.99652 | tower grad norms 0.0281 0.0245 0.025 0.0254 0.0215 0.0146 | 0.168 s/step | spent 0:53:29 left 3:27:28
436
+ [agatha_v2_coco256_agatha_s0] step 36500/100000 loss 0.99461 | tower grad norms 0.0282 0.0234 0.025 0.0255 0.0207 0.0138 | 0.169 s/step | spent 0:53:46 left 3:26:58
437
+ [agatha_v2_coco256_agatha_s0] step 36600/100000 loss 0.99476 | tower grad norms 0.0276 0.0231 0.0251 0.0256 0.0194 0.0143 | 0.168 s/step | spent 0:54:03 left 3:26:28
438
+ [agatha_v2_coco256_agatha_s0] step 36700/100000 loss 0.99417 | tower grad norms 0.0248 0.0208 0.0242 0.0245 0.0204 0.0134 | 0.168 s/step | spent 0:54:20 left 3:25:58
439
+ [agatha_v2_coco256_agatha_s0] step 36800/100000 loss 0.99522 | tower grad norms 0.0265 0.0221 0.0247 0.0251 0.0199 0.0135 | 0.169 s/step | spent 0:54:37 left 3:25:29
440
+ [agatha_v2_coco256_agatha_s0] step 36900/100000 loss 0.99522 | tower grad norms 0.0242 0.0198 0.0229 0.0238 0.0201 0.0133 | 0.169 s/step | spent 0:54:54 left 3:24:59
441
+ [agatha_v2_coco256_agatha_s0] step 37000/100000 loss 0.99463 | tower grad norms 0.0279 0.0222 0.0292 0.03 0.024 0.0159 | 0.168 s/step | spent 0:55:11 left 3:24:30
442
+ [agatha_v2_coco256_agatha_s0] step 37100/100000 loss 0.99549 | tower grad norms 0.0253 0.0202 0.0242 0.0257 0.0196 0.0134 | 0.168 s/step | spent 0:55:27 left 3:24:01
443
+ [agatha_v2_coco256_agatha_s0] step 37200/100000 loss 0.99473 | tower grad norms 0.0296 0.0252 0.0268 0.0268 0.0208 0.0155 | 0.168 s/step | spent 0:55:44 left 3:23:32
444
+ [agatha_v2_coco256_agatha_s0] step 37300/100000 loss 0.99356 | tower grad norms 0.0263 0.0219 0.0254 0.027 0.0207 0.0139 | 0.169 s/step | spent 0:56:01 left 3:23:03
445
+ [agatha_v2_coco256_agatha_s0] step 37400/100000 loss 0.99372 | tower grad norms 0.0264 0.0221 0.0253 0.026 0.021 0.0143 | 0.168 s/step | spent 0:56:18 left 3:22:34
446
+ [agatha_v2_coco256_agatha_s0] step 37500/100000 loss 0.99516 | tower grad norms 0.0262 0.0216 0.0255 0.0268 0.0209 0.0139 | 0.168 s/step | spent 0:56:35 left 3:22:05
447
+ [hook heldout] step 37500 ok in 2.57 s: {"items": 2000, "raw": {"loss": 0.9942010100483895, "bands": [1.1440137335759293, 0.9153982110246731, 0.9163006992833538], "loss_shuffled": 1.004725433319807, "loss_null": 1.0001862834692001, "gap_shuffled": 0.010524423271417617, "gap_shuffled_se": 0.0005651169151879088, "gap_null": 0.0059852734208107, "gap_null_se": 0.0002984633362382758, "gap_shuffled_bands": [0.0002506544454249726, 0.0021477612055645977, 0.03983315511080236]}, "ema": {"loss": 1.009167422696948, "bands": [1.1608672620118499, 0.9312522847508846, 0.9270157528599009], "loss_shuffled": 1.018981810644269, "loss_null": 1.014767670 ...
448
+ [hook sheet] step 37500 ok in 4.43 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step037500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step037500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step037500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step037500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": 0.001031022984534502, "latent_std": 0.9829272031784058, "image_mean": ...
449
+ [hook sphere] step 37500 ok in 21.0 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.06794602329134479, "a90": -0.018840361784659327, "a180": -0.11705168479803027, "b": 0.07979309069075083, "c": 0.040602054928910225, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06306782912251169, "a90": 0.0013693086932178582, "a180": -0.1275049669382412, "b": 0.09072673151083707, "c": 0.030337566565647563, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08576330649604855, "a90": -0.0267742966267505 ...
450
+ [agatha_v2_coco256_agatha_s0] step 37600/100000 loss 0.99468 | tower grad norms 0.0277 0.0233 0.0253 0.0268 0.0192 0.0141 | 0.448 s/step | spent 0:57:19 left 3:23:16
451
+ [agatha_v2_coco256_agatha_s0] step 37700/100000 loss 0.99436 | tower grad norms 0.0259 0.0204 0.0227 0.0234 0.0183 0.0128 | 0.168 s/step | spent 0:57:36 left 3:22:47
452
+ [agatha_v2_coco256_agatha_s0] step 37800/100000 loss 0.99609 | tower grad norms 0.0247 0.0201 0.024 0.0238 0.0191 0.0129 | 0.168 s/step | spent 0:57:53 left 3:22:18
453
+ [agatha_v2_coco256_agatha_s0] step 37900/100000 loss 0.99464 | tower grad norms 0.0262 0.0236 0.0278 0.0277 0.0232 0.0149 | 0.168 s/step | spent 0:58:10 left 3:21:49
454
+ [agatha_v2_coco256_agatha_s0] step 38000/100000 loss 0.99377 | tower grad norms 0.0266 0.0224 0.0238 0.0248 0.0196 0.0145 | 0.168 s/step | spent 0:58:27 left 3:21:20
455
+ [agatha_v2_coco256_agatha_s0] step 38100/100000 loss 0.99316 | tower grad norms 0.0272 0.0208 0.027 0.029 0.0231 0.015 | 0.168 s/step | spent 0:58:44 left 3:20:51
456
+ [agatha_v2_coco256_agatha_s0] step 38200/100000 loss 0.99370 | tower grad norms 0.0282 0.0221 0.0256 0.0275 0.0197 0.0145 | 0.168 s/step | spent 0:59:00 left 3:20:23
457
+ [agatha_v2_coco256_agatha_s0] step 38300/100000 loss 0.99377 | tower grad norms 0.027 0.0218 0.0235 0.0248 0.0187 0.0142 | 0.168 s/step | spent 0:59:17 left 3:19:55
458
+ [agatha_v2_coco256_agatha_s0] step 38400/100000 loss 0.99406 | tower grad norms 0.0243 0.0201 0.0221 0.0238 0.0196 0.0133 | 0.168 s/step | spent 0:59:34 left 3:19:26
459
+ [agatha_v2_coco256_agatha_s0] step 38500/100000 loss 0.99579 | tower grad norms 0.0316 0.0265 0.0264 0.0273 0.0214 0.0145 | 0.168 s/step | spent 0:59:51 left 3:18:58
460
+ [agatha_v2_coco256_agatha_s0] step 38600/100000 loss 0.99295 | tower grad norms 0.028 0.0224 0.0272 0.0285 0.0231 0.015 | 0.168 s/step | spent 1:00:08 left 3:18:30
461
+ [agatha_v2_coco256_agatha_s0] step 38700/100000 loss 0.99405 | tower grad norms 0.0264 0.0222 0.0266 0.0273 0.0219 0.0142 | 0.168 s/step | spent 1:00:24 left 3:18:03
462
+ [agatha_v2_coco256_agatha_s0] step 38800/100000 loss 0.99403 | tower grad norms 0.0265 0.0223 0.0267 0.0269 0.0219 0.0146 | 0.168 s/step | spent 1:00:41 left 3:17:35
463
+ [agatha_v2_coco256_agatha_s0] step 38900/100000 loss 0.99443 | tower grad norms 0.0262 0.0216 0.0248 0.0246 0.0197 0.0136 | 0.169 s/step | spent 1:00:58 left 3:17:07
464
+ [agatha_v2_coco256_agatha_s0] step 39000/100000 loss 0.99499 | tower grad norms 0.0277 0.0222 0.024 0.0248 0.0205 0.0132 | 0.168 s/step | spent 1:01:15 left 3:16:40
465
+ [agatha_v2_coco256_agatha_s0] step 39100/100000 loss 0.99279 | tower grad norms 0.0311 0.0233 0.0282 0.029 0.023 0.0164 | 0.168 s/step | spent 1:01:32 left 3:16:13
466
+ [agatha_v2_coco256_agatha_s0] step 39200/100000 loss 0.99315 | tower grad norms 0.0277 0.0225 0.025 0.0259 0.0204 0.0141 | 0.168 s/step | spent 1:01:49 left 3:15:45
467
+ [agatha_v2_coco256_agatha_s0] step 39300/100000 loss 0.99274 | tower grad norms 0.0269 0.0223 0.0251 0.0258 0.021 0.0149 | 0.169 s/step | spent 1:02:06 left 3:15:18
468
+ [agatha_v2_coco256_agatha_s0] step 39400/100000 loss 0.99346 | tower grad norms 0.0291 0.0239 0.0259 0.0274 0.0205 0.0151 | 0.169 s/step | spent 1:02:22 left 3:14:51
469
+ [agatha_v2_coco256_agatha_s0] step 39500/100000 loss 0.99352 | tower grad norms 0.0247 0.0207 0.024 0.0249 0.02 0.0132 | 0.168 s/step | spent 1:02:39 left 3:14:24
470
+ [agatha_v2_coco256_agatha_s0] step 39600/100000 loss 0.99331 | tower grad norms 0.0281 0.0221 0.0242 0.0258 0.0197 0.0137 | 0.168 s/step | spent 1:02:56 left 3:13:58
471
+ [agatha_v2_coco256_agatha_s0] step 39700/100000 loss 0.99417 | tower grad norms 0.0262 0.0218 0.0281 0.0275 0.0228 0.015 | 0.168 s/step | spent 1:03:13 left 3:13:31
472
+ [agatha_v2_coco256_agatha_s0] step 39800/100000 loss 0.99144 | tower grad norms 0.0283 0.023 0.0263 0.0274 0.0209 0.0145 | 0.168 s/step | spent 1:03:30 left 3:13:04
473
+ [agatha_v2_coco256_agatha_s0] step 39900/100000 loss 0.99343 | tower grad norms 0.0246 0.0204 0.0231 0.024 0.0192 0.0126 | 0.168 s/step | spent 1:03:47 left 3:12:38
474
+ [agatha_v2_coco256_agatha_s0] step 40000/100000 loss 0.99277 | tower grad norms 0.0248 0.0209 0.0239 0.0239 0.0187 0.0126 | 0.168 s/step | spent 1:04:03 left 3:12:11
475
+ [hook heldout] step 40000 ok in 2.58 s: {"items": 2000, "raw": {"loss": 0.9925916863083839, "bands": [1.1424173048386788, 0.9134369175449736, 0.9152849953407941], "loss_shuffled": 1.003037084132433, "loss_null": 0.9985187782049179, "gap_shuffled": 0.010445397824048996, "gap_shuffled_se": 0.0005539167998577369, "gap_null": 0.005927091896533966, "gap_null_se": 0.0002992679605988222, "gap_shuffled_bands": [0.00027397493154700947, 0.0022200467666541796, 0.039344135920759894]}, "ema": {"loss": 1.0038266747891902, "bands": [1.1551798546049183, 0.9254693086562198, 0.922941478455642], "loss_shuffled": 1.013815280035138, "loss_null": 1.00953 ...
476
+ [hook sheet] step 40000 ok in 4.47 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step040000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step040000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step040000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step040000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.007130899466574192, "latent_std": 0.973584771156311, "image_mean": ...
477
+ [hook sphere] step 40000 ok in 20.98 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.0648783933289942, "a90": -0.02249444046130115, "a180": -0.10726234619668724, "b": 0.08591765323630368, "c": 0.05141972163743776, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.05455865466514041, "a90": 0.0019451205844071007, "a180": -0.11106242991468793, "b": 0.10335067652049498, "c": 0.05342073874080179, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08414000296796657, "a90": -0.026578322766527412, ...
478
+ [hook fid] step 40000 ok in 173.1 s: {"prompts": 2000, "raw_g1": {"fid": 85.90093593973478, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.056421703432409306, "kid_std": 0.0025616237564751215, "subset_size_used": 1000, "subsets": 100}, "raw_g3": {"fid": 45.75301377308705, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.008849132026788586, "kid_std": 0.0006797347510201995, "subset_size_used": 1000, "subsets": 100}, "ema_g1": {"fid": 95.08436080868574, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.05139846026891051, "kid_std": 0.0019575498037017578, "subset_size_used": 1000, "subsets": 100}, "ema_g3": {"fid": 57.72401214174988, "n_ ...
479
+ [hook clip] step 40000 ok in 93.18 s: {"items": 500, "raw_g1": {"clip_score": {"mean": 18.082584813004544, "se": 0.161266706530065, "n": 500}, "grounding": {"mean": 0.04462400122201957, "se": 0.002021391788343266, "n": 500, "own_cos": 0.5946540468069335, "shuffled_cos": 0.550030045584914}}, "raw_g3": {"clip_score": {"mean": 21.6563236357106, "se": 0.17475593629041294, "n": 500}, "grounding": {"mean": 0.10544687142847742, "se": 0.003358959098872691, "n": 500, "own_cos": 0.6485122166407505, "shuffled_cos": 0.5430653452122731}}, "ema_g1": {"clip_score": {"mean": 17.282634243569355, "se": 0.16650190507187712, "n": 500}, "grounding": { ...
480
+ [hook checkpoint] step 40000 ok in 8.03 s: {"file": "agatha_v2_coco256_agatha_s0_state_step040000.pt", "bytes": 672811376, "pushed": true}
481
+ [agatha_v2_coco256_agatha_s0] step 40100/100000 loss 0.99241 | tower grad norms 0.0264 0.0234 0.0268 0.0278 0.0215 0.0147 | 3.192 s/step | spent 1:09:23 left 3:26:46
482
+ [agatha_v2_coco256_agatha_s0] step 40200/100000 loss 0.99103 | tower grad norms 0.0243 0.02 0.0239 0.0251 0.0193 0.0129 | 0.168 s/step | spent 1:09:39 left 3:26:14
483
+ [agatha_v2_coco256_agatha_s0] step 40300/100000 loss 0.99439 | tower grad norms 0.0321 0.0243 0.0259 0.0263 0.0219 0.0145 | 0.168 s/step | spent 1:09:56 left 3:25:42
484
+ [agatha_v2_coco256_agatha_s0] step 40400/100000 loss 0.99295 | tower grad norms 0.0258 0.0213 0.0238 0.0248 0.0181 0.0129 | 0.168 s/step | spent 1:10:13 left 3:25:10
485
+ [agatha_v2_coco256_agatha_s0] step 40500/100000 loss 0.99374 | tower grad norms 0.0256 0.0217 0.0243 0.0241 0.0192 0.0134 | 0.168 s/step | spent 1:10:30 left 3:24:38
486
+ [agatha_v2_coco256_agatha_s0] step 40600/100000 loss 0.99173 | tower grad norms 0.0312 0.0245 0.0268 0.0273 0.0222 0.0156 | 0.168 s/step | spent 1:10:47 left 3:24:06
487
+ [agatha_v2_coco256_agatha_s0] step 40700/100000 loss 0.99128 | tower grad norms 0.0282 0.0221 0.0234 0.0256 0.0194 0.0137 | 0.168 s/step | spent 1:11:03 left 3:23:35
488
+ [agatha_v2_coco256_agatha_s0] step 40800/100000 loss 0.99159 | tower grad norms 0.0281 0.0221 0.0246 0.026 0.0219 0.0136 | 0.168 s/step | spent 1:11:20 left 3:23:03
489
+ [agatha_v2_coco256_agatha_s0] step 40900/100000 loss 0.99386 | tower grad norms 0.025 0.0203 0.0253 0.0244 0.0185 0.0132 | 0.168 s/step | spent 1:11:37 left 3:22:32
490
+ [agatha_v2_coco256_agatha_s0] step 41000/100000 loss 0.99172 | tower grad norms 0.0284 0.0226 0.0257 0.0275 0.0212 0.015 | 0.168 s/step | spent 1:11:54 left 3:22:01
491
+ [agatha_v2_coco256_agatha_s0] step 41100/100000 loss 0.99294 | tower grad norms 0.0269 0.0214 0.026 0.0263 0.0216 0.0144 | 0.168 s/step | spent 1:12:11 left 3:21:30
492
+ [agatha_v2_coco256_agatha_s0] step 41200/100000 loss 0.99285 | tower grad norms 0.031 0.0253 0.0299 0.03 0.0233 0.0164 | 0.168 s/step | spent 1:12:28 left 3:20:59
493
+ [agatha_v2_coco256_agatha_s0] step 41300/100000 loss 0.99106 | tower grad norms 0.0275 0.0214 0.0259 0.0253 0.0213 0.0144 | 0.168 s/step | spent 1:12:44 left 3:20:29
494
+ [agatha_v2_coco256_agatha_s0] step 41400/100000 loss 0.99041 | tower grad norms 0.0313 0.0248 0.026 0.0273 0.0214 0.0147 | 0.168 s/step | spent 1:13:01 left 3:19:58
495
+ [agatha_v2_coco256_agatha_s0] step 41500/100000 loss 0.99056 | tower grad norms 0.0269 0.0213 0.0251 0.0266 0.0207 0.0133 | 0.169 s/step | spent 1:13:18 left 3:19:28
496
+ [agatha_v2_coco256_agatha_s0] step 41600/100000 loss 0.99106 | tower grad norms 0.0259 0.0203 0.0239 0.0246 0.0187 0.0131 | 0.168 s/step | spent 1:13:35 left 3:18:58
497
+ [agatha_v2_coco256_agatha_s0] step 41700/100000 loss 0.99196 | tower grad norms 0.0255 0.0211 0.0236 0.0236 0.0191 0.0121 | 0.169 s/step | spent 1:13:52 left 3:18:27
498
+ [agatha_v2_coco256_agatha_s0] step 41800/100000 loss 0.99235 | tower grad norms 0.0257 0.0211 0.0232 0.0248 0.0189 0.0127 | 0.168 s/step | spent 1:14:09 left 3:17:57
499
+ [agatha_v2_coco256_agatha_s0] step 41900/100000 loss 0.99230 | tower grad norms 0.0259 0.0202 0.0236 0.0249 0.0187 0.0132 | 0.168 s/step | spent 1:14:25 left 3:17:28
500
+ [agatha_v2_coco256_agatha_s0] step 42000/100000 loss 0.99201 | tower grad norms 0.0259 0.0209 0.0229 0.0239 0.019 0.0126 | 0.168 s/step | spent 1:14:42 left 3:16:58
501
+ [agatha_v2_coco256_agatha_s0] step 42100/100000 loss 0.99279 | tower grad norms 0.0252 0.0208 0.0243 0.0252 0.0194 0.0135 | 0.169 s/step | spent 1:14:59 left 3:16:28
502
+ [agatha_v2_coco256_agatha_s0] step 42200/100000 loss 0.99231 | tower grad norms 0.0264 0.0219 0.0261 0.0262 0.0212 0.0146 | 0.169 s/step | spent 1:15:16 left 3:15:59
503
+ [agatha_v2_coco256_agatha_s0] step 42300/100000 loss 0.99165 | tower grad norms 0.0264 0.0215 0.0239 0.0241 0.019 0.0131 | 0.169 s/step | spent 1:15:33 left 3:15:29
504
+ [agatha_v2_coco256_agatha_s0] step 42400/100000 loss 0.99274 | tower grad norms 0.0259 0.0217 0.024 0.0248 0.0195 0.0135 | 0.168 s/step | spent 1:15:50 left 3:15:00
505
+ [agatha_v2_coco256_agatha_s0] step 42500/100000 loss 0.99178 | tower grad norms 0.0291 0.0253 0.0257 0.0258 0.0198 0.0149 | 0.168 s/step | spent 1:16:07 left 3:14:31
506
+ [hook heldout] step 42500 ok in 2.57 s: {"items": 2000, "raw": {"loss": 0.9911648867577314, "bands": [1.1410497310278922, 0.911815516137545, 0.9141116577696009], "loss_shuffled": 1.0017949780523776, "loss_null": 0.9970391308814287, "gap_shuffled": 0.010630091294646263, "gap_shuffled_se": 0.0005753054371152865, "gap_null": 0.0058742441236972805, "gap_null_se": 0.00030298051852427305, "gap_shuffled_bands": [0.000255623377782189, 0.002193079371186511, 0.040188260701091806]}, "ema": {"loss": 0.9996603723317384, "bands": [1.1508516768696908, 0.9209136360448174, 0.9196845973270974], "loss_shuffled": 1.0097980411350727, "loss_null": 1.0054 ...
507
+ [hook sheet] step 42500 ok in 4.48 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step042500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step042500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step042500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step042500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.0044710938818752766, "latent_std": 0.9776895046234131, "image_mean ...
508
+ [hook sphere] step 42500 ok in 20.94 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.07331778478452466, "a90": -0.021781301047365313, "a180": -0.124854268521684, "b": 0.08295564770867445, "c": 0.055614956780567916, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06029725976674423, "a90": 0.004497490069866652, "a180": -0.12509200960335512, "b": 0.09830378971700432, "c": 0.03886903658260276, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08311633560568824, "a90": -0.026590533998343477, ...
509
+ [agatha_v2_coco256_agatha_s0] step 42600/100000 loss 0.99311 | tower grad norms 0.0275 0.0227 0.0239 0.0248 0.0185 0.0144 | 0.448 s/step | spent 1:16:51 left 3:15:13
510
+ [agatha_v2_coco256_agatha_s0] step 42700/100000 loss 0.99123 | tower grad norms 0.0273 0.0234 0.025 0.0252 0.0189 0.0124 | 0.168 s/step | spent 1:17:08 left 3:14:43
511
+ [agatha_v2_coco256_agatha_s0] step 42800/100000 loss 0.99026 | tower grad norms 0.0257 0.0215 0.0242 0.0243 0.0192 0.0124 | 0.168 s/step | spent 1:17:25 left 3:14:14
512
+ [agatha_v2_coco256_agatha_s0] step 42900/100000 loss 0.99085 | tower grad norms 0.0262 0.0211 0.0248 0.0265 0.0196 0.0132 | 0.168 s/step | spent 1:17:42 left 3:13:45
513
+ [agatha_v2_coco256_agatha_s0] step 43000/100000 loss 0.98990 | tower grad norms 0.0251 0.0203 0.0249 0.026 0.0184 0.0134 | 0.169 s/step | spent 1:17:59 left 3:13:16
514
+ [agatha_v2_coco256_agatha_s0] step 43100/100000 loss 0.99080 | tower grad norms 0.0247 0.0192 0.0239 0.0249 0.0198 0.013 | 0.169 s/step | spent 1:18:16 left 3:12:47
515
+ [agatha_v2_coco256_agatha_s0] step 43200/100000 loss 0.99298 | tower grad norms 0.0247 0.0199 0.0225 0.0237 0.0183 0.0116 | 0.169 s/step | spent 1:18:32 left 3:12:18
516
+ [agatha_v2_coco256_agatha_s0] step 43300/100000 loss 0.99085 | tower grad norms 0.0283 0.0238 0.0269 0.0274 0.0217 0.0153 | 0.169 s/step | spent 1:18:49 left 3:11:49
517
+ [agatha_v2_coco256_agatha_s0] step 43400/100000 loss 0.99296 | tower grad norms 0.0249 0.0205 0.0245 0.025 0.0189 0.0135 | 0.169 s/step | spent 1:19:06 left 3:11:21
518
+ [agatha_v2_coco256_agatha_s0] step 43500/100000 loss 0.99035 | tower grad norms 0.0276 0.0227 0.0251 0.0263 0.0207 0.0137 | 0.169 s/step | spent 1:19:23 left 3:10:52
519
+ [agatha_v2_coco256_agatha_s0] step 43600/100000 loss 0.99194 | tower grad norms 0.027 0.0223 0.0265 0.0272 0.0206 0.0154 | 0.168 s/step | spent 1:19:40 left 3:10:24
520
+ [agatha_v2_coco256_agatha_s0] step 43700/100000 loss 0.98988 | tower grad norms 0.0274 0.0235 0.0253 0.0266 0.0197 0.0138 | 0.168 s/step | spent 1:19:57 left 3:09:55
521
+ [agatha_v2_coco256_agatha_s0] step 43800/100000 loss 0.99188 | tower grad norms 0.0292 0.0233 0.0247 0.0257 0.0192 0.0139 | 0.168 s/step | spent 1:20:14 left 3:09:27
522
+ [agatha_v2_coco256_agatha_s0] step 43900/100000 loss 0.98993 | tower grad norms 0.0257 0.0212 0.026 0.0263 0.0211 0.0138 | 0.168 s/step | spent 1:20:30 left 3:08:59
523
+ [agatha_v2_coco256_agatha_s0] step 44000/100000 loss 0.99021 | tower grad norms 0.0276 0.0238 0.0268 0.0285 0.0214 0.0149 | 0.168 s/step | spent 1:20:47 left 3:08:31
524
+ [agatha_v2_coco256_agatha_s0] step 44100/100000 loss 0.99040 | tower grad norms 0.0258 0.0217 0.0243 0.0248 0.0202 0.0131 | 0.168 s/step | spent 1:21:04 left 3:08:03
525
+ [agatha_v2_coco256_agatha_s0] step 44200/100000 loss 0.99065 | tower grad norms 0.0264 0.0219 0.0248 0.0258 0.0188 0.0134 | 0.168 s/step | spent 1:21:21 left 3:07:35
526
+ [agatha_v2_coco256_agatha_s0] step 44300/100000 loss 0.98749 | tower grad norms 0.0301 0.0246 0.0274 0.0279 0.0204 0.0147 | 0.168 s/step | spent 1:21:38 left 3:07:07
527
+ [agatha_v2_coco256_agatha_s0] step 44400/100000 loss 0.98925 | tower grad norms 0.026 0.02 0.0242 0.0256 0.0199 0.0132 | 0.168 s/step | spent 1:21:54 left 3:06:39
528
+ [agatha_v2_coco256_agatha_s0] step 44500/100000 loss 0.99051 | tower grad norms 0.0254 0.0211 0.0235 0.0257 0.0189 0.0134 | 0.168 s/step | spent 1:22:11 left 3:06:12
529
+ [agatha_v2_coco256_agatha_s0] step 44600/100000 loss 0.99155 | tower grad norms 0.0264 0.0219 0.0239 0.0247 0.0196 0.014 | 0.168 s/step | spent 1:22:28 left 3:05:44
530
+ [agatha_v2_coco256_agatha_s0] step 44700/100000 loss 0.98968 | tower grad norms 0.0238 0.0191 0.0258 0.0268 0.0198 0.013 | 0.168 s/step | spent 1:22:45 left 3:05:17
531
+ [agatha_v2_coco256_agatha_s0] step 44800/100000 loss 0.98966 | tower grad norms 0.0274 0.0214 0.0261 0.0268 0.0194 0.014 | 0.168 s/step | spent 1:23:02 left 3:04:49
532
+ [agatha_v2_coco256_agatha_s0] step 44900/100000 loss 0.98956 | tower grad norms 0.0269 0.0207 0.0258 0.0265 0.0208 0.0135 | 0.168 s/step | spent 1:23:19 left 3:04:22
533
+ [agatha_v2_coco256_agatha_s0] step 45000/100000 loss 0.98988 | tower grad norms 0.0291 0.0231 0.0232 0.0253 0.0191 0.0132 | 0.168 s/step | spent 1:23:35 left 3:03:55
534
+ [hook heldout] step 45000 ok in 2.57 s: {"items": 2000, "raw": {"loss": 0.9900695872157812, "bands": [1.1399282413097909, 0.9104416586714078, 0.9135383840235362], "loss_shuffled": 1.000771230816841, "loss_null": 0.995885888710618, "gap_shuffled": 0.010701643601059914, "gap_shuffled_se": 0.0005764172648094155, "gap_null": 0.005816301494836807, "gap_null_se": 0.0003015290774555751, "gap_shuffled_bands": [0.00027466111795146846, 0.0022092913388780736, 0.040431404057629204]}, "ema": {"loss": 0.9963440653681755, "bands": [1.147343615149537, 0.9173767356837835, 0.91702705743346], "loss_shuffled": 1.006629373818636, "loss_null": 1.00215704 ...
535
+ [hook sheet] step 45000 ok in 4.53 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step045000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step045000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step045000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step045000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.008647803217172623, "latent_std": 0.9684465527534485, "image_mean" ...
536
+ [hook sphere] step 45000 ok in 20.98 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.0759486923642054, "a90": -0.02316786586263217, "a180": -0.12872951886577866, "b": 0.08243427488191366, "c": 0.05310502136096665, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06576589311812805, "a90": 0.0010510538031149909, "a180": -0.1325828400393711, "b": 0.09928025979390448, "c": 0.06275022074410612, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08168759212433618, "a90": -0.025976030446909673, ...
537
+ [agatha_v2_coco256_agatha_s0] step 45100/100000 loss 0.99099 | tower grad norms 0.0236 0.019 0.0241 0.0245 0.0195 0.0129 | 0.449 s/step | spent 1:24:20 left 3:04:29
538
+ [agatha_v2_coco256_agatha_s0] step 45200/100000 loss 0.99027 | tower grad norms 0.0289 0.0241 0.0275 0.0282 0.0211 0.015 | 0.168 s/step | spent 1:24:37 left 3:04:01
539
+ [agatha_v2_coco256_agatha_s0] step 45300/100000 loss 0.99024 | tower grad norms 0.028 0.0243 0.0255 0.0251 0.0192 0.0142 | 0.168 s/step | spent 1:24:54 left 3:03:34
540
+ [agatha_v2_coco256_agatha_s0] step 45400/100000 loss 0.98934 | tower grad norms 0.0234 0.0195 0.0244 0.0241 0.0188 0.0125 | 0.168 s/step | spent 1:25:11 left 3:03:07
541
+ [agatha_v2_coco256_agatha_s0] step 45500/100000 loss 0.98890 | tower grad norms 0.0248 0.0201 0.0252 0.0255 0.0193 0.0129 | 0.168 s/step | spent 1:25:28 left 3:02:40
542
+ [agatha_v2_coco256_agatha_s0] step 45600/100000 loss 0.98912 | tower grad norms 0.0254 0.0196 0.0241 0.0247 0.0192 0.0124 | 0.168 s/step | spent 1:25:44 left 3:02:13
543
+ [agatha_v2_coco256_agatha_s0] step 45700/100000 loss 0.98988 | tower grad norms 0.0247 0.0201 0.0243 0.0242 0.0193 0.0131 | 0.168 s/step | spent 1:26:01 left 3:01:46
544
+ [agatha_v2_coco256_agatha_s0] step 45800/100000 loss 0.98911 | tower grad norms 0.0274 0.0223 0.0251 0.0255 0.0211 0.0141 | 0.168 s/step | spent 1:26:18 left 3:01:19
545
+ [agatha_v2_coco256_agatha_s0] step 45900/100000 loss 0.98980 | tower grad norms 0.0275 0.0217 0.0251 0.0261 0.0196 0.0128 | 0.168 s/step | spent 1:26:35 left 3:00:52
546
+ [agatha_v2_coco256_agatha_s0] step 46000/100000 loss 0.98977 | tower grad norms 0.025 0.0205 0.0255 0.0255 0.0191 0.014 | 0.168 s/step | spent 1:26:52 left 3:00:25
547
+ [agatha_v2_coco256_agatha_s0] step 46100/100000 loss 0.98938 | tower grad norms 0.0251 0.0202 0.0248 0.0263 0.0197 0.013 | 0.168 s/step | spent 1:27:09 left 2:59:59
548
+ [agatha_v2_coco256_agatha_s0] step 46200/100000 loss 0.98797 | tower grad norms 0.0282 0.0227 0.0282 0.0284 0.023 0.0156 | 0.168 s/step | spent 1:27:26 left 2:59:32
549
+ [agatha_v2_coco256_agatha_s0] step 46300/100000 loss 0.99108 | tower grad norms 0.0248 0.0209 0.0221 0.0233 0.0171 0.0124 | 0.168 s/step | spent 1:27:42 left 2:59:05
550
+ [agatha_v2_coco256_agatha_s0] step 46400/100000 loss 0.98918 | tower grad norms 0.0272 0.0219 0.0256 0.0271 0.0189 0.0136 | 0.168 s/step | spent 1:27:59 left 2:58:39
551
+ [agatha_v2_coco256_agatha_s0] step 46500/100000 loss 0.98886 | tower grad norms 0.0242 0.0198 0.0224 0.0227 0.0173 0.0127 | 0.168 s/step | spent 1:28:16 left 2:58:13
552
+ [agatha_v2_coco256_agatha_s0] step 46600/100000 loss 0.98961 | tower grad norms 0.0296 0.0245 0.0251 0.0256 0.0208 0.0147 | 0.168 s/step | spent 1:28:33 left 2:57:46
553
+ [agatha_v2_coco256_agatha_s0] step 46700/100000 loss 0.98990 | tower grad norms 0.0257 0.0194 0.0239 0.0251 0.0183 0.0127 | 0.168 s/step | spent 1:28:50 left 2:57:20
554
+ [agatha_v2_coco256_agatha_s0] step 46800/100000 loss 0.98999 | tower grad norms 0.0255 0.0208 0.0246 0.0254 0.0202 0.0129 | 0.168 s/step | spent 1:29:07 left 2:56:54
555
+ [agatha_v2_coco256_agatha_s0] step 46900/100000 loss 0.98883 | tower grad norms 0.0238 0.0203 0.0224 0.023 0.0178 0.0121 | 0.168 s/step | spent 1:29:23 left 2:56:28
556
+ [agatha_v2_coco256_agatha_s0] step 47000/100000 loss 0.99017 | tower grad norms 0.0248 0.0209 0.0229 0.0233 0.0188 0.0131 | 0.168 s/step | spent 1:29:40 left 2:56:02
557
+ [agatha_v2_coco256_agatha_s0] step 47100/100000 loss 0.98979 | tower grad norms 0.0283 0.0225 0.0265 0.027 0.0199 0.0143 | 0.168 s/step | spent 1:29:57 left 2:55:36
558
+ [agatha_v2_coco256_agatha_s0] step 47200/100000 loss 0.98950 | tower grad norms 0.0251 0.02 0.0246 0.0252 0.0193 0.0125 | 0.168 s/step | spent 1:30:14 left 2:55:10
559
+ [agatha_v2_coco256_agatha_s0] step 47300/100000 loss 0.98762 | tower grad norms 0.0278 0.0219 0.0243 0.0258 0.0196 0.0133 | 0.168 s/step | spent 1:30:31 left 2:54:44
560
+ [agatha_v2_coco256_agatha_s0] step 47400/100000 loss 0.98952 | tower grad norms 0.0278 0.0205 0.0232 0.0242 0.0174 0.0126 | 0.168 s/step | spent 1:30:48 left 2:54:18
561
+ [agatha_v2_coco256_agatha_s0] step 47500/100000 loss 0.98939 | tower grad norms 0.0273 0.0231 0.0266 0.0275 0.0205 0.0149 | 0.169 s/step | spent 1:31:04 left 2:53:52
562
+ [hook heldout] step 47500 ok in 2.57 s: {"items": 2000, "raw": {"loss": 0.9887935122400522, "bands": [1.1385441364020927, 0.9089453825241609, 0.9128002670776602], "loss_shuffled": 0.9994015937447548, "loss_null": 0.9947267057299614, "gap_shuffled": 0.010608081504702568, "gap_shuffled_se": 0.0005690346072220111, "gap_null": 0.005933193489909172, "gap_null_se": 0.0003065871097589053, "gap_shuffled_bands": [0.0002461266916215229, 0.0021753917793172306, 0.040140780462962586]}, "ema": {"loss": 0.9936771188527346, "bands": [1.14459997374616, 0.9144557761773826, 0.9149119131445796], "loss_shuffled": 1.0040820005238056, "loss_null": 0.99953 ...
563
+ [hook sheet] step 47500 ok in 4.48 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step047500_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step047500_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step047500_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step047500_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.003589596599340439, "latent_std": 0.9748448729515076, "image_mean" ...
564
+ [hook sphere] step 47500 ok in 20.95 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.07624964919861611, "a90": -0.02357392866454802, "a180": -0.12892536973268423, "b": 0.08266127163259065, "c": 0.057493868625526225, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.059190900851421965, "a90": 0.0026234583214433185, "a180": -0.12100526002428723, "b": 0.10021933415127762, "c": 0.053495141538335955, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.0803718049956828, "a90": -0.0257164138722759 ...
565
+ [agatha_v2_coco256_agatha_s0] step 47600/100000 loss 0.98813 | tower grad norms 0.026 0.0209 0.0257 0.0272 0.0199 0.0138 | 0.449 s/step | spent 1:31:49 left 2:54:20
566
+ [agatha_v2_coco256_agatha_s0] step 47700/100000 loss 0.98821 | tower grad norms 0.0255 0.0207 0.0252 0.026 0.0185 0.0126 | 0.169 s/step | spent 1:32:06 left 2:53:54
567
+ [agatha_v2_coco256_agatha_s0] step 47800/100000 loss 0.98880 | tower grad norms 0.028 0.0228 0.0261 0.0273 0.0207 0.0136 | 0.168 s/step | spent 1:32:23 left 2:53:28
568
+ [agatha_v2_coco256_agatha_s0] step 47900/100000 loss 0.98859 | tower grad norms 0.0278 0.0228 0.0249 0.0249 0.0194 0.0136 | 0.169 s/step | spent 1:32:40 left 2:53:03
569
+ [agatha_v2_coco256_agatha_s0] step 48000/100000 loss 0.98832 | tower grad norms 0.028 0.0217 0.0258 0.0268 0.0202 0.0129 | 0.169 s/step | spent 1:32:57 left 2:52:37
570
+ [agatha_v2_coco256_agatha_s0] step 48100/100000 loss 0.98796 | tower grad norms 0.0269 0.0225 0.0248 0.025 0.0186 0.0131 | 0.168 s/step | spent 1:33:13 left 2:52:11
571
+ [agatha_v2_coco256_agatha_s0] step 48200/100000 loss 0.98963 | tower grad norms 0.0255 0.0205 0.0256 0.0252 0.0194 0.0125 | 0.168 s/step | spent 1:33:30 left 2:51:46
572
+ [agatha_v2_coco256_agatha_s0] step 48300/100000 loss 0.98681 | tower grad norms 0.0254 0.0204 0.0237 0.0236 0.0176 0.0124 | 0.168 s/step | spent 1:33:47 left 2:51:20
573
+ [agatha_v2_coco256_agatha_s0] step 48400/100000 loss 0.98830 | tower grad norms 0.0278 0.0215 0.0246 0.0268 0.0182 0.0129 | 0.168 s/step | spent 1:34:04 left 2:50:55
574
+ [agatha_v2_coco256_agatha_s0] step 48500/100000 loss 0.99103 | tower grad norms 0.0258 0.021 0.0233 0.0241 0.0183 0.0132 | 0.168 s/step | spent 1:34:21 left 2:50:29
575
+ [agatha_v2_coco256_agatha_s0] step 48600/100000 loss 0.98822 | tower grad norms 0.0263 0.02 0.0232 0.025 0.0182 0.0125 | 0.168 s/step | spent 1:34:38 left 2:50:04
576
+ [agatha_v2_coco256_agatha_s0] step 48700/100000 loss 0.98976 | tower grad norms 0.0287 0.0222 0.0229 0.0235 0.0167 0.0125 | 0.168 s/step | spent 1:34:54 left 2:49:39
577
+ [agatha_v2_coco256_agatha_s0] step 48800/100000 loss 0.98815 | tower grad norms 0.0243 0.0203 0.0221 0.0231 0.017 0.012 | 0.168 s/step | spent 1:35:11 left 2:49:14
578
+ [agatha_v2_coco256_agatha_s0] step 48900/100000 loss 0.98683 | tower grad norms 0.0254 0.022 0.0232 0.0238 0.0176 0.0125 | 0.168 s/step | spent 1:35:28 left 2:48:49
579
+ [agatha_v2_coco256_agatha_s0] step 49000/100000 loss 0.98974 | tower grad norms 0.0247 0.0207 0.0246 0.0239 0.0188 0.0121 | 0.168 s/step | spent 1:35:45 left 2:48:23
580
+ [agatha_v2_coco256_agatha_s0] step 49100/100000 loss 0.98772 | tower grad norms 0.0249 0.0215 0.0231 0.0237 0.0168 0.0118 | 0.169 s/step | spent 1:36:02 left 2:47:58
581
+ [agatha_v2_coco256_agatha_s0] step 49200/100000 loss 0.99005 | tower grad norms 0.027 0.022 0.0253 0.0259 0.018 0.0133 | 0.168 s/step | spent 1:36:19 left 2:47:34
582
+ [agatha_v2_coco256_agatha_s0] step 49300/100000 loss 0.98999 | tower grad norms 0.0284 0.023 0.025 0.0265 0.0187 0.0148 | 0.169 s/step | spent 1:36:35 left 2:47:09
583
+ [agatha_v2_coco256_agatha_s0] step 49400/100000 loss 0.98777 | tower grad norms 0.0251 0.0201 0.0258 0.0257 0.0189 0.0129 | 0.168 s/step | spent 1:36:52 left 2:46:44
584
+ [agatha_v2_coco256_agatha_s0] step 49500/100000 loss 0.98665 | tower grad norms 0.0244 0.0207 0.0225 0.0234 0.017 0.0116 | 0.168 s/step | spent 1:37:09 left 2:46:19
585
+ [agatha_v2_coco256_agatha_s0] step 49600/100000 loss 0.98747 | tower grad norms 0.026 0.0206 0.0244 0.0259 0.0188 0.0126 | 0.168 s/step | spent 1:37:26 left 2:45:54
586
+ [agatha_v2_coco256_agatha_s0] step 49700/100000 loss 0.99072 | tower grad norms 0.0296 0.0233 0.0251 0.027 0.0193 0.0132 | 0.168 s/step | spent 1:37:43 left 2:45:29
587
+ [agatha_v2_coco256_agatha_s0] step 49800/100000 loss 0.98863 | tower grad norms 0.0313 0.0241 0.0259 0.027 0.0197 0.0144 | 0.168 s/step | spent 1:38:00 left 2:45:05
588
+ [agatha_v2_coco256_agatha_s0] step 49900/100000 loss 0.98824 | tower grad norms 0.0276 0.0209 0.0247 0.0265 0.019 0.0131 | 0.168 s/step | spent 1:38:16 left 2:44:40
589
+ [agatha_v2_coco256_agatha_s0] step 50000/100000 loss 0.98684 | tower grad norms 0.0272 0.0237 0.0258 0.0262 0.019 0.0134 | 0.168 s/step | spent 1:38:33 left 2:44:16
590
+ [hook heldout] step 50000 ok in 2.57 s: {"items": 2000, "raw": {"loss": 0.9876734229773283, "bands": [1.1375199083025993, 0.9074680035238283, 0.9121640113932399], "loss_shuffled": 0.998096381559968, "loss_null": 0.9936989120543003, "gap_shuffled": 0.010422958582639695, "gap_shuffled_se": 0.0005623147763035434, "gap_null": 0.006025489076972008, "gap_null_se": 0.00030951230248076766, "gap_shuffled_bands": [0.00026430255935556236, 0.0021782747403356535, 0.03933700123613604]}, "ema": {"loss": 0.9915075074881315, "bands": [1.1423221589775987, 0.9120700816664888, 0.9132733469209854], "loss_shuffled": 1.0019971336424351, "loss_null": 0.997 ...
591
+ [hook sheet] step 50000 ok in 4.53 s: {"prompts": 8, "seeds": [0, 1, 2, 3], "files": ["/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step050000_raw_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step050000_raw_g3.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step050000_ema_g1.png", "/content/coco256/runs/agatha_v2_coco256_agatha_s0/sheets/agatha_v2_coco256_agatha_s0_step050000_ema_g3.png"], "raw_g1": {"finite": true, "latent_mean": -0.005326401442289352, "latent_std": 0.9730930328369141, "image_mean" ...
592
+ [hook sphere] step 50000 ok in 20.94 s: {"samples": 256, "thresholds": {"a": 0.14978502342341612, "b": 0.3048599402288705}, "raw_g1": {"n": 256, "size": 256, "a": -0.07757875936154576, "a90": -0.019779807734990815, "a180": -0.13537771098810075, "b": 0.08613481194668048, "c": 0.05589262942411118, "c_pairs": 32640, "sphere_like": false}, "raw_g3": {"n": 256, "size": 256, "a": -0.06364668598291538, "a90": -0.003272238170361669, "a180": -0.12402113379546911, "b": 0.09817541246483291, "c": 0.05484722803662785, "c_pairs": 32640, "sphere_like": false}, "ema_g1": {"n": 256, "size": 256, "a": -0.08048647691735841, "a90": -0.02561708593489507 ...
593
+ [hook fid] step 50000 ok in 172.5 s: {"prompts": 2000, "raw_g1": {"fid": 78.38395796824767, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.04673764649934381, "kid_std": 0.002182370195363323, "subset_size_used": 1000, "subsets": 100}, "raw_g3": {"fid": 43.406524504769834, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.007405634777095038, "kid_std": 0.00047614576406396393, "subset_size_used": 1000, "subsets": 100}, "ema_g1": {"fid": 82.24183137085606, "n_real": 40504, "n_fake": 2000, "kid_mean": 0.0480243107028998, "kid_std": 0.0022158037502084975, "subset_size_used": 1000, "subsets": 100}, "ema_g3": {"fid": 46.46724147557586, "n_r ...
594
+ [hook clip] step 50000 ok in 93.06 s: {"items": 500, "raw_g1": {"clip_score": {"mean": 18.194331548616525, "se": 0.16202108310482533, "n": 500}, "grounding": {"mean": 0.0475769885760916, "se": 0.0019883150388673243, "n": 500, "own_cos": 0.5994609751705561, "shuffled_cos": 0.5518839865944645}}, "raw_g3": {"clip_score": {"mean": 21.81861600245198, "se": 0.17558844295985107, "n": 500}, "grounding": {"mean": 0.10984932104188458, "se": 0.0034055360238698803, "n": 500, "own_cos": 0.6572034583605368, "shuffled_cos": 0.5473541373186522}}, "ema_g1": {"clip_score": {"mean": 17.91791611400637, "se": 0.16553803799783726, "n": 500}, "grounding ...
595
+ [hook checkpoint] step 50000 ok in 7.96 s: {"file": "agatha_v2_coco256_agatha_s0_state_step050000.pt", "bytes": 672811376, "pushed": true}