eval_results.json

{
  "ckpt": "/kaggle/working/out/stack_k1_60m/best.pt",
  "arch": "stack",
  "K_train": 1,
  "k_max": 12,
  "val_tokens": 1539584,
  "params": 9962496,
  "uniform_ce": 9.704060527839234,
  "unigram_ce": 7.547556945473469,
  "best_k": 1,
  "gain_first_loop": 1.012203666907749,
  "gain_extra_loops": 0.0,
  "mean_rel_step": 0.28043451470633346,
  "verdict": "TRAINED UNLOOPED (K=1): the k>1 rows show how a model trained at one application behaves when looped anyway, not whether looping pays.",
  "rows": [
    {
      "k": 0,
      "ce": 5.408761565218267,
      "ppl": 223.35480596749395,
      "bpb": 1.8913016483504717,
      "delta_ce": null,
      "flops_per_token": 18259968
    },
    {
      "k": 1,
      "ce": 4.396557898310518,
      "ppl": 81.17098845673196,
      "bpb": 1.5373606508401174,
      "delta_ce": 1.012203666907749,
      "flops_per_token": 21098496,
      "std_ratio": 0.9512100219726562,
      "erank": 172.63084411621094,
      "step_cos": 0.7777038216590881,
      "rel_step": 0.8602083921432495,
      "state_norm": 15.174788475036621
    },
    {
      "k": 2,
      "ce": 4.5994899281640365,
      "ppl": 99.4335844337849,
      "bpb": 1.6083206437954694,
      "delta_ce": -0.20293202985351844,
      "flops_per_token": 23937024,
      "std_ratio": 0.9564117789268494,
      "erank": 152.0776138305664,
      "step_cos": 0.9423013627529144,
      "rel_step": 0.5801940560340881,
      "state_norm": 21.41031837463379
    },
    {
      "k": 3,
      "ce": 4.941250652976713,
      "ppl": 139.94516299387544,
      "bpb": 1.727825379655199,
      "delta_ce": -0.3417607248126764,
      "flops_per_token": 26775552,
      "std_ratio": 0.9550213515758514,
      "erank": 126.67342376708984,
      "step_cos": 0.9775733947753906,
      "rel_step": 0.3683442771434784,
      "state_norm": 27.632020950317383
    },
    {
      "k": 4,
      "ce": 5.265907488037353,
      "ppl": 193.6219386774754,
      "bpb": 1.8413493351660124,
      "delta_ce": -0.32465683506064025,
      "flops_per_token": 29614080,
      "std_ratio": 0.9511829912662506,
      "erank": 109.17454528808594,
      "step_cos": 0.9875146746635437,
      "rel_step": 0.27501752972602844,
      "state_norm": 33.76785087585449
    },
    {
      "k": 5,
      "ce": 5.547618621125902,
      "ppl": 256.625704638506,
      "bpb": 1.9398563083325306,
      "delta_ce": -0.28171113308854867,
      "flops_per_token": 32452608,
      "std_ratio": 0.945938229560852,
      "erank": 88.04708099365234,
      "step_cos": 0.9916830360889435,
      "rel_step": 0.2244347706437111,
      "state_norm": 39.811710357666016
    },
    {
      "k": 6,
      "ce": 5.789976763812497,
      "ppl": 327.00542592830055,
      "bpb": 2.024602575888849,
      "delta_ce": -0.24235814268659528,
      "flops_per_token": 35291136,
      "std_ratio": 0.9405999779701233,
      "erank": 86.94815063476562,
      "step_cos": 0.9938283264636993,
      "rel_step": 0.19342376291751862,
      "state_norm": 45.64400100708008
    },
    {
      "k": 7,
      "ce": 5.99996355549616,
      "ppl": 403.4140909984358,
      "bpb": 2.0980294334891267,
      "delta_ce": -0.2099867916836633,
      "flops_per_token": 38129664,
      "std_ratio": 0.9329713881015778,
      "erank": 64.25496482849121,
      "step_cos": 0.9951082766056061,
      "rel_step": 0.1726725548505783,
      "state_norm": 53.134456634521484
    },
    {
      "k": 8,
      "ce": 6.18351374157727,
      "ppl": 484.6920503676553,
      "bpb": 2.1622121054934955,
      "delta_ce": -0.18355018608110996,
      "flops_per_token": 40968192,
      "std_ratio": 0.9266401827335358,
      "erank": 56.75149154663086,
      "step_cos": 0.9959644377231598,
      "rel_step": 0.157696433365345,
      "state_norm": 59.62197303771973
    },
    {
      "k": 9,
      "ce": 6.345236095432273,
      "ppl": 569.7718943785347,
      "bpb": 2.218762158723424,
      "delta_ce": -0.16172235385500233,
      "flops_per_token": 43806720,
      "std_ratio": 0.9184004366397858,
      "erank": 48.32929611206055,
      "step_cos": 0.9965897798538208,
      "rel_step": 0.14616148173809052,
      "state_norm": 67.28807830810547
    },
    {
      "k": 10,
      "ce": 6.488679977838169,
      "ppl": 657.6546712541515,
      "bpb": 2.268920711280938,
      "delta_ce": -0.14344388240589634,
      "flops_per_token": 46645248,
      "std_ratio": 0.9087829887866974,
      "erank": 38.35821723937988,
      "step_cos": 0.9970792233943939,
      "rel_step": 0.13675487786531448,
      "state_norm": 75.25280380249023
    },
    {
      "k": 11,
      "ce": 6.616616099706154,
      "ppl": 747.4116465669175,
      "bpb": 2.3136566078914447,
      "delta_ce": -0.1279361218679851,
      "flops_per_token": 49483776,
      "std_ratio": 0.8985040187835693,
      "erank": 32.420021057128906,
      "step_cos": 0.9974798560142517,
      "rel_step": 0.12871047109365463,
      "state_norm": 83.94036483764648
    },
    {
      "k": 12,
      "ce": 6.731254219889919,
      "ppl": 838.1978914258499,
      "bpb": 2.3537425431010153,
      "delta_ce": -0.11463812018376451,
      "flops_per_token": 52322304,
      "std_ratio": 0.8866880834102631,
      "erank": 26.89052963256836,
      "step_cos": 0.9978146255016327,
      "rel_step": 0.12159556895494461,
      "state_norm": 93.48854446411133
    }
  ]
}