gpu-primary-output/summary.json

{
  "paper_setup": {
    "vocab": 5,
    "primary_k": 2,
    "alpha": 1.0
  },
  "run_setup": {
    "output": "/data/gpu-primary-output",
    "steps": 500,
    "length": 64,
    "vocab": 5,
    "seeds": 5,
    "device": "cuda",
    "orders": [
      2
    ]
  },
  "wall_seconds": 223.4062225818634,
  "rows": [
    {
      "k": 2,
      "bos": false,
      "seed": 0,
      "beta": [
        0.8580751049349579,
        0.821533196854088
      ],
      "bos_mass": null,
      "nll": 2.2259960636175333,
      "kl_to_true": 0.9963021462182176,
      "kl_to_bayes": 0.7706829684681136,
      "bayes_kl_to_true": 0.26272999132318264,
      "attention_overlap_slope": 0.8398105753614834,
      "attention_overlap_r2": 0.9995735884693955,
      "positions": 2240
    },
    {
      "k": 2,
      "bos": false,
      "seed": 1,
      "beta": [
        0.7813418229519343,
        0.5874054560908514
      ],
      "bos_mass": null,
      "nll": 2.239154091536007,
      "kl_to_true": 0.8914109711229098,
      "kl_to_bayes": 0.6483643243405118,
      "bayes_kl_to_true": 0.2647279366222993,
      "attention_overlap_slope": 0.6843692331425872,
      "attention_overlap_r2": 0.9815924405244475,
      "positions": 2240
    },
    {
      "k": 2,
      "bos": false,
      "seed": 2,
      "beta": [
        0.9325293521882272,
        0.7229287141071734
      ],
      "bos_mass": null,
      "nll": 2.2878528096420587,
      "kl_to_true": 0.9700856527272188,
      "kl_to_bayes": 0.7119090667599893,
      "bayes_kl_to_true": 0.2661596335013201,
      "attention_overlap_slope": 0.8277578714700348,
      "attention_overlap_r2": 0.9852416231304755,
      "positions": 2240
    },
    {
      "k": 2,
      "bos": false,
      "seed": 3,
      "beta": [
        0.8055885050829208,
        0.7202856606462539
      ],
      "bos_mass": null,
      "nll": 2.21744811755106,
      "kl_to_true": 1.0060883187258387,
      "kl_to_bayes": 0.7016385027486404,
      "bayes_kl_to_true": 0.25795664002150487,
      "attention_overlap_slope": 0.7629491026112536,
      "attention_overlap_r2": 0.9970769972942697,
      "positions": 2240
    },
    {
      "k": 2,
      "bos": false,
      "seed": 4,
      "beta": [
        0.5939132996761789,
        0.7506196262624412
      ],
      "bos_mass": null,
      "nll": 2.2992463870303923,
      "kl_to_true": 1.1495594385365024,
      "kl_to_bayes": 0.9131012755864052,
      "bayes_kl_to_true": 0.25537628251057604,
      "attention_overlap_slope": 0.6723500032427813,
      "attention_overlap_r2": 0.9876638299065911,
      "positions": 2240
    },
    {
      "k": 2,
      "bos": true,
      "seed": 0,
      "beta": [
        2.6753169722580106,
        2.524039958967024
      ],
      "bos_mass": 1069.3730847240818,
      "nll": 1.5351920162036736,
      "kl_to_true": 0.2692611215846471,
      "kl_to_bayes": 0.004756522223734605,
      "bayes_kl_to_true": 0.26272999132318264,
      "attention_overlap_slope": 2.599705061765402,
      "attention_overlap_r2": 0.9992376385766464,
      "positions": 2240
    },
    {
      "k": 2,
      "bos": true,
      "seed": 1,
      "beta": [
        2.7711495122275958,
        2.70421161362507
      ],
      "bos_mass": 1043.3951931987665,
      "nll": 1.537563619100544,
      "kl_to_true": 0.2670327896446933,
      "kl_to_bayes": 0.002875007858342953,
      "bayes_kl_to_true": 0.2647279366222993,
      "attention_overlap_slope": 2.737679042047381,
      "attention_overlap_r2": 0.9998604126989626,
      "positions": 2240
    },
    {
      "k": 2,
      "bos": true,
      "seed": 2,
      "beta": [
        2.7764362037965613,
        2.7744367663912617
      ],
      "bos_mass": 1110.9417493832684,
      "nll": 1.551900404967508,
      "kl_to_true": 0.2721866141877306,
      "kl_to_bayes": 0.0031677553645135825,
      "bayes_kl_to_true": 0.2661596335013201,
      "attention_overlap_slope": 2.7754367601904972,
      "attention_overlap_r2": 0.9999998787532285,
      "positions": 2240
    },
    {
      "k": 2,
      "bos": true,
      "seed": 3,
      "beta": [
        2.8543394056860145,
        2.8691916175493772
      ],
      "bos_mass": 1122.3681192527608,
      "nll": 1.5615310611503912,
      "kl_to_true": 0.2621058802727228,
      "kl_to_bayes": 0.00346397651910272,
      "bayes_kl_to_true": 0.25795664002150487,
      "attention_overlap_slope": 2.861763418840777,
      "attention_overlap_r2": 0.9999936834971035,
      "positions": 2240
    },
    {
      "k": 2,
      "bos": true,
      "seed": 4,
      "beta": [
        2.6599593598446023,
        2.597604378925108
      ],
      "bos_mass": 1043.5114042630291,
      "nll": 1.5463042238735973,
      "kl_to_true": 0.2607410366724894,
      "kl_to_bayes": 0.003973705649362713,
      "bayes_kl_to_true": 0.25537628251057604,
      "attention_overlap_slope": 2.6287486278924987,
      "attention_overlap_r2": 0.9998706468634581,
      "positions": 2240
    }
  ],
  "mutation_tests": [
    {
      "k": 2,
      "seed": 0,
      "learned_nll": 1.6564684714070328,
      "untrained_uniform_overlap_nll": 1.6560642706133422,
      "hard_match_nll": 11.38202445066547,
      "shuffled_context_nll": 1.7275352944206195,
      "untrained_worse": false,
      "hard_match_worse": true,
      "shuffled_worse": true
    },
    {
      "k": 2,
      "seed": 1,
      "learned_nll": 2.093263527416452,
      "untrained_uniform_overlap_nll": 2.0958453712693985,
      "hard_match_nll": 13.72265086280333,
      "shuffled_context_nll": 2.1099935623839987,
      "untrained_worse": true,
      "hard_match_worse": true,
      "shuffled_worse": true
    },
    {
      "k": 2,
      "seed": 2,
      "learned_nll": 2.041227172099717,
      "untrained_uniform_overlap_nll": 2.0675759502009052,
      "hard_match_nll": 12.420809425412369,
      "shuffled_context_nll": 2.151110589976682,
      "untrained_worse": true,
      "hard_match_worse": true,
      "shuffled_worse": true
    },
    {
      "k": 2,
      "seed": 3,
      "learned_nll": 2.121490669801476,
      "untrained_uniform_overlap_nll": 2.1403534758305303,
      "hard_match_nll": 12.838975232795969,
      "shuffled_context_nll": 2.192333527448208,
      "untrained_worse": true,
      "hard_match_worse": true,
      "shuffled_worse": true
    },
    {
      "k": 2,
      "seed": 4,
      "learned_nll": 2.1026130342424394,
      "untrained_uniform_overlap_nll": 2.1216206632762753,
      "hard_match_nll": 13.295075237503912,
      "shuffled_context_nll": 2.170215629188961,
      "untrained_worse": true,
      "hard_match_worse": true,
      "shuffled_worse": true
    }
  ],
  "aggregate": {
    "k2_bos0": {
      "nll": {
        "mean": 2.25393949387541,
        "std": 0.03326827773301641
      },
      "kl_to_true": {
        "mean": 1.0026893054661374,
        "std": 0.08374849365645665
      },
      "kl_to_bayes": {
        "mean": 0.7491392275807321,
        "std": 0.0907145752511971
      },
      "bayes_kl_to_true": {
        "mean": 0.2613900967957766,
        "std": 0.004089977019869316
      },
      "attention_overlap_slope": {
        "mean": 0.757447357165628,
        "std": 0.06977035071099172
      },
      "attention_overlap_r2": {
        "mean": 0.9902296958650358,
        "std": 0.006931963407362833
      }
    },
    "k2_bos1": {
      "nll": {
        "mean": 1.5464982650591428,
        "std": 0.00962148520218784
      },
      "kl_to_true": {
        "mean": 0.2662654884724566,
        "std": 0.004299865268431111
      },
      "kl_to_bayes": {
        "mean": 0.0036473935230113145,
        "std": 0.0006628605062857784
      },
      "bayes_kl_to_true": {
        "mean": 0.2613900967957766,
        "std": 0.004089977019869316
      },
      "attention_overlap_slope": {
        "mean": 2.720666582147311,
        "std": 0.09620614663017038
      },
      "attention_overlap_r2": {
        "mean": 0.9997924520778799,
        "std": 0.00028357397672899094
      }
    }
  }
}