{
  "benchmark_type": "reference_only_same_scale_base_lm",
  "important_caveat": "Dense public models use different corpora, tokenizers, and training schedules. These numbers are not controlled training-budget comparisons.",
  "settings": {
    "max_wikitext_chars": 50000,
    "max_lambada_docs": 200,
    "max_mc_docs": 200
  },
  "results": [
    {
      "model": "SymbolicLight-V1-0.8B",
      "params": 873668135,
      "device": "cuda",
      "dtype": "fp16",
      "metrics": {
        "wikitext2": {
          "nll": 40508.8044,
          "tokens": 12388,
          "ppl": 26.3114,
          "bits_per_byte": 1.165,
          "chars": 50000
        },
        "lambada_ppl": {
          "nll": 64891.4496,
          "tokens": 18218,
          "ppl": 35.2315,
          "bits_per_byte": 1.4056,
          "docs": 200
        },
        "sciq": {
          "accuracy": 0.52,
          "correct": 104,
          "total": 200
        },
        "arc_easy": {
          "accuracy": 0.415,
          "correct": 83,
          "total": 200
        },
        "hellaswag": {
          "accuracy": 0.36,
          "correct": 72,
          "total": 200
        },
        "elapsed_sec": 210.3
      }
    },
    {
      "model": "gpt2-large",
      "params": 774030080,
      "device": "cuda",
      "dtype": "fp16",
      "metrics": {
        "wikitext2": {
          "nll": 36412.3386,
          "tokens": 11308,
          "ppl": 25.0294,
          "bits_per_byte": 1.0472,
          "chars": 50000
        },
        "lambada_ppl": {
          "nll": 59116.0558,
          "tokens": 17378,
          "ppl": 30.0174,
          "bits_per_byte": 1.2805,
          "docs": 200
        },
        "sciq": {
          "accuracy": 0.525,
          "correct": 105,
          "total": 200
        },
        "arc_easy": {
          "accuracy": 0.4,
          "correct": 80,
          "total": 200
        },
        "hellaswag": {
          "accuracy": 0.395,
          "correct": 79,
          "total": 200
        },
        "elapsed_sec": 92.05
      }
    },
    {
      "model": "EleutherAI/pythia-1b",
      "params": 1011781632,
      "device": "cuda",
      "dtype": "fp16",
      "metrics": {
        "wikitext2": {
          "nll": 33414.1723,
          "tokens": 11354,
          "ppl": 18.9716,
          "bits_per_byte": 0.9609,
          "chars": 50000
        },
        "lambada_ppl": {
          "nll": 52602.2452,
          "tokens": 16843,
          "ppl": 22.7165,
          "bits_per_byte": 1.1394,
          "docs": 200
        },
        "sciq": {
          "accuracy": 0.565,
          "correct": 113,
          "total": 200
        },
        "arc_easy": {
          "accuracy": 0.435,
          "correct": 87,
          "total": 200
        },
        "hellaswag": {
          "accuracy": 0.395,
          "correct": 79,
          "total": 200
        },
        "elapsed_sec": 58.8
      }
    }
  ]
}