Super-squash branch 'main' using huggingface_hub

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,102 @@
+---
+license: apache-2.0
+datasets:
+- JeanKaddour/minipile
+- BEE-spoke-data/wikipedia-20230901.en-deduped
+- BEE-spoke-data/knowledge-inoc-concat-v1
+language:
+- en
+inference:
+  parameters:
+    max_new_tokens: 64
+    do_sample: true
+    temperature: 0.8
+    repetition_penalty: 1.05
+    no_repeat_ngram_size: 4
+    epsilon_cutoff: 0.0006
+    renormalize_logits: true
+widget:
+  - text: My name is El Microondas the Wise, and
+    example_title: El Microondas
+  - text: Kennesaw State University is a public
+    example_title: Kennesaw State University
+  - text: >-
+      Bungie Studios is an American video game developer. They are most famous
+      for developing the award winning Halo series of video games. They also
+      made Destiny. The studio was founded
+    example_title: Bungie
+  - text: The Mona Lisa is a world-renowned painting created by
+    example_title: Mona Lisa
+  - text: >-
+      The Harry Potter series, written by J.K. Rowling, begins with the book
+      titled
+    example_title: Harry Potter Series
+  - text: >-
+      Question: I have cities, but no houses. I have mountains, but no trees. I
+      have water, but no fish. What am I?
+
+      Answer:
+    example_title: Riddle
+  - text: The process of photosynthesis involves the conversion of
+    example_title: Photosynthesis
+  - text: >-
+      Jane went to the store to buy some groceries. She picked up apples,
+      oranges, and a loaf of bread. When she got home, she realized she forgot
+    example_title: Story Continuation
+  - text: >-
+      Problem 2: If a train leaves Station A at 9:00 AM and travels at 60 mph,
+      and another train leaves Station B at 10:00 AM and travels at 80 mph, when
+      will they meet if the distance between the stations is 300 miles?
+
+      To determine
+    example_title: Math Problem
+  - text: In the context of computer programming, an algorithm is
+    example_title: Algorithm Definition
+pipeline_tag: text-generation
+---
+
+
+# BEE-spoke-data/mega-ar-126m-4k
+
+
+This may not be the _best_ language model, but it is a language model! It's interesting for several reasons, not the least of which is that it's not technically a transformer.
+
+Details:
+
+- 768 hidden size, 12 layers
+- no MEGA chunking, 4096 context length
+- EMA dimension 16, shared dimension 192
+- tokenizer: GPT NeoX
+- train-from-scratch
+
+
+For more info on MEGA (_& what some of the params above mean_), check out the [model docs](https://huggingface.co/docs/transformers/main/en/model_doc/mega#mega) or the [original paper](https://arxiv.org/abs/2209.10655)
+
+## Usage
+
+Usage is the same as any other small textgen model. 
+
+Given the model's small size and architecture, it's probably best to leverage its longer context by adding input context to "see more" rather than "generate more".
+
+## evals
+
+Initial data:
+
+`hf-causal-experimental (pretrained=BEE-spoke-data/mega-ar-126m-4k,revision=main,trust_remote_code=True,dtype='float'), limit: None, provide_description: False, num_fewshot: 0, batch_size: 4`
+
+|     Task     |Version| Metric | Value |   |Stderr|
+|--------------|------:|--------|------:|---|-----:|
+|arc_easy      |      0|acc     | 0.4415|±  |0.0102|
+|              |       |acc_norm| 0.3969|±  |0.0100|
+|boolq         |      1|acc     | 0.5749|±  |0.0086|
+|lambada_openai|      0|ppl     |94.9912|±  |3.9682|
+|              |       |acc     | 0.2408|±  |0.0060|
+|openbookqa    |      0|acc     | 0.1660|±  |0.0167|
+|              |       |acc_norm| 0.2780|±  |0.0201|
+|piqa          |      0|acc     | 0.5974|±  |0.0114|
+|              |       |acc_norm| 0.5914|±  |0.0115|
+|winogrande    |      0|acc     | 0.4830|±  |0.0140|
+
+
+
+---

config.json

@@ -0,0 +1,47 @@
+{
+  "activation": "silu",
+  "add_lm_hidden_dense_layer": false,
+  "add_token_type_embeddings": false,
+  "architectures": [
+    "MegaForCausalLM"
+  ],
+  "attention_activation": "softmax",
+  "attention_probs_dropout_prob": 0,
+  "bidirectional": false,
+  "bos_token_id": 0,
+  "chunk_size": -1,
+  "classifier_dropout": null,
+  "dropout_prob": 0,
+  "ema_beta_range": 0.02,
+  "ema_delta_alpha_range": 0.2,
+  "ema_gamma_omega_range": 1.0,
+  "ema_projection_size": 16,
+  "eos_token_id": 0,
+  "hidden_dropout_prob": 0,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "is_decoder": true,
+  "max_positions": 4096,
+  "model_type": "mega",
+  "nffn_activation_dropout_prob": 0,
+  "nffn_hidden_size": 1536,
+  "norm_affine": true,
+  "normalization_type": "layernorm",
+  "normalize_before_ffn": true,
+  "normalize_before_mega": true,
+  "num_attention_heads": 1,
+  "num_hidden_layers": 12,
+  "pad_token_id": null,
+  "relative_positional_bias": "rotary",
+  "shared_representation_size": 192,
+  "torch_dtype": "float32",
+  "transformers_version": "4.36.2",
+  "truncation": null,
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "use_chunking": false,
+  "use_feature_dropout": false,
+  "use_normalized_ffn": true,
+  "vocab_size": 50304
+}

evals/mega-ar-126m-4k-evals/json_object_1.json

@@ -0,0 +1,56 @@
+{
+  "results": {
+    "arc_easy": {
+      "acc": 0.4414983164983165,
+      "acc_stderr": 0.010189314382749946,
+      "acc_norm": 0.3968855218855219,
+      "acc_norm_stderr": 0.01003923680058321
+    },
+    "boolq": {
+      "acc": 0.5749235474006116,
+      "acc_stderr": 0.008646316159373179
+    },
+    "lambada_openai": {
+      "ppl": 94.9912418583616,
+      "ppl_stderr": 3.9681997912266134,
+      "acc": 0.24083058412575198,
+      "acc_stderr": 0.005957132284367873
+    },
+    "openbookqa": {
+      "acc": 0.166,
+      "acc_stderr": 0.016656616876531142,
+      "acc_norm": 0.278,
+      "acc_norm_stderr": 0.020055833888070914
+    },
+    "piqa": {
+      "acc": 0.5973884657236126,
+      "acc_stderr": 0.011442395233488698,
+      "acc_norm": 0.5914036996735582,
+      "acc_norm_stderr": 0.01146924038724515
+    },
+    "winogrande": {
+      "acc": 0.48303078137332284,
+      "acc_stderr": 0.014044390401612976
+    }
+  },
+  "versions": {
+    "arc_easy": 0,
+    "boolq": 1,
+    "lambada_openai": 0,
+    "openbookqa": 0,
+    "piqa": 0,
+    "winogrande": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=BEE-spoke-data/mega-ar-126m-4k,revision=main,trust_remote_code=True,dtype='float'",
+    "num_fewshot": 0,
+    "batch_size": "4",
+    "batch_sizes": [],
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/mega-ar-126m-4k-evals/json_object_2.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "arc_challenge": {
+      "acc": 0.20051194539249148,
+      "acc_stderr": 0.011700318050499384,
+      "acc_norm": 0.22098976109215018,
+      "acc_norm_stderr": 0.012124929206818258
+    }
+  },
+  "versions": {
+    "arc_challenge": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=BEE-spoke-data/mega-ar-126m-4k,revision=main,trust_remote_code=True,dtype='float'",
+    "num_fewshot": 25,
+    "batch_size": "4",
+    "batch_sizes": [],
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/mega-ar-126m-4k-evals/json_object_3.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc": 0.2685258964143426,
+      "acc_stderr": 0.008847941337419094,
+      "acc_norm": 0.2752988047808765,
+      "acc_norm_stderr": 0.008917257773359156
+    }
+  },
+  "versions": {
+    "hellaswag": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=BEE-spoke-data/mega-ar-126m-4k,revision=main,trust_remote_code=True,dtype='float'",
+    "num_fewshot": 10,
+    "batch_size": "4",
+    "batch_sizes": [],
+    "device": "cuda",
+    "no_cache": false,
+    "limit": 0.25,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/mega-ar-126m-4k-evals/json_object_4.json

@@ -0,0 +1,25 @@
+{
+  "results": {
+    "truthfulqa_mc": {
+      "mc1": 0.24724602203182375,
+      "mc1_stderr": 0.015102404797359654,
+      "mc2": 0.45134269722856163,
+      "mc2_stderr": 0.015590807604578908
+    }
+  },
+  "versions": {
+    "truthfulqa_mc": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=BEE-spoke-data/mega-ar-126m-4k,revision=main,trust_remote_code=True,dtype='float'",
+    "num_fewshot": 0,
+    "batch_size": "4",
+    "batch_sizes": [],
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/mega-ar-126m-4k-evals/json_object_5.json

@@ -0,0 +1,417 @@
+{
+  "results": {
+    "hendrycksTest-abstract_algebra": {
+      "acc": 0.16,
+      "acc_stderr": 0.07483314773547882,
+      "acc_norm": 0.16,
+      "acc_norm_stderr": 0.07483314773547882
+    },
+    "hendrycksTest-anatomy": {
+      "acc": 0.16,
+      "acc_stderr": 0.0748331477354788,
+      "acc_norm": 0.16,
+      "acc_norm_stderr": 0.0748331477354788
+    },
+    "hendrycksTest-astronomy": {
+      "acc": 0.28,
+      "acc_stderr": 0.0916515138991168,
+      "acc_norm": 0.28,
+      "acc_norm_stderr": 0.0916515138991168
+    },
+    "hendrycksTest-business_ethics": {
+      "acc": 0.36,
+      "acc_stderr": 0.09797958971132711,
+      "acc_norm": 0.36,
+      "acc_norm_stderr": 0.09797958971132711
+    },
+    "hendrycksTest-clinical_knowledge": {
+      "acc": 0.24,
+      "acc_stderr": 0.08717797887081348,
+      "acc_norm": 0.24,
+      "acc_norm_stderr": 0.08717797887081348
+    },
+    "hendrycksTest-college_biology": {
+      "acc": 0.24,
+      "acc_stderr": 0.08717797887081347,
+      "acc_norm": 0.24,
+      "acc_norm_stderr": 0.08717797887081347
+    },
+    "hendrycksTest-college_chemistry": {
+      "acc": 0.24,
+      "acc_stderr": 0.08717797887081347,
+      "acc_norm": 0.24,
+      "acc_norm_stderr": 0.08717797887081347
+    },
+    "hendrycksTest-college_computer_science": {
+      "acc": 0.2,
+      "acc_stderr": 0.08164965809277261,
+      "acc_norm": 0.2,
+      "acc_norm_stderr": 0.08164965809277261
+    },
+    "hendrycksTest-college_mathematics": {
+      "acc": 0.28,
+      "acc_stderr": 0.09165151389911678,
+      "acc_norm": 0.28,
+      "acc_norm_stderr": 0.09165151389911678
+    },
+    "hendrycksTest-college_medicine": {
+      "acc": 0.24,
+      "acc_stderr": 0.08717797887081345,
+      "acc_norm": 0.24,
+      "acc_norm_stderr": 0.08717797887081345
+    },
+    "hendrycksTest-college_physics": {
+      "acc": 0.2,
+      "acc_stderr": 0.08164965809277261,
+      "acc_norm": 0.2,
+      "acc_norm_stderr": 0.08164965809277261
+    },
+    "hendrycksTest-computer_security": {
+      "acc": 0.32,
+      "acc_stderr": 0.09521904571390466,
+      "acc_norm": 0.32,
+      "acc_norm_stderr": 0.09521904571390466
+    },
+    "hendrycksTest-conceptual_physics": {
+      "acc": 0.24,
+      "acc_stderr": 0.08717797887081347,
+      "acc_norm": 0.24,
+      "acc_norm_stderr": 0.08717797887081347
+    },
+    "hendrycksTest-econometrics": {
+      "acc": 0.2,
+      "acc_stderr": 0.08164965809277261,
+      "acc_norm": 0.2,
+      "acc_norm_stderr": 0.08164965809277261
+    },
+    "hendrycksTest-electrical_engineering": {
+      "acc": 0.12,
+      "acc_stderr": 0.06633249580710801,
+      "acc_norm": 0.12,
+      "acc_norm_stderr": 0.06633249580710801
+    },
+    "hendrycksTest-elementary_mathematics": {
+      "acc": 0.16,
+      "acc_stderr": 0.0748331477354788,
+      "acc_norm": 0.16,
+      "acc_norm_stderr": 0.0748331477354788
+    },
+    "hendrycksTest-formal_logic": {
+      "acc": 0.24,
+      "acc_stderr": 0.08717797887081345,
+      "acc_norm": 0.24,
+      "acc_norm_stderr": 0.08717797887081345
+    },
+    "hendrycksTest-global_facts": {
+      "acc": 0.12,
+      "acc_stderr": 0.066332495807108,
+      "acc_norm": 0.12,
+      "acc_norm_stderr": 0.066332495807108
+    },
+    "hendrycksTest-high_school_biology": {
+      "acc": 0.32,
+      "acc_stderr": 0.09521904571390467,
+      "acc_norm": 0.32,
+      "acc_norm_stderr": 0.09521904571390467
+    },
+    "hendrycksTest-high_school_chemistry": {
+      "acc": 0.36,
+      "acc_stderr": 0.09797958971132711,
+      "acc_norm": 0.36,
+      "acc_norm_stderr": 0.09797958971132711
+    },
+    "hendrycksTest-high_school_computer_science": {
+      "acc": 0.16,
+      "acc_stderr": 0.0748331477354788,
+      "acc_norm": 0.16,
+      "acc_norm_stderr": 0.0748331477354788
+    },
+    "hendrycksTest-high_school_european_history": {
+      "acc": 0.2,
+      "acc_stderr": 0.08164965809277261,
+      "acc_norm": 0.2,
+      "acc_norm_stderr": 0.08164965809277261
+    },
+    "hendrycksTest-high_school_geography": {
+      "acc": 0.36,
+      "acc_stderr": 0.09797958971132713,
+      "acc_norm": 0.36,
+      "acc_norm_stderr": 0.09797958971132713
+    },
+    "hendrycksTest-high_school_government_and_politics": {
+      "acc": 0.32,
+      "acc_stderr": 0.09521904571390465,
+      "acc_norm": 0.32,
+      "acc_norm_stderr": 0.09521904571390465
+    },
+    "hendrycksTest-high_school_macroeconomics": {
+      "acc": 0.32,
+      "acc_stderr": 0.09521904571390466,
+      "acc_norm": 0.32,
+      "acc_norm_stderr": 0.09521904571390466
+    },
+    "hendrycksTest-high_school_mathematics": {
+      "acc": 0.36,
+      "acc_stderr": 0.09797958971132713,
+      "acc_norm": 0.36,
+      "acc_norm_stderr": 0.09797958971132713
+    },
+    "hendrycksTest-high_school_microeconomics": {
+      "acc": 0.12,
+      "acc_stderr": 0.06633249580710801,
+      "acc_norm": 0.12,
+      "acc_norm_stderr": 0.06633249580710801
+    },
+    "hendrycksTest-high_school_physics": {
+      "acc": 0.32,
+      "acc_stderr": 0.09521904571390466,
+      "acc_norm": 0.32,
+      "acc_norm_stderr": 0.09521904571390466
+    },
+    "hendrycksTest-high_school_psychology": {
+      "acc": 0.2,
+      "acc_stderr": 0.08164965809277261,
+      "acc_norm": 0.2,
+      "acc_norm_stderr": 0.08164965809277261
+    },
+    "hendrycksTest-high_school_statistics": {
+      "acc": 0.44,
+      "acc_stderr": 0.10132456102380442,
+      "acc_norm": 0.44,
+      "acc_norm_stderr": 0.10132456102380442
+    },
+    "hendrycksTest-high_school_us_history": {
+      "acc": 0.36,
+      "acc_stderr": 0.09797958971132711,
+      "acc_norm": 0.36,
+      "acc_norm_stderr": 0.09797958971132711
+    },
+    "hendrycksTest-high_school_world_history": {
+      "acc": 0.2,
+      "acc_stderr": 0.08164965809277262,
+      "acc_norm": 0.2,
+      "acc_norm_stderr": 0.08164965809277262
+    },
+    "hendrycksTest-human_aging": {
+      "acc": 0.28,
+      "acc_stderr": 0.09165151389911678,
+      "acc_norm": 0.28,
+      "acc_norm_stderr": 0.09165151389911678
+    },
+    "hendrycksTest-human_sexuality": {
+      "acc": 0.24,
+      "acc_stderr": 0.08717797887081345,
+      "acc_norm": 0.24,
+      "acc_norm_stderr": 0.08717797887081345
+    },
+    "hendrycksTest-international_law": {
+      "acc": 0.2,
+      "acc_stderr": 0.08164965809277261,
+      "acc_norm": 0.2,
+      "acc_norm_stderr": 0.08164965809277261
+    },
+    "hendrycksTest-jurisprudence": {
+      "acc": 0.44,
+      "acc_stderr": 0.10132456102380442,
+      "acc_norm": 0.44,
+      "acc_norm_stderr": 0.10132456102380442
+    },
+    "hendrycksTest-logical_fallacies": {
+      "acc": 0.36,
+      "acc_stderr": 0.09797958971132711,
+      "acc_norm": 0.36,
+      "acc_norm_stderr": 0.09797958971132711
+    },
+    "hendrycksTest-machine_learning": {
+      "acc": 0.24,
+      "acc_stderr": 0.08717797887081345,
+      "acc_norm": 0.24,
+      "acc_norm_stderr": 0.08717797887081345
+    },
+    "hendrycksTest-management": {
+      "acc": 0.28,
+      "acc_stderr": 0.09165151389911677,
+      "acc_norm": 0.28,
+      "acc_norm_stderr": 0.09165151389911677
+    },
+    "hendrycksTest-marketing": {
+      "acc": 0.32,
+      "acc_stderr": 0.09521904571390465,
+      "acc_norm": 0.32,
+      "acc_norm_stderr": 0.09521904571390465
+    },
+    "hendrycksTest-medical_genetics": {
+      "acc": 0.36,
+      "acc_stderr": 0.09797958971132711,
+      "acc_norm": 0.36,
+      "acc_norm_stderr": 0.09797958971132711
+    },
+    "hendrycksTest-miscellaneous": {
+      "acc": 0.32,
+      "acc_stderr": 0.09521904571390466,
+      "acc_norm": 0.32,
+      "acc_norm_stderr": 0.09521904571390466
+    },
+    "hendrycksTest-moral_disputes": {
+      "acc": 0.2,
+      "acc_stderr": 0.08164965809277261,
+      "acc_norm": 0.2,
+      "acc_norm_stderr": 0.08164965809277261
+    },
+    "hendrycksTest-moral_scenarios": {
+      "acc": 0.12,
+      "acc_stderr": 0.06633249580710801,
+      "acc_norm": 0.12,
+      "acc_norm_stderr": 0.06633249580710801
+    },
+    "hendrycksTest-nutrition": {
+      "acc": 0.16,
+      "acc_stderr": 0.0748331477354788,
+      "acc_norm": 0.16,
+      "acc_norm_stderr": 0.0748331477354788
+    },
+    "hendrycksTest-philosophy": {
+      "acc": 0.16,
+      "acc_stderr": 0.0748331477354788,
+      "acc_norm": 0.16,
+      "acc_norm_stderr": 0.0748331477354788
+    },
+    "hendrycksTest-prehistory": {
+      "acc": 0.12,
+      "acc_stderr": 0.066332495807108,
+      "acc_norm": 0.12,
+      "acc_norm_stderr": 0.066332495807108
+    },
+    "hendrycksTest-professional_accounting": {
+      "acc": 0.28,
+      "acc_stderr": 0.09165151389911677,
+      "acc_norm": 0.28,
+      "acc_norm_stderr": 0.09165151389911677
+    },
+    "hendrycksTest-professional_law": {
+      "acc": 0.24,
+      "acc_stderr": 0.08717797887081347,
+      "acc_norm": 0.24,
+      "acc_norm_stderr": 0.08717797887081347
+    },
+    "hendrycksTest-professional_medicine": {
+      "acc": 0.36,
+      "acc_stderr": 0.09797958971132711,
+      "acc_norm": 0.36,
+      "acc_norm_stderr": 0.09797958971132711
+    },
+    "hendrycksTest-professional_psychology": {
+      "acc": 0.28,
+      "acc_stderr": 0.09165151389911678,
+      "acc_norm": 0.28,
+      "acc_norm_stderr": 0.09165151389911678
+    },
+    "hendrycksTest-public_relations": {
+      "acc": 0.2,
+      "acc_stderr": 0.08164965809277261,
+      "acc_norm": 0.2,
+      "acc_norm_stderr": 0.08164965809277261
+    },
+    "hendrycksTest-security_studies": {
+      "acc": 0.24,
+      "acc_stderr": 0.08717797887081347,
+      "acc_norm": 0.24,
+      "acc_norm_stderr": 0.08717797887081347
+    },
+    "hendrycksTest-sociology": {
+      "acc": 0.12,
+      "acc_stderr": 0.066332495807108,
+      "acc_norm": 0.12,
+      "acc_norm_stderr": 0.066332495807108
+    },
+    "hendrycksTest-us_foreign_policy": {
+      "acc": 0.16,
+      "acc_stderr": 0.0748331477354788,
+      "acc_norm": 0.16,
+      "acc_norm_stderr": 0.0748331477354788
+    },
+    "hendrycksTest-virology": {
+      "acc": 0.2,
+      "acc_stderr": 0.08164965809277262,
+      "acc_norm": 0.2,
+      "acc_norm_stderr": 0.08164965809277262
+    },
+    "hendrycksTest-world_religions": {
+      "acc": 0.36,
+      "acc_stderr": 0.09797958971132711,
+      "acc_norm": 0.36,
+      "acc_norm_stderr": 0.09797958971132711
+    }
+  },
+  "versions": {
+    "hendrycksTest-abstract_algebra": 1,
+    "hendrycksTest-anatomy": 1,
+    "hendrycksTest-astronomy": 1,
+    "hendrycksTest-business_ethics": 1,
+    "hendrycksTest-clinical_knowledge": 1,
+    "hendrycksTest-college_biology": 1,
+    "hendrycksTest-college_chemistry": 1,
+    "hendrycksTest-college_computer_science": 1,
+    "hendrycksTest-college_mathematics": 1,
+    "hendrycksTest-college_medicine": 1,
+    "hendrycksTest-college_physics": 1,
+    "hendrycksTest-computer_security": 1,
+    "hendrycksTest-conceptual_physics": 1,
+    "hendrycksTest-econometrics": 1,
+    "hendrycksTest-electrical_engineering": 1,
+    "hendrycksTest-elementary_mathematics": 1,
+    "hendrycksTest-formal_logic": 1,
+    "hendrycksTest-global_facts": 1,
+    "hendrycksTest-high_school_biology": 1,
+    "hendrycksTest-high_school_chemistry": 1,
+    "hendrycksTest-high_school_computer_science": 1,
+    "hendrycksTest-high_school_european_history": 1,
+    "hendrycksTest-high_school_geography": 1,
+    "hendrycksTest-high_school_government_and_politics": 1,
+    "hendrycksTest-high_school_macroeconomics": 1,
+    "hendrycksTest-high_school_mathematics": 1,
+    "hendrycksTest-high_school_microeconomics": 1,
+    "hendrycksTest-high_school_physics": 1,
+    "hendrycksTest-high_school_psychology": 1,
+    "hendrycksTest-high_school_statistics": 1,
+    "hendrycksTest-high_school_us_history": 1,
+    "hendrycksTest-high_school_world_history": 1,
+    "hendrycksTest-human_aging": 1,
+    "hendrycksTest-human_sexuality": 1,
+    "hendrycksTest-international_law": 1,
+    "hendrycksTest-jurisprudence": 1,
+    "hendrycksTest-logical_fallacies": 1,
+    "hendrycksTest-machine_learning": 1,
+    "hendrycksTest-management": 1,
+    "hendrycksTest-marketing": 1,
+    "hendrycksTest-medical_genetics": 1,
+    "hendrycksTest-miscellaneous": 1,
+    "hendrycksTest-moral_disputes": 1,
+    "hendrycksTest-moral_scenarios": 1,
+    "hendrycksTest-nutrition": 1,
+    "hendrycksTest-philosophy": 1,
+    "hendrycksTest-prehistory": 1,
+    "hendrycksTest-professional_accounting": 1,
+    "hendrycksTest-professional_law": 1,
+    "hendrycksTest-professional_medicine": 1,
+    "hendrycksTest-professional_psychology": 1,
+    "hendrycksTest-public_relations": 1,
+    "hendrycksTest-security_studies": 1,
+    "hendrycksTest-sociology": 1,
+    "hendrycksTest-us_foreign_policy": 1,
+    "hendrycksTest-virology": 1,
+    "hendrycksTest-world_religions": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=BEE-spoke-data/mega-ar-126m-4k,revision=main,trust_remote_code=True,dtype='float'",
+    "num_fewshot": 5,
+    "batch_size": "4",
+    "batch_sizes": [],
+    "device": "cuda",
+    "no_cache": false,
+    "limit": 0.25,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.36.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6797e3725f934e91332190eb2b9765e4137041f61c8276ca2d071b245a0c2215
+size 504560824

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,211 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}