Update README.md
Browse files
README.md
CHANGED
|
@@ -43,4 +43,28 @@ for _ in range(20):
|
|
| 43 |
next_token = torch.multinomial(torch.softmax(logits, dim=-1), 1).unsqueeze(0)
|
| 44 |
input_ids = torch.cat([input_ids, next_token], dim=1)
|
| 45 |
print(tokenizer.decode(input_ids[0]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
```
|
|
|
|
| 43 |
next_token = torch.multinomial(torch.softmax(logits, dim=-1), 1).unsqueeze(0)
|
| 44 |
input_ids = torch.cat([input_ids, next_token], dim=1)
|
| 45 |
print(tokenizer.decode(input_ids[0]))
|
| 46 |
+
```
|
| 47 |
+
|
| 48 |
+
Eval:
|
| 49 |
+
```
|
| 50 |
+
$ lm_eval --model hf \
|
| 51 |
+
--model_args pretrained=michaelbzhu/test-3.2B-base,trust_remote_code=True \
|
| 52 |
+
--tasks mmlu_college_medicine,hellaswag,lambada_openai,arc_easy,winogrande,arc_challenge,openbookqa \
|
| 53 |
+
--device cuda:0 \
|
| 54 |
+
--batch_size 16
|
| 55 |
+
|
| 56 |
+
| Tasks |Version|Filter|n-shot| Metric | | Value | |Stderr|
|
| 57 |
+
|----------------|------:|------|-----:|----------|---|------:|---|-----:|
|
| 58 |
+
|arc_challenge | 1|none | 0|acc |↑ | 0.2363|± |0.0124|
|
| 59 |
+
| | |none | 0|acc_norm |↑ | 0.2637|± |0.0129|
|
| 60 |
+
|arc_easy | 1|none | 0|acc |↑ | 0.5758|± |0.0101|
|
| 61 |
+
| | |none | 0|acc_norm |↑ | 0.4996|± |0.0103|
|
| 62 |
+
|hellaswag | 1|none | 0|acc |↑ | 0.3827|± |0.0049|
|
| 63 |
+
| | |none | 0|acc_norm |↑ | 0.4846|± |0.0050|
|
| 64 |
+
|lambada_openai | 1|none | 0|acc |↑ | 0.4238|± |0.0069|
|
| 65 |
+
| | |none | 0|perplexity|↓ |14.7850|± |0.4335|
|
| 66 |
+
|college_medicine| 1|none | 0|acc |↑ | 0.2370|± |0.0324|
|
| 67 |
+
|openbookqa | 1|none | 0|acc |↑ | 0.2180|± |0.0185|
|
| 68 |
+
| | |none | 0|acc_norm |↑ | 0.3180|± |0.0208|
|
| 69 |
+
|winogrande | 1|none | 0|acc |↑ | 0.5367|± |0.0140|
|
| 70 |
```
|