svlandeg commited on
Commit ·
a2fb3c8
1
Parent(s): e5efb4b
fix typos
Browse files- nanochat/loss_eval.py +2 -2
- scripts/chat_eval.py +1 -1
nanochat/loss_eval.py
CHANGED
|
@@ -9,9 +9,9 @@ import torch.distributed as dist
|
|
| 9 |
def evaluate_bpb(model, batches, steps, token_bytes):
|
| 10 |
"""
|
| 11 |
Instead of the naive 'mean loss', this function returns the bits per byte (bpb),
|
| 12 |
-
which is a tokenization vocab size-
|
| 13 |
apples:apples if you change the vocab size. The way this works is that instead of just
|
| 14 |
-
calculating the average loss as usual, you calculate the sum loss, and
|
| 15 |
also the sum bytes (of all the target tokens), and divide. This normalizes the loss by
|
| 16 |
the number of bytes that the target tokens represent.
|
| 17 |
|
|
|
|
| 9 |
def evaluate_bpb(model, batches, steps, token_bytes):
|
| 10 |
"""
|
| 11 |
Instead of the naive 'mean loss', this function returns the bits per byte (bpb),
|
| 12 |
+
which is a tokenization vocab size-independent metric, meaning you are still comparing
|
| 13 |
apples:apples if you change the vocab size. The way this works is that instead of just
|
| 14 |
+
calculating the average loss as usual, you calculate the sum loss, and independently
|
| 15 |
also the sum bytes (of all the target tokens), and divide. This normalizes the loss by
|
| 16 |
the number of bytes that the target tokens represent.
|
| 17 |
|
scripts/chat_eval.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
Evaluate the Chat model.
|
| 3 |
-
All the generic code lives here, and all the
|
| 4 |
code lives in nanochat directory and is imported from here.
|
| 5 |
|
| 6 |
Example runs:
|
|
|
|
| 1 |
"""
|
| 2 |
Evaluate the Chat model.
|
| 3 |
+
All the generic code lives here, and all the evaluation-specific
|
| 4 |
code lives in nanochat directory and is imported from here.
|
| 5 |
|
| 6 |
Example runs:
|