svlandeg commited on
Commit
a2fb3c8
·
1 Parent(s): e5efb4b

fix typos

Browse files
Files changed (2) hide show
  1. nanochat/loss_eval.py +2 -2
  2. scripts/chat_eval.py +1 -1
nanochat/loss_eval.py CHANGED
@@ -9,9 +9,9 @@ import torch.distributed as dist
9
  def evaluate_bpb(model, batches, steps, token_bytes):
10
  """
11
  Instead of the naive 'mean loss', this function returns the bits per byte (bpb),
12
- which is a tokenization vocab size-indepedent metric, meaning you are still comparing
13
  apples:apples if you change the vocab size. The way this works is that instead of just
14
- calculating the average loss as usual, you calculate the sum loss, and indepependently
15
  also the sum bytes (of all the target tokens), and divide. This normalizes the loss by
16
  the number of bytes that the target tokens represent.
17
 
 
9
  def evaluate_bpb(model, batches, steps, token_bytes):
10
  """
11
  Instead of the naive 'mean loss', this function returns the bits per byte (bpb),
12
+ which is a tokenization vocab size-independent metric, meaning you are still comparing
13
  apples:apples if you change the vocab size. The way this works is that instead of just
14
+ calculating the average loss as usual, you calculate the sum loss, and independently
15
  also the sum bytes (of all the target tokens), and divide. This normalizes the loss by
16
  the number of bytes that the target tokens represent.
17
 
scripts/chat_eval.py CHANGED
@@ -1,6 +1,6 @@
1
  """
2
  Evaluate the Chat model.
3
- All the generic code lives here, and all the evlauation-specific
4
  code lives in nanochat directory and is imported from here.
5
 
6
  Example runs:
 
1
  """
2
  Evaluate the Chat model.
3
+ All the generic code lives here, and all the evaluation-specific
4
  code lives in nanochat directory and is imported from here.
5
 
6
  Example runs: