shard_dataset: drop the removed drop_last param's leftover body check (NameError at first call)
Browse files- train/shard_dataset.py +0 -2
train/shard_dataset.py
CHANGED
|
@@ -90,8 +90,6 @@ def make_dataset(store, seq_len, seed, start_sample=0, count=None, shuffle=True)
|
|
| 90 |
import torch
|
| 91 |
|
| 92 |
n_samples = store.total_tokens // seq_len
|
| 93 |
-
if drop_last is False:
|
| 94 |
-
raise ValueError("partial trailing windows are not supported; drop_last must stay True")
|
| 95 |
perm = None
|
| 96 |
if shuffle:
|
| 97 |
# Seed must be identical on every resume or the two runs read different data orders. Stored in
|
|
|
|
| 90 |
import torch
|
| 91 |
|
| 92 |
n_samples = store.total_tokens // seq_len
|
|
|
|
|
|
|
| 93 |
perm = None
|
| 94 |
if shuffle:
|
| 95 |
# Seed must be identical on every resume or the two runs read different data orders. Stored in
|