Cion-lab commited on
Commit
d5f46da
·
verified ·
1 Parent(s): e687c2c

shard_dataset: drop the removed drop_last param's leftover body check (NameError at first call)

Browse files
Files changed (1) hide show
  1. train/shard_dataset.py +0 -2
train/shard_dataset.py CHANGED
@@ -90,8 +90,6 @@ def make_dataset(store, seq_len, seed, start_sample=0, count=None, shuffle=True)
90
  import torch
91
 
92
  n_samples = store.total_tokens // seq_len
93
- if drop_last is False:
94
- raise ValueError("partial trailing windows are not supported; drop_last must stay True")
95
  perm = None
96
  if shuffle:
97
  # Seed must be identical on every resume or the two runs read different data orders. Stored in
 
90
  import torch
91
 
92
  n_samples = store.total_tokens // seq_len
 
 
93
  perm = None
94
  if shuffle:
95
  # Seed must be identical on every resume or the two runs read different data orders. Stored in