Andrej Karpathy commited on
Commit ·
bdcc030
1
Parent(s): 22a71aa
oops legacy spurious line now
Browse files- scripts/base_train.py +0 -1
scripts/base_train.py
CHANGED
|
@@ -208,7 +208,6 @@ if resuming:
|
|
| 208 |
|
| 209 |
# -----------------------------------------------------------------------------
|
| 210 |
# Initialize the DataLoaders for train/val
|
| 211 |
-
tokens_dir = os.path.join(base_dir, "tokenized_data")
|
| 212 |
dataloader_resume_state_dict = None if not resuming else meta_data["dataloader_state_dict"]
|
| 213 |
train_loader = tokenizing_distributed_data_loader_with_state_bos_bestfit(tokenizer, args.device_batch_size, args.max_seq_len, split="train", device=device, resume_state_dict=dataloader_resume_state_dict)
|
| 214 |
build_val_loader = lambda: tokenizing_distributed_data_loader_bos_bestfit(tokenizer, args.device_batch_size, args.max_seq_len, split="val", device=device)
|
|
|
|
| 208 |
|
| 209 |
# -----------------------------------------------------------------------------
|
| 210 |
# Initialize the DataLoaders for train/val
|
|
|
|
| 211 |
dataloader_resume_state_dict = None if not resuming else meta_data["dataloader_state_dict"]
|
| 212 |
train_loader = tokenizing_distributed_data_loader_with_state_bos_bestfit(tokenizer, args.device_batch_size, args.max_seq_len, split="train", device=device, resume_state_dict=dataloader_resume_state_dict)
|
| 213 |
build_val_loader = lambda: tokenizing_distributed_data_loader_bos_bestfit(tokenizer, args.device_batch_size, args.max_seq_len, split="val", device=device)
|