drzeeIslam commited on
Commit
9e75e52
·
verified ·
1 Parent(s): 0d47e08

Update train.py

Browse files
Files changed (1) hide show
  1. train.py +6 -1
train.py CHANGED
@@ -1,3 +1,8 @@
 
 
 
 
 
1
  from datasets import load_dataset
2
  from transformers import AutoTokenizer, DataCollatorForLanguageModeling, Trainer, TrainingArguments, AutoModelForMaskedLM
3
 
@@ -37,4 +42,4 @@ trainer = Trainer(
37
  )
38
 
39
  # Start training
40
- trainer.train()
 
1
+ import os
2
+ os.environ["HF_HOME"] = "./hf_cache"
3
+ os.environ["TRANSFORMERS_CACHE"] = "./hf_cache/transformers"
4
+ os.environ["HF_DATASETS_CACHE"] = "./hf_cache/datasets"
5
+
6
  from datasets import load_dataset
7
  from transformers import AutoTokenizer, DataCollatorForLanguageModeling, Trainer, TrainingArguments, AutoModelForMaskedLM
8
 
 
42
  )
43
 
44
  # Start training
45
+ trainer.train()