Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
|
@@ -9,7 +9,7 @@ import requests as rq
|
|
| 9 |
import gc
|
| 10 |
from tokenizers import ByteLevelBPETokenizer
|
| 11 |
|
| 12 |
-
dataset = load_dataset("nroggendorff/openhermes", split="train")
|
| 13 |
|
| 14 |
def get_training_corpus():
|
| 15 |
for i in range(0, len(dataset), 1000):
|
|
@@ -129,4 +129,6 @@ trained_tokenizer = trainer.tokenizer
|
|
| 129 |
|
| 130 |
repo_id = "makeshift-mayo"
|
| 131 |
trained_model.push_to_hub(repo_id)
|
| 132 |
-
trained_tokenizer.push_to_hub(repo_id)
|
|
|
|
|
|
|
|
|
| 9 |
import gc
|
| 10 |
from tokenizers import ByteLevelBPETokenizer
|
| 11 |
|
| 12 |
+
dataset = load_dataset("nroggendorff/openhermes", split="train").select(range(int(1e+5)))
|
| 13 |
|
| 14 |
def get_training_corpus():
|
| 15 |
for i in range(0, len(dataset), 1000):
|
|
|
|
| 129 |
|
| 130 |
repo_id = "makeshift-mayo"
|
| 131 |
trained_model.push_to_hub(repo_id)
|
| 132 |
+
trained_tokenizer.push_to_hub(repo_id)
|
| 133 |
+
|
| 134 |
+
raise RuntimeError("The script was finished.")
|