add
Browse files- scripts/train.py +1 -1
scripts/train.py
CHANGED
|
@@ -35,7 +35,7 @@ def main():
|
|
| 35 |
|
| 36 |
# Load dataset
|
| 37 |
dataset = load_dataset("allenai/tulu-v2-sft-mixture", split="train")
|
| 38 |
-
|
| 39 |
def tokenize_function(examples):
|
| 40 |
texts = []
|
| 41 |
for message_list in examples["messages"]:
|
|
|
|
| 35 |
|
| 36 |
# Load dataset
|
| 37 |
dataset = load_dataset("allenai/tulu-v2-sft-mixture", split="train")
|
| 38 |
+
texts = []
|
| 39 |
def tokenize_function(examples):
|
| 40 |
texts = []
|
| 41 |
for message_list in examples["messages"]:
|