Fix doctrine dataset load (explicit jsonl)
Browse files- train_willay.py +5 -1
train_willay.py
CHANGED
|
@@ -74,7 +74,11 @@ def to_msgs(u, a):
|
|
| 74 |
|
| 75 |
|
| 76 |
def main():
|
| 77 |
-
doctrine = load_dataset(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
refusal = Dataset.from_list([to_msgs(u, a) for u, a in REFUSALS])
|
| 79 |
# upsample the refusal set so the specialist behavior dominates
|
| 80 |
combined = concatenate_datasets([doctrine, refusal, refusal, refusal])
|
|
|
|
| 74 |
|
| 75 |
|
| 76 |
def main():
|
| 77 |
+
doctrine = load_dataset(
|
| 78 |
+
"json",
|
| 79 |
+
data_files="hf://datasets/SZLHOLDINGS/szl-1-doctrine-sft/szl_dataset.jsonl",
|
| 80 |
+
split="train",
|
| 81 |
+
)
|
| 82 |
refusal = Dataset.from_list([to_msgs(u, a) for u, a in REFUSALS])
|
| 83 |
# upsample the refusal set so the specialist behavior dominates
|
| 84 |
combined = concatenate_datasets([doctrine, refusal, refusal, refusal])
|