betterwithage commited on
Commit
c3e05e1
·
verified ·
1 Parent(s): 1188123

Fix doctrine dataset load (explicit jsonl)

Browse files
Files changed (1) hide show
  1. train_willay.py +5 -1
train_willay.py CHANGED
@@ -74,7 +74,11 @@ def to_msgs(u, a):
74
 
75
 
76
  def main():
77
- doctrine = load_dataset("SZLHOLDINGS/szl-1-doctrine-sft", split="train")
 
 
 
 
78
  refusal = Dataset.from_list([to_msgs(u, a) for u, a in REFUSALS])
79
  # upsample the refusal set so the specialist behavior dominates
80
  combined = concatenate_datasets([doctrine, refusal, refusal, refusal])
 
74
 
75
 
76
  def main():
77
+ doctrine = load_dataset(
78
+ "json",
79
+ data_files="hf://datasets/SZLHOLDINGS/szl-1-doctrine-sft/szl_dataset.jsonl",
80
+ split="train",
81
+ )
82
  refusal = Dataset.from_list([to_msgs(u, a) for u, a in REFUSALS])
83
  # upsample the refusal set so the specialist behavior dominates
84
  combined = concatenate_datasets([doctrine, refusal, refusal, refusal])