ligaments-dev commited on
Commit
4778444
·
verified ·
1 Parent(s): 938e32b

Filter None text values to fix TypeError in join

Browse files
Files changed (1) hide show
  1. train.py +6 -2
train.py CHANGED
@@ -40,7 +40,12 @@ print(f"Rows: {len(ds)}, Columns: {ds.column_names}")
40
  print("Grouping conversations...")
41
  conversations = defaultdict(list)
42
  for row in ds:
43
- conversations[row["conversation_id"]].append(row)
 
 
 
 
 
44
 
45
  for conv_id in conversations:
46
  conversations[conv_id].sort(key=lambda x: x["date_time"])
@@ -55,7 +60,6 @@ for conv_id, turns in conversations.items():
55
  for turn in turns:
56
  role = "user" if turn["speaker"] == "client" else "assistant"
57
  if role == current_role:
58
- # Merge consecutive same-role turns
59
  current_content.append(turn["text"])
60
  else:
61
  if current_role is not None:
 
40
  print("Grouping conversations...")
41
  conversations = defaultdict(list)
42
  for row in ds:
43
+ text = row["text"] if row["text"] is not None else ""
44
+ conversations[row["conversation_id"]].append({
45
+ "speaker": row["speaker"],
46
+ "date_time": row["date_time"],
47
+ "text": text,
48
+ })
49
 
50
  for conv_id in conversations:
51
  conversations[conv_id].sort(key=lambda x: x["date_time"])
 
60
  for turn in turns:
61
  role = "user" if turn["speaker"] == "client" else "assistant"
62
  if role == current_role:
 
63
  current_content.append(turn["text"])
64
  else:
65
  if current_role is not None: