Filter None text values to fix TypeError in join
Browse files
train.py
CHANGED
|
@@ -40,7 +40,12 @@ print(f"Rows: {len(ds)}, Columns: {ds.column_names}")
|
|
| 40 |
print("Grouping conversations...")
|
| 41 |
conversations = defaultdict(list)
|
| 42 |
for row in ds:
|
| 43 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
for conv_id in conversations:
|
| 46 |
conversations[conv_id].sort(key=lambda x: x["date_time"])
|
|
@@ -55,7 +60,6 @@ for conv_id, turns in conversations.items():
|
|
| 55 |
for turn in turns:
|
| 56 |
role = "user" if turn["speaker"] == "client" else "assistant"
|
| 57 |
if role == current_role:
|
| 58 |
-
# Merge consecutive same-role turns
|
| 59 |
current_content.append(turn["text"])
|
| 60 |
else:
|
| 61 |
if current_role is not None:
|
|
|
|
| 40 |
print("Grouping conversations...")
|
| 41 |
conversations = defaultdict(list)
|
| 42 |
for row in ds:
|
| 43 |
+
text = row["text"] if row["text"] is not None else ""
|
| 44 |
+
conversations[row["conversation_id"]].append({
|
| 45 |
+
"speaker": row["speaker"],
|
| 46 |
+
"date_time": row["date_time"],
|
| 47 |
+
"text": text,
|
| 48 |
+
})
|
| 49 |
|
| 50 |
for conv_id in conversations:
|
| 51 |
conversations[conv_id].sort(key=lambda x: x["date_time"])
|
|
|
|
| 60 |
for turn in turns:
|
| 61 |
role = "user" if turn["speaker"] == "client" else "assistant"
|
| 62 |
if role == current_role:
|
|
|
|
| 63 |
current_content.append(turn["text"])
|
| 64 |
else:
|
| 65 |
if current_role is not None:
|