Data saving

app.py

@@ -56,6 +56,18 @@ class BasicAgent:
             all_steps = self.agent.master_agent.memory.steps
             new_rows = []  # List to store new rows
             
+            def serialize_value(value):
+                """Convert complex objects to serializable format"""
+                if hasattr(value, 'dict'):
+                    return value.dict()
+                elif hasattr(value, '__dict__'):
+                    return str(value.__dict__)
+                elif isinstance(value, (list, tuple)):
+                    return [serialize_value(item) for item in value]
+                elif isinstance(value, dict):
+                    return {k: serialize_value(v) for k, v in value.items()}
+                return value
+            
             for step in all_steps:
                 if isinstance(step, ActionStep):
                     step_class = "ActionStep"
@@ -72,13 +84,20 @@ class BasicAgent:
 
                 step_dict = step.dict()
                 # Create a new row with default None values
-                new_row = {col: None for col in df_agent_steps.columns}
+                new_row = {col: "None" for col in df_agent_steps.columns}
                 # Update with actual values
                 new_row['task_id'] = task_id
                 new_row['step_class'] = step_class
+                
+                # Serialize complex objects before adding to DataFrame
                 for key, value in step_dict.items():
                     if key in df_agent_steps.columns:
-                        new_row[key] = value
+                        try:
+                            new_row[key] = serialize_value(value)
+                        except Exception as e:
+                            print(f"Warning: Could not serialize {key}, using string representation: {e}")
+                            new_row[key] = str(value)
+                
                 new_rows.append(new_row)
             
             # Append all new rows at once
@@ -121,6 +140,11 @@ def save_dataset_to_hub(df: pd.DataFrame, dataset_name: str) -> tuple[bool, str]
             
         print(f"Saving {len(df)} steps to {dataset_name}...")
         
+        # Convert complex types to strings before creating dataset
+        for col in df.columns:
+            if df[col].dtype == 'object':
+                df[col] = df[col].apply(lambda x: str(x) if pd.notnull(x) else None)
+        
         # Convert to dataset
         dataset = datasets.Dataset.from_pandas(df)
         
@@ -129,7 +153,23 @@ def save_dataset_to_hub(df: pd.DataFrame, dataset_name: str) -> tuple[bool, str]
         dataset.info.features = {
             'task_id': datasets.Value('string'),
             'step_class': datasets.Value('string'),
-            # Add other feature definitions as needed
+            'model_input_messages': datasets.Value('string'),
+            'tool_calls': datasets.Value('string'),
+            'start_time': datasets.Value('string'),
+            'end_time': datasets.Value('string'),
+            'step_number': datasets.Value('int64'),
+            'error': datasets.Value('string'),
+            'duration': datasets.Value('float64'),
+            'model_output_message': datasets.Value('string'),
+            'model_output': datasets.Value('string'),
+            'observations': datasets.Value('string'),
+            'observations_images': datasets.Value('string'),
+            'action_output': datasets.Value('string'),
+            'plan': datasets.Value('string'),
+            'task': datasets.Value('string'),
+            'task_images': datasets.Value('string'),
+            'system_prompt': datasets.Value('string'),
+            'final_answer': datasets.Value('string')
         }
         
         # Save to hub with token