Update script_for_automation.py

script_for_automation.py

@@ -352,7 +352,20 @@ def format_json(json_data, truncate_length=500):
         # If it's not valid JSON, return the string as it is
         return json_data[:truncate_length] + "..." if len(json_data) > truncate_length else json_data
 
+import yaml
+
+def sanitize_json_for_yaml(data):
+    if isinstance(data, dict):
+        return {key: sanitize_json_for_yaml(value) for key, value in data.items()}
+    elif isinstance(data, list):
+        return [sanitize_json_for_yaml(item) for item in data]
+    elif isinstance(data, tuple):  # Convert tuples to lists
+        return list(data)
+    else:
+        return data  # Keep other types as-is
+
 def generate_markdown_output(df):
+    # Start the markdown output string
     markdown = ""
 
     # 1. Input Transcript
@@ -367,11 +380,10 @@ def generate_markdown_output(df):
         "Recipe ID", "Testing Strategy", "Schema Processing Model", "Pre-Processing Strategy", 
         "Pre-Processing Text", "Pre-Processing Model", "Prompting Strategy"
     ]
-    # Generate table
     recipe_table = "| " + " | ".join(recipe_columns) + " |\n"
     recipe_table += "| " + " | ".join(["-" * len(col) for col in recipe_columns]) + " |\n"
     for _, row in df.iterrows():
-        recipe_table += "| " + " | ".join([str(row[col]) for col in recipe_columns]) + " |\n"
+        recipe_table += f"| {row['Recipe_ID']} | {row['Testing_Strategy_Text']} | {row['Schema_Processing_Model']} | {row['Pre_Processing_Strategy']} | {row['Pre_Processing_Text']} | {row['Pre_Processing_Model']} | {row['Prompting_Strategy']} |\n"
     markdown += recipe_table + "\n"
 
     # 3. Prompts
@@ -380,54 +392,45 @@ def generate_markdown_output(df):
     prompt_table = "| " + " | ".join(prompt_columns) + " |\n"
     prompt_table += "| " + " | ".join(["-" * len(col) for col in prompt_columns]) + " |\n"
     for _, row in df.iterrows():
-        prompt_table += "| " + " | ".join([str(row[col]) for col in prompt_columns]) + " |\n"
+        prompt_table += f"| {row['Plantings_and_Fields_Prompt']} | {row['Interactions_Prompt']} | {row['Treatments_Prompt']} |\n"
     markdown += prompt_table + "\n"
 
-    # 4. Gold Standard vs Machine Generated Key-Values
-    markdown += "\n## Gold Standard vs Machine Generated Key-Values\n"
+    # 4. Side-by-Side Comparisons
+    markdown += "\n## Gold Standard vs Machine Generated Key-Values\n"  # Add space before header for consistency
     markdown += "| Key | Gold Standard | Machine Generated |\n"
     markdown += "|-----|---------------|-------------------|\n"
     for _, row in df.iterrows():
-        # Truncate or format the JSON-like data
-        gold_standard = format_json(row['Gold_Standard_Key_Values'])
-        machine_generated = format_json(row['Machine_Generated_Key_Values'])
-        markdown += f"| {row['Recipe_ID']} | {gold_standard} | {machine_generated} |\n"
+        markdown += f"| {row['Recipe_ID']} | {row['Gold_Standard_Key_Values']} | {row['Machine_Generated_Key_Values']} |\n"
     markdown += "\n"
 
     # 5. Differences
-    markdown += "\n## Differences\n"
+    markdown += "\n## Differences\n"  # Add space before header for consistency
     markdown += "| Key | Difference |\n"
     markdown += "|-----|------------|\n"
     for _, row in df.iterrows():
         differences = row['Differences']
         if isinstance(differences, list):
-            if len(differences) > 0:
-                for diff in differences:
-                    if isinstance(diff, dict) and 'values_changed' in diff:
-                        for path, change in diff['values_changed'].items():
-                            if 'old_value' in change and 'new_value' in change:
-                                markdown += f"| {path} | {change['old_value']} → {change['new_value']} |\n"
-            else:
-                markdown += f"| {row['Recipe_ID']} | No differences found |\n"
+            for diff in differences:
+                if isinstance(diff, dict) and 'values_changed' in diff:
+                    for path, change in diff['values_changed'].items():
+                        if 'old_value' in change and 'new_value' in change:
+                            markdown += f"| {path} | {change['old_value']} → {change['new_value']} |\n"
         else:
-            markdown += f"| {row['Recipe_ID']} | No differences found |\n"
-    markdown += "\n"
+            markdown += "| No differences found | |\n"
 
     # 6. YAML Comparisons
-    markdown = "\n## Gold Standard vs Machine Generated YAML\n"
-    
+    markdown += "\n## Gold Standard vs Machine Generated YAML\n"  # Add space before header for consistency
     for _, row in df.iterrows():
-        # Ensure YAML data is properly loaded and formatted
         gold_yaml = yaml.safe_dump(yaml.safe_load(row['Gold_Standard_YAML']), default_flow_style=False)
         machine_yaml = yaml.safe_dump(yaml.safe_load(row['Machine_Generated_YAML']), default_flow_style=False)
 
-        # Add comparison to markdown
         markdown += f"**Recipe ID {row['Recipe_ID']}**:\n\n"
         markdown += "**Gold Standard YAML:**\n"
         markdown += f"```yaml\n{gold_yaml}\n```\n"
         markdown += "**Machine Generated YAML:**\n"
         markdown += f"```yaml\n{machine_yaml}\n```\n\n"
 
+    # Ensure clean separation
     markdown += "---\n\n"
     return markdown