Update script_for_automation.py

script_for_automation.py

@@ -340,73 +340,94 @@ def sanitize_json_for_yaml(data):
     else:
         return data  # Keep other types as-is
 
+def format_json(json_data, truncate_length=500):
+    try:
+        # Try to load the JSON data
+        parsed_data = json.loads(json_data)
+        # Convert it into a pretty-printed string
+        formatted_json = json.dumps(parsed_data, indent=2)
+        # Truncate if it's too long
+        return formatted_json[:truncate_length] + "..." if len(formatted_json) > truncate_length else formatted_json
+    except json.JSONDecodeError:
+        # If it's not valid JSON, return the string as it is
+        return json_data[:truncate_length] + "..." if len(json_data) > truncate_length else json_data
+
 def generate_markdown_output(df):
-    # Start the markdown output string
     markdown = ""
 
     # 1. Input Transcript
-    markdown += "## Input Transcript\n"
+    markdown += "\n## Input Transcript\n"  # Add space before header for consistency
     for _, row in df.iterrows():
         truncated_input = row['Input_Transcript'][:500] + "..." if len(row['Input_Transcript']) > 500 else row['Input_Transcript']
         markdown += f"**Recipe ID {row['Recipe_ID']}**:\n```\n{truncated_input}\n```\n\n"
 
     # 2. Recipe Fields
-    markdown += "## Recipe Fields\n"
+    markdown += "\n## Recipe Fields\n"  # Add space before header for consistency
     recipe_columns = [
         "Recipe ID", "Testing Strategy", "Schema Processing Model", "Pre-Processing Strategy", 
         "Pre-Processing Text", "Pre-Processing Model", "Prompting Strategy"
     ]
+    # Generate table
     recipe_table = "| " + " | ".join(recipe_columns) + " |\n"
     recipe_table += "| " + " | ".join(["-" * len(col) for col in recipe_columns]) + " |\n"
     for _, row in df.iterrows():
-        recipe_table += f"| {row['Recipe_ID']} | {row['Testing_Strategy_Text']} | {row['Schema_Processing_Model']} | {row['Pre_Processing_Strategy']} | {row['Pre_Processing_Text']} | {row['Pre_Processing_Model']} | {row['Prompting_Strategy']} |\n"
+        recipe_table += "| " + " | ".join([str(row[col]) for col in recipe_columns]) + " |\n"
     markdown += recipe_table + "\n"
 
     # 3. Prompts
-    markdown += "## Prompts\n"
+    markdown += "\n## Prompts\n"  # Add space before header for consistency
     prompt_columns = ["Plantings and Fields Prompt", "Interactions Prompt", "Treatments Prompt"]
     prompt_table = "| " + " | ".join(prompt_columns) + " |\n"
     prompt_table += "| " + " | ".join(["-" * len(col) for col in prompt_columns]) + " |\n"
     for _, row in df.iterrows():
-        prompt_table += f"| {row['Plantings_and_Fields_Prompt']} | {row['Interactions_Prompt']} | {row['Treatments_Prompt']} |\n"
+        prompt_table += "| " + " | ".join([str(row[col]) for col in prompt_columns]) + " |\n"
     markdown += prompt_table + "\n"
 
-    # 4. Side-by-Side Comparisons
-    markdown += "## Gold Standard vs Machine Generated Key-Values\n"
+    # 4. Gold Standard vs Machine Generated Key-Values
+    markdown += "\n## Gold Standard vs Machine Generated Key-Values\n"
     markdown += "| Key | Gold Standard | Machine Generated |\n"
     markdown += "|-----|---------------|-------------------|\n"
     for _, row in df.iterrows():
-        markdown += f"| {row['Recipe_ID']} | {row['Gold_Standard_Key_Values']} | {row['Machine_Generated_Key_Values']} |\n"
+        # Truncate or format the JSON-like data
+        gold_standard = format_json(row['Gold_Standard_Key_Values'])
+        machine_generated = format_json(row['Machine_Generated_Key_Values'])
+        markdown += f"| {row['Recipe_ID']} | {gold_standard} | {machine_generated} |\n"
     markdown += "\n"
 
     # 5. Differences
-    markdown += "## Differences\n"
+    markdown += "\n## Differences\n"
     markdown += "| Key | Difference |\n"
     markdown += "|-----|------------|\n"
     for _, row in df.iterrows():
         differences = row['Differences']
         if isinstance(differences, list):
-            for diff in differences:
-                if isinstance(diff, dict) and 'values_changed' in diff:
-                    for path, change in diff['values_changed'].items():
-                        if 'old_value' in change and 'new_value' in change:
-                            markdown += f"| {path} | {change['old_value']} → {change['new_value']} |\n"
+            if len(differences) > 0:
+                for diff in differences:
+                    if isinstance(diff, dict) and 'values_changed' in diff:
+                        for path, change in diff['values_changed'].items():
+                            if 'old_value' in change and 'new_value' in change:
+                                markdown += f"| {path} | {change['old_value']} → {change['new_value']} |\n"
+            else:
+                markdown += f"| {row['Recipe_ID']} | No differences found |\n"
         else:
-            markdown += "| No differences found | |\n"
+            markdown += f"| {row['Recipe_ID']} | No differences found |\n"
+    markdown += "\n"
 
     # 6. YAML Comparisons
-    markdown += "## Gold Standard vs Machine Generated YAML\n"
+    markdown = "\n## Gold Standard vs Machine Generated YAML\n"
+    
     for _, row in df.iterrows():
+        # Ensure YAML data is properly loaded and formatted
         gold_yaml = yaml.safe_dump(yaml.safe_load(row['Gold_Standard_YAML']), default_flow_style=False)
         machine_yaml = yaml.safe_dump(yaml.safe_load(row['Machine_Generated_YAML']), default_flow_style=False)
 
+        # Add comparison to markdown
         markdown += f"**Recipe ID {row['Recipe_ID']}**:\n\n"
         markdown += "**Gold Standard YAML:**\n"
         markdown += f"```yaml\n{gold_yaml}\n```\n"
         markdown += "**Machine Generated YAML:**\n"
         markdown += f"```yaml\n{machine_yaml}\n```\n\n"
 
-    # Ensure clean separation
     markdown += "---\n\n"
     return markdown