Update script_for_automation.py

script_for_automation.py

@@ -329,52 +329,51 @@ def get_data_ready(recipe_dict, input_data_piece):
 
     print("DID THAT NOW")
     return processed_data
-        
-import yaml
-import json
+
 
 def generate_markdown_output(df):
     # Start the markdown output string
     markdown = ""
 
-    # Input Transcript Section
-    markdown += "### Input Transcript\n"
-    markdown += "Since the input transcript might be very long, it is truncated here for readability:\n\n"
-
+    # 1. Input Transcript
+    markdown += "## Input Transcript\n"
     for _, row in df.iterrows():
-        truncated_input = (row['Input_Transcript'][:500] + '...') if len(row['Input_Transcript']) > 500 else row['Input_Transcript']
-        markdown += f"**Recipe ID {row['Recipe_ID']}**:\n\n{truncated_input}\n\n"
-
-    # Recipe Fields Section
-    markdown += "\n### Recipe Fields (Basic Information)\n"
-    markdown += "| Recipe ID | Testing Strategy | Schema Processing Model | Pre-Processing Strategy | Pre-Processing Text | Pre-Processing Model | Prompting Strategy |\n"
-    markdown += "|-----------|------------------|-------------------------|--------------------------|---------------------|----------------------|-------------------|\n"
-
+        truncated_input = row['Input_Transcript'][:500] + "..." if len(row['Input_Transcript']) > 500 else row['Input_Transcript']
+        markdown += f"**Recipe ID {row['Recipe_ID']}**:\n```\n{truncated_input}\n```\n\n"
+
+    # 2. Recipe Fields
+    markdown += "## Recipe Fields\n"
+    recipe_columns = [
+        "Recipe ID", "Testing Strategy", "Schema Processing Model", "Pre-Processing Strategy", 
+        "Pre-Processing Text", "Pre-Processing Model", "Prompting Strategy"
+    ]
+    recipe_table = "| " + " | ".join(recipe_columns) + " |\n"
+    recipe_table += "| " + " | ".join(["-" * len(col) for col in recipe_columns]) + " |\n"
     for _, row in df.iterrows():
-        markdown += f"| {str(row['Recipe_ID']).ljust(10)} | {str(row['Testing_Strategy_Text']).ljust(20)} | {str(row['Schema_Processing_Model']).ljust(25)} | {str(row['Pre_Processing_Strategy']).ljust(23)} | {str(row['Pre_Processing_Text']).ljust(20)} | {str(row['Pre_Processing_Model']).ljust(20)} | {str(row['Prompting_Strategy']).ljust(25)} |\n"
-
-    # Prompts Section
-    markdown += "\n### Prompts\n"
-    markdown += "| Plantings and Fields Prompt | Interactions Prompt | Treatments Prompt |\n"
-    markdown += "|-----------------------------|---------------------|-------------------|\n"
-
+        recipe_table += f"| {row['Recipe_ID']} | {row['Testing_Strategy_Text']} | {row['Schema_Processing_Model']} | {row['Pre_Processing_Strategy']} | {row['Pre_Processing_Text']} | {row['Pre_Processing_Model']} | {row['Prompting_Strategy']} |\n"
+    markdown += recipe_table + "\n"
+
+    # 3. Prompts
+    markdown += "## Prompts\n"
+    prompt_columns = ["Plantings and Fields Prompt", "Interactions Prompt", "Treatments Prompt"]
+    prompt_table = "| " + " | ".join(prompt_columns) + " |\n"
+    prompt_table += "| " + " | ".join(["-" * len(col) for col in prompt_columns]) + " |\n"
     for _, row in df.iterrows():
-        markdown += f"| {str(row['Plantings_and_Fields_Prompt']).ljust(30)} | {str(row['Interactions_Prompt']).ljust(20)} | {str(row['Treatments_Prompt']).ljust(20)} |\n"
+        prompt_table += f"| {row['Plantings_and_Fields_Prompt']} | {row['Interactions_Prompt']} | {row['Treatments_Prompt']} |\n"
+    markdown += prompt_table + "\n"
 
-    # Side-by-Side Comparison
-    markdown += "\n### Gold Standard vs Machine Generated Key-Values\n"
+    # 4. Side-by-Side Comparisons
+    markdown += "## Gold Standard vs Machine Generated Key-Values\n"
     markdown += "| Key | Gold Standard | Machine Generated |\n"
     markdown += "|-----|---------------|-------------------|\n"
-
     for _, row in df.iterrows():
-        markdown += f"| {str(row['Recipe_ID']).ljust(10)} | {str(row['Gold_Standard_Key_Values']).ljust(25)} | {str(row['Machine_Generated_Key_Values']).ljust(25)} |\n"
+        markdown += f"| {row['Recipe_ID']} | {row['Gold_Standard_Key_Values']} | {row['Machine_Generated_Key_Values']} |\n"
+    markdown += "\n"
 
-    # Differences Section
-    markdown += "\n### Differences\n"
-    markdown += "The following differences were found between the gold standard and the machine-generated output:\n\n"
+    # 5. Differences
+    markdown += "## Differences\n"
     markdown += "| Key | Difference |\n"
     markdown += "|-----|------------|\n"
-
     for _, row in df.iterrows():
         differences = row['Differences']
         if isinstance(differences, list):
@@ -382,24 +381,24 @@ def generate_markdown_output(df):
                 if isinstance(diff, dict) and 'values_changed' in diff:
                     for path, change in diff['values_changed'].items():
                         if 'old_value' in change and 'new_value' in change:
-                            markdown += f"| {str(path).ljust(20)} | {str(change['old_value']).ljust(20)} -> {str(change['new_value']).ljust(20)} |\n"
-                        else:
-                            markdown += f"| {str(path).ljust(20)} | (Missing old/new value) |\n"
-                else:
-                    markdown += f"| (Invalid diff) | |\n"
+                            markdown += f"| {path} | {change['old_value']} → {change['new_value']} |\n"
         else:
-            markdown += f"| (No differences) | |\n"
-
-    # YAML Comparison Section
-    markdown += "\n### Gold Standard vs Machine Generated YAML\n"
-    markdown += "| Gold Standard YAML | Machine Generated YAML |\n"
-    markdown += "|--------------------|------------------------|\n"
+            markdown += "| No differences found | |\n"
 
+    # 6. YAML Comparisons
+    markdown += "## Gold Standard vs Machine Generated YAML\n"
     for _, row in df.iterrows():
-        gold_yaml = yaml.dump(yaml.safe_load(row['Gold_Standard_YAML']), default_flow_style=False)
-        machine_yaml = yaml.dump(yaml.safe_load(row['Machine_Generated_YAML']), default_flow_style=False)
-        markdown += f"| ```yaml\n{gold_yaml}``` | ```yaml\n{machine_yaml}``` |\n"
+        gold_yaml = yaml.safe_dump(yaml.safe_load(row['Gold_Standard_YAML']), default_flow_style=False)
+        machine_yaml = yaml.safe_dump(yaml.safe_load(row['Machine_Generated_YAML']), default_flow_style=False)
+
+        markdown += f"**Recipe ID {row['Recipe_ID']}**:\n\n"
+        markdown += "**Gold Standard YAML:**\n"
+        markdown += f"```yaml\n{gold_yaml}\n```\n"
+        markdown += "**Machine Generated YAML:**\n"
+        markdown += f"```yaml\n{machine_yaml}\n```\n\n"
 
+    # Ensure clean separation
+    markdown += "---\n\n"
     return markdown
     
 def drive_process():