import requests from jsondiff import diff import yaml import pandas as pd import os import shutil import json from datetime import datetime # The purpose of this script is to automate running a bunch of tests # This script will take an input folder # The input folder should contain: # 1. A file containing a list of the recipe parameters # 2. A file containing the input data for each of the schemas # 3. .... # Steps to do this that we will outline then perform # First, get the gold standard JSONs from baserow # Next, get the recipe parameter list from the input folder # Iterate through the recipe parameter list one at a time # In the iteration, first fill out a surveystack submission - is this possible with the current surveystack API? # Next, save the surveystack submission ID (?) # Use the iteration parameters to then get the three JSONs back from chatgpt # Compare the JSONs to the gold standard JSONs # Print out the differences in a .csv # Print out a side by side of the yaml # store all these together # continue through iterations # create downloadables of the results BASEROW_API_KEY = os.getenv("BASEROW_API_KEY") from process_data import process_specifications def get_baserow_url(table_id): print("GETTING BASEROW URL") BASEROW_API_BASE = "https://baserow.f11804a1.federatedcomputer.net/api" return f"{BASEROW_API_BASE}/database/rows/table/{table_id}/?user_field_names=true" def get_baserow_data(): # This is to get the gold standards from baserow # We will also get the input data print("GETTING BASEROW DATA") TABLE_ID = "560" BASEROW_URL = get_baserow_url(TABLE_ID) headers = { "Authorization": f"Token {os.environ['BASEROW_API_KEY']}", "Content-Type": "application/json" } print("STARTING TO TRY RESPONSE REQUEST") try: response = requests.get(BASEROW_URL, headers=headers) print("GOT") response.raise_for_status() rows = response.json() results = rows.get("results", []) print("PARSING ROWS NOW") for row in results: print(f"Row ID: {row.get('id')}, Data: {row}") if row.get("id") == 2: liz_carrot_plantings_gold_standard = row.get("Plantings and Fields - Gold Standard") liz_carrot_interactions_gold_standard = row.get("Interactions - Gold Standard") liz_carrot_trials_gold_standard = row.get("Trials - Gold Standard") liz_carrot_input_data_raw_interview = row.get("Raw Interview") liz_carrot_otter_summary_preprocessing = row.get("Otter Summary") liz_carrot_greg_summary_preprocessing = row.get("Post-Interview Summary") elif row.get("id") == 3: ben_soybean_plantings_gold_standard = row.get("Plantings and Fields - Gold Standard") ben_soybean_interactions_gold_standard = row.get("Interactions - Gold Standard") ben_soybean_trials_gold_standard = row.get("Trials - Gold Standard") ben_soybean_input_data_raw_interview = row.get("Raw Interview") ben_soybean_otter_summary_preprocessing = row.get("Otter Summary") ben_soybean_greg_summary_preprocessing = row.get("Post-Interview Summary") elif row.get("id") == 5: wally_squash_plantings_gold_standard = row.get("Plantings and Fields - Gold Standard") wally_squash_interactions_gold_standard = row.get("Interactions - Gold Standard") wally_squash_trials_gold_standard = row.get("Trials - Gold Standard") wally_squash_input_data_raw_interview = row.get("Raw Interview") wally_squash_otter_summary_preprocessing = row.get("Otter Summary") wally_squash_greg_summary_preprocessing = row.get("Post-Interview Summary") gold_standards = { "liz_carrot": { "planting": liz_carrot_plantings_gold_standard, "interactions": liz_carrot_interactions_gold_standard, "trials": liz_carrot_trials_gold_standard, }, "ben_soybean": { "planting": ben_soybean_plantings_gold_standard, "interactions": ben_soybean_interactions_gold_standard, "trials": ben_soybean_trials_gold_standard, }, "wally_squash": { "planting": wally_squash_plantings_gold_standard, "interactions": wally_squash_interactions_gold_standard, "trials": wally_squash_trials_gold_standard, } } # How to retrieve this data # liz_carrot_planting = gold_standards["liz_carrot"]["planting"] input_data = { "liz_carrot": { "raw_interview": liz_carrot_input_data_raw_interview, "otter_summary": liz_carrot_otter_summary_preprocessing, "greg_summary": liz_carrot_greg_summary_preprocessing }, "ben_soybean": { "raw_interview": ben_soybean_input_data_raw_interview, "otter_summary": ben_soybean_otter_summary_preprocessing, "greg_summary": ben_soybean_greg_summary_preprocessing }, "wally_squash": { "raw_interview": wally_squash_input_data_raw_interview, "otter_summary": wally_squash_otter_summary_preprocessing, "greg_summary": wally_squash_greg_summary_preprocessing } } print("BASEROW DATA DONE GOT") print("GOLD STANDARDS HERE") print(gold_standards) print("INPUT DATA HERE") print(input_data) return gold_standards, input_data except requests.exceptions.RequestException as e: print(f"Failed to fetch rows: {e}") def get_recipes(): print("GETTING RECIPES FROM BASEROW NOW") #TABLE_ID = "588" #TABLE_ID = "578" #TABLE_ID = "580" This table contains only one row for testing purposes TABLE_ID = "589" BASEROW_URL = get_baserow_url(TABLE_ID) headers = { "Authorization": f"Token {os.environ['BASEROW_API_KEY']}", "Content-Type": "application/json" } print("TRYING TO GET A RESPONSE") try: response = requests.get(BASEROW_URL, headers=headers) response.raise_for_status() rows = response.json() results = rows.get("results", []) my_recipes = [] print("PARSING ROWS") for row in results: print(f"Row ID: {row.get('id')}, Data: {row}") recipe_id = row.get("Recipe ID") testing_strategy_text = row.get("Testing Strategy for Set") schema_processing_model = row.get("Schema Processing Model", {}).get("value", None) pre_processing_strategy = row.get("Pre-Processing Strategy", [{}])[0].get("value", None) pre_processing_text = row.get("Pre-Prompt Text") pre_processing_model = row.get("Preprocessing Model", {}).get("value", None) prompting_strategy = row.get("Prompting Strategy", [{}])[0].get("value", None) plantings_and_fields_prompt = row.get("Plantings and Fields Prompting Text") interactions_prompt = row.get("Interactions Prompting Text") treatments_prompt = row.get("Treatments Prompting Text") recipe_dict = { "recipe_id": recipe_id, "testing_strategy_text": testing_strategy_text, "schema_processing_model": schema_processing_model, "pre_processing_strategy": pre_processing_strategy, "pre_processing_text": pre_processing_text, "pre_processing_model": pre_processing_model, "prompting_strategy": prompting_strategy, "plantings_and_fields_prompt": plantings_and_fields_prompt, "interactions_prompt": interactions_prompt, "treatments_prompt": treatments_prompt } my_recipes.append(recipe_dict) print("FINISHED GETTING THE RECIPE DATA") print("RECIPES HERE") print(my_recipes) return my_recipes except requests.exceptions.RequestException as e: print(f"Failed to fetch rows: {e}") def fill_out_survey(recipe_dict, input_data): print("filling out survey") survey_id = "673b4994aef86f0533b3546c" base_url = "https://app.surveystack.io/api/submissions" if recipe_dict.get("pre_processing_text") is None: pre_processing = False pre_process = "no" pre_process_model_version = "None" else: pre_processing = True pre_process = recipe_dict # Set the prompting strategy to be a variable from the list # Do this here if pre_processing: submission_data = { "survey": survey_id, "data": { "inputstyle": "big-block-input-text", "onelonginputtext": input_data, "schema_prompt": { "firstschemaprompt": recipe_dict["plantings_and_fields_prompt"], "secondschemaprompt": recipe_dict["interactions_prompt"], "thirdschemaprompt": recipe_dict["treatments_prompt"], }, }, "parameters": { "modelversion": recipe_dict["schema_processing_model"], "preprocessdata": ["yes"], "promptstyle": recipe_dict["prompting_strategy"], "preprocessmodelversion": recipe_dict["prompting_strategy"], "multiplepreprompts": "no", "prepromptstyle": recipe_dict["pre_processing_strategy"], "preprocessingprompt1": recipe_dict["pre_processing_text"], "preprocessingprompt2": "", "preprocessingprompt3": "" } } else: submission_data = { "survey": survey_id, "data": { "inputstyle": "big-block-input-text", "onelonginputtext": input_data, "schema_prompt": { "firstschemaprompt": recipe_dict["plantings_and_fields_prompt"], "secondschemaprompt": recipe_dict["interactions_prompt"], "thirdschemaprompt": recipe_dict["treatments_prompt"], }, }, "parameters": { "modelversion": recipe_dict["schema_processing_model"], "preprocessdata": ["no"], "promptstyle": recipe_dict["prompting_strategy"], "preprocessmodelversion": None, "multiplepreprompts": "no", "prepromptstyle": None, "preprocessingprompt1": None, "preprocessingprompt2": None, "preprocessingprompt3": None } } headers = { "Content-Type": "application/json", } print("GETTING SURVEY RESPONSE") try: response = requests.post(base_url, headers=headers, data=json.dumps(submission_data)) response.raise_for_status() if response.status_code == 200: print("Submission successful to SurveyStack!") print(response.json()) return submission_data else: print(f"Failed to submit: {response.status_code} - {response.text}") except requests.exceptions.RequestException as e: print(f"An error occurred while submitting the data: {e}") def get_data_ready(recipe_dict, input_data_piece): ## Input chunk structure # "raw_interview": liz_carrot_input_data_raw_interview, # # # recipe_dict = { # "recipe_id": recipe_id, # "testing_strategy_text": testing_strategy_text, # "schema_processing_model", schema_processing_model, # "pre_processing_strategy", pre_processing_strategy, # "pre_processing_text", pre_processing_text, # "pre_processing_model", pre_processing_model, # "prompting_strategy", prompting_strategy, # "plantings_and_fields_prompt", plantings_and_fields_prompt, # "interactions_prompt", interactions_prompt, # "treatments_prompt", treatments_prompt # } # print("GETTING DATA READY") processed_data = {} processed_data["prompts"] = {} processed_data["inputstyle"] = 'big-block-input-text' processed_data["input_text"] = input_data_piece processed_data["prompts"]["firstschemaprompt"] = recipe_dict["plantings_and_fields_prompt"] processed_data["prompts"]["secondschemaprompt"] = recipe_dict["interactions_prompt"] processed_data["prompts"]["thirdschemaprompt"] = recipe_dict["treatments_prompt"] processed_data["parameters"] = {} processed_data["parameters"]["modelversion"] = recipe_dict["schema_processing_model"] processed_data["parameters"]["promptstyle"] = recipe_dict["prompting_strategy"] if (recipe_dict["pre_processing_strategy"] == "None") and (recipe_dict["pre_processing_model"] == "No preprocessing"): processed_data["parameters"]["preprocessdata"] = "no" else: processed_data["parameters"]["preprocessdata"] = "yes" processed_data["parameters"]["preprocessmodelversion"] = recipe_dict["pre_processing_model"] processed_data["parameters"]["multiplepreprompts"] = "no" processed_data["parameters"]["prepromptstyle"] = recipe_dict["pre_processing_strategy"] processed_data["parameters"]["preprocessingprompt1"] = recipe_dict["pre_processing_text"] processed_data["parameters"]["preprocessingprompt2"] = "" processed_data["parameters"]["preprocessingprompt3"] = "" print("DID THAT NOW") return processed_data def format_json(json_data, truncate_length=500): try: # Try to load the JSON data parsed_data = json.loads(json_data) # Convert it into a pretty-printed string formatted_json = json.dumps(parsed_data, indent=2) # Truncate if it's too long return formatted_json[:truncate_length] + "..." if len(formatted_json) > truncate_length else formatted_json except json.JSONDecodeError: # If it's not valid JSON, return the string as it is return json_data[:truncate_length] + "..." if len(json_data) > truncate_length else json_data # Custom method to handle all objects def custom_serializer(obj): if isinstance(obj, Enum): return obj.name # Or obj.value, depending on what you need if isinstance(obj, Soil): return obj.to_dict() if isinstance(obj, Yield): return obj.to_dict() return obj.__dict__ # Default case: use the __dict__ method for other custom objects def sanitize_json_for_yaml(data): if isinstance(data, dict): return {key: sanitize_json_for_yaml(value) for key, value in data.items()} elif isinstance(data, list): return [sanitize_json_for_yaml(item) for item in data] elif isinstance(data, tuple): # Convert tuples to lists return list(data) else: return data # Keep other types as-is def generate_markdown_output(df): # Start the markdown output string markdown = "" # 1. Input Transcript markdown += "\n## Input Transcript\n" for _, row in df.iterrows(): truncated_input = row['Input_Transcript'][:500] + "..." if len(row['Input_Transcript']) > 500 else row['Input_Transcript'] markdown += f"**Recipe ID {row['Recipe_ID']}**:\n```\n{truncated_input}\n```\n\n" # 2. Recipe Fields markdown += "\n## Recipe Fields\n" recipe_columns = [ "Recipe ID", "Testing Strategy", "Schema Processing Model", "Pre-Processing Strategy", "Pre-Processing Text", "Pre-Processing Model", "Prompting Strategy" ] recipe_table = "| " + " | ".join(recipe_columns) + " |\n" recipe_table += "| " + " | ".join(["-" * len(col) for col in recipe_columns]) + " |\n" for _, row in df.iterrows(): recipe_table += f"| {row['Recipe_ID']} | {row['Testing_Strategy_Text']} | {row['Schema_Processing_Model']} | {row['Pre_Processing_Strategy']} | {row['Pre_Processing_Text']} | {row['Pre_Processing_Model']} | {row['Prompting_Strategy']} |\n" markdown += recipe_table + "\n" # 3. Differences markdown += "\n## Differences\n" for _, row in df.iterrows(): markdown += f"\n### Recipe ID: {row['Recipe_ID']}\n" differences = row['Differences'] # Loop through the differences list for key, value in differences.items(): markdown += f"#### {key.capitalize()}\n" for item in value: markdown += f" - {item}\n" # 4. Prompts markdown += "\n## Prompts\n" prompt_columns = ["Plantings and Fields Prompt", "Interactions Prompt", "Treatments Prompt"] prompt_table = "| " + " | ".join(prompt_columns) + " |\n" prompt_table += "| " + " | ".join(["-" * len(col) for col in prompt_columns]) + " |\n" for _, row in df.iterrows(): prompt_table += f"| {row['Plantings_and_Fields_Prompt']} | {row['Interactions_Prompt']} | {row['Treatments_Prompt']} |\n" markdown += prompt_table + "\n" # 5. Side-by-Side JSON Comparisons markdown += "\n## Gold Standard vs Machine Generated JSON\n" for _, row in df.iterrows(): markdown += f"\n### Recipe ID: {row['Recipe_ID']}\n" for key in ["planting", "interactions", "trials"]: gold = json.dumps(row['Gold_Standard_JSON'].get(key, {}), indent=2) machine = json.dumps(row['Machine_Generated_JSON'].get(key, {}), default=custom_serializer, indent=2) markdown += f"#### {key.capitalize()}\n" markdown += f"**Gold Standard JSON**:\n```json\n{gold}\n```\n" markdown += f"**Machine Generated JSON**:\n```json\n{machine}\n```\n" # 6. Side-by-Side YAML Comparisons markdown += "\n## Gold Standard vs Machine Generated YAML\n" for _, row in df.iterrows(): markdown += f"\n### Recipe ID: {row['Recipe_ID']}\n" for key in ["planting", "interactions", "trials"]: gold = yaml.dump(row['Gold_Standard_JSON'].get(key, {}), default_flow_style=False, sort_keys=True) machine = yaml.dump(row['Machine_Generated_JSON'].get(key, {}), default_flow_style=False, sort_keys=True) markdown += f"#### {key.capitalize()}\n" markdown += f"**Gold Standard YAML**:\n```yaml\n{gold}\n```\n" markdown += f"**Machine Generated YAML**:\n```yaml\n{machine}\n```\n" return markdown def drive_process(): # this is to drive the processing process print("We are starting to DRIVE PROCESS") # Get the data from baserow (gold standards JSON and Input data) gold_standards, input_data = get_baserow_data() # Get the recipes from baserow too my_recipes = get_recipes() # Input chunk structure # "liz_carrot": { # "raw_interview": liz_carrot_input_data_raw_interview, # "otter_summary": liz_carrot_otter_summary_preprocessing, # "greg_summary": liz_carrot_greg_summary_preprocessing # }, print("Making the OUTPUT STUFF") output_folder = "output_results_" +datetime.now().strftime("%Y%m%d_%H%M%S") os.makedirs(output_folder, exist_ok=True) print("GOING THROUGH RECIPES NOW") for recipe_dict in my_recipes: for key, input_chunks in input_data.items(): output_rows = [] print("RECIPE INFO") print(key) print(recipe_dict["recipe_id"]) # Get the input data based on the recipe if recipe_dict["pre_processing_strategy"] == "Otter.ai Summary": input_data_piece = input_chunks["otter_summary"] elif recipe_dict["pre_processing_strategy"] == "Greg Summary": input_data_piece = input_chunks["greg_summary"] else: input_data_piece = input_chunks["raw_interview"] print("DECIDED INPUT DATA") print(input_data_piece) # Fill out a Surveystack submission # This isn't accepted by the data #fill_out_survey(recipe_dict, input_data) # Prepare the data for the structured output setup proc_spec = get_data_ready(recipe_dict, input_data_piece) print("Gold Standard") # Get the gold standard for this input_chunk (key = liz_carrot, ben_soybean, wally_squash) gold_standard_json = gold_standards[key] # "liz_carrot": { # "planting": liz_carrot_plantings_gold_standard, # "interactions": liz_carrot_interactions_gold_standard, # "trials": liz_carrot_trials_gold_standard, # }, gold_standard_planting_json = json.loads(gold_standard_json["planting"]) gold_standard_interactions_json = json.loads(gold_standard_json["interactions"]) gold_standard_trials_json = json.loads(gold_standard_json["trials"]) print("Gold standard json after loading") print(gold_standard_planting_json) print("PROCESSING SPECIFICATIONS!!!!!!!!!!!!!!!") processed_farm_activity_json, processed_interactions_json, processed_trials_json = process_specifications(proc_spec) # THIS SHOULD ONLY BE USED FOR TESTING #processed_farm_activity_json = gold_standard_planting_json #processed_interactions_json = gold_standard_interactions_json #processed_trials_json = gold_standard_trials_json processed_farm_activity_json = json.loads(processed_farm_activity_json) processed_interactions_json = json.loads(processed_interactions_json) processed_trials_json = json.loads(processed_trials_json) print("Processed and loaded 1st json from machine gen") print(processed_farm_activity_json) # Compare the generated JSON to the gold standard differences_planting = list(diff(gold_standard_planting_json, processed_farm_activity_json)) differences_interactions = list(diff(gold_standard_interactions_json, processed_interactions_json)) differences_trials = list(diff(gold_standard_trials_json, processed_trials_json)) print("Diff planting") print(differences_planting) # Convert to yaml completed_gold_standard_planting_json = sanitize_json_for_yaml(gold_standard_planting_json) completed_gold_standard_interactions_json = sanitize_json_for_yaml(gold_standard_interactions_json) completed_gold_standard_trials_json = sanitize_json_for_yaml(gold_standard_trials_json) completed_processed_farm_activity_json = sanitize_json_for_yaml(processed_farm_activity_json) completed_processed_interactions_json = sanitize_json_for_yaml(processed_interactions_json) completed_processed_trials_json = sanitize_json_for_yaml(processed_trials_json) json_diff = { "planting": differences_planting, "interactions": differences_interactions, "trials": differences_trials } gold_standard_json = { "planting": completed_gold_standard_planting_json, "interactions": completed_gold_standard_interactions_json, "trials": completed_gold_standard_trials_json } comparison_json = { "planting": completed_processed_farm_activity_json, "interactions": completed_processed_interactions_json, "trials": completed_processed_trials_json } recipe_id = recipe_dict.get("recipe_id", "N/A") output_rows.append({ "Recipe_ID": recipe_id, "Testing_Strategy_Text": recipe_dict.get("testing_strategy_text", "N/A"), "Schema_Processing_Model": recipe_dict.get("schema_processing_model", "N/A"), "Pre_Processing_Strategy": recipe_dict.get("pre_processing_strategy", "N/A"), "Pre_Processing_Text": recipe_dict.get("pre_processing_text", "N/A"), "Pre_Processing_Model": recipe_dict.get("pre_processing_model", "N/A"), "Prompting_Strategy": recipe_dict.get("prompting_strategy", "N/A"), "Plantings_and_Fields_Prompt": recipe_dict.get("plantings_and_fields_prompt", "N/A"), "Interactions_Prompt": recipe_dict.get("interactions_prompt", "N/A"), "Treatments_Prompt": recipe_dict.get("treatments_prompt", "N/A"), "Input_Transcript": input_chunks, "Gold_Standard_JSON": gold_standard_json, "Machine_Generated_JSON": comparison_json, "Differences": json_diff }) df = pd.DataFrame(output_rows) print("dataframe done now onto markdown") markdown_output = generate_markdown_output(df) recipe_folder = os.path.join(output_folder, f"recipe_{recipe_dict['recipe_id']}") os.makedirs(recipe_folder, exist_ok=True) # Save markdown to file markdown_file = os.path.join(recipe_folder, f"recipe_{recipe_dict['recipe_id']}_data_{key}_output.md") with open(markdown_file, 'w') as f: f.write(markdown_output) # Save JSON files json_file_gold = os.path.join(recipe_folder, f"recipe_{recipe_dict['recipe_id']}_data_{key}_gold_standard.json") json_file_generated = os.path.join(recipe_folder, f"recipe_{recipe_dict['recipe_id']}_data_{key}_generated.json") with open(json_file_gold, 'w') as f: json.dump(gold_standard_json, f, indent=2) with open(json_file_generated, 'w') as f: json.dump(comparison_json, f, indent=2) # Optionally save differences as a separate file differences_file = os.path.join(recipe_folder, f"recipe_{recipe_dict['recipe_id']}_data_{key}_differences.json") with open(differences_file, 'w') as f: f.write(str(differences_file)) print("ZIPPING UP WHOLE THING") # Zip the entire output folder zip_filename = f"{output_folder}.zip" shutil.make_archive(output_folder, 'zip', output_folder) # Cleanup by removing the unzipped folder after zipping it shutil.rmtree(output_folder) # Return the zip file for downloading return zip_filename return output_folder