Update process_data.py

process_data.py

@@ -87,36 +87,22 @@ client = OpenAI()
 #    'prompt' = ['prompts': ["prompt1": "value", "prompt2", "value"]]
 #}
 
-
-def generate_json(input_data, parameters): 
+def generate_json(processed_data): 
     """
     Function to prompt OpenAI API to generate structured JSON output.
 
     Args: 
-        input_data: (dict) The input data, preprocessed, from the user. Aka what will fill in the JSON
-            input_data["input_text"] = the preprocessed input text
-            input_data["input_context"] = depending on levers, empty or what is put in front of the prompt
-        parameters: (dict) All of the individual parameters and "flippers" 
-            parameters["model_version"] = (str) what model should be used 
-            parameters["chaining"] = (bool) whether or not the preprocessed input context should be chained (given to multiple models)
-            parameters["context_pre_prompt"], parameters["summary_pre_prompt"], parameters["conversation_pre_prompt"], parameters["example_pre_prompt"] = (str) all of the pre-prompts, separated
-            parameters["combined_pre_prompt"] = (str) concatenated individual pre-prompts
 
     Returns: 
         3 processed data-filled JSON objects: farm_pretty_json, interactions_pretty_json, trial_pretty_json
     """
     print("Generating JSON Whole!")
-    input_text = input_data["input_text"]
-    model_version = parameters["model_version"]
+    input_text = processed_data["input_text"]
+    model_version = processed_data["parameters"]["model_version"]
 
-    farm_prompt = "Extract the farm information."
-    interactions_prompt = "Extract the interactions information."
-    trial_prompt = "Extract the trial information."
-    
-    if parameters["combined_pre_prompt"]:
-        farm_prompt = parameters["combined_pre_prompt"] + farm_prompt
-        interactions_prompt = parameters["combined_pre_prompt"] + interactions_prompt
-        trial_prompt = parameters["combined_pre_prompt"] + trial_prompt
+    farm_prompt = processed_data["prompts"]["first_schema_prompt"]
+    interactions_prompt = processed_data["prompts"]["second_schema_prompt"]
+    trial_prompt = processed_data["prompts"]["third_schema_prompt"]
 
         
     try:
@@ -178,8 +164,9 @@ def generate_json(input_data, parameters):
     except Exception as e:
         return {"error": "Failed to generate valid JSON. " + str(e)}
 
+
 # This is for the step-wise JSON creation
-def generate_json_pieces(input_data, parameters): 
+def generate_json_pieces(processed_data): 
     """
     This is primarily for one of the flippers, which allows each individual JSON section to be created individually, then concatenates them all together. 
     It is proposed that perhaps the individual calls to the model will be more robust than giving the model all the data at once. 
@@ -204,18 +191,14 @@ def generate_json_pieces(input_data, parameters):
     print("Generating JSON Pieces!")
     
     print("INPUT DATA")
-    print(input_data)
-    print("PARAMS")
-    print(parameters)
+    print(processed_data)
     
-    specification = input_data["input_text"]
-    model_version = parameters["model_version"]
+    model_version = processed_data["parameters"]["model_version"]
 
-    print("Specification and Model Version")
-    print(specification)
+    print("Model Version")
     print(model_version)
     
-    if parameters["pre_prompt"] and parameters["chaining"]:
+    if processed_data["parameters"]["pre_process"] == "yes":
         print("Pre prompt is true")
         field_data_input = input_data["input_text_pieces"]["pre_processed_pieces"]["field_data_input"]
         planting_data_input = input_data["input_text_pieces"]["pre_processed_pieces"]["planting_data_input"]
@@ -245,31 +228,18 @@ def generate_json_pieces(input_data, parameters):
 
     # Fix these prompts for all 
     print("Setting prompts")
-    field_prompt = "Extract the field information."
-    plant_prompt = "Extract the planting information."
-    log_prompt = "Extract the log information."
-    soil_prompt = "Extract the soil information."
-    yield_prompt = "Extract the yield information."
-
-    interaction_prompt = "Extract the interaction information"
-    person_prompt = "Please provide a list of people involved in this interaction, with each person's name, role, and any other relevant details."
-
-    trial_prompt = "Extract the trial information"
-    treatment_prompt = "Please provide a list of different treatments (strips or blocks with the same conditions applied) performed by the partner."
-
-
-    if parameters["combined_pre_prompt"]:
-        field_prompt = parameters["combined_pre_prompt"] + field_prompt
-        plant_prompt = parameters["combined_pre_prompt"] + plant_prompt
-        log_prompt = parameters["combined_pre_prompt"] + log_prompt
-        soil_prompt = parameters["combined_pre_prompt"] + soil_prompt
-        yield_prompt = parameters["combined_pre_prompt"] + yield_prompt
-
-        interaction_prompt = parameters["combined_pre_prompt"] + interaction_prompt
-        person_prompt = parameters["combined_pre_prompt"] + person_prompt
+    field_prompt = processed_data["prompts"]["first_schema_prompt_one"]
+    plant_prompt = processed_data["prompts"]["first_schema_prompt_two"]
+    log_prompt = processed_data["prompts"]["first_schema_prompt_three"]
+    soil_prompt = processed_data["prompts"]["first_schema_prompt_four"]
+    yield_prompt = processed_data["prompts"]["first_schema_prompt_five"]
+       
+    interaction_prompt = processed_data["prompts"]["second_schema_prompt_one"]
+    person_prompt = processed_data["prompts"]["second_schema_prompt_two"]
+        
+    trial_prompt = processed_data["prompts"]["third_schema_prompt_one"]
+    treatment_prompt = processed_data["prompts"]["third_schema_prompt_two"]
 
-        trial_prompt = parameters["combined_pre_prompt"] + trial_prompt
-        treatment_prompt = parameters["combined_pre_prompt"] + treatment_prompt
 
     try:
         # Call OpenAI API to generate structured output based on prompt
@@ -496,22 +466,11 @@ def generate_json_pieces(input_data, parameters):
         return {"error": "Failed to generate valid JSON. " + str(e)}
     
 
-def pre_processing(input_data, parameters):
+def pre_processing(processed_data):
     """
     In the event there's a pre-prompt, process the pre-prompts and input text accordingly
 
     Args: 
-        input_data: (dict) The input data, preprocessed, from the user. Aka what will fill in the JSON
-            input_data["input_text"] = (str) the preprocessed input text
-            input_data["input_context"] = (str) depending on levers, empty or what is put in front of the prompt
-            input_data["input_text_pieces"] = (dict) containing the individual split up prompt pieces: field_data_input, planting_data_input, log_data_input, soil_data_input, yield_data_input
-            
-        parameters: (dict) All of the individual parameters and "flippers" 
-            parameters["model_version"] = (str) what model should be used 
-            parameters["chaining"] = (bool) whether or not the preprocessed input context should be chained (given to multiple models)
-            parameters["context_pre_prompt"], parameters["summary_pre_prompt"], parameters["conversation_pre_prompt"], parameters["example_pre_prompt"] = (str) all of the pre-prompts, separated
-            parameters["combined_pre_prompt"] = (str) concatenated individual pre-prompts
-            parameters["additional_json_pieces_options"] = (str) "Explicit specific pieces" or "Parse from one big input text" to indicate whether it's many function calls on one input text or many function calls on smaller pieces of input texts
 
     Returns: 
         (dict) input_data
@@ -519,46 +478,49 @@ def pre_processing(input_data, parameters):
         input_data["input_text"] = (str) input text
     """
     print("Starting preprocessing")
-    if input_data["stepwise_json_creation"][0] == "stepwisejsoncreation":
+
+
+    pre_processing_list = [processed_data.get('parameters', {}).get('pre_prompt_first', None), processed_data.get('parameters', {}).get('pre_prompt_second', None), processed_data.get('parameters', {}).get('pre_prompt_third', None)]
+
+    print("Preprocessing list")
+    print(pre_processing_list)
+    
+    if processed_data["inputstyle"] == "individual-pieces-input-text":
         print("Stepwise Creation")
-        input_data["input_text_pieces"]["pre_processed_pieces"] = {}
+        processed_data["input_text_pieces"]["pre_processed_pieces"] = {}
+
+        for text_label, text_body in processed_data["input_text_pieces"].items():
+                    if 'data_input' in text_label:
+                        processed_data["parameters"]["pre_prompt_first"]
+                        # change this to pre-processing list
+                        for pre_prompt in pre_processing_list:
+                            if pre_process_task:
+                                print("Text Label")
+                                print(text_label)
+                                print("Prompt followed by data entered")
+                                print(parameter_value)
+                                print(text_body)
+                                response = client.chat.completions.create(
+                                                model=processed_data["parameters"]["model_version"],
+                                                messages=[
+                                                    {"role": "system", "content": pre_process_task},
+                                                    {"role": "user", "content": text_body}
+                                                ]
+                                            )
+                                            
+                                response_text = response.choices[0].message.content
+                                print("Response text")
+                                print(response_text)
+                                
+                                input_data["input_text_pieces"]["pre_processed_pieces"][text_label] = response_text
+
+            return input_data
         
-        if parameters["chaining"]:
-            print("Chaining")
-            for text_label, text_body in input_data["input_text_pieces"].items():
-                if 'data_input' in text_label:
-                    for parameter_name, parameter_value in parameters.items():
-                        if 'pre_prompt' in parameter_name and parameter_value and not isinstance(parameter_value, bool) and text_body:
-                            print("Text Label")
-                            print(text_label)
-                            print("Prompt followed by data entered")
-                            print(parameter_value)
-                            print(text_body)
-                            response = client.chat.completions.create(
-                                            model=parameters["model_version"],
-                                            messages=[
-                                                {"role": "system", "content": parameter_value},
-                                                {"role": "user", "content": text_body}
-                                            ]
-                                        )
-                                        
-                            response_text = response.choices[0].message.content
-                            print("Response text")
-                            print(response_text)
-                            
-                            input_data["input_text_pieces"]["pre_processed_pieces"][text_label] = response_text
-
-        return input_data
+    if processed_data["inputstyle"] == "big-block-input-text":
         
-    if input_data["stepwise_json_creation"][0] == "singlejsoncreation":    
-        if parameters["chaining"]:
-    
-            input_text = input_data["input_text"]
-            pre_processing_list = [parameters["context_pre_prompt"], parameters["summary_pre_prompt"], parameters["conversation_pre_prompt"], parameters["example_pre_prompt"]]
-    
-            print("PreProcessingList")
-            print(pre_processing_list)
-            for pre_prompt in pre_processing_list:
+        input_text = input_data["input_text"]
+
+        for pre_prompt in pre_processing_list:
                 try:
                     print("Pre-Processing: ")
                     if pre_prompt:
@@ -589,25 +551,13 @@ def pre_processing(input_data, parameters):
 
             input_data["input_text"] = input_text
             return input_data
-
+        
     
-def process_specifications(input_data, parameters):
+def process_specifications(processed_data):
     """
     Once the parameters and data are processed, do the pre-processing and then generate JSONs
 
     Args: 
-        input_data: (dict) The input data, preprocessed, from the user. Aka what will fill in the JSON
-            input_data["input_text"] = (str) the preprocessed input text
-            input_data["input_context"] = (str) depending on levers, empty or what is put in front of the prompt
-            input_data["input_text_pieces"] = (dict) containing the individual split up prompt pieces: field_data_input, planting_data_input, log_data_input, soil_data_input, yield_data_input
-            
-        parameters: (dict) All of the individual parameters and "flippers" 
-            parameters["pre_prompt"] = (bool) whether or not there is a pre-prompt to process through pre_processing()
-            parameters["model_version"] = (str) what model should be used 
-            parameters["chaining"] = (bool) whether or not the preprocessed input context should be chained (given to multiple models)
-            parameters["context_pre_prompt"], parameters["summary_pre_prompt"], parameters["conversation_pre_prompt"], parameters["example_pre_prompt"] = (str) all of the pre-prompts, separated
-            parameters["combined_pre_prompt"] = (str) concatenated individual pre-prompts
-            parameters["additional_json_pieces_options"] = (str) "Explicit specific pieces" or "Parse from one big input text" to indicate whether it's many function calls on one input text or many function calls on smaller pieces of input texts
 
     Returns: 
         3 processed data-filled JSON objects: farm_pretty_json, interactions_pretty_json, trial_pretty_json
@@ -619,172 +569,26 @@ def process_specifications(input_data, parameters):
     print(parameters)
     
     # here is where parsing and other things will happen before 
-    if input_data["stepwise_json_creation"][0] == "stepwisejsoncreation":
-        print("You are continuing with stepwise json creation")
-        if parameters["pre_prompt"] == True:
+
+    if processed_data["inputstyle"] == "individual-pieces-input-text":
+        print("You are continuing with step-wise creation with your individual text pieces")
+        if processed_data["parameters"]["pre_process"] == "yes":
             print("You are continuing with pre_prompt processing")
-            processed_input = pre_processing(input_data, parameters)
+            processed_input = pre_processing(processed_data)
         else: 
             print("You have elsed into no pre-processing")
-            processed_input = input_data
-        return generate_json_pieces(processed_input, parameters)
-    elif input_data["stepwise_json_creation"][0] == "singlejsoncreation":
+            processed_input = processed_data
+        return generate_json_pieces(processed_input)
+    elif processed_data["inputstyle"] == "big-block-input-text":
         print("You are elifing into single json creation")
-        #input_data["input_context"] = "EMPTY"
-        if parameters["pre_prompt"] == True:
+        if processed_data["parameters"]["pre_process"] == "yes":
             print("You are preprocessing now")
-            processed_input = pre_processing(input_data, parameters)
-        else: 
+            processed_input = pre_processing(input_data)
+        else:
             print("You do not have any preprocessing now")
             processed_input = input_data
-        return generate_json(processed_input, parameters)
-    
-    
-def parse_survey_stack_parameters(data):
-    """
-    Parse the incoming parameters from the parameter survey
-
-    Args: 
-        data: (json) JSON retrieved from surveystack API after retrieving survey info/details
-
-    Returns:
-        processed_data (dict)
-            processed_data["pre_prompt"] = (bool) whether or not there is a pre-prompt to process through pre_processing()
-            processed_data["model_version"] = (str) what model should be used 
-            processed_data["chaining"] = (bool) whether or not the preprocessed input context should be chained (given to multiple models)
-            processed_data["context_pre_prompt"], parameters["summary_pre_prompt"], parameters["conversation_pre_prompt"], parameters["example_pre_prompt"] = (str) all of the pre-prompts, separated
-            processed_data["combined_pre_prompt"] = (str) concatenated individual pre-prompts
-            processed_data["additional_json_pieces_options"] = (str) "Explicit specific pieces" or "Parse from one big input text" to indicate whether it's many function calls on one input text or many function calls on smaller pieces of input texts
-    """
-    processed_data = {}
-
-    processed_data["model_version"] = data[0]['data']['modelversion']['value'][0]
-
-    print("DATA: ")
-    print(data)
+        return generate_json(processed_input)
     
-    try:
-
-        print("Extracting parameters")
-
-        pre_promp_parameters = data[0]['data']['group_2']
-
-        if pre_promp_parameters['preprompt']['value'][0] == 'continue_preprompts':
-            processed_data["pre_prompt"] = True
-    
-            # Accessing context and other prompts, with defaults in case they are None
-            processed_data["context_pre_prompt"] = pre_promp_parameters.get('contextpreprompt', {}).get('value', None)
-            processed_data["summary_pre_prompt"] = pre_promp_parameters.get('summarypreprompt', {}).get('value', None)
-            processed_data["conversation_pre_prompt"] = pre_promp_parameters.get('conversationpreprompt', {}).get('value', None)
-            processed_data["example_pre_prompt"] = pre_promp_parameters.get('examplepreprompt', {}).get('value', None)
-            
-            # Check if chaining is set to "yes" or "no"
-            chaining_value = pre_promp_parameters.get('prepromptchaining', {}).get('value', [None])[0]
-            
-            if chaining_value == "no":
-                # Combine prompts if chaining is "no"
-                combined_prompt = " ".join(
-                    filter(None, [
-                        processed_data["context_pre_prompt"], 
-                        processed_data["summary_pre_prompt"], 
-                        processed_data["conversation_pre_prompt"], 
-                        processed_data["example_pre_prompt"]
-                    ])
-                )
-                processed_data["chaining"] = False
-                processed_data["combined_pre_prompt"] = combined_prompt
-            else:
-                # Set combined_pre_prompt to None if chaining is enabled
-                processed_data["chaining"] = True
-                processed_data["combined_pre_prompt"] = None
-        else:
-            # Set fields to None if preprompt is not "continue_preprompts"
-            processed_data["pre_prompt"] = False
-            processed_data["context_pre_prompt"] = None
-            processed_data["summary_pre_prompt"] = None
-            processed_data["conversation_pre_prompt"] = None
-            processed_data["example_pre_prompt"] = None
-            processed_data["chaining"] = False
-            processed_data["combined_pre_prompt"] = None
-            
-    except Exception as e:
-        print(f"An error occurred: {e}")
-
-    print("Done Extracting parameters:")
-    print(str(processed_data))
-    return processed_data
-
-def parse_survey_stack_data(data):
-    """
-    Parse the incoming data from the survey stack survey
-
-    Args: 
-        data: (json) JSON retrieved from surveystack API after retrieving survey info/details
-
-    Returns: 
-        processed_data
-            processed_data["input_text"] = (str) the raw input text
-    """
-    print("PROCESSING SURVEY STACK DATA")
-    processed_data = {}
-
-    print("JUST PRINTING OUT THE DATA FOR YA")
-
-    print(data)
-
-    
-    processed_data["stepwise_json_creation"] = data[0]['data']['stepwisejsoncreation']['value']
-    print("STEPWISE?: " + str(processed_data["stepwise_json_creation"]))
-
-    if processed_data["stepwise_json_creation"][0] == "stepwisejsoncreation":
-        print("IN THE STEP")
-        farm_management_inputs = data[0]['data']['group_4']
-        print("FARM MANAGEMENT INPUTS" + str(farm_management_inputs))
-        
-        processed_data["input_text_pieces"] = {}
-        processed_data["input_text_pieces"]["field_data_input"] = farm_management_inputs.get('field_data_input', {}).get('value', None)
-        processed_data["input_text_pieces"]["planting_data_input"] = farm_management_inputs.get('planting_data_input', {}).get('value', None)
-        processed_data["input_text_pieces"]["log_data_input"] = farm_management_inputs.get('log_data_input', {}).get('value', None)
-        processed_data["input_text_pieces"]["soil_data_input"] = farm_management_inputs.get('soil_data_input', {}).get('value', None)
-        processed_data["input_text_pieces"]["yield_data_input"] = farm_management_inputs.get('yield_data_input', {}).get('value', None)
-        processed_data["input_text"] = "EMPTY"
-
-        print("NEXT SCHEMA INPUTS")
-        interactions_inputs = data[0]['data']['group_5']
-        print("INTERACTIONS INPUTS" + str(interactions_inputs))
-        processed_data["input_text_pieces"]["interaction_data_input"] = interactions_inputs.get('interaction_data_input', {}).get('value', None)
-        processed_data["input_text_pieces"]["person_data_input"] = interactions_inputs.get('person_data_input', {}).get('value', None)
-
-        print("NEXT SCHEMA INPUTS 2")
-        trials_inputs = data[0]['data']['group_6']
-        print("TRIALS INPUTS" + str(trials_inputs))
-        processed_data["input_text_pieces"]["trial_data_input"] = trials_inputs.get('trial_data_input', {}).get('value', None)
-        processed_data["input_text_pieces"]["treatment_data_input"] = trials_inputs.get('treatment_data_input', {}).get('value', None)
-
-        
-    elif processed_data["stepwise_json_creation"][0] == "singlejsoncreation":
-        print("IN THE SINGLE")
-        processed_data["input_text"] = data[0]['data']['onelonginputtext']['value']
-        print(processed_data["input_text"])
-        
-        processed_data["input_text_pieces"] = {}
-        processed_data["input_text_pieces"]["field_data_input"] = "EMPTY"
-        processed_data["input_text_pieces"]["planting_data_input"] = "EMPTY"
-        processed_data["input_text_pieces"]["log_data_input"] = "EMPTY"
-        processed_data["input_text_pieces"]["soil_data_input"] = "EMPTY"
-        processed_data["input_text_pieces"]["yield_data_input"] = "EMPTY"
-
-        processed_data["input_text_pieces"]["interaction_data_input"] = "EMPTY"
-        processed_data["input_text_pieces"]["person_data_input"] = "EMPTY"
-
-        processed_data["input_text_pieces"]["trial_data_input"] = "EMPTY"
-        processed_data["input_text_pieces"]["treatment_data_input"] = "EMPTY"
-
-    print("RETURNING DATA")
-    print(processed_data)
-        
-    return processed_data
-
 def parse_survey_stack(data):
     """
     Parse the incoming data from the survey stack survey
@@ -847,7 +651,7 @@ def parse_survey_stack(data):
         processed_data["prompts"]["second_schema_prompt_two"] = parameter_inputs.get('secondschemaprompt2', {}).get('value', None)
         processed_data["prompts"]["third_schema_prompt_one"] = parameter_inputs.get('thirdschemaprompt1', {}).get('value', None)
         processed_data["prompts"]["third_schema_prompt_two"] = parameter_inputs.get('thirdschemaprompt2', {}).get('value', None)
-        
+
     elif processed_data["inputstyle"] == "big-block-input-text":
         print("IN THE SINGLE")
         processed_data["input_text"] = data[0]['data']['onelonginputtext']['value']
@@ -867,7 +671,7 @@ def parse_survey_stack(data):
 
     if processed_data["parameters"]["pre_process"] == "yes":
         processed_data["parameters"]["multiple_pre_prompts"] = data[0]['data']['multiplepreprompts']['value'] # this is a value in string words
-        processed_data["parameters"]["pre_prompt_style"] = data[0]['data']['prepromptstyle']['value'] # this is a list
+        processed_data["parameters"]["preprompt_style"] = data[0]['data']['prepromptstyle']['value'] # this is a list
         processed_data["parameters"]["pre_prompt_first"] = data[0]['data']['preprocessingprompt1']['value']
         processed_data["parameters"]["pre_prompt_second"] = data[0]['data']['preprocessingprompt2'].get('value', None)
         processed_data["parameters"]["pre_prompt_third"] = data[0]['data']['preprocessingprompt3'].get('value', None)