3

src/backend/evaluate_model.py

@@ -86,7 +86,7 @@ class Evaluator:
             # avg_summary_len = self.summary_generator.avg_length
             # answer_rate = self.summary_generator.answer_rate
             '''开始评估模型的结果'''
-            self.humanlike = self.eval_model.evaluate_humanlike(self.generated_summaries_df, envs.HUMAN_DATA)
+            self.humanlike = self.eval_model.evaluate_humanlike(self.generated_summaries_df, envs.HUMAN_DATA, f"generation_results/{self.model}.csv")
             '''原始指标'''
             # self.hallucination_scores, self.eval_results = self.eval_model.evaluate_hallucination(
                 # self.generated_summaries_df)

src/backend/model_operations.py

@@ -33,7 +33,7 @@ logging.basicConfig(level=logging.INFO,
 
 # Load spacy model for word tokenization
 nlp = spacy.load("en_core_web_sm")
-
+nlp1 = spacy.load("en_core_web_trf")
 os.environ["HUGGINGFACE_API_KEY"] =  envs.TOKEN
 os.environ["OPENAI_API_KEY"] = "sk-None-tanhMyavhUtpX2G1kmPuT3BlbkFJGEhM5jmyGyhrTd3LdHDI"
 
@@ -46,7 +46,8 @@ def load_evaluation_model(model_path):
     Returns:
         CrossEncoder: The evaluation model
     """
-    model = CrossEncoder(model_path)
+    # model = CrossEncoder(model_path)
+    model = ""
     return model
 
 
@@ -121,10 +122,13 @@ class SummaryGenerator:
             print(f"Total: {len(sheet_names)}")  
             print(sheet_names)  
             
-            item_ID, questions_ID, user_prompt, response = [], [], [], []  
+            Experiment_ID, Questions_ID, Item_ID, Condition, User_prompt, Response, Factor_2, Stimuli_1 = [], [], [], [], [] ,[], [], []
             
-            for i, sheet_name in enumerate(sheet_names[0:1], start=1):  
+            for i, sheet_name in enumerate(sheet_names, start=1):  
                 # 读取每个工作表  
+                # if i > 2 and i ==1:
+                #     continue
+                print(i, sheet_name)
                 df_sheet = pd.read_excel(xls, sheet_name=sheet_name)  
                 
                 # 假设第一列是'Prompt0'，但这里我们使用列名来避免硬编码  
@@ -132,18 +136,37 @@ class SummaryGenerator:
                     prompt_column = df_sheet['Prompt0']  
                 else:  
                     # 如果'Prompt0'列不存在，则跳过该工作表或进行其他处理  
-                    continue  
+                    continue 
+                if i == 3 :
+                    word1_list = df_sheet['Stimuli-2'] 
+                    word2_list = df_sheet['Stimuli-3'] 
+                    V2_column = []
+                    for jj in range(len(word1_list)):
+                        V2_column.append(word1_list[jj] + '_' + word2_list[jj])
+                    # print(V2_column)
+                elif i == 9:
+                    V2_column = df_sheet['V2'] #SL, LS
+                elif i == 4 or i == 6 :
+                    V2_column = df_sheet['Stimuli-2'] #Stimuli-2
+                else:
+                    V2_column = [""] * len(prompt_column)
+                q_column = df_sheet["ID"]
+                Item_column = df_sheet["Item"]
+                Condition_column = df_sheet["Condition"]
+                Stimuli_1_column = df_sheet["Stimuli-1"]
+                if 'Stimuli-2' in df_sheet.columns:  
+                    Stimuli_2_column = df_sheet["Stimuli-2"]
                 
                 # 遍历Prompt0列的值  
-                for j, prompt_value in enumerate(tqdm(prompt_column, desc=f"Processing {sheet_name}"), start=1):  
+                for j, prompt_value in enumerate(tqdm(prompt_column[0:2], desc=f"Processing {sheet_name}"), start=0):  
                     ID = 'E' + str(i)  
-                    q_ID = ID + '_' + str(j)  
+                    # q_ID = ID + '_' + str(j)  
                       
-                    # print(ID, q_ID, prompt_value) 
-                    for i in range(2):
-                        system_prompt = envs.SYSTEM_PROMPT
+                    # print(ID, q_ID, prompt_value)
+                    system_prompt = envs.SYSTEM_PROMPT 
+                    _user_prompt = prompt_value
+                    for ii in range(2):
                     # user_prompt = f"{envs.USER_PROMPT}\nPassage:\n{_source}"
-                        _user_prompt = prompt_value
                         while True:
                             try:
                                 '''调用'''
@@ -171,19 +194,58 @@ class SummaryGenerator:
                                     _response = ""
                                     exceptions.append(i)
                                     break
-
-                        item_ID.append(ID)
-                        questions_ID.append(q_ID)
-                        user_prompt.append(_user_prompt)
-                        response.append(_response)
-                        print(_response)
+                        if  i == 5:
+                            print(_response)
+
+                            _response1, _response2 = _response.split('\n\n')
+                            Experiment_ID.append(ID)
+                            Questions_ID.append(q_column[j])
+                            User_prompt.append(_user_prompt)
+                            
+                            Response.append(_response2)
+                           
+                            Factor_2.append(V2_column[j])
+                            Stimuli_1.append(Stimuli_2_column[j])
+                            Item_ID.append(Item_column[j])
+                            Condition.append(Condition_column[j])
+
+                            # the first sentence in the response is saved as E51
+                            Experiment_ID.append(ID + '1')
+                            Questions_ID.append(str(q_column[j]) + '1')
+                            User_prompt.append(_user_prompt)
+                            Response.append(_response1)
+
+                            
+                            
+                            Factor_2.append(V2_column[j])
+                            Stimuli_1.append(Stimuli_1_column[j])
+                            Item_ID.append(Item_column[j])
+                            Condition.append(Condition_column[j])
+                            
+                        else:
+                            Experiment_ID.append(ID)
+                            Questions_ID.append(q_column[j])
+                            User_prompt.append(_user_prompt)
+                            
+                            Response.append(_response)
+                            if i == 6:
+                                Factor_2.append(Condition_column[j])
+                                Stimuli_1.append(V2_column[j])
+                            else:
+                                Factor_2.append(V2_column[j])
+                                Stimuli_1.append(Stimuli_1_column[j])
+                            Item_ID.append(Item_column[j])
+                            Condition.append(Condition_column[j])
+                            print(_response)
+
+                        
                         # exit()
 
                     # Sleep to prevent hitting rate limits too frequently
                         time.sleep(1)
 
-            self.summaries_df = pd.DataFrame(list(zip(item_ID, questions_ID, user_prompt, response)),
-                                            columns=["Experiment", "Question_ID", "User_prompt", "Response"])
+            self.summaries_df = pd.DataFrame(list(zip(Experiment_ID, Questions_ID, Item_ID, Condition, User_prompt, Response, Factor_2, Stimuli_1)),
+                                            columns=["Experiment", "Question_ID", "Item", "Condition", "User_prompt", "Response","Factor 2","Stimuli 1"])
 
             if save_path is not None:
                 print(f'Save summaries to {save_path}')
@@ -419,85 +481,486 @@ class EvaluationModel:
     def code_results(self, summaries_df):
         '''code results from LLM's response'''
         output = []
-        '''item1'''
-        # print(len(summaries_df['Experiment']),len(summaries_df['Response']))
-        # exit()
-        '''人类数据需要处理Item3'''
-        item3 = pd.read_csv('/Users/tangtang/Desktop/leaderboard/src/datasets/Experiment_3_Items.csv')
-        item2word = {}
-        for j in range(len(item3['Item'])):
-            item2word[item3['Item'][j]] = [item3['Field 2'][j], item3['Field 3'][j]]
+        '''database for Exp4'''
+        item4 = pd.read_csv(envs.ITEM_4_DATA)
+        wordpair2code = {}
+        for j in range(len(item4['Coding'])):
+            wordpair2code[item4['Pair'][j]] = item4['Coding'][j]
+        '''verb for Exp5'''
+        item5 = pd.read_csv(envs.ITEM_5_DATA)
+        # item corresponding to verb, same item id corresponding to verb pair
+        item2verb2 = {}
+        item2verb1 = {}
+
+        Stimuli1, Stimuli2 = {}, {}
+        for j in range(len(item5['Item'])):
+            item2verb1[item5['Item'][j]] = item5['Verb1'][j]
+            item2verb2[item5['Item'][j]] = item5['Verb2'][j]
+            Stimuli1[item5['ID'][j]] = item5['Stimuli-1'][j]
+            Stimuli2[item5['ID'][j]] = item5['Stimuli-2'][j]
+
 
         male_keyword = ["he", "his", "himself"]
         female_keyword = ["she", "her", "herself"]
-        for i in range(len(summaries_df['Experiment'])):
+        print(len(summaries_df["Experiment"]))
+        for i in range(len(summaries_df["Experiment"])):
             # vote_1_1, vote_1_2, vote_1_3 = 0, 0, 0
+            # print()
+            if pd.isna(summaries_df["Response"][i]): 
+                output.append("Other")
+                continue
+            rs = summaries_df["Response"][i].strip().lower()
+            '''Exp1'''
             if summaries_df["Experiment"][i] == "E1":
-                if summaries_df["Response"][i].strip() == "Round":
+                print("E1", rs)
+                rs = rs.replace('"','')
+                if rs == "round":
                     # vote_1_1 += 1
                     output.append("Round")
-                elif summaries_df["Response"][i].strip() == "Spiky":
-                    output.append("Round")
+                elif rs == "spiky":
+                    output.append("Spiky")
                 else:
-                    output.append("NA")
-            # print()
+                    output.append("Other")
+            
 
-            '''item2'''
-            # vote_2_1, vote_2_2, vote_2_3 = 0, 0, 0
+                '''Exp2'''
 
-            if summaries_df["Experiment"][i] == "E2":
-                rs = summaries_df["Response"][i].strip()
+            elif summaries_df["Experiment"][i] == "E2":
+                # rs = summaries_df["Response"][i].strip()
                 rs = rs.split(' ')
+                print("E2", rs)
                 male, female = 0, 0
                 for word in rs:
-                    if word in female_keyword and male != 1:
+                    if word in female_keyword and male == 0:
                         female = 1
                         output.append("Female")
                         break
-                    if word in male_keyword and female != 1:
+                    if word in male_keyword and female == 0:
                         male = 1
                         output.append("Male")
                         break
                 if male == 0 and female == 0 :
-                    output.append("NA")
-            '''item3'''
-
-            if summaries_df["Experiment"][i] == "E3":
-                rs = summaries_df["Response"][i].strip()
-                id = summaries_df["Item"][i].strip()
-                if '2' in rs:
-                    item2word[id][0]
+                    output.append("Other")
+                    
+                '''Exp3'''
+            elif summaries_df["Experiment"][i] == "E3":
+                # rs = summaries_df["Response"][i].strip()
+                print("E3", rs)
+                if pd.isna(summaries_df["Factor 2"][i]): 
+                    output.append("Other")
+                else:
+                    if summaries_df["Factor 2"][i].strip() == "LS":
+                        if "2" in rs:
+                            output.append("Long")
+                        elif "3" in rs:
+                            output.append("Short")
+                        else:
+                            output.append("Other")
+                    if summaries_df["Factor 2"][i].strip() == "SL":
+                        if "2" in rs:
+                            output.append("Short")
+                        elif "3" in rs:
+                            output.append("Long")
+                        else:
+                            output.append("Other")      
+                '''Exp4'''
+            
+            elif summaries_df["Experiment"][i] == "E4":
+                # rs = summaries_df["Response"][i].strip()
+                target = summaries_df["Factor 2"][i].strip().lower()
+                pair = target + "_" + rs
+                print("E4:", pair)
+                if pair in wordpair2code.keys():
+                    output.append(wordpair2code[pair])
+                else:
+                    output.append("Other")
+
+                '''Exp5'''
+            elif summaries_df["Experiment"][i] == "E5" or summaries_df["Experiment"][i] == "E51":
+                # sentence = summaries_df["Response"][i].strip()
+                item_id = summaries_df["Item"][i]
+                question_id = summaries_df["Question_ID"][i]
+
+                sti1, sti2 = "", ""
+
+                if summaries_df["Experiment"][i] == "E51":
+                    sti1 = Stimuli1[question_id[0:-1]].lower().replace("...", "")
+                    sti2 = Stimuli2[question_id[0:-1]].lower().replace("...", "")
+                    verb = item2verb1[item_id].lower()
+
+                    sentence = sti1 + " " + rs.replace(sti1, "")
+                    print("E5", verb, sentence)
+                if summaries_df["Experiment"][i] == "E5":
+                    sti1 = Stimuli1[question_id].lower().replace("...", "")
+                # print(sti1)
+                    sti2 = Stimuli2[question_id].lower().replace("...", "")
+
+                    verb = item2verb2[item_id].lower()
+                    sentence = sti2.replace("...","") + " " + rs.replace(sti2, "")
+                    print("E5", verb, sentence)
+                
+ 
+                doc = nlp1(sentence.replace("  "," "))
+                # print(doc)
+                # print()
+                verb_token = None
+                for token in doc:
+                    # print(token.lemma_)
+                    if token.lemma_ == verb:
+                        verb_token = token
+                        break
+                # exit()
+                if verb_token is None:
+                    output.append("Other")
+                    print("E5 The target verb is missing from the sentence.")
+                else:
+                    pobj, dative = None, None
+                    # print(verb_token.children)
+                    # exit()
+                    for child in verb_token.children:
+                        print(child)
+                        if (child.dep_ == 'dative' and child.pos_ == "ADP") or (child.text == "to" and child.dep_ == 'prep' and child.pos_ == "ADP"):
+                            pobj = child.text
+                        if child.dep_ == 'dative':
+                            dative = child.text
+                    print("E5", pobj, dative)
+                    # exit()
+
+                    if pobj:
+                        output.append("PO")
+                    elif dative:
+                        output.append("DO")
+                    else:
+                        print("Other", sentence, pobj, dative)
+                        # exit()
+                        output.append("Other")
+
+                '''Exp6'''
+
+            elif summaries_df["Experiment"][i] == "E6":
+                sentence = summaries_df["Stimuli 1"][i].strip().lower()
+                print("E6", sentence) 
+                doc = nlp1(sentence)  
+                subject = "None"
+                obj = "None"
+                # 遍历依存关系，寻找主语和宾语  
+                for token in doc:
+                    if token.dep_ == "nsubj":
+                        subject = token.text
+                    elif token.dep_ == "dobj":
+                        obj = token.text
+                print("E6", subject, obj)
+                if subject in rs and obj in rs:
+                    print(rs, subject, obj, "Other")
+                    output.append("Other")
+                elif subject in rs:
+                    print(rs, subject, obj, "VP")
+                    output.append("VP")
+                elif obj in rs:
+                    print(rs, subject, obj, "NP")
+                    output.append("NP")
+                else:
+                    print(rs, subject, obj, "Other")
+                    output.append("Other") 
 
                 
-            '''item4'''
-            '''item5'''
-            '''item6'''
-
-            '''item7'''
-            if summaries_df["Experiment"][i] == "E7":
-                rs = summaries_df["Response"][i].strip()
-                if rs == "No":
+
+
+                '''Exp7'''
+            elif summaries_df["Experiment"][i] == "E7":
+                # rs = summaries_df["Response"][i].strip().lower()
+                print("E7",rs)
+                if rs == "no":
                     output.append("0")
-                elif rs == "Yes":
+                elif rs == "yes":
                     output.append("1")
                 else:
-                    output.append("NA")
-            '''item8'''
-            if summaries_df["Experiment"][i] == "E8":
-                rs = summaries_df["Response"][i].strip()
-                if rs == "Something is wrong with the question":
+                    output.append("Other")
+
+                '''Exp8'''
+            elif summaries_df["Experiment"][i] == "E8":
+                # rs = summaries_df["Response"][i].strip()
+                
+                if "something is wrong with the question" in rs:
+                    output.append("1")
+                else:
+                    output.append("0")
+
+                '''Exp9'''
+            elif summaries_df["Experiment"][i] == "E9":
+                male, female = 0, 0
+                
+                # rs = summaries_df["Response"][i].strip()
+                if "because" in rs:
+                    rs = rs.replace("because because","because").split("because")[1]
+                else:
+                    rs = rs
+                condition = summaries_df["Factor 2"][i].strip()
+                rs = rs.split(" ")
+                for w in rs:
+                    if w in male_keyword and female != 1:
+                        male = 1
+                        break
+                    if w in female_keyword and male != 1:
+                        female = 1
+                        break
+                print("E9", "condition", condition, "male", male, "female", female)
+                if  male == 0 and female == 0:
+                    output.append('Other')
+                else:
+                    if male == 1 and female==0:
+                        if condition == "MF":
+                            output.append("Subject")
+                        elif condition == "FM":
+                            output.append("Object")
+                        else:
+                            output.append("Other")
+                    elif female == 1 and male ==0:
+                        if condition == "MF":
+                            output.append("Object")
+                        elif condition == "FM":
+                            output.append("Subject")
+                        else:
+                            output.append("Other")
+
+                '''Exp10'''
+            elif summaries_df["Experiment"][i] == "E10":
+                # rs = summaries_df["Response"][i].strip()
+                if rs == "yes":
                     output.append("1")
                 else:
                     output.append("0")
+            else:
+                print("can;t find the Exp:", summaries_df["Experiment"][i])
+                output.append("NA")
+            # print(output)
+        # exit()
+        '''human'''
+        self.data = pd.DataFrame(list(zip(summaries_df["Experiment"], summaries_df["Question_ID"], summaries_df["Item"],  summaries_df["Response"], summaries_df["Factor 2"], summaries_df["Stimuli 1"], summaries_df["Coding"], output)),
+                                            columns=["Experiment", "Question_ID", "Item",  "Response", "Factor 2", "Simulate 1","Original_Coding","Coding"])
+        # '''LLM'''
+        # self.data = pd.DataFrame(list(zip(summaries_df["Experiment"], summaries_df["Question_ID"], summaries_df["Item"],  summaries_df["Response"], summaries_df["Factor 2"], summaries_df["Stimuli 1"], output)),
+        #                                     columns=["Experiment", "Question_ID", "Item",  "Response", "Factor 2", "Simulate 1","Coding"])
+        print(self.data.head()) 
+        
+        return self.data
+    def code_results_llm(self, summaries_df):
+        '''code results from LLM's response'''
+        output = []
+        '''database for Exp4'''
+        item4 = pd.read_csv(envs.ITEM_4_DATA)
+        wordpair2code = {}
+        for j in range(len(item4['Coding'])):
+            wordpair2code[item4['Pair'][j]] = item4['Coding'][j]
+        '''verb for Exp5'''
+        item5 = pd.read_csv(envs.ITEM_5_DATA)
+        # item corresponding to verb, same item id corresponding to verb pair
+        item2verb2 = {}
+        item2verb1 = {}
+
+        Stimuli1, Stimuli2 = {}, {}
+        for j in range(len(item5['Item'])):
+            item2verb1[item5['Item'][j]] = item5['Verb1'][j]
+            item2verb2[item5['Item'][j]] = item5['Verb2'][j]
+            Stimuli1[item5['ID'][j]] = item5['Stimuli-1'][j]
+            Stimuli2[item5['ID'][j]] = item5['Stimuli-2'][j]
 
 
+        male_keyword = ["he", "his", "himself"]
+        female_keyword = ["she", "her", "herself"]
+        print(len(summaries_df["Experiment"]))
+        for i in range(len(summaries_df["Experiment"])):
+            # vote_1_1, vote_1_2, vote_1_3 = 0, 0, 0
+            # print()
+            if pd.isna(summaries_df["Response"][i]): 
+                output.append("Other")
+                continue
+            rs = summaries_df["Response"][i].strip().lower()
+            '''Exp1'''
+            if summaries_df["Experiment"][i] == "E1":
+                print("E1", rs)
+                rs = rs.replace('"','')
+                if rs == "round":
+                    # vote_1_1 += 1
+                    output.append("Round")
+                elif rs == "spiky":
+                    output.append("Spiky")
+                else:
+                    output.append("Other")
+            
 
-            '''item9'''
-            if summaries_df["Experiment"][i] == "E9":
+                '''Exp2'''
+
+            elif summaries_df["Experiment"][i] == "E2":
+                # rs = summaries_df["Response"][i].strip()
+                rs = rs.split(' ')
+                print("E2", rs)
                 male, female = 0, 0
-                rs = summaries_df["Response"][i].strip()
+                for word in rs:
+                    if word in female_keyword and male == 0:
+                        female = 1
+                        output.append("Female")
+                        break
+                    if word in male_keyword and female == 0:
+                        male = 1
+                        output.append("Male")
+                        break
+                if male == 0 and female == 0 :
+                    output.append("Other")
+                    
+                '''Exp3'''
+            elif summaries_df["Experiment"][i] == "E3":
+                # rs = summaries_df["Response"][i].strip()
+                print("E3", rs)
+                rs = rs.replace('"', '')
+                pair = summaries_df["Factor 2"][i] 
+                word1, word2 = pair.split('_')
+
+                if rs == word1:
+                    if len(word1) > len(word2):
+                        output.append("Long")
+                    else:
+                        output.append("Short")
+                elif rs == word2:
+                    if len(word1) > len(word2):
+                        output.append("Short")
+                    else:
+                        output.append("Long")
+                else:
+                    output.append("Other")
+
+                '''Exp4'''
+            
+            elif summaries_df["Experiment"][i] == "E4":
+                # rs = summaries_df["Response"][i].strip()
+                meaning_word = rs.split(";")[4].replace(" ",'')
+                target = summaries_df["Factor 2"][i].strip().lower()
+                pair = target + "_" + meaning_word
+                print("E4:", pair)
+                if pair in wordpair2code.keys():
+                    output.append(wordpair2code[pair])
+                else:
+                    output.append("Other")
+
+                '''Exp5'''
+            elif summaries_df["Experiment"][i] == "E5" or summaries_df["Experiment"][i] == "E51":
+                # sentence = summaries_df["Response"][i].strip()
+                item_id = summaries_df["Item"][i]
+                question_id = summaries_df["Question_ID"][i]
+
+                sti1, sti2 = "", ""
+
+                if summaries_df["Experiment"][i] == "E51":
+                    sti1 = Stimuli1[question_id[0:-1]].lower().replace("...", "")
+                    sti2 = Stimuli2[question_id[0:-1]].lower().replace("...", "")
+                    verb = item2verb1[item_id].lower()
+
+                    sentence = sti1 + " " + rs.replace(sti1, "")
+                    print("E5", verb, sentence)
+                if summaries_df["Experiment"][i] == "E5":
+                    sti1 = Stimuli1[question_id].lower().replace("...", "")
+                # print(sti1)
+                    sti2 = Stimuli2[question_id].lower().replace("...", "")
+
+                    verb = item2verb2[item_id].lower()
+                    sentence = sti2.replace("...","") + " " + rs.replace(sti2, "")
+                    print("E5", verb, sentence)
+                
+ 
+                doc = nlp1(sentence.replace("  "," "))
+                # print(doc)
+                # print()
+                verb_token = None
+                for token in doc:
+                    # print(token.lemma_)
+                    if token.lemma_ == verb:
+                        verb_token = token
+                        break
+                # exit()
+                if verb_token is None:
+                    output.append("Other")
+                    print("E5 The target verb is missing from the sentence.")
+                else:
+                    pobj, dative = None, None
+                    # print(verb_token.children)
+                    # exit()
+                    for child in verb_token.children:
+                        print(child)
+                        if (child.dep_ == 'dative' and child.pos_ == "ADP") or (child.text == "to" and child.dep_ == 'prep' and child.pos_ == "ADP"):
+                            pobj = child.text
+                        if child.dep_ == 'dative':
+                            dative = child.text
+                    print("E5", pobj, dative)
+                    # exit()
+
+                    if pobj:
+                        output.append("PO")
+                    elif dative:
+                        output.append("DO")
+                    else:
+                        print("Other", sentence, pobj, dative)
+                        # exit()
+                        output.append("Other")
+
+                '''Exp6'''
+
+            elif summaries_df["Experiment"][i] == "E6":
+                sentence = summaries_df["Stimuli 1"][i].strip().lower()
+                print("E6", sentence) 
+                doc = nlp1(sentence)  
+                subject = "None"
+                obj = "None"
+                # 遍历依存关系，寻找主语和宾语  
+                for token in doc:
+                    if token.dep_ == "nsubj":
+                        subject = token.text
+                    elif token.dep_ == "dobj":
+                        obj = token.text
+                print("E6", subject, obj)
+                if subject in rs and obj in rs:
+                    print(rs, subject, obj, "Other")
+                    output.append("Other")
+                elif subject in rs:
+                    print(rs, subject, obj, "VP")
+                    output.append("VP")
+                elif obj in rs:
+                    print(rs, subject, obj, "NP")
+                    output.append("NP")
+                else:
+                    print(rs, subject, obj, "Other")
+                    output.append("Other") 
+
+                
+
+
+                '''Exp7'''
+            elif summaries_df["Experiment"][i] == "E7":
+                # rs = summaries_df["Response"][i].strip().lower()
+                rs = rs.replace(".", "").replace(",", "")
+                print("E7",rs)
+                if rs == "no":
+                    output.append("0")
+                elif rs == "yes":
+                    output.append("1")
+                else:
+                    output.append("Other")
+
+                '''Exp8'''
+            elif summaries_df["Experiment"][i] == "E8":
+                # rs = summaries_df["Response"][i].strip()
+                print("E8",rs)
+                if "something is wrong with the question" in rs:
+                    output.append("1")
+                else:
+                    output.append("0")
+
+                '''Exp9'''
+            elif summaries_df["Experiment"][i] == "E9":
+                male, female = 0, 0
+                
+                # rs = summaries_df["Response"][i].strip()
                 if "because" in rs:
-                    rs = rs.split("because")[1]
+                    rs = rs.replace("because because","because").split("because")[1]
                 else:
                     rs = rs
                 condition = summaries_df["Factor 2"][i].strip()
@@ -507,9 +970,11 @@ class EvaluationModel:
                         male = 1
                         break
                     if w in female_keyword and male != 1:
+                        female = 1
                         break
+                print("E9", "condition", condition, "male", male, "female", female)
                 if  male == 0 and female == 0:
-                    output.append('NA')
+                    output.append('Other')
                 else:
                     if male == 1 and female==0:
                         if condition == "MF":
@@ -517,36 +982,70 @@ class EvaluationModel:
                         elif condition == "FM":
                             output.append("Object")
                         else:
-                            output.append("NA")
+                            output.append("Other")
                     elif female == 1 and male ==0:
                         if condition == "MF":
                             output.append("Object")
                         elif condition == "FM":
                             output.append("Subject")
                         else:
-                            output.append("NA")
+                            output.append("Other")
 
-            '''item10'''
-            if summaries_df["Experiment"][i] == "E10":
-                rs = summaries_df["Response"][i].strip()
-                if rs == "Yes":
+                '''Exp10'''
+            elif summaries_df["Experiment"][i] == "E10":
+                # rs = summaries_df["Response"][i].strip()
+                rs = rs.replace(".", "")
+                if rs == "yes":
                     output.append("1")
                 else:
-                    output.append("0")                
-
+                    output.append("0")
+            else:
+                print("can;t find the Exp:", summaries_df["Experiment"][i])
+                output.append("NA")
+            # print(output)
+        # exit()
+        '''human'''
+        # self.data = pd.DataFrame(list(zip(summaries_df["Experiment"], summaries_df["Question_ID"], summaries_df["Item"],  summaries_df["Response"], summaries_df["Factor 2"], summaries_df["Stimuli 1"], summaries_df["Coding"], output)),
+        #                                     columns=["Experiment", "Question_ID", "Item",  "Response", "Factor 2", "Simulate 1","Original_Coding","Coding"])
+        '''LLM'''
+        self.data = pd.DataFrame(list(zip(summaries_df["Experiment"], summaries_df["Question_ID"], summaries_df["Item"],  summaries_df["Response"], summaries_df["Factor 2"], summaries_df["Stimuli 1"], output)),
+                                            columns=["Experiment", "Question_ID", "Item",  "Response", "Factor 2", "Simulate 1","Coding"])
+        print(self.data.head()) 
+        
+        return self.data
+        
 
         
             
                 
         '''是不是有不同的问题，如何计算'''
-    def evaluate_humanlike(self, summaries_df, human_data_path):
+    def evaluate_humanlike(self, summaries_df, human_data_path, result_save_path):
         '''
         evaluate humanlike score
         1. code the result
         2. comput the similaritirs between human and model
         process model responses'''
-        huamn_df = pd.read_csv(human_data_path)
-        self.code_results(summaries_df)
+
+        '''coding human data'''
+        # self.huamn_df = pd.read_csv(human_data_path)
+        # self.data = self.code_results(self.huamn_df)
+        # save_path = human_data_path.replace('.csv','_coding.csv')
+        # if save_path is not None:
+        #     print(f'Save human coding results to {save_path}')
+        #     fpath = Path(save_path)
+        #     fpath.parent.mkdir(parents=True, exist_ok=True)
+        #     self.data.to_csv(fpath) 
+
+        '''coding llm data'''
+        save_path = result_save_path.replace('.csv','_coding.csv')
+        self.llm_df = self.code_results_llm(summaries_df)
+        if save_path is not None:
+            print(f'Save LLM coding results to {save_path}')
+            fpath = Path(save_path)
+            fpath.parent.mkdir(parents=True, exist_ok=True)
+            self.llm_df.to_csv(fpath)
+        # exit() 
+
         return 9.00