methods/searchain/eval_SCwIR.py

from openai import OpenAI
import json
import os
import string
import regex
import time
from collections import Counter
import joblib
from tqdm import tqdm
import pandas as pd
import pandas as pd
import socket

from flask import Flask, request, jsonify

app = Flask(__name__)



HOST = '127.0.0.1'
PORT = 50002

client = OpenAI(
        api_key=os.environ.get("CHATANYWHERE_API_KEY", ""),
        base_url='https://api.chatanywhere.tech/v1', 
    )


import re

def extract_between_asterisks(text):
    # 使用正则表达式匹配 `**` 之间的内容
    pattern = r'\*\*(.*?)\*\*'
    matches = re.findall(pattern, text)
    return matches




def eval_em(p_ls, g_ls):
    assert len(p_ls) == len(g_ls)
    cnt = 0
    for idx in range(len(p_ls)):
        pred = p_ls[idx]
        gold = g_ls[idx]
        if gold in pred:
            cnt += 1
    return cnt/len(p_ls)



def excute(questions, answers):
    
    ans_ls = []

    query_ls = []

    acc_ls = []

    acc = 0
    
    cnt = 0
    for idx in range(len(questions)):

        print(cnt)
        q = questions[idx]
        answer = answers[idx]
        
        query_ls.append(q)
        ans_ls.append(answer)
        
        feedback_tmp_ls = []
        
        round_count = 0  # The final answer for the [Question] can only be yes or no, marked with **
        message_keys_list = [{"role": "user", "content":
                    """Construct a global reasoning chain for this complex [Question] : " {} " You should generate a query to the search engine based on
                        what you already know at each step of the reasoning chain, starting with [Query].
                        If you know the answer for [Query], generate it starting with [Answer].
                        You can try to generate the final answer for the [Question] by referring to the [Query]-[Answer] pairs, starting with [Final
                        Content]. The final answer for the [Question] must be either "yes" or "no", and it should be highlighted using double asterisks (**).
                        If you don't know the answer, generate a query to search engine based on what you already know and do not know, starting with
                        [Unsolved Query].
                        For example:
                        [Question]: "Where do greyhound buses that are in the birthplace of Spirit If...'s performer leave from? "
                        [Query 1]: Who is the performer of Spirit If... ?
                        If you don't know the answer:
                        [Unsolved Query]: Who is the performer of Spirit If... ?
                        If you know the answer:
                        [Answer 1]: The performer of Spirit If... is Kevin Drew.
                        [Query 2]: Where was Kevin Drew born?
                        If you don't know the answer:
                        [Unsolved Query]: Where was Kevin Drew born?
                        If you know the answer:
                        [Answer 2]: Toronto.
                        [Query 3]: Where do greyhound buses in Toronto leave from?
                        If you don't know the answer:
                        [Unsolved Query]: Where do greyhound buses in Toronto leave from?
                        If you know the answer:
                        [Answer 3]: Toronto Coach Terminal.
                        [Final Content]: The performer of Spirit If... is Kevin Drew [1]. Kevin Drew was born in Toronto [2]. Greyhound buses in
                        Toronto leave from Toronto
                        Coach Terminal [3]. So the final answer is Toronto Coach Terminal.
                        
                        [Question]:"Which magazine was started first Arthur’s Magazine or First for Women?"
                        [Query 1]: When was Arthur’s Magazine started?
                        [Answer 1]: 1844.
                        [Query 2]: When was First for Women started?
                        [Answer 2]: 1989
                        [Final Content]: Arthur’s Magazine started in 1844 [1]. First for Women started in 1989 [2]. So Arthur’s Magazine was started
                        first. So the answer is Arthur’s Magazi
                        [Question]: {}
                    """.format(q,q)}]
        feedback_answer = 'continue'
        predict_answer = ''
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((HOST, PORT))
        while round_count < 5 and not feedback_answer == 'end':
            print('round is {}'.format(round_count))
            try:
                for attempt in range(5):
                    try:
                        rsp = client.chat.completions.create(
                            model='gpt-4o-mini',
                            messages=message_keys_list,
                            temperature=0
                        )
                        
                        # 检查返回结果是否有效
                        if rsp.choices[0].message.content is not None:
                            input_str = rsp.choices[0].message.content
                            break  # 获取到有效结果，跳出循环
                        else:
                            print(f"尝试 {attempt + 1}/{5}：返回内容无效，重试中...")
                
                    except Exception as e:
                        print(f"尝试 {attempt + 1}/{5} 失败: {e}")
                        
                round_count += 1
                
                # input_str = rsp.choices[0].message.content

                message_keys_list.append({"role": "assistant", "content": input_str})
                
                print('solving......')
                predict_answer += input_str
                sock.send(input_str.encode())
                print('send message {}'.format(input_str))
                
                feedback = sock.recv(10240).decode()
                print('feedback is '+feedback)
                
                if feedback == 'end':
                    break
                #[Query]:xxxx<SEP>[Answer]:xxxx<SEP>[Reference]:xxxx<SEP>
                feedback_list = feedback.split('<SEP>')
                if not 'Unsolved Query' in feedback:
                    new_prompt = """
                    According to this Reference, the answer for "{}" should be "{}",  
                    you can change your answer based on the Reference and continue constructing the reasoning chain to give the final answer for [Question]:{}
                    Reference: {}
                    """.format(feedback_list[0],feedback_list[1],q,feedback_list[2])
                else:
                    new_prompt = """
                    According to this Reference, the answer for "{}" should be "{}",  
                    you can give your answer based on the Reference and continue constructing the reasoning chain to give the final answer for [Question]：{}
                    Reference: {}
                    """.format(feedback_list[0],feedback_list[1],q,feedback_list[2])
                message_keys_list.append({"role": "user", "content":new_prompt})
                feedback_tmp_ls.append((feedback_list[0], feedback_list[2]))

                
            except:
                print('start_idx is {}'.format(k))
                sock.send('end'.encode())
                sock.close()
                return k
            
            
        if not feedback_answer == 'end':
            sock.send('end'.encode())
        sock.close()
        
        print(message_keys_list)
        
        last_assistant = None
        for entry in reversed(message_keys_list):
            if entry['role'] == 'assistant':
                last_assistant = entry
                break
        
        pred_ans = last_assistant['content'].split('[Final Content]')[-1].lower()
        
        # pred_ls.append(pred_ans.split('.')[-2])
        
        # if answer is True:
        #     answer = 'yes'
        # else:
        #     answer = 'no'
        
        if answer in extract_between_asterisks(pred_ans):
            acc += 1
            acc_ls.append(1)
        else:
            acc_ls.append(0)
            
        cnt += 1

    return jsonify({
            "query_ls": query_ls,
            "ans_ls": ans_ls,
            "acc_ls": acc_ls
        })


@app.route('/execute', methods=["GET"])
def api_search():
    if request.method == "GET":

        queries = request.args.getlist("query")  
        answers = request.args.getlist("answers") 

        return excute(queries, answers)
        
    else:
        return '', 405
    

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=50003)