methods/searchain/eval.py

from transformers import pipeline
import torch
import pandas as pd

from transformers import AutoTokenizer
import requests
from openai import OpenAI



instructions = [

"As a query optimization expert, systematically evaluate the provided query Q, identify ambiguities, inefficiencies, and potential improvements, and refine it to ensure maximum clarity, precision, and alignment with the user's intent, while maintaining semantic accuracy and contextual coherence in the output.",

"As a query optimization expert, meticulously evaluate the provided query Q, identify and resolve ambiguities or inefficiencies, and refine it to achieve exceptional clarity, precision, and alignment with the user's intent, ensuring the output is concise, semantically robust, contextually accurate, and optimized for maximum effectiveness, readability, and relevance, while preserving the original meaning and enhancing overall utility."
    
]



dataset_path = '/home/icml01/multi_rag/RAG/opro-para/data/strategyqa/strategyqa_dev.csv'

raw_data = pd.read_csv(dataset_path)
raw_queries = list(raw_data['question'])
true_answers = list(raw_data['answer'])
true_answers = ['yes' if row else 'no' for row in true_answers]

acc_dic = {}

def send_request(FLASK_URL, questions, answers):
    
    payload = {
        "questions": questions,
        "answers": answers
    }
    url = "http://127.0.0.1:50003/execute"

  
    params = {
        "query": questions, 
        "answers": answers
    }

    # 发送请求
    response = requests.get(url, params=params, timeout=None)
    
    response_data = response.json()

    query_ls = response_data.get("query_ls", [])
    ans_ls = response_data.get("ans_ls", [])
    acc_ls = response_data.get("acc_ls", [])
    return query_ls, ans_ls, acc_ls


def get_acc(instruction, queries, answers):
    url = "http://127.0.0.1:8894/api/update_instruction" 
    new_instruction = {"instruction": instruction}

    requests.post(url, json=new_instruction)

    FLASK_URL = "http://127.0.0.1:50003/execute"
    query_ls, ans_ls, acc_ls = send_request(FLASK_URL, raw_queries, true_answers)
    
    average = sum(acc_ls) / len(acc_ls)
    return average

cnt = 0
for instruction in instructions:
    print(f"========= evaluating round: {cnt} ==========")
    
    acc = get_acc(instruction, raw_queries, true_answers)
    print(f"INS: {instruction}, accuracy: {acc}")
    
    acc_dic[instruction] = acc
    cnt += 1