review_content_analysis/analysis.py

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import os
import re
import time
import json
import matplotlib.pyplot as plt
from openai import OpenAI
import multiprocessing

FONT_SIZE = 20

COLORS = ['#26547c', '#06d6a0', '#ef476f', '#ffd166']

openai_api_key = os.getenv("OPENAI_KEY")
# print(openai.api_key)

base_dir = '/home/v-qinlinzhao/agent4reviews/simulated_review/reviews'
save_base_dir = '/home/v-qinlinzhao/agent4reviews/simulated_review/classified_reason/'

with open('iter_prompt.txt', 'r') as f:
    iter_prompt = f.read()
    
with open('classification_prompt.txt', 'r') as f:
    classification_prompt = f.read()

with open('reason_library.txt', 'r') as f:
    reason_library = f.read()

def get_gpt_response(prompt):
    client = OpenAI(api_key=openai_api_key)
    messages = [{'role': 'user', 'content': prompt}]
    completion = client.chat.completions.create(
            model="gpt-4-1106-preview",
            messages=messages,
            temperature=0.7,
            max_tokens=2000,
        )

    response = completion.choices[0].message.content
    response = response.strip()
    
    # time.sleep(5)
    return response

def extract_review_from_real_data():
    base_dir = '/home/v-qinlinzhao/agent4reviews/real_review/original_data'
    result_dir = '/home/v-qinlinzhao/agent4reviews/real_review/extracted_real_review/'
    # 目录为 ICLR202X/notes/xxx.json
    # 将其中所有的json文件的review提取处理
    for root, dirs, files in os.walk(base_dir):
        for file in files:
            if file.endswith('.json'):
                with open(os.path.join(root, file), 'r') as f:
                    data = json.load(f)
                    reviews = []
                    data = data['details']['replies']
                    id = []
                    for d in data:
                        if d['id'] not in id:
                            id.append(d['id'])
                            # 2020-2021
                            if 'content' in d and 'review' in d['content']:
                                reviews.append(d['content']['review'])
                            # 2022
                            if 'content' in d and 'main_review' in d['content']:
                                reviews.append(d['content']['main_review'])
                            # 2023
                            if 'content' in d and 'strength_and_weaknesses' in d['content']:
                                reviews.append(d['content']['strength_and_weaknesses'])
                                
                    # 将每个review分别存入到json文件中，命名格式为 {当前文件名}_{序号}.json
                    # 同时保持每个文件在原目录下相对路径    
                    relative_dir = os.path.relpath(root, base_dir)
                    result_file_dir = os.path.join(result_dir, relative_dir)
                    os.makedirs(result_file_dir, exist_ok=True) 
                       
                    file_base_name = os.path.splitext(file)[0]
                    
                    for i, review in enumerate(reviews): 
                        result_file_name = f"{file_base_name}_{i}.json"
                        result_file_path = os.path.join(result_file_dir, result_file_name)
                        
                        with open(result_file_path, 'w') as result_file:
                            json.dump({"review": review}, result_file, ensure_ascii=False, indent=4)

def extract_meta_review_from_simulated_data():
    base_dir = '/home/v-qinlinzhao/agent4reviews/simulated_review/full_paper_discussion'
    result_dir = '/home/v-qinlinzhao/agent4reviews/simulated_review/meta_review/'
    # 目录为 ICLR202X/notes/xxx.json
    # 将其中所有的json文件的review提取处理
    for root, dirs, files in os.walk(base_dir):
        for file in files:
            if file.endswith('.json'):
                with open(os.path.join(root, file), 'r') as f:
                    data = json.load(f)
                    # review在data['messages']中最后一个元素中的"content"中
                    review = data['messages'][-1]['content']
                # write review into file, keep the abstract path 
                relative_dir = os.path.relpath(root, base_dir)
                result_file_dir = os.path.join(result_dir, relative_dir)
                os.makedirs(result_file_dir, exist_ok=True)
                result_file_path = os.path.join(result_file_dir, file)
                with open(result_file_path, 'w') as result_file:
                    json.dump({"meta_review": review}, result_file, ensure_ascii=False, indent=4)
                    
# Select 1% of the data randomly, let GPT-4 summarize the reasons, and add them to the reason library if there are reasons that do not exist
def construct_reason_library():
    
    base_dir = '/home/v-qinlinzhao/agent4reviews/paper_review_and_rebuttal/selected_files/'
    
    json_files = []
    for root, dirs, files in os.walk(base_dir):
        for file in files:
            if file.endswith('.json'):
                json_files.append(os.path.join(root, file))

    for file in json_files:
        with open(file, 'r') as f:
            data = json.load(f)
            review = data['review']
            prompt = iter_prompt.format(review=review, 
                                        reason_library=reason_library)
            ans = get_gpt_response(prompt)
            print(ans)

def analyze_reason_in_batch(json_files):
    
    for file in json_files:
        with open(file, 'r') as f:
            data = json.load(f)
            review = data['review']
            prompt = classification_prompt.format(review=review)
            res = get_gpt_response(prompt)
            
            # 解析res的输出，将accept和reject的原因分别提取出来，写成json格式
            # 依据该字符串分别抽取Accept和Reject的原因
            reason_dict = {}
            if 'Reject' in res:
                accept_reason = re.search(r"Accept: (.+?);", res)
            else:
                accept_reason = re.search(r"Accept: (.+)", res)
                
            reject_reason = re.search(r"Reject: (.+)", res)
            # print(reject_reason)
            if accept_reason:
                accept_reason = accept_reason.group(1).split(',')
                reason_dict['accept'] = []
                for r in accept_reason:
                    r = r.strip()
                    if r in ['1', '2', '3', '4', '5']:
                        reason_dict['accept'].append(r)
            if reject_reason:
                reject_reason = reject_reason.group(1).split(',')
                reason_dict['reject'] = []
                for r in reject_reason:
                    r = r.strip()
                    if r in ['1', '2', '3', '4', '5', '6', '7']:
                        reason_dict['reject'].append(r)
    
            # print(res)
            relative_path = os.path.relpath(file, base_dir)
            save_path = os.path.join(save_base_dir, relative_path)
            save_dir = os.path.dirname(save_path)
       
            # 首先找到原来目录的目录结构，然后在save_dir中按照该目录保存结果保存结果
            if not os.path.exists(save_dir):
                os.makedirs(save_dir)
            with open(save_path, 'w') as f:
                json.dump(reason_dict, f, indent=4)

def convert_txt_to_json():
    base_dir = '/home/v-qinlinzhao/agent4reviews/simulated_review/classified_meta_review_reason'
    reason_count = {}
    reason_total_count = {'accept': {}, 'reject': {}}

    def process_directory(path, reason_dict):
        # 迭代path下的内容
        for item in os.listdir(path):
            item_path = os.path.join(path, item)
            if os.path.isdir(item_path):
                # 如果是目录，递归处理
                reason_dict[item] = {}
                process_directory(item_path, reason_dict[item])
            elif item.endswith('.txt'):
                # 去除txt后缀
                item_name = item.replace('.txt', '')
                reason_dict[item_name] = {'accept': {}, 'reject': {}}
                # 如果是txt文件，处理文件内容
                with open(item_path, 'r') as f:
                    content = f.read()
                    # "Accept: 1,2,3; Reject: 3,4,7"
                    # 依据该字符串分别抽取Accept和Reject的原因
                    if 'Reject' in content:
                        accept_reason = re.search(r"Accept: (.+?);", content)
                    else:
                        accept_reason = re.search(r"Accept: (.+)", content)
                    reject_reason = re.search(r"Reject: (.+)", content)
                    # print(reject_reason)
                    if accept_reason:
                        accept_reason = accept_reason.group(1).split(',')
                        reason_dict[item_name]['accept'] = []
                        for r in accept_reason:
                            r = r.strip()
                            if r in ['1', '2', '3', '4', '5']:
                                if r not in reason_total_count['accept']:
                                    reason_total_count['accept'][r] = 0
                                reason_total_count['accept'][r] += 1
                                reason_dict[item_name]['accept'].append(r)
                    if reject_reason:
                        reject_reason = reject_reason.group(1).split(',')
                        reason_dict[item_name]['reject'] = []
                        for r in reject_reason:
                            r = r.strip()
                            if r in ['1', '2', '3', '4', '5', '6', '7']:
                                if r not in reason_total_count['reject']:
                                    reason_total_count['reject'][r] = 0
                                reason_total_count['reject'][r] += 1
                                reason_dict[item_name]['reject'].append(r)

    process_directory(base_dir, reason_count)

    # 将统计结果写入文件
    with open('reason.json', 'w') as f:
        json.dump(reason_count, f, indent=4)
    
    # 计算accept 和 reject中每一类原因的占比
    # reason_percentage = {'accept': {}, 'reject': {}}
    # for key, value in reason_total_count.items():
    #     total = sum(value.values())
    #     for k, v in value.items():
    #         reason_percentage[key][k] = v / total
        
    # with open('reason_count.json', 'w') as f:
    #     json.dump(reason_total_count, f, indent=4)
        
    # with open('reason_percentage.json', 'w') as f:
    #     json.dump(reason_percentage, f, indent=4)

def count_reasons():
    with open('../reason_result/reason.json', 'r') as f:
        reason_count = json.load(f)

    count = {}
    for year, year_dict in reason_count.items():
        count[year] = {}
        for model, model_dict in year_dict.items():
            count[year][model] = {}
            for type, type_dict in model_dict.items():
                count[year][model][type] = {}
                count[year][model][type]['accept'] = {}
                count[year][model][type]['reject'] = {}
                # 只在type层面做统计就好了
                for paper_id, paper_id_dict in type_dict.items():
                    for review_id, review_id_dict in paper_id_dict.items():
                        print(year, model, type, paper_id, review_id, review_id_dict)
                        # {'accept': {'1': 1, '2': 1, '5': 1}, 'reject': {'3': 1, '4': 1, '5': 1, '7': 1}}
                        if 'accept' in review_id_dict:
                            for accept_reason in review_id_dict['accept']:
                                if accept_reason not in count[year][model][type]['accept']  \
                                                and accept_reason in ['1', '2', '3', '4', '5']:
                                    count[year][model][type]['accept'][accept_reason] = 0
                                count[year][model][type]['accept'][accept_reason] += 1
                        if 'reject' in review_id_dict: 
                            for reject_reason in review_id_dict['reject']:
                                if reject_reason not in count[year][model][type]['reject'] \
                                                and reject_reason in ['1', '2', '3', '4', '5', '6', '7']:
                                    count[year][model][type]['reject'][reject_reason] = 0
                                count[year][model][type]['reject'][reject_reason] += 1   

    with open('reason_count.json', 'w') as f:
        json.dump(count, f, indent=4)

def calcu_reason_percentage_every_year():
    with open('../reason_result/reason_count.json', 'r') as f:
        reason_count = json.load(f)

    distribution = {}
    for year, year_dict in reason_count.items():
        distribution[year] = {}
        for model, model_dict in year_dict.items():
            distribution[year][model] = {}
            for type, type_dict in model_dict.items():
                distribution[year][model][type] = {}
                distribution[year][model][type]['accept'] = {}
                distribution[year][model][type]['reject'] = {}
                # 统计百分比，先将accept下面的count加起来，然后得到每个百分比
                accept_sum = sum(type_dict['accept'].values())
                for reason, count in type_dict['accept'].items():
                    distribution[year][model][type]['accept'][reason] = count / accept_sum
                reject_sum = sum(type_dict['reject'].values())
                for reason, count in type_dict['reject'].items():
                    distribution[year][model][type]['reject'][reason] = count / reject_sum
    
    with open('reason_percentage.json', 'w') as f:
        json.dump(distribution, f, indent=4)
 
def calcu_reason_percentage():
    # 以每种类别为单位，计算每种类别下的accept和reject的百分比
    with open('../reason_result/reason_count.json', 'r') as f:
        reason_count = json.load(f)
        
    count_dict = {}
    for year, year_dict in reason_count.items():
        for model, model_dict in year_dict.items():
            for type, type_dict in model_dict.items():
                count_dict[type] = {'accept': {}, 'reject': {}}
                # 得到所有year和model的accept和reject的count
                accept_count = type_dict['accept']
                reject_count = type_dict['reject']
                # 将accept中每一类原因进行累加
                for reason, count in accept_count.items():
                    if reason not in count_dict[type]['accept']:
                        count_dict[type]['accept'][reason] = 0
                    count_dict[type]['accept'][reason] += count
                for reason, count in reject_count.items():
                    if reason not in count_dict[type]['reject']:
                        count_dict[type]['reject'][reason] = 0
                    count_dict[type]['reject'][reason] += count
    # 计算count_dict中accept和reject其中原因的百分比
    reason_percentage = {}
    for type, type_dict in count_dict.items():
        reason_percentage[type] = {'accept': {}, 'reject': {}}
        accept_sum = sum(type_dict['accept'].values())
        for reason, count in type_dict['accept'].items():
            reason_percentage[type]['accept'][reason] = count / accept_sum
        reject_sum = sum(type_dict['reject'].values())
        for reason, count in type_dict['reject'].items():
            reason_percentage[type]['reject'][reason] = count / reject_sum
    
    with open('reason_percentage.json', 'w') as f:
        json.dump(reason_percentage, f, indent=4)

def draw_bar_chart(accept_or_reject, ax, type, name1, name2):
    # accept_or_reject = 'accept'
    
    x = {
        "accept": ['Novelty', 'Significance', 'Theoretical', 'Clarity', 'Future'],
        "reject": ['Novelty', 'Theoretical', 'Validation', 'Practicality', 'Limitations', 'Presentation', 'Related Work']
    }
    x_range = range(1, len(x[accept_or_reject])+1)
    
    # 画出每一年的type1 和 type2两种type的比例图
    with open('../reason_result/reason_percentage.json', 'r') as f:
        reason_percentage = json.load(f)
    
        # 取出其中的type1和type2两种type
        type1 = reason_percentage[name1][accept_or_reject]
        type2 = reason_percentage[name2][accept_or_reject]
        
        # 按照key排序
        type1 = dict(sorted(type1.items(), key=lambda x: int(x[0])))
        type2 = dict(sorted(type2.items(), key=lambda x: int(x[0])))
        # dict中key应该是1-7，如果有的Key没有，就加上这个key，value设置为0
        for i in x_range:
            if str(i) not in type1:
                type1[str(i)] = 0
            if str(i) not in type2:
                type2[str(i)] = 0
        
        width = 0.35  # 柱子的宽度
        
        # fig, ax = plt.subplots()
        ax.bar([i - width/2 for i in x_range], type1.values(), width, label=name1, color=COLORS[0], alpha=0.3)
        ax.bar([i + width/2 for i in x_range], type2.values(), width, label=name2, color=COLORS[1], alpha=0.3)
        
        ax.legend()
        ax.set_xlabel('Reason', fontsize=FONT_SIZE)
        # ax.set_ylabel('Percentage', fontsize=FONT_SIZE)
        ax.set_title(type, fontsize=FONT_SIZE)
        ax.set_xticks(x_range)  # 设置x轴刻度为整数
        ax.set_xticklabels(x[accept_or_reject], rotation=30)
        
        # plt.savefig(f'reason_distribution_{type}.png')
        # plt.close()                  
                
def draw_bar_chart_baseline(ax, baseline_or_ground, accept_or_reject):
    # if baseline_or_ground == 'Baseline':
    #     with open('../simulated_review/reason_result/reason_percentage.json', 'r') as f:
    #         reason_percentage = json.load(f)
    #         type_data = reason_percentage['BASELINE'][accept_or_reject]
    # elif baseline_or_ground == 'Ground Truth':
    with open('reason_percentage.json', 'r') as f:
        reason_percentage = json.load(f)
        type_data = reason_percentage[baseline_or_ground][accept_or_reject]
    
    
    x = {
        "accept": ['Novelty', 'Significance', 'Theoretical', 'Clarity', 'Future'],
        "reject": ['Novelty', 'Theoretical', 'Validation', 'Practicality', 'Limitations', 'Presentation', 'Related Work']
    }
    x_range = range(1, len(x[accept_or_reject])+1)
        
    # 按照key排序
    type_data = dict(sorted(type_data.items(), key=lambda x: int(x[0])))
    
    # dict中key应该是1-7，如果有的Key没有，就加上这个key，value设置为0
    for i in x_range:
        if str(i) not in type_data:
            type_data[str(i)] = 0
    
    # 画图，将单一类型画到图上，选取颜色，设置透明度
    width = 0.35  # 柱子的宽度
    
    # fig, ax = plt.subplots()
    ax.bar(x_range, type_data.values(), width, label=accept_or_reject, color=COLORS[0], alpha=0.7)
    
    ax.legend()
    ax.set_xlabel('Reason', fontsize=FONT_SIZE)
    # ax.set_ylabel('Percentage', fontsize=FONT_SIZE)
    ax.set_title(baseline_or_ground, fontsize=FONT_SIZE)
    ax.set_xticks(x_range)  # 设置x轴刻度为整数
    ax.set_xticklabels(x[accept_or_reject], rotation=30)
    
    # plt.savefig(f'{baseline_or_ground}_{accept_or_reject}_reason_distribution.pdf')
    # plt.close()

def draw_reason_distribution(accept_or_reject):
    type2name = {'accept': 'Acceptance', 'reject': 'Rejection'}
    
    fig, axs = plt.subplots(1, 3, figsize=(15, 5))
    fig.suptitle(f'Distribution of {type2name[accept_or_reject]} Reasons', fontsize=FONT_SIZE)
    
    # authoritarian_ACx1 inclusive_ACx1 conformist_ACx1
    draw_bar_chart_baseline(axs[0], 'authoritarian_ACx1', accept_or_reject)
    draw_bar_chart_baseline(axs[1], 'inclusive_ACx1', accept_or_reject)
    draw_bar_chart_baseline(axs[2], 'conformist_ACx1', accept_or_reject)
    
    # draw_bar_chart_baseline(axs[0], 'Baseline', accept_or_reject)
    # draw_bar_chart_baseline(axs[1], 'Ground Truth', accept_or_reject)
    
    # for i, (key, value) in enumerate(types.items()):
    #     if i == 3:
    #         break
    #     draw_bar_chart(accept_or_reject, axs[i], key, value[0], value[1])

    axs[0].set_ylabel('Percentage', fontsize=FONT_SIZE)
    
    plt.tight_layout()
    plt.savefig(f'reason_distribution_AC_{accept_or_reject}.pdf')
    plt.close()
        

if __name__ == "__main__":
    # analysis_pipeline()
    # convert_txt_to_json()
    draw_reason_distribution('reject')
    
    
# if __name__ == "__main__":
#     # get current path
#     # print(os.getcwd())
#     print("Start analysis...")
        
#     json_files = []
#     for root, dirs, files in os.walk(base_dir):
#         for file in files:
#             if file.endswith('.json'):
#                 json_files.append(os.path.join(root, file))
    
#     # json_files = [f for f in json_files]
#     # print(json_files)
    
#     # 将其平均分为6份，每份分配给一个进程
#     n = len(json_files)
#     n_per_process = n // 6
#     processes = []
#     for i in range(6):
#         start = i * n_per_process
#         end = (i + 1) * n_per_process
#         if i == 5:
#             end = n
#         p = multiprocessing.Process(target=analyze_reason_in_batch, args=(json_files[start:end], ))
#         processes.append(p)
#         p.start()