src/check_client/fact_checker.py

# -*- coding: utf-8 -*-

"""
@Author             : Bao
@Date               : 2020/8/12
@Desc               :
@Last modified by   : Bao
@Last modified date : 2020/8/20
"""

import os
import sys
import logging
import torch
import numpy as np
from tqdm import tqdm
import tensorflow as tf
import ujson as json
import argparse
import cjjpy as cjj
from itertools import repeat
from torch.utils.data import DataLoader, SequentialSampler
from transformers import (
    BertConfig, BertTokenizer, AutoTokenizer,
    RobertaConfig, RobertaTokenizer,
)

try:
    from .modules.data_processor import DataProcessor
    from .plm_checkers import BertChecker, RobertaChecker
    from .utils import read_json_lines, compute_metrics, set_seed
    from .train import do_evaluate
    from ..eval_client.fever_scorer import FeverScorer
except:
    sys.path.append(cjj.AbsParentDir(__file__, '.'))
    sys.path.append(cjj.AbsParentDir(__file__, '..'))
    from eval_client.fever_scorer import FeverScorer
    from modules.data_processor import DataProcessor
    from plm_checkers import BertChecker, RobertaChecker
    from utils import read_json_lines, compute_metrics, set_seed
    from train import do_evaluate

MODEL_MAPPING = {
    'bert': (BertConfig, BertTokenizer, BertChecker),
    'roberta': (RobertaConfig, RobertaTokenizer, RobertaChecker),
}

logger = logging.getLogger(__name__)
label2id = {"SUPPORTS": 2, "REFUTES": 0, 'NOT ENOUGH INFO': 1}
id2label = {v: k for k, v in label2id.items()}


class FactChecker:
    def __init__(self, args, fc_ckpt_dir=None, mask_rate=0.):
        self.data_processor = None
        self.tokenizer = None
        self.model = None
        self.args = args
        self.ckpt = args.fc_dir if fc_ckpt_dir is None else fc_ckpt_dir
        self.mask_rate = mask_rate

        logger.info('Initializing fact checker.')
        self._prepare_ckpt(self.args.model_name_or_path, self.ckpt)
        self.load_model()

    def _prepare_ckpt(self, model_name_or_path, ckpt_dir):
        tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
        tokenizer.save_pretrained(ckpt_dir)

    def load_model(self):
        if self.model is None:
            self.data_processor = DataProcessor(
                self.args.model_name_or_path,
                self.args.max_seq1_length,
                self.args.max_seq2_length,
                self.args.max_num_questions,
                self.args.cand_k,
                mask_rate=self.mask_rate
            )

            _, tokenizer_class, model_class = MODEL_MAPPING[self.args.model_type]
            self.tokenizer = tokenizer_class.from_pretrained(
                self.ckpt,
                do_lower_case=self.args.do_lower_case
            )
            self.model = model_class.from_pretrained(
                self.ckpt,
                from_tf=bool(".ckpt" in self.ckpt),
                logic_lambda=self.args.logic_lambda,
                prior=self.args.prior,
            )
            self.model = torch.nn.DataParallel(self.model)

    def _check(self, inputs: list, batch_size=32, verbose=True):
        dataset = self.data_processor.convert_inputs_to_dataset(inputs, self.tokenizer, verbose=verbose)
        sampler = SequentialSampler(dataset)
        dataloader = DataLoader(dataset, sampler=sampler, batch_size=batch_size)

        with torch.no_grad():
            self.model.to(self.args.device)
            self.model.eval()
            iter = tqdm(dataloader, desc="Fact Checking") if verbose else dataloader
            _, y_predicted, z_predicted, m_attn, mask = \
                do_evaluate(iter, self.model, self.args, during_training=False, with_label=False)

        return y_predicted, z_predicted, m_attn, mask

    def check_from_file(self, in_filename, out_filename, batch_size, verbose=False):
        if 'test' in in_filename:
            raw_inp = f'{os.environ["PJ_HOME"]}/data/fever/shared_task_test.jsonl'
        else:
            raw_inp = None
        tf.io.gfile.makedirs(os.path.dirname(out_filename))
        inputs = list(read_json_lines(in_filename))
        y_predicted, z_predicted, m_attn, mask = self._check(inputs, batch_size)

        z_predicted = repeat(None) if z_predicted is None else z_predicted
        m_attn = repeat(None) if m_attn is None else m_attn
        ordered_results = {}
        with_label = inputs[0].get('label') is not None

        if with_label:
            label_truth = [label2id[x['label']] for x in inputs]
            _, acc_results = compute_metrics(label_truth, y_predicted, z_predicted, mask)
        else:
            acc_results = {}

        for i, (inp, y, z, attn, _mask) in \
                enumerate(zip(inputs, y_predicted, z_predicted, m_attn, mask)):
            result = {'id': inp['id'],
                      'predicted_label': id2label[y],
                      'predicted_evidence': inp.get('predicted_evidence', [])}
            if verbose:
                if i < 5:
                    print("{}\t{}\t{}".format(inp.get("id", i), inp["claim"], y))
                if z is not None and attn is not None:
                    result.update({
                        'z_prob': z[:torch.tensor(_mask).sum()],
                        'm_attn': attn[:torch.tensor(_mask).sum()],
                    })
            ordered_results[inp['id']] = result

        with tf.io.gfile.GFile(out_filename, 'w') as fout:
            if raw_inp:
                with tf.io.gfile.GFile(raw_inp) as f:
                    for line in f:
                        raw_js = json.loads(line)
                        fout.write(json.dumps(ordered_results[raw_js['id']]) + '\n')
            else:
                for k in ordered_results:
                    fout.write(json.dumps(ordered_results[k]) + '\n')

        if ('dev' in in_filename or 'val' in in_filename) and with_label:
            scorer = FeverScorer()
            fever_results = scorer.get_scores(out_filename)
            fever_results.update(acc_results)

            print(fever_results)
            return fever_results

    def check_from_batch(self, inputs: list, verbose=False):
        y_predicted, z_predicted, m_attn, mask = self._check(inputs, len(inputs), verbose)
        return y_predicted, z_predicted, m_attn


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--input', '-i', required=True, type=str,
                        choices=['val', 'eval', 'test', 'demo'])
    parser.add_argument('--output', '-o', default='none', type=str)
    parser.add_argument('--ckpt', '-c', required=True, type=str)
    parser.add_argument('--model_type', default='roberta', type=str,
                        choices=['roberta', 'bert'])
    parser.add_argument('--model_name_or_path', default='roberta-large', type=str)
    parser.add_argument('--verbose', '-v', action='store_true', default=False,
                        help='whether output phrasal veracity or not')
    parser.add_argument('--logic_lambda', '-l', required=True, type=float)
    parser.add_argument('--prior', default='random', type=str, choices=['nli', 'uniform', 'logic', 'random'],
                        help='type of prior distribution')
    parser.add_argument('--mask_rate', '-m', default=0., type=float)

    parser.add_argument('--cand_k', '-k', default=3, type=int)
    parser.add_argument('--max_seq1_length', default=256, type=int)
    parser.add_argument('--max_seq2_length', default=128, type=int)
    parser.add_argument('--max_num_questions', default=8, type=int)
    parser.add_argument('--do_lower_case', action='store_true', default=False)
    parser.add_argument('--batch_size', '-b', default=64, type=int)
    parser.add_argument('--seed', default=42)
    parser.add_argument('--n_gpu', default=4)

    args = parser.parse_args()

    set_seed(args)

    args.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    if args.output == 'none':
        args.ckpt = args.ckpt[:-1] if args.ckpt.endswith('/') else args.ckpt
        base_name = os.path.basename(args.ckpt)
        args.output = f'{os.environ["PJ_HOME"]}/results/fact_checking/AAAI22/{args.input}.{args.model_name_or_path}_m{args.mask_rate}_l{args.logic_lambda}_{base_name}_{args.prior}.predictions.jsonl'

    assert args.output.endswith('predictions.jsonl'), \
        f"{args.output} must end with predictions.jsonl"

    args.input = f'{os.environ["PJ_HOME"]}/data/fact_checking/v5/{args.input}.json'

    checker = FactChecker(args, args.ckpt, args.mask_rate)
    fever_results = checker.check_from_file(args.input, args.output, args.batch_size, args.verbose)
    cjj.lark(f"{args.output}: {fever_results}")