fairseq/examples/noisychannel/rerank_utils.py

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import math
import os
import re
import subprocess
from contextlib import redirect_stdout

from fairseq import options
from fairseq_cli import eval_lm, preprocess


def reprocess(fle):
    # takes in a file of generate.py translation generate_output
    # returns a source dict and hypothesis dict, where keys are the ID num (as a string)
    # and values and the corresponding source and translation. There may be several translations
    # per source, so the values for hypothesis_dict are lists.
    # parses output of generate.py

    with open(fle, "r") as f:
        txt = f.read()

    """reprocess generate.py output"""
    p = re.compile(r"[STHP][-]\d+\s*")
    hp = re.compile(r"(\s*[-]?\d+[.]?\d+\s*)|(\s*(-inf)\s*)")
    source_dict = {}
    hypothesis_dict = {}
    score_dict = {}
    target_dict = {}
    pos_score_dict = {}
    lines = txt.split("\n")

    for line in lines:
        line += "\n"
        prefix = re.search(p, line)
        if prefix is not None:
            assert len(prefix.group()) > 2, "prefix id not found"
            _, j = prefix.span()
            id_num = prefix.group()[2:]
            id_num = int(id_num)
            line_type = prefix.group()[0]
            if line_type == "H":
                h_txt = line[j:]
                hypo = re.search(hp, h_txt)
                assert (
                    hypo is not None
                ), "regular expression failed to find the hypothesis scoring"
                _, i = hypo.span()
                score = hypo.group()
                if id_num in hypothesis_dict:
                    hypothesis_dict[id_num].append(h_txt[i:])
                    score_dict[id_num].append(float(score))
                else:
                    hypothesis_dict[id_num] = [h_txt[i:]]
                    score_dict[id_num] = [float(score)]

            elif line_type == "S":
                source_dict[id_num] = line[j:]
            elif line_type == "T":
                target_dict[id_num] = line[j:]
            elif line_type == "P":
                pos_scores = (line[j:]).split()
                pos_scores = [float(x) for x in pos_scores]
                if id_num in pos_score_dict:
                    pos_score_dict[id_num].append(pos_scores)
                else:
                    pos_score_dict[id_num] = [pos_scores]

    return source_dict, hypothesis_dict, score_dict, target_dict, pos_score_dict


def reprocess_nbest(fle):
    """reprocess interactive.py output"""
    with open(fle, "r") as f:
        txt = f.read()

    source_dict = {}
    hypothesis_dict = {}
    score_dict = {}
    target_dict = {}
    pos_score_dict = {}
    lines = txt.split("\n")

    hp = re.compile(r"[-]?\d+[.]?\d+")
    j = -1

    for _i, line in enumerate(lines):
        line += "\n"
        line_type = line[0]

        if line_type == "H":
            hypo = re.search(hp, line)
            _, start_index = hypo.span()
            score = hypo.group()
            if j in score_dict:
                score_dict[j].append(float(score))
                hypothesis_dict[j].append(line[start_index:].strip("\t"))
            else:
                score_dict[j] = [float(score)]
                hypothesis_dict[j] = [line[start_index:].strip("\t")]
        elif line_type == "O":
            j += 1
            source_dict[j] = line[2:]
            # we don't have the targets for interactive.py
            target_dict[j] = "filler"

        elif line_type == "P":
            pos_scores = [float(pos_score) for pos_score in line.split()[1:]]
            if j in pos_score_dict:
                pos_score_dict[j].append(pos_scores)
            else:
                pos_score_dict[j] = [pos_scores]

    assert source_dict.keys() == hypothesis_dict.keys()
    assert source_dict.keys() == pos_score_dict.keys()
    assert source_dict.keys() == score_dict.keys()

    return source_dict, hypothesis_dict, score_dict, target_dict, pos_score_dict


def write_reprocessed(
    sources,
    hypos,
    targets,
    source_outfile,
    hypo_outfile,
    target_outfile,
    right_to_left=False,
    prefix_len=None,
    bpe_symbol=None,
    target_prefix_frac=None,
    source_prefix_frac=None,
):

    """writes nbest hypothesis for rescoring"""
    assert not (
        prefix_len is not None and target_prefix_frac is not None
    ), "in writing reprocessed, only one type of prefix may be used"
    assert not (
        prefix_len is not None and source_prefix_frac is not None
    ), "in writing reprocessed, only one type of prefix may be used"
    assert not (
        target_prefix_frac is not None and source_prefix_frac is not None
    ), "in writing reprocessed, only one type of prefix may be used"

    with open(source_outfile, "w") as source_file, open(
        hypo_outfile, "w"
    ) as hypo_file, open(target_outfile, "w") as target_file:

        assert len(sources) == len(hypos), "sources and hypos list length mismatch"
        if right_to_left:
            for i in range(len(sources)):
                for j in range(len(hypos[i])):
                    if prefix_len is None:
                        hypo_file.write(make_right_to_left(hypos[i][j]) + "\n")
                    else:
                        raise NotImplementedError()
                    source_file.write(make_right_to_left(sources[i]) + "\n")
                    target_file.write(make_right_to_left(targets[i]) + "\n")
        else:
            for i in sorted(sources.keys()):
                for j in range(len(hypos[i])):
                    if prefix_len is not None:
                        shortened = (
                            get_prefix_no_bpe(hypos[i][j], bpe_symbol, prefix_len)
                            + "\n"
                        )
                        hypo_file.write(shortened)
                        source_file.write(sources[i])
                        target_file.write(targets[i])
                    elif target_prefix_frac is not None:
                        num_words, shortened, num_bpe_tokens = calc_length_from_frac(
                            hypos[i][j], target_prefix_frac, bpe_symbol
                        )
                        shortened += "\n"
                        hypo_file.write(shortened)
                        source_file.write(sources[i])
                        target_file.write(targets[i])
                    elif source_prefix_frac is not None:
                        num_words, shortened, num_bpe_tokensn = calc_length_from_frac(
                            sources[i], source_prefix_frac, bpe_symbol
                        )
                        shortened += "\n"
                        hypo_file.write(hypos[i][j])
                        source_file.write(shortened)
                        target_file.write(targets[i])
                    else:
                        hypo_file.write(hypos[i][j])
                        source_file.write(sources[i])
                        target_file.write(targets[i])


def calc_length_from_frac(bpe_sentence, prefix_frac, bpe_symbol):
    # return number of words, (not bpe tokens) that we want
    no_bpe_sen = remove_bpe(bpe_sentence, bpe_symbol)
    len_sen = len(no_bpe_sen.split())

    num_words = math.ceil(len_sen * prefix_frac)
    prefix = get_prefix_no_bpe(bpe_sentence, bpe_symbol, num_words)
    num_bpe_tokens = len(prefix.split())
    return num_words, prefix, num_bpe_tokens


def get_prefix(sentence, prefix_len):
    """assuming no bpe, gets the prefix of the sentence with prefix_len words"""
    tokens = sentence.strip("\n").split()
    if prefix_len >= len(tokens):
        return sentence.strip("\n")
    else:
        return " ".join(tokens[:prefix_len])


def get_prefix_no_bpe(sentence, bpe_symbol, prefix_len):
    if bpe_symbol is None:
        return get_prefix(sentence, prefix_len)
    else:
        return " ".join(get_prefix_from_len(sentence.split(), bpe_symbol, prefix_len))


def get_prefix_from_len(sentence, bpe_symbol, prefix_len):
    """get the prefix of sentence with bpe, with prefix len in terms of words, not bpe tokens"""
    bpe_count = sum([bpe_symbol.strip(" ") in t for t in sentence[:prefix_len]])
    if bpe_count == 0:
        return sentence[:prefix_len]
    else:
        return sentence[:prefix_len] + get_prefix_from_len(
            sentence[prefix_len:], bpe_symbol, bpe_count
        )


def get_num_bpe_tokens_from_len(sentence, bpe_symbol, prefix_len):
    """given a prefix length in terms of words, return the number of bpe tokens"""
    prefix = get_prefix_no_bpe(sentence, bpe_symbol, prefix_len)
    assert len(remove_bpe(prefix, bpe_symbol).split()) <= prefix_len
    return len(prefix.split(" "))


def make_right_to_left(line):
    tokens = line.split()
    tokens.reverse()
    new_line = " ".join(tokens)
    return new_line


def remove_bpe(line, bpe_symbol):
    line = line.replace("\n", "")
    line = (line + " ").replace(bpe_symbol, "").rstrip()
    return line + ("\n")


def remove_bpe_dict(pred_dict, bpe_symbol):
    new_dict = {}
    for i in pred_dict:
        if type(pred_dict[i]) == list:
            new_list = [remove_bpe(elem, bpe_symbol) for elem in pred_dict[i]]
            new_dict[i] = new_list
        else:
            new_dict[i] = remove_bpe(pred_dict[i], bpe_symbol)
    return new_dict


def parse_bleu_scoring(line):
    p = re.compile(r"(BLEU4 = )\d+[.]\d+")
    res = re.search(p, line)
    assert res is not None, line
    return float(res.group()[8:])


def get_full_from_prefix(hypo_prefix, hypos):
    """given a hypo prefix, recover the first hypo from the list of complete hypos beginning with that prefix"""
    for hypo in hypos:
        hypo_prefix = hypo_prefix.strip("\n")
        len_prefix = len(hypo_prefix)
        if hypo[:len_prefix] == hypo_prefix:
            return hypo
    # no match found
    raise Exception()


def get_score(
    a,
    b,
    c,
    target_len,
    bitext_score1,
    bitext_score2=None,
    lm_score=None,
    lenpen=None,
    src_len=None,
    tgt_len=None,
    bitext1_backwards=False,
    bitext2_backwards=False,
    normalize=False,
):
    if bitext1_backwards:
        bitext1_norm = src_len
    else:
        bitext1_norm = tgt_len
    if bitext_score2 is not None:
        if bitext2_backwards:
            bitext2_norm = src_len
        else:
            bitext2_norm = tgt_len
    else:
        bitext2_norm = 1
        bitext_score2 = 0
    if normalize:
        score = (
            a * bitext_score1 / bitext1_norm
            + b * bitext_score2 / bitext2_norm
            + c * lm_score / src_len
        )
    else:
        score = a * bitext_score1 + b * bitext_score2 + c * lm_score

    if lenpen is not None:
        score /= (target_len) ** float(lenpen)

    return score


class BitextOutput(object):
    def __init__(
        self,
        output_file,
        backwards,
        right_to_left,
        bpe_symbol,
        prefix_len=None,
        target_prefix_frac=None,
        source_prefix_frac=None,
    ):
        """process output from rescoring"""
        source, hypo, score, target, pos_score = reprocess(output_file)
        if backwards:
            self.hypo_fracs = source_prefix_frac
        else:
            self.hypo_fracs = target_prefix_frac

        # remove length penalty so we can use raw scores
        score, num_bpe_tokens = get_score_from_pos(
            pos_score, prefix_len, hypo, bpe_symbol, self.hypo_fracs, backwards
        )
        source_lengths = {}
        target_lengths = {}

        assert hypo.keys() == source.keys(), "key mismatch"
        if backwards:
            tmp = hypo
            hypo = source
            source = tmp
        for i in source:
            # since we are reranking, there should only be one hypo per source sentence
            if backwards:
                len_src = len(source[i][0].split())
                # record length without <eos>
                if len_src == num_bpe_tokens[i][0] - 1:
                    source_lengths[i] = num_bpe_tokens[i][0] - 1
                else:
                    source_lengths[i] = num_bpe_tokens[i][0]

                target_lengths[i] = len(hypo[i].split())

                source[i] = remove_bpe(source[i][0], bpe_symbol)
                target[i] = remove_bpe(target[i], bpe_symbol)
                hypo[i] = remove_bpe(hypo[i], bpe_symbol)

                score[i] = float(score[i][0])
                pos_score[i] = pos_score[i][0]

            else:
                len_tgt = len(hypo[i][0].split())
                # record length without <eos>
                if len_tgt == num_bpe_tokens[i][0] - 1:
                    target_lengths[i] = num_bpe_tokens[i][0] - 1
                else:
                    target_lengths[i] = num_bpe_tokens[i][0]

                source_lengths[i] = len(source[i].split())

                if right_to_left:
                    source[i] = remove_bpe(make_right_to_left(source[i]), bpe_symbol)
                    target[i] = remove_bpe(make_right_to_left(target[i]), bpe_symbol)
                    hypo[i] = remove_bpe(make_right_to_left(hypo[i][0]), bpe_symbol)
                    score[i] = float(score[i][0])
                    pos_score[i] = pos_score[i][0]
                else:
                    assert (
                        len(hypo[i]) == 1
                    ), "expected only one hypothesis per source sentence"
                    source[i] = remove_bpe(source[i], bpe_symbol)
                    target[i] = remove_bpe(target[i], bpe_symbol)
                    hypo[i] = remove_bpe(hypo[i][0], bpe_symbol)
                    score[i] = float(score[i][0])
                    pos_score[i] = pos_score[i][0]

        self.rescore_source = source
        self.rescore_hypo = hypo
        self.rescore_score = score
        self.rescore_target = target
        self.rescore_pos_score = pos_score
        self.backwards = backwards
        self.right_to_left = right_to_left
        self.target_lengths = target_lengths
        self.source_lengths = source_lengths


class BitextOutputFromGen(object):
    def __init__(
        self,
        predictions_bpe_file,
        bpe_symbol=None,
        nbest=False,
        prefix_len=None,
        target_prefix_frac=None,
    ):
        if nbest:
            (
                pred_source,
                pred_hypo,
                pred_score,
                pred_target,
                pred_pos_score,
            ) = reprocess_nbest(predictions_bpe_file)
        else:
            pred_source, pred_hypo, pred_score, pred_target, pred_pos_score = reprocess(
                predictions_bpe_file
            )

        assert len(pred_source) == len(pred_hypo)
        assert len(pred_source) == len(pred_score)
        assert len(pred_source) == len(pred_target)
        assert len(pred_source) == len(pred_pos_score)

        # remove length penalty so we can use raw scores
        pred_score, num_bpe_tokens = get_score_from_pos(
            pred_pos_score, prefix_len, pred_hypo, bpe_symbol, target_prefix_frac, False
        )

        self.source = pred_source
        self.target = pred_target
        self.score = pred_score
        self.pos_score = pred_pos_score
        self.hypo = pred_hypo
        self.target_lengths = {}
        self.source_lengths = {}

        self.no_bpe_source = remove_bpe_dict(pred_source.copy(), bpe_symbol)
        self.no_bpe_hypo = remove_bpe_dict(pred_hypo.copy(), bpe_symbol)
        self.no_bpe_target = remove_bpe_dict(pred_target.copy(), bpe_symbol)

        # indexes to match those from the rescoring models
        self.rescore_source = {}
        self.rescore_target = {}
        self.rescore_pos_score = {}
        self.rescore_hypo = {}
        self.rescore_score = {}
        self.num_hypos = {}
        self.backwards = False
        self.right_to_left = False

        index = 0

        for i in sorted(pred_source.keys()):
            for j in range(len(pred_hypo[i])):

                self.target_lengths[index] = len(self.hypo[i][j].split())
                self.source_lengths[index] = len(self.source[i].split())

                self.rescore_source[index] = self.no_bpe_source[i]
                self.rescore_target[index] = self.no_bpe_target[i]
                self.rescore_hypo[index] = self.no_bpe_hypo[i][j]
                self.rescore_score[index] = float(pred_score[i][j])
                self.rescore_pos_score[index] = pred_pos_score[i][j]
                self.num_hypos[index] = len(pred_hypo[i])
                index += 1


def get_score_from_pos(
    pos_score_dict, prefix_len, hypo_dict, bpe_symbol, hypo_frac, backwards
):
    score_dict = {}
    num_bpe_tokens_dict = {}
    assert prefix_len is None or hypo_frac is None
    for key in pos_score_dict:
        score_dict[key] = []
        num_bpe_tokens_dict[key] = []
        for i in range(len(pos_score_dict[key])):
            if prefix_len is not None and not backwards:
                num_bpe_tokens = get_num_bpe_tokens_from_len(
                    hypo_dict[key][i], bpe_symbol, prefix_len
                )
                score_dict[key].append(sum(pos_score_dict[key][i][:num_bpe_tokens]))
                num_bpe_tokens_dict[key].append(num_bpe_tokens)
            elif hypo_frac is not None:
                num_words, shortened, hypo_prefix_len = calc_length_from_frac(
                    hypo_dict[key][i], hypo_frac, bpe_symbol
                )
                score_dict[key].append(sum(pos_score_dict[key][i][:hypo_prefix_len]))
                num_bpe_tokens_dict[key].append(hypo_prefix_len)
            else:
                score_dict[key].append(sum(pos_score_dict[key][i]))
                num_bpe_tokens_dict[key].append(len(pos_score_dict[key][i]))
    return score_dict, num_bpe_tokens_dict


class LMOutput(object):
    def __init__(
        self,
        lm_score_file,
        lm_dict=None,
        prefix_len=None,
        bpe_symbol=None,
        target_prefix_frac=None,
    ):
        (
            lm_sentences,
            lm_sen_scores,
            lm_sen_pos_scores,
            lm_no_bpe_sentences,
            lm_bpe_tokens,
        ) = parse_lm(
            lm_score_file,
            prefix_len=prefix_len,
            bpe_symbol=bpe_symbol,
            target_prefix_frac=target_prefix_frac,
        )

        self.sentences = lm_sentences
        self.score = lm_sen_scores
        self.pos_score = lm_sen_pos_scores
        self.lm_dict = lm_dict
        self.no_bpe_sentences = lm_no_bpe_sentences
        self.bpe_tokens = lm_bpe_tokens


def parse_lm(input_file, prefix_len=None, bpe_symbol=None, target_prefix_frac=None):
    """parse output of eval_lm"""
    with open(input_file, "r") as f:
        text = f.readlines()
        text = text[7:]
        cleaned_text = text[:-2]

        sentences = {}
        sen_scores = {}
        sen_pos_scores = {}
        no_bpe_sentences = {}
        num_bpe_tokens_dict = {}
        for _i, line in enumerate(cleaned_text):
            tokens = line.split()
            if tokens[0].isdigit():
                line_id = int(tokens[0])
                scores = [float(x[1:-1]) for x in tokens[2::2]]
                sentences[line_id] = " ".join(tokens[1::2][:-1]) + "\n"
                if bpe_symbol is not None:
                    # exclude <eos> symbol to match output from generate.py
                    bpe_sen = " ".join(tokens[1::2][:-1]) + "\n"
                    no_bpe_sen = remove_bpe(bpe_sen, bpe_symbol)
                    no_bpe_sentences[line_id] = no_bpe_sen

                if prefix_len is not None:
                    num_bpe_tokens = get_num_bpe_tokens_from_len(
                        bpe_sen, bpe_symbol, prefix_len
                    )
                    sen_scores[line_id] = sum(scores[:num_bpe_tokens])
                    num_bpe_tokens_dict[line_id] = num_bpe_tokens
                elif target_prefix_frac is not None:
                    num_words, shortened, target_prefix_len = calc_length_from_frac(
                        bpe_sen, target_prefix_frac, bpe_symbol
                    )
                    sen_scores[line_id] = sum(scores[:target_prefix_len])
                    num_bpe_tokens_dict[line_id] = target_prefix_len
                else:
                    sen_scores[line_id] = sum(scores)
                    num_bpe_tokens_dict[line_id] = len(scores)

                sen_pos_scores[line_id] = scores

    return sentences, sen_scores, sen_pos_scores, no_bpe_sentences, num_bpe_tokens_dict


def get_directories(
    data_dir_name,
    num_rescore,
    gen_subset,
    fw_name,
    shard_id,
    num_shards,
    sampling=False,
    prefix_len=None,
    target_prefix_frac=None,
    source_prefix_frac=None,
):
    nbest_file_id = (
        "nbest_"
        + str(num_rescore)
        + "_subset_"
        + gen_subset
        + "_fw_name_"
        + fw_name
        + "_shard_"
        + str(shard_id)
        + "_of_"
        + str(num_shards)
    )

    if sampling:
        nbest_file_id += "_sampling"

    # the directory containing all information for this nbest list
    pre_gen = (
        os.path.join(os.path.dirname(__file__))
        + "/rerank_data/"
        + data_dir_name
        + "/"
        + nbest_file_id
    )
    # the directory to store the preprocessed nbest list, for left to right rescoring
    left_to_right_preprocessed_dir = pre_gen + "/left_to_right_preprocessed"
    if source_prefix_frac is not None:
        left_to_right_preprocessed_dir = (
            left_to_right_preprocessed_dir + "/prefix_frac" + str(source_prefix_frac)
        )
    # the directory to store the preprocessed nbest list, for right to left rescoring
    right_to_left_preprocessed_dir = pre_gen + "/right_to_left_preprocessed"
    # the directory to store the preprocessed nbest list, for backwards rescoring
    backwards_preprocessed_dir = pre_gen + "/backwards"
    if target_prefix_frac is not None:
        backwards_preprocessed_dir = (
            backwards_preprocessed_dir + "/prefix_frac" + str(target_prefix_frac)
        )
    elif prefix_len is not None:
        backwards_preprocessed_dir = (
            backwards_preprocessed_dir + "/prefix_" + str(prefix_len)
        )

    # the directory to store the preprocessed nbest list, for rescoring with P(T)
    lm_preprocessed_dir = pre_gen + "/lm_preprocessed"

    return (
        pre_gen,
        left_to_right_preprocessed_dir,
        right_to_left_preprocessed_dir,
        backwards_preprocessed_dir,
        lm_preprocessed_dir,
    )


def lm_scoring(
    preprocess_directory,
    bpe_status,
    gen_output,
    pre_gen,
    cur_lm_dict,
    cur_lm_name,
    cur_language_model,
    cur_lm_bpe_code,
    batch_size,
    lm_score_file,
    target_lang,
    source_lang,
    prefix_len=None,
):
    if prefix_len is not None:
        assert (
            bpe_status == "different"
        ), "bpe status must be different to use prefix len"
    if bpe_status == "no bpe":
        # run lm on output without bpe
        write_reprocessed(
            gen_output.no_bpe_source,
            gen_output.no_bpe_hypo,
            gen_output.no_bpe_target,
            pre_gen + "/rescore_data_no_bpe.de",
            pre_gen + "/rescore_data_no_bpe.en",
            pre_gen + "/reference_file_no_bpe",
        )

        preprocess_lm_param = [
            "--only-source",
            "--trainpref",
            pre_gen + "/rescore_data_no_bpe." + target_lang,
            "--srcdict",
            cur_lm_dict,
            "--destdir",
            preprocess_directory,
        ]
        preprocess_parser = options.get_preprocessing_parser()
        input_args = preprocess_parser.parse_args(preprocess_lm_param)
        preprocess.main(input_args)

        eval_lm_param = [
            preprocess_directory,
            "--path",
            cur_language_model,
            "--output-word-probs",
            "--batch-size",
            str(batch_size),
            "--max-tokens",
            "1024",
            "--sample-break-mode",
            "eos",
            "--gen-subset",
            "train",
        ]

        eval_lm_parser = options.get_eval_lm_parser()
        input_args = options.parse_args_and_arch(eval_lm_parser, eval_lm_param)

        with open(lm_score_file, "w") as f:
            with redirect_stdout(f):
                eval_lm.main(input_args)

    elif bpe_status == "shared":
        preprocess_lm_param = [
            "--only-source",
            "--trainpref",
            pre_gen + "/rescore_data." + target_lang,
            "--srcdict",
            cur_lm_dict,
            "--destdir",
            preprocess_directory,
        ]
        preprocess_parser = options.get_preprocessing_parser()
        input_args = preprocess_parser.parse_args(preprocess_lm_param)
        preprocess.main(input_args)

        eval_lm_param = [
            preprocess_directory,
            "--path",
            cur_language_model,
            "--output-word-probs",
            "--batch-size",
            str(batch_size),
            "--sample-break-mode",
            "eos",
            "--gen-subset",
            "train",
        ]

        eval_lm_parser = options.get_eval_lm_parser()
        input_args = options.parse_args_and_arch(eval_lm_parser, eval_lm_param)

        with open(lm_score_file, "w") as f:
            with redirect_stdout(f):
                eval_lm.main(input_args)

    elif bpe_status == "different":
        rescore_file = pre_gen + "/rescore_data_no_bpe"
        rescore_bpe = pre_gen + "/rescore_data_new_bpe"

        rescore_file += "."
        rescore_bpe += "."

        write_reprocessed(
            gen_output.no_bpe_source,
            gen_output.no_bpe_hypo,
            gen_output.no_bpe_target,
            rescore_file + source_lang,
            rescore_file + target_lang,
            pre_gen + "/reference_file_no_bpe",
            bpe_symbol=None,
        )

        # apply LM bpe to nbest list
        bpe_src_param = [
            "-c",
            cur_lm_bpe_code,
            "--input",
            rescore_file + target_lang,
            "--output",
            rescore_bpe + target_lang,
        ]
        subprocess.call(
            [
                "python",
                os.path.join(
                    os.path.dirname(__file__), "subword-nmt/subword_nmt/apply_bpe.py"
                ),
            ]
            + bpe_src_param,
            shell=False,
        )
        # uncomment to use fastbpe instead of subword-nmt bpe
        # bpe_src_param = [rescore_bpe+target_lang, rescore_file+target_lang, cur_lm_bpe_code]
        # subprocess.call(["/private/home/edunov/fastBPE/fast", "applybpe"] + bpe_src_param, shell=False)

        preprocess_dir = preprocess_directory

        preprocess_lm_param = [
            "--only-source",
            "--trainpref",
            rescore_bpe + target_lang,
            "--srcdict",
            cur_lm_dict,
            "--destdir",
            preprocess_dir,
        ]
        preprocess_parser = options.get_preprocessing_parser()
        input_args = preprocess_parser.parse_args(preprocess_lm_param)
        preprocess.main(input_args)

        eval_lm_param = [
            preprocess_dir,
            "--path",
            cur_language_model,
            "--output-word-probs",
            "--batch-size",
            str(batch_size),
            "--max-tokens",
            "1024",
            "--sample-break-mode",
            "eos",
            "--gen-subset",
            "train",
        ]

        eval_lm_parser = options.get_eval_lm_parser()
        input_args = options.parse_args_and_arch(eval_lm_parser, eval_lm_param)

        with open(lm_score_file, "w") as f:
            with redirect_stdout(f):
                eval_lm.main(input_args)


def rescore_file_name(
    nbest_dir,
    prefix_len,
    scorer_name,
    lm_file=False,
    target_prefix_frac=None,
    source_prefix_frac=None,
    backwards=None,
):
    if lm_file:
        score_file = nbest_dir + "/lm_score_translations_model_" + scorer_name + ".txt"
    else:
        score_file = nbest_dir + "/" + scorer_name + "_score_translations.txt"
    if backwards:
        if prefix_len is not None:
            score_file += "prefix_len" + str(prefix_len)
        elif target_prefix_frac is not None:
            score_file += "target_prefix_frac" + str(target_prefix_frac)
    else:
        if source_prefix_frac is not None:
            score_file += "source_prefix_frac" + str(source_prefix_frac)
    return score_file