fairseq/examples/megatron_11b/detok.py

#!/usr/bin/env python3 -u
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import argparse
import fileinput

import sacremoses


def main():
    parser = argparse.ArgumentParser(description="")
    parser.add_argument("files", nargs="*", help="input files")
    args = parser.parse_args()

    detok = sacremoses.MosesDetokenizer()

    for line in fileinput.input(args.files, openhook=fileinput.hook_compressed):
        print(
            detok.detokenize(line.strip().split(" "))
            .replace(" @", "")
            .replace("@ ", "")
            .replace(" =", "=")
            .replace("= ", "=")
            .replace(" – ", "–")
        )


if __name__ == "__main__":
    main()