diff --git a/data/fairseq/examples/MMPT/mmpt/datasets/__init__.py b/data/fairseq/examples/MMPT/mmpt/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2578235e1771fdc7e6fcfb66a519cbe891d7e254 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/datasets/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +from .mmdataset import * + +try: + from .fairseqmmdataset import * +except ImportError: + pass diff --git a/data/fairseq/examples/MMPT/mmpt/datasets/fairseqmmdataset.py b/data/fairseq/examples/MMPT/mmpt/datasets/fairseqmmdataset.py new file mode 100644 index 0000000000000000000000000000000000000000..02c49141db69c44663bd438b947c268d06f8aa2b --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/datasets/fairseqmmdataset.py @@ -0,0 +1,57 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +""" +TODO (huxu): fairseq wrapper class for all dataset you defined: mostly MMDataset. +""" + +from collections import OrderedDict + +from torch.utils.data import Dataset +from torch.utils.data.dataloader import default_collate +from fairseq.data import FairseqDataset, data_utils + + +class FairseqMMDataset(FairseqDataset): + """ + A wrapper class for MMDataset for fairseq. + """ + + def __init__(self, mmdataset): + if not isinstance(mmdataset, Dataset): + raise TypeError("mmdataset must be of type `torch.utils.data.dataset`.") + self.mmdataset = mmdataset + + def set_epoch(self, epoch, **unused): + super().set_epoch(epoch) + self.epoch = epoch + + def __getitem__(self, idx): + with data_utils.numpy_seed(43211, self.epoch, idx): + return self.mmdataset[idx] + + def __len__(self): + return len(self.mmdataset) + + def collater(self, samples): + if hasattr(self.mmdataset, "collator"): + return self.mmdataset.collator(samples) + if len(samples) == 0: + return {} + if isinstance(samples[0], dict): + batch = OrderedDict() + for key in samples[0]: + if samples[0][key] is not None: + batch[key] = default_collate([sample[key] for sample in samples]) + return batch + else: + return default_collate(samples) + + def size(self, index): + """dummy implementation: we don't use --max-tokens""" + return 1 + + def num_tokens(self, index): + """dummy implementation: we don't use --max-tokens""" + return 1 diff --git a/data/fairseq/examples/MMPT/mmpt/datasets/mmdataset.py b/data/fairseq/examples/MMPT/mmpt/datasets/mmdataset.py new file mode 100644 index 0000000000000000000000000000000000000000..3d07283f917a430a8d9b1226c8fa6ab71450e8a9 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/datasets/mmdataset.py @@ -0,0 +1,111 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import torch + +from collections import OrderedDict + +from torch.utils.data import Dataset +from torch.utils.data.dataloader import default_collate + +from ..utils import set_seed + + +class MMDataset(Dataset): + """ + A generic multi-modal dataset. + Args: + `meta_processor`: a meta processor, + handling loading meta data and return video_id and text_id. + `video_processor`: a video processor, + handling e.g., decoding, loading .np files. + `text_processor`: a text processor, + handling e.g., tokenization. + `aligner`: combine the video and text feature + as one training example. + """ + + def __init__( + self, + meta_processor, + video_processor, + text_processor, + align_processor, + ): + self.split = meta_processor.split + self.meta_processor = meta_processor + self.video_processor = video_processor + self.text_processor = text_processor + self.align_processor = align_processor + + def __len__(self): + return len(self.meta_processor) + + def __getitem__(self, idx): + if self.split == "test": + set_seed(idx) + video_id, text_id = self.meta_processor[idx] + video_feature = self.video_processor(video_id) + text_feature = self.text_processor(text_id) + output = self.align_processor(video_id, video_feature, text_feature) + # TODO (huxu): the following is for debug purpose. + output.update({"idx": idx}) + return output + + def collater(self, samples): + """This collator is deprecated. + set self.collator = MMDataset.collater. + see collator in FairseqMMDataset. + """ + + if len(samples) == 0: + return {} + if isinstance(samples[0], dict): + batch = OrderedDict() + for key in samples[0]: + if samples[0][key] is not None: + batch[key] = default_collate( + [sample[key] for sample in samples]) + # if torch.is_tensor(batch[key]): + # print(key, batch[key].size()) + # else: + # print(key, len(batch[key])) + return batch + else: + return default_collate(samples) + + def print_example(self, output): + print("[one example]", output["video_id"]) + if ( + hasattr(self.align_processor, "subsampling") + and self.align_processor.subsampling is not None + and self.align_processor.subsampling > 1 + ): + for key in output: + if torch.is_tensor(output[key]): + output[key] = output[key][0] + + # search tokenizer to translate ids back. + tokenizer = None + if hasattr(self.text_processor, "tokenizer"): + tokenizer = self.text_processor.tokenizer + elif hasattr(self.align_processor, "tokenizer"): + tokenizer = self.align_processor.tokenizer + if tokenizer is not None: + caps = output["caps"].tolist() + if isinstance(caps[0], list): + caps = caps[0] + print("caps", tokenizer.decode(caps)) + print("caps", tokenizer.convert_ids_to_tokens(caps)) + + for key, value in output.items(): + if torch.is_tensor(value): + if len(value.size()) >= 3: # attention_mask. + print(key, value.size()) + print(key, "first", value[0, :, :]) + print(key, "last", value[-1, :, :]) + else: + print(key, value) + print("[end of one example]") diff --git a/data/fairseq/examples/MMPT/mmpt/evaluators/evaluator.py b/data/fairseq/examples/MMPT/mmpt/evaluators/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..94d9c5ec9a6e84434dbced8d5647754c5a571570 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/evaluators/evaluator.py @@ -0,0 +1,54 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import os +import glob +import numpy as np + +from . import metric as metric_path +from . import predictor as predictor_path + + +class Evaluator(object): + """ + perform evaluation on a single (downstream) task. + make this both offline and online. + TODO(huxu) saving evaluation results. + """ + + def __init__(self, config, eval_dataloader=None): + if config.metric is None: + raise ValueError("config.metric is", config.metric) + metric_cls = getattr(metric_path, config.metric) + self.metric = metric_cls(config) + if config.predictor is None: + raise ValueError("config.predictor is", config.predictor) + predictor_cls = getattr(predictor_path, config.predictor) + self.predictor = predictor_cls(config) + self.eval_dataloader = eval_dataloader + + def __call__(self): + try: + print(self.predictor.pred_dir) + for pred_file in glob.glob( + self.predictor.pred_dir + "/*_merged.npy"): + outputs = np.load(pred_file) + results = self.metric.compute_metrics(outputs) + self.metric.print_computed_metrics(results) + + outputs = np.load(os.path.join( + self.predictor.pred_dir, "merged.npy")) + results = self.metric.compute_metrics(outputs) + return {"results": results, "metric": self.metric} + except FileNotFoundError: + print("\n[missing]", self.predictor.pred_dir) + return {} + + def evaluate(self, model, eval_dataloader=None, output_file="merged"): + if eval_dataloader is None: + eval_dataloader = self.eval_dataloader + outputs = self.predictor.predict_loop( + model, eval_dataloader, output_file) + results = self.metric.compute_metrics(**outputs) + return results diff --git a/data/fairseq/examples/MMPT/mmpt/evaluators/metric.py b/data/fairseq/examples/MMPT/mmpt/evaluators/metric.py new file mode 100644 index 0000000000000000000000000000000000000000..163724bb250cb1b7057b3fa4d75a9fafa9c181f5 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/evaluators/metric.py @@ -0,0 +1,313 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import numpy as np +import json + + +class Metric(object): + def __init__(self, config, metric_names): + self.metric_names = metric_names + + def best_metric(self, metric): + return metric[self.metric_names[0]] + + def save_metrics(self, fn, metrics): + with open(fn, "w") as fw: + json.dump(fw, metrics) + + def print_computed_metrics(self, metrics): + raise NotImplementedError + + +class RetrievalMetric(Metric): + """ + this is modified from `howto100m/metrics.py`. + History of changes: + refactor as a class. + add metric_key in __init__ + """ + + def __init__(self, config, metric_names=["R1", "R5", "R10", "MR"]): + super().__init__(config, metric_names) + self.error = False # TODO(huxu): add to config to print error. + + def compute_metrics(self, outputs, texts, **kwargs): + x = outputs + sx = np.sort(-x, axis=1) + d = np.diag(-x) + d = d[:, np.newaxis] + ind = sx - d + ind = np.where(ind == 0) + ind = ind[1] + metrics = {} + metrics["R1"] = float(np.sum(ind == 0)) / len(ind) + metrics["R5"] = float(np.sum(ind < 5)) / len(ind) + metrics["R10"] = float(np.sum(ind < 10)) / len(ind) + metrics["MR"] = np.median(ind) + 1 + + max_idx = np.argmax(outputs, axis=1) + if self.error: + # print top-20 errors. + error = [] + for ex_idx in range(20): + error.append((texts[ex_idx], texts[max_idx[ex_idx]])) + metrics["error"] = error + return metrics + + def print_computed_metrics(self, metrics): + r1 = metrics["R1"] + r5 = metrics["R5"] + r10 = metrics["R10"] + mr = metrics["MR"] + print( + "R@1: {:.4f} - R@5: {:.4f} - R@10: {:.4f} - Median R: {}".format( + r1, r5, r10, mr + ) + ) + if "error" in metrics: + print(metrics["error"]) + + +class DiDeMoMetric(Metric): + """ + History of changes: + python 2.x to python 3.x. + merge utils.py into eval to save one file. + reference: https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/eval.py + Code to evaluate your results on the DiDeMo dataset. + """ + def __init__(self, config, metric_names=["rank1", "rank5", "miou"]): + super().__init__(config, metric_names) + + def compute_metrics(self, outputs, targets, **kwargs): + assert len(outputs) == len(targets) + rank1, rank5, miou = self._eval_predictions(outputs, targets) + metrics = { + "rank1": rank1, + "rank5": rank5, + "miou": miou + } + return metrics + + def print_computed_metrics(self, metrics): + rank1 = metrics["rank1"] + rank5 = metrics["rank5"] + miou = metrics["miou"] + # print("Average rank@1: %f" % rank1) + # print("Average rank@5: %f" % rank5) + # print("Average iou: %f" % miou) + + print( + "Average rank@1: {:.4f} Average rank@5: {:.4f} Average iou: {:.4f}".format( + rank1, rank5, miou + ) + ) + + def _iou(self, pred, gt): + intersection = max(0, min(pred[1], gt[1]) + 1 - max(pred[0], gt[0])) + union = max(pred[1], gt[1]) + 1 - min(pred[0], gt[0]) + return float(intersection)/union + + def _rank(self, pred, gt): + return pred.index(tuple(gt)) + 1 + + def _eval_predictions(self, segments, data): + ''' + Inputs: + segments: For each item in the ground truth data, rank possible video segments given the description and video. + In DiDeMo, there are 21 posible moments extracted for each video so the list of video segments will be of length 21. + The first video segment should be the video segment that best corresponds to the text query. + There are 4180 sentence in the validation data, so when evaluating a model on the val dataset, + segments should be a list of lenght 4180, and each item in segments should be a list of length 21. + data: ground truth data + ''' + average_ranks = [] + average_iou = [] + for s, d in zip(segments, data): + pred = s[0] + ious = [self._iou(pred, t) for t in d['times']] + average_iou.append(np.mean(np.sort(ious)[-3:])) + ranks = [self._rank(s, t) for t in d['times'] if tuple(t) in s] # if t in s] is added for s, e not in prediction. + average_ranks.append(np.mean(np.sort(ranks)[:3])) + rank1 = np.sum(np.array(average_ranks) <= 1)/float(len(average_ranks)) + rank5 = np.sum(np.array(average_ranks) <= 5)/float(len(average_ranks)) + miou = np.mean(average_iou) + + # print("Average rank@1: %f" % rank1) + # print("Average rank@5: %f" % rank5) + # print("Average iou: %f" % miou) + return rank1, rank5, miou + + +class NLGMetric(Metric): + def __init__( + self, + config, + metric_names=[ + "Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4", + "METEOR", "ROUGE_L", "CIDEr" + ] + ): + super().__init__(config, metric_names) + # please install NLGEval from `https://github.com/Maluuba/nlg-eval` + from nlgeval import NLGEval + self.nlg = NLGEval() + + def compute_metrics(self, outputs, targets, **kwargs): + return self.nlg.compute_metrics( + hyp_list=outputs, ref_list=targets) + + def print_computed_metrics(self, metrics): + Bleu_1 = metrics["Bleu_1"] + Bleu_2 = metrics["Bleu_2"] + Bleu_3 = metrics["Bleu_3"] + Bleu_4 = metrics["Bleu_4"] + METEOR = metrics["METEOR"] + ROUGE_L = metrics["ROUGE_L"] + CIDEr = metrics["CIDEr"] + + print( + "Bleu_1: {:.4f} - Bleu_2: {:.4f} - Bleu_3: {:.4f} - Bleu_4: {:.4f} - METEOR: {:.4f} - ROUGE_L: {:.4f} - CIDEr: {:.4f}".format( + Bleu_1, Bleu_2, Bleu_3, Bleu_4, METEOR, ROUGE_L, CIDEr + ) + ) + + +class QAMetric(Metric): + def __init__( + self, + config, + metric_names=["acc"] + ): + super().__init__(config, metric_names) + + def compute_metrics(self, outputs, targets, **kwargs): + from sklearn.metrics import accuracy_score + return {"acc": accuracy_score(targets, outputs)} + + def print_computed_metrics(self, metrics): + print("acc: {:.4f}".format(metrics["acc"])) + + +class COINActionSegmentationMetric(Metric): + """ + COIN dataset listed 3 repos for Action Segmentation. + Action Sets, NeuralNetwork-Viterbi, TCFPN-ISBA. + The first and second are the same. + https://github.com/alexanderrichard/action-sets/blob/master/eval.py + + Future reference for the third: + `https://github.com/Zephyr-D/TCFPN-ISBA/blob/master/utils/metrics.py` + """ + def __init__(self, config, metric_name=["frame_acc"]): + super().__init__(config, metric_name) + + def compute_metrics(self, outputs, targets): + n_frames = 0 + n_errors = 0 + n_errors = sum(outputs != targets) + n_frames = len(targets) + return {"frame_acc": 1.0 - float(n_errors) / n_frames} + + def print_computed_metrics(self, metrics): + fa = metrics["frame_acc"] + print("frame accuracy:", fa) + + +class CrossTaskMetric(Metric): + def __init__(self, config, metric_names=["recall"]): + super().__init__(config, metric_names) + + def compute_metrics(self, outputs, targets, **kwargs): + """refactored from line 166: + https://github.com/DmZhukov/CrossTask/blob/master/train.py""" + + recalls = self._get_recalls(Y_true=targets, Y_pred=outputs) + results = {} + for task, rec in recalls.items(): + results[str(task)] = rec + + avg_recall = np.mean(list(recalls.values())) + results["recall"] = avg_recall + return results + + def print_computed_metrics(self, metrics): + print('Recall: {0:0.3f}'.format(metrics["recall"])) + for task in metrics: + if task != "recall": + print('Task {0}. Recall = {1:0.3f}'.format( + task, metrics[task])) + + def _get_recalls(self, Y_true, Y_pred): + """refactored from + https://github.com/DmZhukov/CrossTask/blob/master/train.py""" + + step_match = {task: 0 for task in Y_true.keys()} + step_total = {task: 0 for task in Y_true.keys()} + for task, ys_true in Y_true.items(): + ys_pred = Y_pred[task] + for vid in set(ys_pred.keys()).intersection(set(ys_true.keys())): + y_true = ys_true[vid] + y_pred = ys_pred[vid] + step_total[task] += (y_true.sum(axis=0) > 0).sum() + step_match[task] += (y_true*y_pred).sum() + recalls = { + task: step_match[task] / n for task, n in step_total.items()} + return recalls + + +class ActionRecognitionMetric(Metric): + def __init__( + self, + config, + metric_names=["acc", "acc_splits", "r1_splits", "r5_splits", "r10_splits"] + ): + super().__init__(config, metric_names) + + def compute_metrics(self, outputs, targets, splits, **kwargs): + all_video_embd = outputs + labels = targets + split1, split2, split3 = splits + accs = [] + r1s = [] + r5s = [] + r10s = [] + for split in range(3): + if split == 0: + s = split1 + elif split == 1: + s = split2 + else: + s = split3 + + X_pred = all_video_embd[np.where(s == 2)[0]] + label_test = labels[np.where(s == 2)[0]] + logits = X_pred + X_pred = np.argmax(X_pred, axis=1) + acc = np.sum(X_pred == label_test) / float(len(X_pred)) + accs.append(acc) + # compute recall. + sorted_pred = (-logits).argsort(axis=-1) + label_test_sp = label_test.reshape(-1, 1) + + r1 = np.mean((sorted_pred[:, :1] == label_test_sp).sum(axis=1), axis=0) + r5 = np.mean((sorted_pred[:, :5] == label_test_sp).sum(axis=1), axis=0) + r10 = np.mean((sorted_pred[:, :10] == label_test_sp).sum(axis=1), axis=0) + r1s.append(r1) + r5s.append(r5) + r10s.append(r10) + + return {"acc": accs[0], "acc_splits": accs, "r1_splits": r1s, "r5_splits": r5s, "r10_splits": r10s} + + def print_computed_metrics(self, metrics): + for split, acc in enumerate(metrics["acc_splits"]): + print("Top 1 accuracy on split {}: {}; r1 {}; r5 {}; r10 {}".format( + split + 1, acc, + metrics["r1_splits"][split], + metrics["r5_splits"][split], + metrics["r10_splits"][split], + ) + ) diff --git a/data/fairseq/examples/MMPT/mmpt/losses/__init__.py b/data/fairseq/examples/MMPT/mmpt/losses/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8dc32c96d2d8aed25c59e69e9d9a2ff24a9a2a47 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/losses/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +from .loss import * +from .nce import * + +try: + from .fairseqmmloss import * +except ImportError: + pass + +try: + from .expnce import * +except ImportError: + pass diff --git a/data/fairseq/examples/MMPT/mmpt/losses/fairseqmmloss.py b/data/fairseq/examples/MMPT/mmpt/losses/fairseqmmloss.py new file mode 100644 index 0000000000000000000000000000000000000000..a95e5ecf45d90098c1719487bb9a11c36be7c507 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/losses/fairseqmmloss.py @@ -0,0 +1,63 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +""" +TODO (huxu): a general fairseq criterion for all your pre-defined losses. +""" + +from fairseq.criterions import FairseqCriterion, register_criterion +from fairseq.logging import metrics + + +@register_criterion("mmloss") +class MMCriterion(FairseqCriterion): + def __init__(self, task): + super().__init__(task) + # TODO (huxu): wrap forward call of loss_fn and eval_fn into task. + self.mmtask = task.mmtask + + def forward(self, model, sample): + """Compute the loss for the given sample. + Returns a tuple with three elements: + 1) the loss + 2) the sample size, which is used as the denominator for the gradient + 3) logging outputs to display while training + """ + outputs = self.mmtask(model, sample) + + loss, loss_scalar, max_len, batch_size, sample_size = ( + outputs["loss"], + outputs["loss_scalar"], + outputs["max_len"], + outputs["batch_size"], + outputs["sample_size"], + ) + + logging_output = { + "loss": loss_scalar, + "ntokens": max_len * batch_size, # dummy report. + "nsentences": batch_size, # dummy report. + "sample_size": sample_size, + } + + return loss, 1, logging_output + + @staticmethod + def reduce_metrics(logging_outputs) -> None: + """Aggregate logging outputs from data parallel training.""" + """since we use NCE, our actual batch_size is 1 per GPU. + Then we take the mean of each worker.""" + loss_sum = sum(log.get("loss", 0.0) for log in logging_outputs) + sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) + metrics.log_scalar("loss", loss_sum / sample_size, round=3) + + @staticmethod + def logging_outputs_can_be_summed() -> bool: + """ + Whether the logging outputs returned by `forward` can be summed + across workers prior to calling `reduce_metrics`. Setting this + to True will improves distributed training speed. + """ + return True diff --git a/data/fairseq/examples/MMPT/mmpt/losses/loss.py b/data/fairseq/examples/MMPT/mmpt/losses/loss.py new file mode 100644 index 0000000000000000000000000000000000000000..99c05d067edac220f9e53080f09f0b40d7dc1e8d --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/losses/loss.py @@ -0,0 +1,87 @@ +# Copyright (c) Facebook, Inc. All Rights Reserved + +import torch + +from torch import nn + + +class Loss(object): + def __call__(self, *args, **kwargs): + raise NotImplementedError + + +# Dummy Loss for testing. +class DummyLoss(Loss): + def __init__(self): + self.loss = nn.CrossEntropyLoss() + + def __call__(self, logits, targets, **kwargs): + return self.loss(logits, targets) + + +class DummyK400Loss(Loss): + """dummy k400 loss for MViT.""" + def __init__(self): + self.loss = nn.CrossEntropyLoss() + + def __call__(self, logits, targets, **kwargs): + return self.loss( + logits, torch.randint(0, 400, (logits.size(0),), device=logits.device)) + + +class CrossEntropy(Loss): + def __init__(self): + self.loss = nn.CrossEntropyLoss() + + def __call__(self, logits, targets, **kwargs): + return self.loss(logits.reshape(-1, logits.size(-1)), targets.reshape(-1)) + + +class ArgmaxCrossEntropy(Loss): + def __init__(self): + self.loss = nn.CrossEntropyLoss() + + def __call__(self, logits, targets, **kwargs): + return self.loss(logits, targets.argmax(dim=1)) + + +class BCE(Loss): + def __init__(self): + self.loss = nn.BCEWithLogitsLoss() + + def __call__(self, logits, targets, **kwargs): + targets = targets.squeeze(0) + return self.loss(logits, targets) + + +class NLGLoss(Loss): + def __init__(self): + self.loss = nn.CrossEntropyLoss() + + def __call__(self, logits, text_label, **kwargs): + targets = text_label[text_label != -100] + return self.loss(logits, targets) + + +class MSE(Loss): + def __init__(self): + self.loss = nn.MSELoss() + + def __call__(self, logits, targets, **kwargs): + return self.loss(logits, targets) + + +class L1(Loss): + def __init__(self): + self.loss = nn.L1Loss() + + def __call__(self, logits, targets, **kwargs): + return self.loss(logits, targets) + + +class SmoothL1(Loss): + def __init__(self): + self.loss = nn.SmoothL1Loss() + + def __call__(self, logits, targets, **kwargs): + return self.loss(logits, targets) diff --git a/data/fairseq/examples/MMPT/mmpt/losses/nce.py b/data/fairseq/examples/MMPT/mmpt/losses/nce.py new file mode 100644 index 0000000000000000000000000000000000000000..ed7be8d372e371bb0e0d6166f76e01d3466d2306 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/losses/nce.py @@ -0,0 +1,156 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +""" +softmax-based NCE loss, used by this project. +""" + +import torch + +from torch import nn + +from .loss import Loss + + +class NCE(Loss): + def __init__(self): + # TODO (huxu): define temperature. + self.loss = nn.CrossEntropyLoss() + + def __call__(self, align_scores, **kargs): + # note: we reuse the same shape as cls head in BERT (batch_size, 2) + # but NCE only needs one logits. + # (so we drop all weights in the second neg logits.) + align_scores = align_scores[:, :1] + # duplicate negative examples + batch_size = align_scores.size(0) // 2 + pos_scores = align_scores[:batch_size] + neg_scores = align_scores[batch_size:].view(1, batch_size).repeat( + batch_size, 1) + scores = torch.cat([pos_scores, neg_scores], dim=1) + return self.loss( + scores, + torch.zeros( + (batch_size,), + dtype=torch.long, + device=align_scores.device), + ) + + +class T2VContraLoss(Loss): + """NCE for MM joint space, on softmax text2video matrix. + """ + def __init__(self): + # TODO (huxu): define temperature. + self.loss = nn.CrossEntropyLoss() + + def __call__(self, pooled_video, pooled_text, **kargs): + batch_size = pooled_video.size(0) + logits = torch.mm(pooled_text, pooled_video.transpose(1, 0)) + targets = torch.arange( + batch_size, + dtype=torch.long, + device=pooled_video.device) + return self.loss(logits, targets) + + +class V2TContraLoss(Loss): + """NCE for MM joint space, with softmax on video2text matrix.""" + + def __init__(self): + # TODO (huxu): define temperature. + self.loss = nn.CrossEntropyLoss() + + def __call__(self, pooled_video, pooled_text, **kargs): + batch_size = pooled_video.size(0) + logits = torch.mm(pooled_video, pooled_text.transpose(1, 0)) + targets = torch.arange( + batch_size, + dtype=torch.long, + device=pooled_video.device) + return self.loss(logits, targets) + + +class MMContraLoss(Loss): + def __init__(self): + self.loss = nn.CrossEntropyLoss() + + def __call__(self, pooled_video, pooled_text, **kwargs): + logits_per_video = pooled_video @ pooled_text.t() + logits_per_text = pooled_text @ pooled_video.t() + + targets = torch.arange( + pooled_video.size(0), + dtype=torch.long, + device=pooled_video.device) + loss_video = self.loss(logits_per_video, targets) + loss_text = self.loss(logits_per_text, targets) + return loss_video + loss_text + + +class MTM(Loss): + """Combination of MFM and MLM.""" + + def __init__(self): + self.loss = nn.CrossEntropyLoss() + + def __call__( + self, + video_logits, + text_logits, + video_label, + text_label, + **kwargs + ): + text_logits = torch.cat([ + text_logits, + torch.zeros( + (text_logits.size(0), 1), device=text_logits.device) + ], dim=1) + vt_logits = torch.cat([video_logits, text_logits], dim=0) + # loss for video. + video_label = torch.zeros( + (video_logits.size(0),), + dtype=torch.long, + device=video_logits.device + ) + + # loss for text. + text_label = text_label.reshape(-1) + labels_mask = text_label != -100 + selected_text_label = text_label[labels_mask] + + vt_label = torch.cat([video_label, selected_text_label], dim=0) + return self.loss(vt_logits, vt_label) + + +class MFMMLM(Loss): + """Combination of MFM and MLM.""" + + def __init__(self): + self.loss = nn.CrossEntropyLoss() + + def __call__( + self, + video_logits, + text_logits, + video_label, + text_label, + **kwargs + ): + # loss for video. + video_label = torch.zeros( + (video_logits.size(0),), + dtype=torch.long, + device=video_logits.device + ) + masked_frame_loss = self.loss(video_logits, video_label) + + # loss for text. + text_label = text_label.reshape(-1) + labels_mask = text_label != -100 + selected_text_label = text_label[labels_mask] + masked_lm_loss = self.loss(text_logits, selected_text_label) + return masked_frame_loss + masked_lm_loss diff --git a/data/fairseq/examples/MMPT/mmpt/models/__init__.py b/data/fairseq/examples/MMPT/mmpt/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..825250cd007f5e072b6c9d2376445b955a7aa71e --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/models/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +from .mmfusion import * +from .transformermodel import * +from .mmfusionnlg import * + +try: + from .fairseqmmmodel import * +except ImportError: + pass + +try: + from .expmmfusion import * +except ImportError: + pass diff --git a/data/fairseq/examples/MMPT/mmpt/models/fairseqmmmodel.py b/data/fairseq/examples/MMPT/mmpt/models/fairseqmmmodel.py new file mode 100644 index 0000000000000000000000000000000000000000..b7dd643693dee8cfc20ca77d6cea798d07eaf15a --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/models/fairseqmmmodel.py @@ -0,0 +1,51 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from fairseq.models import ( + BaseFairseqModel, + register_model, + register_model_architecture +) + + +@register_model("mmmodel") +class FairseqMMModel(BaseFairseqModel): + """a fairseq wrapper of model built by `task`.""" + + @classmethod + def build_model(cls, args, task): + return FairseqMMModel(task.mmtask.model) + + def __init__(self, mmmodel): + super().__init__() + self.mmmodel = mmmodel + + def forward(self, *args, **kwargs): + return self.mmmodel(*args, **kwargs) + + def upgrade_state_dict_named(self, state_dict, name): + + super().upgrade_state_dict_named(state_dict, name) + + keys_to_delete = [] + + for key in state_dict: + if key not in self.state_dict(): + keys_to_delete.append(key) + for key in keys_to_delete: + print("[INFO]", key, "not used anymore.") + del state_dict[key] + + # copy any newly defined parameters. + for key in self.state_dict(): + if key not in state_dict: + print("[INFO] adding", key) + state_dict[key] = self.state_dict()[key] + + +# a dummy arch, we config the model. +@register_model_architecture("mmmodel", "mmarch") +def mmarch(args): + pass diff --git a/data/fairseq/examples/MMPT/mmpt/models/mmfusion.py b/data/fairseq/examples/MMPT/mmpt/models/mmfusion.py new file mode 100644 index 0000000000000000000000000000000000000000..2509e26b67b467c3b18c76630881e66cf334a350 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/models/mmfusion.py @@ -0,0 +1,926 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright (c) Facebook, Inc. All Rights Reserved + + +import torch + +from torch import nn + +try: + from transformers import AutoConfig, AutoTokenizer +except ImportError: + pass + +from . import transformermodel + + +class MMPTModel(nn.Module): + """An e2e wrapper of inference model. + """ + @classmethod + def from_pretrained(cls, config, checkpoint="checkpoint_best.pt"): + import os + from ..utils import recursive_config + from ..tasks import Task + config = recursive_config(config) + mmtask = Task.config_task(config) + checkpoint_path = os.path.join(config.eval.save_path, checkpoint) + mmtask.build_model(checkpoint=checkpoint_path) + # TODO(huxu): make the video encoder configurable. + from ..processors.models.s3dg import S3D + video_encoder = S3D('pretrained_models/s3d_dict.npy', 512) + video_encoder.load_state_dict( + torch.load('pretrained_models/s3d_howto100m.pth')) + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained( + config.dataset.bert_name, use_fast=config.dataset.use_fast + ) + from ..processors import Aligner + aligner = Aligner(config.dataset) + return ( + MMPTModel(config, mmtask.model, video_encoder), + tokenizer, + aligner + ) + + def __init__(self, config, model, video_encoder, **kwargs): + super().__init__() + self.max_video_len = config.dataset.max_video_len + self.video_encoder = video_encoder + self.model = model + + def forward(self, video_frames, caps, cmasks, return_score=False): + bsz = video_frames.size(0) + assert bsz == 1, "only bsz=1 is supported now." + seq_len = video_frames.size(1) + video_frames = video_frames.view(-1, *video_frames.size()[2:]) + vfeats = self.video_encoder(video_frames.permute(0, 4, 1, 2, 3)) + vfeats = vfeats['video_embedding'] + vfeats = vfeats.view(bsz, seq_len, vfeats.size(-1)) + padding = torch.zeros( + bsz, self.max_video_len - seq_len, vfeats.size(-1)) + vfeats = torch.cat([vfeats, padding], dim=1) + vmasks = torch.cat([ + torch.ones((bsz, seq_len), dtype=torch.bool), + torch.zeros((bsz, self.max_video_len - seq_len), dtype=torch.bool) + ], + dim=1 + ) + output = self.model(caps, cmasks, vfeats, vmasks) + if return_score: + output = {"score": torch.bmm( + output["pooled_video"][:, None, :], + output["pooled_text"][:, :, None] + ).squeeze(-1).squeeze(-1)} + return output + + +class MMFusion(nn.Module): + """a MMPT wrapper class for MMBert style models. + TODO: move isolated mask to a subclass. + """ + def __init__(self, config, **kwargs): + super().__init__() + transformer_config = AutoConfig.from_pretrained( + config.dataset.bert_name) + self.hidden_size = transformer_config.hidden_size + self.is_train = False + if config.dataset.train_path is not None: + self.is_train = True + # 0 means no iso; 1-12 means iso up to that layer. + self.num_hidden_layers = transformer_config.num_hidden_layers + self.last_iso_layer = 0 + if config.dataset.num_iso_layer is not None: + self.last_iso_layer = config.dataset.num_iso_layer - 1 + 1 + + if config.model.mm_encoder_cls is not None: + mm_encoder_cls = getattr(transformermodel, config.model.mm_encoder_cls) + model_config = AutoConfig.from_pretrained(config.dataset.bert_name) + model_config.max_video_len = config.dataset.max_video_len + # TODO: a general way to add parameter for a model. + model_config.use_seg_emb = config.model.use_seg_emb + self.mm_encoder = mm_encoder_cls.from_pretrained( + config.dataset.bert_name, config=model_config) + elif config.model.video_encoder_cls is not None\ + and config.model.text_encoder_cls is not None: + video_encoder_cls = getattr(transformermodel, config.model.video_encoder_cls) + model_config = AutoConfig.from_pretrained(config.dataset.bert_name) + model_config.max_video_len = config.dataset.max_video_len + # TODO: make each model a set of config class. + if hasattr(model_config, "num_layers"): + model_config.num_layers = config.model.num_hidden_video_layers + else: + model_config.num_hidden_layers = config.model.num_hidden_video_layers + self.video_encoder = video_encoder_cls.from_pretrained( + config.dataset.bert_name, config=model_config) + # exact same NLP model from Huggingface. + text_encoder_cls = getattr(transformermodel, config.model.text_encoder_cls) + self.text_encoder = text_encoder_cls.from_pretrained( + config.dataset.bert_name) + else: + raise ValueError("the encoder must be either MM or two backbones.") + + def forward( + self, + caps, + cmasks, + vfeats, + vmasks, + **kwargs + ): + raise NotImplementedError( + "Please derive MMFusion module." + ) + + def _mm_on_the_fly( + self, + cmasks, + vmasks, + attention_mask + ): + """helper function for mask, seg_ids and token_type_ids.""" + if attention_mask is None: + attention_mask = self._mm_attention_mask(cmasks, vmasks) + + """ + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + """ + token_type_ids = torch.cat( + [ + torch.zeros( + (vmasks.size(0), vmasks.size(1) + 2), + dtype=torch.long, + device=vmasks.device, + ), + torch.ones( + (cmasks.size(0), cmasks.size(1) - 2), + dtype=torch.long, + device=cmasks.device, + ), + ], + dim=1, + ) + return attention_mask, token_type_ids + + def _mm_attention_mask(self, cmasks, vmasks): + assert cmasks.size(0) == vmasks.size(0), "{}, {}, {}, {}".format( + str(cmasks.size()), + str(vmasks.size()), + str(cmasks.size(0)), + str(vmasks.size(0)), + ) + + mm_mask = torch.cat([cmasks[:, :1], vmasks, cmasks[:, 1:]], dim=1) + if self.last_iso_layer == 0: + # hard attention mask. + return mm_mask + else: + # a gpu iso mask; 0 : num_iso_layer is isolated; + # num_iso_layer: are MM-fused. + # make an iso layer + batch_size = cmasks.size(0) + iso_mask = self._make_iso_mask(batch_size, cmasks, vmasks) + mm_mask = mm_mask[:, None, :].repeat(1, mm_mask.size(-1), 1) + iso_mm_masks = [] + # hard attention mask. + iso_mask = iso_mask[:, None, :, :].repeat( + 1, self.last_iso_layer, 1, 1) + iso_mm_masks.append(iso_mask) + if self.last_iso_layer < self.num_hidden_layers: + mm_mask = mm_mask[:, None, :, :].repeat( + 1, self.num_hidden_layers - self.last_iso_layer, 1, 1 + ) + iso_mm_masks.append(mm_mask) + iso_mm_masks = torch.cat(iso_mm_masks, dim=1) + return iso_mm_masks + + def _make_iso_mask(self, batch_size, cmasks, vmasks): + cls_self_mask = torch.cat( + [ + torch.ones( + (batch_size, 1), dtype=torch.bool, device=cmasks.device), + torch.zeros( + (batch_size, cmasks.size(1) + vmasks.size(1) - 1), + dtype=torch.bool, device=cmasks.device) + ], dim=1) + + iso_video_mask = torch.cat( + [ + # [CLS] is not used. + torch.zeros( + (batch_size, 1), dtype=torch.bool, device=cmasks.device + ), + vmasks, + # assume to be 1. + cmasks[:, 1:2], + # 2 means [CLS] + [SEP] + torch.zeros( + (batch_size, cmasks.size(1) - 2), + dtype=torch.bool, + device=cmasks.device, + ), + ], + dim=1, + ) + iso_text_mask = torch.cat( + [ + torch.zeros( + (batch_size, 2 + vmasks.size(1)), + dtype=torch.bool, + device=cmasks.device, + ), # [CLS] is not used. + cmasks[:, 2:], # assume to be 1. + ], + dim=1, + ) + cls_self_mask = cls_self_mask[:, None, :] + iso_video_mask = iso_video_mask[:, None, :].repeat( + 1, vmasks.size(1) + 1, 1) + iso_text_mask = iso_text_mask[:, None, :].repeat( + 1, cmasks.size(1) - 2, 1) + return torch.cat([cls_self_mask, iso_video_mask, iso_text_mask], dim=1) + + def _pooling_vt_layer( + self, + layered_sequence_output, + cmasks, + vmasks + ): + layer_idx = self.last_iso_layer \ + if self.last_iso_layer > 0 else self.num_hidden_layers + hidden_state = layered_sequence_output[layer_idx] + # also output pooled_video and pooled_text. + batch_size = cmasks.size(0) + # pool the modality. + text_offset = vmasks.size(1) + 2 # [CLS] + [SEP] + # video tokens + [SEP] + video_outputs = hidden_state[:, 1:text_offset] + video_attention_mask = torch.cat( + [ + vmasks, + torch.ones( + (batch_size, 1), dtype=torch.bool, device=vmasks.device), + ], + dim=1, + ) + assert video_outputs.size(1) == video_attention_mask.size(1) + pooled_video = torch.sum( + video_outputs * video_attention_mask.unsqueeze(-1), dim=1 + ) / video_attention_mask.sum(1, keepdim=True) + # pooled_video = torch.mean(video_outputs[0], dim=1) + + # text tokens + [SEP] + text_attention_mask = cmasks[:, 2:] + text_outputs = hidden_state[:, text_offset:] + assert text_outputs.size(1) == text_attention_mask.size(1) + pooled_text = torch.sum( + text_outputs * text_attention_mask.unsqueeze(-1), dim=1 + ) / text_attention_mask.sum(1, keepdim=True) + return pooled_video, pooled_text + + +class MMFusionMFMMLM(MMFusion): + """forward function for MFM and MLM.""" + def forward( + self, + caps, + cmasks, + vfeats, + vmasks, + attention_mask=None, + video_label=None, + text_label=None, + **kwargs + ): + output_hidden_states = False if self.is_train else True + + target_vfeats, non_masked_frame_mask = None, None + if video_label is not None: + target_vfeats = vfeats.masked_select( + video_label.unsqueeze(-1)).view( + -1, vfeats.size(-1) + ) + # mask video token. + vfeats[video_label] = 0.0 + non_masked_frame_mask = vmasks.clone() + non_masked_frame_mask[video_label] = False + + attention_mask, token_type_ids = self._mm_on_the_fly( + cmasks, vmasks, attention_mask) + + outputs = self.mm_encoder( + input_ids=caps, + input_video_embeds=vfeats, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + masked_frame_labels=video_label, + target_video_hidden_states=target_vfeats, + non_masked_frame_mask=non_masked_frame_mask, + masked_lm_labels=text_label, + output_hidden_states=output_hidden_states, + ) + + video_logits, text_logits = outputs[0], outputs[1] + + if self.is_train: # return earlier for training. + return { + "video_logits": video_logits, + "text_logits": text_logits, + } + + pooled_video, pooled_text = self._pooling_vt_layer( + outputs[2], cmasks, vmasks) + return {"pooled_video": pooled_video, "pooled_text": pooled_text} + + +class MMFusionMTM(MMFusionMFMMLM): + def __init__(self, config, **kwargs): + super().__init__(config) + """ + For reproducibility: + self.mm_encoder will be initialized then discarded. + """ + from .transformermodel import MMBertForMTM + model_config = AutoConfig.from_pretrained(config.dataset.bert_name) + model_config.max_video_len = config.dataset.max_video_len + model_config.use_seg_emb = config.model.use_seg_emb + self.mm_encoder = MMBertForMTM.from_pretrained( + config.dataset.bert_name, config=model_config) + + +class MMFusionShare(MMFusion): + """A retrival wrapper using mm_encoder as both video/text backbone. + TODO: move formally. + """ + def forward( + self, + caps, + cmasks, + vfeats, + vmasks, + attention_mask=None, + video_label=None, + text_label=None, + output_hidden_states=False, + **kwargs + ): + pooled_video = self.forward_video( + vfeats, + vmasks, + caps, + cmasks, + output_hidden_states + ) + + pooled_text = self.forward_text( + caps, + cmasks, + output_hidden_states + ) + + return {"pooled_video": pooled_video, "pooled_text": pooled_text} + + def forward_video( + self, + vfeats, + vmasks, + caps, + cmasks, + output_hidden_states=False, + **kwargs + ): + input_ids = caps[:, :2] + + attention_mask = torch.cat([ + cmasks[:, :1], + vmasks, + cmasks[:, 1:2] + ], dim=1) + + token_type_ids = torch.zeros( + (vmasks.size(0), vmasks.size(1) + 2), + dtype=torch.long, + device=vmasks.device) + + outputs = self.mm_encoder( + input_ids=input_ids, + input_video_embeds=vfeats, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + output_hidden_states=True + ) + video_outputs = outputs[0] + + if output_hidden_states: + return video_outputs + + batch_size = cmasks.size(0) + + video_attention_mask = torch.cat( + [ + torch.zeros( + (batch_size, 1), dtype=torch.bool, device=vmasks.device), + vmasks, + torch.ones( + (batch_size, 1), dtype=torch.bool, device=vmasks.device), + ], + dim=1, + ) + assert video_outputs.size(1) == video_attention_mask.size(1) + + video_attention_mask = video_attention_mask.type(video_outputs.dtype) \ + / video_attention_mask.sum(1, keepdim=True) + + pooled_video = torch.bmm( + video_outputs.transpose(2, 1), + video_attention_mask.unsqueeze(2) + ).squeeze(-1) + return pooled_video # video_outputs + + def forward_text( + self, + caps, + cmasks, + output_hidden_states=False, + **kwargs + ): + input_ids = torch.cat([ + caps[:, :1], caps[:, 2:], + ], dim=1) + + attention_mask = torch.cat([ + cmasks[:, :1], + cmasks[:, 2:] + ], dim=1) + + token_type_ids = torch.cat([ + torch.zeros( + (cmasks.size(0), 1), + dtype=torch.long, + device=cmasks.device), + torch.ones( + (cmasks.size(0), cmasks.size(1) - 2), + dtype=torch.long, + device=cmasks.device) + ], dim=1) + + outputs = self.mm_encoder( + input_ids=input_ids, + input_video_embeds=None, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + output_hidden_states=True + ) + text_outputs = outputs[0] + + if output_hidden_states: + return text_outputs + + batch_size = caps.size(0) + # text tokens + [SEP] + text_attention_mask = torch.cat([ + torch.zeros( + (batch_size, 1), dtype=torch.bool, device=cmasks.device), + cmasks[:, 2:] + ], dim=1) + + assert text_outputs.size(1) == text_attention_mask.size(1) + + text_attention_mask = text_attention_mask.type(text_outputs.dtype) \ + / text_attention_mask.sum(1, keepdim=True) + + pooled_text = torch.bmm( + text_outputs.transpose(2, 1), + text_attention_mask.unsqueeze(2) + ).squeeze(-1) + return pooled_text # text_outputs + + +class MMFusionSeparate(MMFusionShare): + def forward_video( + self, + vfeats, + vmasks, + caps, + cmasks, + output_hidden_states=False, + **kwargs + ): + input_ids = caps[:, :2] + + attention_mask = torch.cat([ + cmasks[:, :1], + vmasks, + cmasks[:, 1:2] + ], dim=1) + + token_type_ids = torch.zeros( + (vmasks.size(0), vmasks.size(1) + 2), + dtype=torch.long, + device=vmasks.device) + + outputs = self.video_encoder( + input_ids=input_ids, + input_video_embeds=vfeats, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + output_hidden_states=True + ) + video_outputs = outputs[0] + + if output_hidden_states: + return video_outputs + + batch_size = cmasks.size(0) + + video_attention_mask = torch.cat( + [ + torch.zeros( + (batch_size, 1), dtype=torch.bool, device=vmasks.device), + vmasks, + torch.ones( + (batch_size, 1), dtype=torch.bool, device=vmasks.device), + ], + dim=1, + ) + assert video_outputs.size(1) == video_attention_mask.size(1) + + video_attention_mask = video_attention_mask.type(video_outputs.dtype) \ + / video_attention_mask.sum(1, keepdim=True) + + pooled_video = torch.bmm( + video_outputs.transpose(2, 1), + video_attention_mask.unsqueeze(2) + ).squeeze(-1) + return pooled_video # video_outputs + + def forward_text( + self, + caps, + cmasks, + output_hidden_states=False, + **kwargs + ): + input_ids = torch.cat([ + caps[:, :1], caps[:, 2:], + ], dim=1) + + attention_mask = torch.cat([ + cmasks[:, :1], + cmasks[:, 2:] + ], dim=1) + # different from sharing, we use all-0 type. + token_type_ids = torch.zeros( + (cmasks.size(0), cmasks.size(1) - 1), + dtype=torch.long, + device=cmasks.device) + + outputs = self.text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + output_hidden_states=True + ) + text_outputs = outputs[0] + + if output_hidden_states: + return text_outputs + + batch_size = caps.size(0) + # text tokens + [SEP] + text_attention_mask = torch.cat([ + torch.zeros( + (batch_size, 1), dtype=torch.bool, device=cmasks.device), + cmasks[:, 2:] + ], dim=1) + + assert text_outputs.size(1) == text_attention_mask.size(1) + + text_attention_mask = text_attention_mask.type(text_outputs.dtype) \ + / text_attention_mask.sum(1, keepdim=True) + + pooled_text = torch.bmm( + text_outputs.transpose(2, 1), + text_attention_mask.unsqueeze(2) + ).squeeze(-1) + return pooled_text # text_outputs + + +class MMFusionJoint(MMFusion): + """fine-tuning wrapper for retrival task.""" + + def forward( + self, + caps, + cmasks, + vfeats, + vmasks, + attention_mask=None, + video_label=None, + text_label=None, + **kwargs + ): + # TODO (huxu): other ways to do negative examples; move the following + # into your criterion forward. + output_hidden_states = True + + attention_mask, token_type_ids = self._mm_on_the_fly( + cmasks, vmasks, attention_mask) + + separate_forward_split = ( + None if self.is_train else vmasks.size(1) + 2 + ) # [CLS] + [SEP] + + outputs = self.mm_encoder( + input_ids=caps, + input_video_embeds=vfeats, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + output_hidden_states=output_hidden_states, + separate_forward_split=separate_forward_split, + ) + + pooled_video, pooled_text = self._pooling_vt_layer( + outputs[2], cmasks, vmasks) + return {"pooled_video": pooled_video, "pooled_text": pooled_text} + + +class MMFusionActionSegmentation(MMFusion): + """Fine-tuning wrapper for action segmentation. + TODO: rename this for VLM. + """ + def forward( + self, + caps, + cmasks, + vfeats, + vmasks, + attention_mask=None, + **kwargs + ): + # ActionLocalization assume of batch_size=1, squeeze it. + caps = caps.view(-1, caps.size(-1)) + cmasks = cmasks.view(-1, cmasks.size(-1)) + vfeats = vfeats.view(-1, vfeats.size(2), vfeats.size(3)) + vmasks = vmasks.view(-1, vmasks.size(-1)) + + # this may not cover all shapes of attention_mask. + attention_mask = attention_mask.view( + -1, attention_mask.size(2), attention_mask.size(3)) \ + if attention_mask is not None else None + + # TODO (huxu): other ways to do negative examples; move the following + # into your criterion forward. + output_hidden_states = True + + # video forwarding, text is dummy; never use attention_mask. + attention_mask, token_type_ids = self._mm_on_the_fly( + cmasks, vmasks, attention_mask) + + logits = self.mm_encoder( + input_ids=caps, + input_video_embeds=vfeats, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + output_hidden_states=output_hidden_states, + ) + return {"logits": logits[0][:, 1:vmasks.size(1)+1]} + + +class MMFusionActionLocalization(MMFusion): + """fine-tuning model for retrival task.""" + + def __init__(self, config, **kwargs): + super().__init__(config) + tokenizer = AutoTokenizer.from_pretrained( + config.dataset.bert_name) + self.cls_token_id = tokenizer.cls_token_id + self.sep_token_id = tokenizer.sep_token_id + self.pad_token_id = tokenizer.pad_token_id + + def forward( + self, + caps, + cmasks, + vfeats, + vmasks, + attention_mask=None, + **kwargs + ): + # ActionLocalization assume of batch_size=1, squeeze it. + caps = caps.squeeze(0) + cmasks = cmasks.squeeze(0) + vfeats = vfeats.squeeze(0) + vmasks = vmasks.squeeze(0) + attention_mask = attention_mask.squeeze(0) if attention_mask is not None else None + + # TODO (huxu): other ways to do negative examples; move the following + # into your criterion forward. + output_hidden_states = True + + # a len1 dummy video token. + dummy_vfeats = torch.zeros( + (caps.size(0), 1, vfeats.size(-1)), device=vfeats.device, dtype=vfeats.dtype) + dummy_vmasks = torch.ones( + (caps.size(0), 1), dtype=torch.bool, + device=vfeats.device) + + dummy_caps = torch.LongTensor( + [[self.cls_token_id, self.sep_token_id, + self.pad_token_id, self.sep_token_id]], + ).to(caps.device).repeat(vfeats.size(0), 1) + dummy_cmasks = torch.BoolTensor( + [[0, 1, 0, 1]] # pad are valid for attention. + ).to(caps.device).repeat(vfeats.size(0), 1) + + # video forwarding, text is dummy; never use attention_mask. + attention_mask, token_type_ids = self._mm_on_the_fly( + dummy_cmasks, vmasks, None) + + outputs = self.mm_encoder( + input_ids=dummy_caps, + input_video_embeds=vfeats, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + output_hidden_states=output_hidden_states, + ) + + layer_idx = self.last_iso_layer \ + if self.last_iso_layer > 0 else self.num_hidden_layers + + video_seq = outputs[2][layer_idx][:, 1:vmasks.size(1)+1].masked_select( + vmasks.unsqueeze(-1) + ).view(-1, self.hidden_size) + + # text forwarding, video is dummy + attention_mask, token_type_ids = self._mm_on_the_fly( + cmasks, dummy_vmasks, None) + + outputs = self.mm_encoder( + input_ids=caps, + input_video_embeds=dummy_vfeats, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + output_hidden_states=output_hidden_states, + ) + + _, pooled_text = self._pooling_vt_layer( + outputs[2], cmasks, dummy_vmasks) + # this line is not right. + logits = torch.mm(video_seq, pooled_text.transpose(1, 0)) + return {"logits": logits} + + +# --------------- MMFusionSeparate for end tasks --------------- + +class MMFusionSeparateActionSegmentation(MMFusionSeparate): + """Fine-tuning wrapper for action segmentation.""" + def forward( + self, + caps, + cmasks, + vfeats, + vmasks, + attention_mask=None, + **kwargs + ): + # ActionLocalization assume of batch_size=1, squeeze it. + caps = caps.view(-1, caps.size(-1)) + cmasks = cmasks.view(-1, cmasks.size(-1)) + vfeats = vfeats.view(-1, vfeats.size(2), vfeats.size(3)) + vmasks = vmasks.view(-1, vmasks.size(-1)) + logits = self.forward_video( + vfeats, + vmasks, + caps, + cmasks, + output_hidden_states=True + ) + return {"logits": logits[:, 1:vmasks.size(1)+1]} + + +class MMFusionSeparateActionLocalization(MMFusionSeparate): + def __init__(self, config, **kwargs): + super().__init__(config) + tokenizer = AutoTokenizer.from_pretrained( + config.dataset.bert_name) + self.cls_token_id = tokenizer.cls_token_id + self.sep_token_id = tokenizer.sep_token_id + self.pad_token_id = tokenizer.pad_token_id + + def forward( + self, + caps, + cmasks, + vfeats, + vmasks, + **kwargs + ): + # ActionLocalization assume of batch_size=1, squeeze it. + caps = caps.squeeze(0) + cmasks = cmasks.squeeze(0) + vfeats = vfeats.squeeze(0) + vmasks = vmasks.squeeze(0) + + # TODO (huxu): other ways to do negative examples; move the following + # into your criterion forward. + dummy_caps = torch.LongTensor( + [[self.cls_token_id, self.sep_token_id, + self.pad_token_id, self.sep_token_id]], + ).to(caps.device).repeat(vfeats.size(0), 1) + dummy_cmasks = torch.BoolTensor( + [[0, 1, 0, 1]] # pad are valid for attention. + ).to(caps.device).repeat(vfeats.size(0), 1) + + outputs = self.forward_video( + vfeats, + vmasks, + dummy_caps, + dummy_cmasks, + output_hidden_states=True + ) + + video_seq = outputs[:, 1:vmasks.size(1)+1].masked_select( + vmasks.unsqueeze(-1) + ).view(-1, self.hidden_size) + + pooled_text = self.forward_text( + caps, + cmasks, + output_hidden_states=False + ) + + # this line is not right. + logits = torch.mm(video_seq, pooled_text.transpose(1, 0)) + return {"logits": logits} + + +class MMFusionShareActionLocalization(MMFusionShare): + def __init__(self, config, **kwargs): + super().__init__(config) + tokenizer = AutoTokenizer.from_pretrained( + config.dataset.bert_name) + self.cls_token_id = tokenizer.cls_token_id + self.sep_token_id = tokenizer.sep_token_id + self.pad_token_id = tokenizer.pad_token_id + + def forward( + self, + caps, + cmasks, + vfeats, + vmasks, + **kwargs + ): + # ActionLocalization assume of batch_size=1, squeeze it. + caps = caps.squeeze(0) + cmasks = cmasks.squeeze(0) + vfeats = vfeats.squeeze(0) + vmasks = vmasks.squeeze(0) + + # TODO (huxu): other ways to do negative examples; move the following + # into your criterion forward. + dummy_caps = torch.LongTensor( + [[self.cls_token_id, self.sep_token_id, + self.pad_token_id, self.sep_token_id]], + ).to(caps.device).repeat(vfeats.size(0), 1) + dummy_cmasks = torch.BoolTensor( + [[0, 1, 0, 1]] # pad are valid for attention. + ).to(caps.device).repeat(vfeats.size(0), 1) + + outputs = self.forward_video( + vfeats, + vmasks, + dummy_caps, + dummy_cmasks, + output_hidden_states=True + ) + + video_seq = outputs[:, 1:vmasks.size(1)+1].masked_select( + vmasks.unsqueeze(-1) + ).view(-1, self.hidden_size) + + pooled_text = self.forward_text( + caps, + cmasks, + output_hidden_states=False + ) + + # this line is not right. + logits = torch.mm(video_seq, pooled_text.transpose(1, 0)) + return {"logits": logits} diff --git a/data/fairseq/examples/MMPT/mmpt/models/mmfusionnlg.py b/data/fairseq/examples/MMPT/mmpt/models/mmfusionnlg.py new file mode 100644 index 0000000000000000000000000000000000000000..9207e77dab3025d7a26efcce0795183de1d34fc7 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/models/mmfusionnlg.py @@ -0,0 +1,999 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors, Facebook AI Research authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright (c) Facebook, Inc. All Rights Reserved + + +import torch + +from torch.nn import functional as F + +from typing import Optional, Iterable + +try: + from transformers import BertPreTrainedModel + from transformers.modeling_bert import BertOnlyMLMHead + + from transformers.file_utils import ModelOutput + from transformers.modeling_outputs import CausalLMOutput + from transformers.generation_utils import ( + BeamHypotheses, + top_k_top_p_filtering + ) +except ImportError: + pass + +from .mmfusion import MMFusion +from .transformermodel import MMBertModel +from ..modules import VideoTokenMLP + + +class MMFusionNLG(MMFusion): + def __init__(self, config, **kwargs): + super().__init__(config) + if config.model.max_decode_length is not None: + self.max_length = min( + config.model.max_decode_length, + config.dataset.max_len - config.dataset.max_video_len - 3 + ) + else: + self.max_length = \ + config.dataset.max_len - config.dataset.max_video_len - 3 + self.gen_param = config.gen_param if config.gen_param is not None \ + else {} + + def forward( + self, + caps, + cmasks, + vfeats, + vmasks, + attention_mask, + video_label=None, + text_label=None, + **kwargs + ): + """use pre-trained LM header for generation.""" + attention_mask, token_type_ids = self._mm_on_the_fly( + cmasks, vmasks, attention_mask) + + outputs = self.mm_encoder( + input_ids=caps, + input_video_embeds=vfeats, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + masked_lm_labels=text_label, + ) + return {"logits": outputs[0]} + + @torch.no_grad() + def generate( + self, + caps, cmasks, vfeats, vmasks, + attention_mask=None, + bos_token_id=None, + eos_token_id=None, + **kwargs + ): + # a simplified interface from + # https://huggingface.co/transformers/v3.4.0/_modules/transformers/generation_utils.html#GenerationMixin.generate + + # caps now only have + # [CLS], [SEP] (for video) and [CLS] (as bos_token) + assert caps.size(1) == 3 + + attention_mask, token_type_ids = self._mm_on_the_fly( + cmasks, vmasks, attention_mask) + + output = self.mm_encoder.generate( + input_ids=caps, + input_video_embeds=vfeats, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + max_length=self.max_length, + **self.gen_param + ) + return output + + +class MMBertForNLG(BertPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.bert = MMBertModel(config) + self.videomlp = VideoTokenMLP(config) + # we do not use `BertGenerationOnlyLMHead` + # because we can reuse pretraining. + self.cls = BertOnlyMLMHead(config) + self.hidden_size = config.hidden_size + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def forward( + self, + input_ids=None, + input_video_embeds=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + masked_lm_labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + # similar to MMBertForMFMMLM without MFM. + video_tokens = self.videomlp(input_video_embeds) + outputs = self.bert( + input_ids, + video_tokens, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + prediction_scores = None + if masked_lm_labels is not None: + text_offset = input_video_embeds.size(1) + 1 # [CLS] + # recover caps format: [CLS] [SEP] text [SEP] + text_sequence_output = torch.cat( + [sequence_output[:, :1], sequence_output[:, text_offset:]], + dim=1 + ) + + # only compute select tokens to training to speed up. + hidden_size = text_sequence_output.size(-1) + # masked_lm_labels = masked_lm_labels.reshape(-1) + labels_mask = masked_lm_labels != -100 + + selected_text_output = text_sequence_output.masked_select( + labels_mask.unsqueeze(-1) + ).view(-1, hidden_size) + prediction_scores = self.cls(selected_text_output) + + if not return_dict: + output = ( + prediction_scores, + ) + outputs[2:] + return output + + # for generation. + text_offset = input_video_embeds.size(1) + 2 # [CLS] + text_sequence_output = sequence_output[:, text_offset:] + prediction_scores = self.cls(text_sequence_output) + return CausalLMOutput( + loss=None, + logits=prediction_scores, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + input_video_embeds, + attention_mask=None, + token_type_ids=None, + **model_kwargs + ): + # must return a dictionary. + seq_len = input_ids.size(1) + input_video_embeds.size(1) + if attention_mask is not None: + if len(attention_mask.size()) == 4: + attention_mask = attention_mask[:, :, :seq_len, :seq_len] + elif len(attention_mask.size()) == 3: + attention_mask = attention_mask[:, :seq_len, :seq_len] + else: + attention_mask = attention_mask[:, :seq_len] + if token_type_ids is not None: + token_type_ids = token_type_ids[:, :seq_len] + + return { + "input_ids": input_ids, + "input_video_embeds": input_video_embeds, + "attention_mask": attention_mask, + "token_type_ids": token_type_ids, + } + + @torch.no_grad() + def generate( + self, + input_ids: Optional[torch.LongTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + max_length: Optional[int] = None, + min_length: Optional[int] = None, + do_sample: Optional[bool] = None, + early_stopping: Optional[bool] = None, + num_beams: Optional[int] = None, + temperature: Optional[float] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + repetition_penalty: Optional[float] = None, + bad_words_ids: Optional[Iterable[int]] = None, + bos_token_id: Optional[int] = None, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + length_penalty: Optional[float] = None, + no_repeat_ngram_size: Optional[int] = None, + num_return_sequences: Optional[int] = None, + attention_mask: Optional[torch.LongTensor] = None, + decoder_start_token_id: Optional[int] = None, + use_cache: Optional[bool] = None, + **model_kwargs + ) -> torch.LongTensor: + r""" + Generates sequences for models with a language modeling head. The method currently supports greedy decoding, + beam-search decoding, sampling with temperature, sampling with top-k or nucleus sampling. + Adapted in part from `Facebook's XLM beam search code + `__. + Apart from :obj:`input_ids` and :obj:`attention_mask`, all the arguments below will default to the value of the + attribute of the same name inside the :class:`~transformers.PretrainedConfig` of the model. The default values + indicated are the default values of those config. + Most of these parameters are explained in more detail in `this blog post + `__. + Parameters: + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + The sequence used as a prompt for the generation. If :obj:`None` the method initializes + it as an empty :obj:`torch.LongTensor` of shape :obj:`(1,)`. + decoder_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + initial input_ids for the decoder of encoder-decoder type models. If :obj:`None` then only + decoder_start_token_id is passed as the first token to the decoder. + max_length (:obj:`int`, `optional`, defaults to 20): + The maximum length of the sequence to be generated. + min_length (:obj:`int`, `optional`, defaults to 10): + The minimum length of the sequence to be generated. + do_sample (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether or not to use sampling ; use greedy decoding otherwise. + early_stopping (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to stop the beam search when at least ``num_beams`` sentences are finished per batch or not. + num_beams (:obj:`int`, `optional`, defaults to 1): + Number of beams for beam search. 1 means no beam search. + temperature (:obj:`float`, `optional`, defaults tp 1.0): + The value used to module the next token probabilities. + top_k (:obj:`int`, `optional`, defaults to 50): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + top_p (:obj:`float`, `optional`, defaults to 1.0): + If set to float < 1, only the most probable tokens with probabilities that add up to ``top_p`` or + higher are kept for generation. + repetition_penalty (:obj:`float`, `optional`, defaults to 1.0): + The parameter for repetition penalty. 1.0 means no penalty. See `this paper + `__ for more details. + pad_token_id (:obj:`int`, `optional`): + The id of the `padding` token. + bos_token_id (:obj:`int`, `optional`): + The id of the `beginning-of-sequence` token. + eos_token_id (:obj:`int`, `optional`): + The id of the `end-of-sequence` token. + length_penalty (:obj:`float`, `optional`, defaults to 1.0): + Exponential penalty to the length. 1.0 means no penalty. + Set to values < 1.0 in order to encourage the model to generate shorter sequences, to a value > 1.0 in + order to encourage the model to produce longer sequences. + no_repeat_ngram_size (:obj:`int`, `optional`, defaults to 0): + If set to int > 0, all ngrams of that size can only occur once. + bad_words_ids(:obj:`List[int]`, `optional`): + List of token ids that are not allowed to be generated. In order to get the tokens of the words that + should not appear in the generated text, use :obj:`tokenizer.encode(bad_word, add_prefix_space=True)`. + num_return_sequences(:obj:`int`, `optional`, defaults to 1): + The number of independently computed returned sequences for each element in the batch. + attention_mask (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values are in ``[0, 1]``, 1 for + tokens that are not masked, and 0 for masked tokens. + If not provided, will default to a tensor the same shape as :obj:`input_ids` that masks the pad token. + `What are attention masks? <../glossary.html#attention-mask>`__ + decoder_start_token_id (:obj:`int`, `optional`): + If an encoder-decoder model starts decoding with a different token than `bos`, the id of that token. + use_cache: (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not the model should use the past last key/values attentions (if applicable to the model) to + speed up decoding. + model_kwargs: + Additional model specific kwargs will be forwarded to the :obj:`forward` function of the model. + Return: + :obj:`torch.LongTensor` of shape :obj:`(batch_size * num_return_sequences, sequence_length)`: + The generated sequences. The second dimension (sequence_length) is either equal to :obj:`max_length` or + shorter if all batches finished early due to the :obj:`eos_token_id`. + Examples:: + tokenizer = AutoTokenizer.from_pretrained('distilgpt2') # Initialize tokenizer + model = AutoModelWithLMHead.from_pretrained('distilgpt2') # Download model and configuration from S3 and cache. + outputs = model.generate(max_length=40) # do greedy decoding + print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True))) + tokenizer = AutoTokenizer.from_pretrained('openai-gpt') # Initialize tokenizer + model = AutoModelWithLMHead.from_pretrained('openai-gpt') # Download model and configuration from S3 and cache. + input_context = 'The dog' + input_ids = tokenizer.encode(input_context, return_tensors='pt') # encode input context + outputs = model.generate(input_ids=input_ids, num_beams=5, num_return_sequences=3, temperature=1.5) # generate 3 independent sequences using beam search decoding (5 beams) with sampling from initial context 'The dog' + for i in range(3): # 3 output sequences were generated + print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True))) + tokenizer = AutoTokenizer.from_pretrained('distilgpt2') # Initialize tokenizer + model = AutoModelWithLMHead.from_pretrained('distilgpt2') # Download model and configuration from S3 and cache. + input_context = 'The dog' + input_ids = tokenizer.encode(input_context, return_tensors='pt') # encode input context + outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3, do_sample=True) # generate 3 candidates using sampling + for i in range(3): # 3 output sequences were generated + print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True))) + tokenizer = AutoTokenizer.from_pretrained('ctrl') # Initialize tokenizer + model = AutoModelWithLMHead.from_pretrained('ctrl') # Download model and configuration from S3 and cache. + input_context = 'Legal My neighbor is' # "Legal" is one of the control codes for ctrl + input_ids = tokenizer.encode(input_context, return_tensors='pt') # encode input context + outputs = model.generate(input_ids=input_ids, max_length=50, temperature=0.7, repetition_penalty=1.2) # generate sequences + print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True))) + tokenizer = AutoTokenizer.from_pretrained('gpt2') # Initialize tokenizer + model = AutoModelWithLMHead.from_pretrained('gpt2') # Download model and configuration from S3 and cache. + input_context = 'My cute dog' # "Legal" is one of the control codes for ctrl + bad_words_ids = [tokenizer.encode(bad_word, add_prefix_space=True) for bad_word in ['idiot', 'stupid', 'shut up']] + input_ids = tokenizer.encode(input_context, return_tensors='pt') # encode input context + outputs = model.generate(input_ids=input_ids, max_length=100, do_sample=True, bad_words_ids=bad_words_ids) # generate sequences without allowing bad_words to be generated + """ + + # We cannot generate if the model does not have a LM head + if self.get_output_embeddings() is None: + raise AttributeError( + "You tried to generate sequences with a model that does not have a LM Head." + "Please use another model class (e.g. `OpenAIGPTLMHeadModel`, `XLNetLMHeadModel`, `GPT2LMHeadModel`, `CTRLLMHeadModel`, `T5WithLMHeadModel`, `TransfoXLLMHeadModel`, `XLMWithLMHeadModel`, `BartForConditionalGeneration` )" + ) + + max_length = max_length if max_length is not None else self.config.max_length + min_length = min_length if min_length is not None else self.config.min_length + do_sample = do_sample if do_sample is not None else self.config.do_sample + early_stopping = early_stopping if early_stopping is not None else self.config.early_stopping + use_cache = use_cache if use_cache is not None else self.config.use_cache + num_beams = num_beams if num_beams is not None else self.config.num_beams + temperature = temperature if temperature is not None else self.config.temperature + top_k = top_k if top_k is not None else self.config.top_k + top_p = top_p if top_p is not None else self.config.top_p + repetition_penalty = repetition_penalty if repetition_penalty is not None else self.config.repetition_penalty + bos_token_id = bos_token_id if bos_token_id is not None else self.config.bos_token_id + pad_token_id = pad_token_id if pad_token_id is not None else self.config.pad_token_id + eos_token_id = eos_token_id if eos_token_id is not None else self.config.eos_token_id + length_penalty = length_penalty if length_penalty is not None else self.config.length_penalty + no_repeat_ngram_size = ( + no_repeat_ngram_size if no_repeat_ngram_size is not None else self.config.no_repeat_ngram_size + ) + bad_words_ids = bad_words_ids if bad_words_ids is not None else self.config.bad_words_ids + num_return_sequences = ( + num_return_sequences if num_return_sequences is not None else self.config.num_return_sequences + ) + decoder_start_token_id = ( + decoder_start_token_id if decoder_start_token_id is not None else self.config.decoder_start_token_id + ) + + if input_ids is not None: + batch_size = input_ids.shape[0] # overriden by the input batch_size + else: + batch_size = 1 + + assert isinstance(max_length, int) and max_length > 0, "`max_length` should be a strictly positive integer." + assert isinstance(min_length, int) and min_length >= 0, "`min_length` should be a positive integer." + assert isinstance(do_sample, bool), "`do_sample` should be a boolean." + assert isinstance(early_stopping, bool), "`early_stopping` should be a boolean." + assert isinstance(use_cache, bool), "`use_cache` should be a boolean." + assert isinstance(num_beams, int) and num_beams > 0, "`num_beams` should be a strictly positive integer." + assert temperature > 0, "`temperature` should be strictly positive." + assert isinstance(top_k, int) and top_k >= 0, "`top_k` should be a positive integer." + assert 0 <= top_p <= 1, "`top_p` should be between 0 and 1." + assert repetition_penalty >= 1.0, "`repetition_penalty` should be >= 1." + assert input_ids is not None or ( + isinstance(bos_token_id, int) and bos_token_id >= 0 + ), "If input_ids is not defined, `bos_token_id` should be a positive integer." + assert pad_token_id is None or ( + isinstance(pad_token_id, int) and (pad_token_id >= 0) + ), "`pad_token_id` should be a positive integer." + assert (eos_token_id is None) or ( + isinstance(eos_token_id, int) and (eos_token_id >= 0) + ), "`eos_token_id` should be a positive integer." + assert length_penalty > 0, "`length_penalty` should be strictly positive." + assert ( + isinstance(no_repeat_ngram_size, int) and no_repeat_ngram_size >= 0 + ), "`no_repeat_ngram_size` should be a positive integer." + assert ( + isinstance(num_return_sequences, int) and num_return_sequences > 0 + ), "`num_return_sequences` should be a strictly positive integer." + assert ( + bad_words_ids is None or isinstance(bad_words_ids, list) and isinstance(bad_words_ids[0], list) + ), "`bad_words_ids` is either `None` or a list of lists of tokens that should not be generated" + + if input_ids is None: + assert isinstance(bos_token_id, int) and bos_token_id >= 0, ( + "you should either supply a context to complete as `input_ids` input " + "or a `bos_token_id` (integer >= 0) as a first token to start the generation." + ) + input_ids = torch.full( + (batch_size, 1), + bos_token_id, + dtype=torch.long, + device=next(self.parameters()).device, + ) + else: + assert input_ids.dim() == 2, "Input prompt should be of shape (batch_size, sequence length)." + + # not allow to duplicate outputs when greedy decoding + if do_sample is False: + if num_beams == 1: + # no_beam_search greedy generation conditions + assert ( + num_return_sequences == 1 + ), "Greedy decoding will always produce the same output for num_beams == 1 and num_return_sequences > 1. Please set num_return_sequences = 1" + + else: + # beam_search greedy generation conditions + assert ( + num_beams >= num_return_sequences + ), "Greedy beam search decoding cannot return more sequences than it has beams. Please set num_beams >= num_return_sequences" + + # create attention mask if necessary + # TODO (PVP): this should later be handled by the forward fn() in each model in the future see PR 3140 + if (attention_mask is None) and (pad_token_id is not None) and (pad_token_id in input_ids): + attention_mask = input_ids.ne(pad_token_id).long() + elif attention_mask is None: + attention_mask = input_ids.new_ones(input_ids.shape) + + # set pad_token_id to eos_token_id if not set. Important that this is done after + # attention_mask is created + if pad_token_id is None and eos_token_id is not None: + print( + "Setting `pad_token_id` to {} (first `eos_token_id`) to generate sequence".format(eos_token_id) + ) + pad_token_id = eos_token_id + + # vocab size + if hasattr(self.config, "vocab_size"): + vocab_size = self.config.vocab_size + elif ( + self.config.is_encoder_decoder + and hasattr(self.config, "decoder") + and hasattr(self.config.decoder, "vocab_size") + ): + vocab_size = self.config.decoder.vocab_size + else: + raise ValueError("either self.config.vocab_size or self.config.decoder.vocab_size needs to be defined") + + # set effective batch size and effective batch multiplier according to do_sample + if do_sample: + effective_batch_size = batch_size * num_return_sequences + effective_batch_mult = num_return_sequences + else: + effective_batch_size = batch_size + effective_batch_mult = 1 + + if self.config.is_encoder_decoder: + if decoder_start_token_id is None: + # see if BOS token can be used for decoder_start_token_id + if bos_token_id is not None: + decoder_start_token_id = bos_token_id + elif ( + hasattr(self.config, "decoder") + and hasattr(self.config.decoder, "bos_token_id") + and self.config.decoder.bos_token_id is not None + ): + decoder_start_token_id = self.config.decoder.bos_token_id + else: + raise ValueError( + "decoder_start_token_id or bos_token_id has to be defined for encoder-decoder generation" + ) + + assert hasattr(self, "get_encoder"), "{} should have a 'get_encoder' function defined".format(self) + assert callable(self.get_encoder), "{} should be a method".format(self.get_encoder) + + # get encoder and store encoder outputs + encoder = self.get_encoder() + encoder_outputs: ModelOutput = encoder(input_ids, attention_mask=attention_mask, return_dict=True) + + # Expand input ids if num_beams > 1 or num_return_sequences > 1 + if num_return_sequences > 1 or num_beams > 1: + # TODO: make this a call-back function. + # input_ids=caps, + # input_video_embeds=vfeats, + # attention_mask=attention_mask, + # token_type_ids=token_type_ids, + input_video_embeds = model_kwargs.pop("input_video_embeds", None) + token_type_ids = model_kwargs.pop("token_type_ids", None) + + input_ids_len = input_ids.shape[-1] + input_ids = input_ids.unsqueeze(1).expand( + batch_size, effective_batch_mult * num_beams, input_ids_len) + + input_video_embeds_len, input_video_embeds_hidden = input_video_embeds.size(1), input_video_embeds.size(2) + input_video_embeds = input_video_embeds.unsqueeze(1).expand( + batch_size, effective_batch_mult * num_beams, input_video_embeds_len, input_video_embeds_hidden) + + attention_mask_from_len, attention_mask_to_len = attention_mask.size(1), attention_mask.size(2) + attention_mask = attention_mask.unsqueeze(1).expand( + batch_size, effective_batch_mult * num_beams, attention_mask_from_len, attention_mask_to_len + ) + + token_type_ids_len = token_type_ids.size(1) + token_type_ids = token_type_ids.unsqueeze(1).expand( + batch_size, effective_batch_mult * num_beams, token_type_ids_len + ) + + # contiguous ... + input_ids = input_ids.contiguous().view( + effective_batch_size * num_beams, input_ids_len + ) # shape: (batch_size * num_return_sequences * num_beams, cur_len) + + input_video_embeds = input_video_embeds.contiguous().view( + effective_batch_size * num_beams, input_video_embeds_len, input_video_embeds_hidden) + + attention_mask = attention_mask.contiguous().view( + effective_batch_size * num_beams, attention_mask_from_len, attention_mask_to_len + ) # shape: (batch_size * num_return_sequences * num_beams, cur_len) + + token_type_ids = token_type_ids.contiguous().view( + effective_batch_size * num_beams, token_type_ids_len + ) + + model_kwargs["input_video_embeds"] = input_video_embeds + model_kwargs["token_type_ids"] = token_type_ids + + if self.config.is_encoder_decoder: + device = next(self.parameters()).device + if decoder_input_ids is not None: + # give initial decoder input ids + input_ids = decoder_input_ids.repeat(effective_batch_size * num_beams, 1).to(device) + else: + # create empty decoder input_ids + input_ids = torch.full( + (effective_batch_size * num_beams, 1), + decoder_start_token_id, + dtype=torch.long, + device=device, + ) + cur_len = input_ids.shape[-1] + + assert ( + batch_size == encoder_outputs.last_hidden_state.shape[0] + ), f"expected encoder_outputs.last_hidden_state to have 1st dimension bs={batch_size}, got {encoder_outputs.last_hidden_state.shape[0]} " + + # expand batch_idx to assign correct encoder output for expanded input_ids (due to num_beams > 1 and num_return_sequences > 1) + expanded_batch_idxs = ( + torch.arange(batch_size) + .view(-1, 1) + .repeat(1, num_beams * effective_batch_mult) + .view(-1) + .to(input_ids.device) + ) + + # expand encoder_outputs + encoder_outputs["last_hidden_state"] = encoder_outputs.last_hidden_state.index_select( + 0, expanded_batch_idxs + ) + + # save encoder_outputs in `model_kwargs` + model_kwargs["encoder_outputs"] = encoder_outputs + + else: + cur_len = input_ids.shape[-1] + + assert ( + cur_len < max_length + ), f"The context has {cur_len} number of tokens, but `max_length` is only {max_length}. Please make sure that `max_length` is bigger than the number of tokens, by setting either `generate(max_length=...,...)` or `config.max_length = ...`" + + if num_beams > 1: + output = self._generate_beam_search( + input_ids, + cur_len=cur_len, + max_length=max_length, + min_length=min_length, + do_sample=do_sample, + early_stopping=early_stopping, + temperature=temperature, + top_k=top_k, + top_p=top_p, + repetition_penalty=repetition_penalty, + no_repeat_ngram_size=no_repeat_ngram_size, + bad_words_ids=bad_words_ids, + pad_token_id=pad_token_id, + eos_token_id=eos_token_id, + batch_size=effective_batch_size, + num_return_sequences=num_return_sequences, + length_penalty=length_penalty, + num_beams=num_beams, + vocab_size=vocab_size, + attention_mask=attention_mask, + use_cache=use_cache, + model_kwargs=model_kwargs, + ) + else: + output = self._generate_no_beam_search( + input_ids, + cur_len=cur_len, + max_length=max_length, + min_length=min_length, + do_sample=do_sample, + temperature=temperature, + top_k=top_k, + top_p=top_p, + repetition_penalty=repetition_penalty, + no_repeat_ngram_size=no_repeat_ngram_size, + bad_words_ids=bad_words_ids, + pad_token_id=pad_token_id, + eos_token_id=eos_token_id, + batch_size=effective_batch_size, + attention_mask=attention_mask, + use_cache=use_cache, + model_kwargs=model_kwargs, + ) + + return output + + def _generate_beam_search( + self, + input_ids, + cur_len, + max_length, + min_length, + do_sample, + early_stopping, + temperature, + top_k, + top_p, + repetition_penalty, + no_repeat_ngram_size, + bad_words_ids, + pad_token_id, + eos_token_id, + batch_size, + num_return_sequences, + length_penalty, + num_beams, + vocab_size, + attention_mask, + use_cache, + model_kwargs, + ): + """Generate sequences for each example with beam search.""" + + # generated hypotheses + generated_hyps = [ + BeamHypotheses(num_beams, max_length, length_penalty, early_stopping=early_stopping) + for _ in range(batch_size) + ] + + # scores for each sentence in the beam + beam_scores = torch.zeros((batch_size, num_beams), dtype=torch.float, device=input_ids.device) + + # for greedy decoding it is made sure that only tokens of the first beam are considered to avoid sampling the exact same tokens three times + if do_sample is False: + beam_scores[:, 1:] = -1e9 + beam_scores = beam_scores.view(-1) # shape (batch_size * num_beams,) + + # cache compute states + past = None + + # done sentences + done = [False for _ in range(batch_size)] + + while cur_len < max_length: + model_inputs = self.prepare_inputs_for_generation( + input_ids, past=past, attention_mask=attention_mask, use_cache=use_cache, **model_kwargs + ) + outputs = self(**model_inputs, return_dict=True) # (batch_size * num_beams, cur_len, vocab_size) + next_token_logits = outputs.logits[:, -1, :] # (batch_size * num_beams, vocab_size) + + # if model has past, then set the past variable to speed up decoding + if "past_key_values" in outputs: + past = outputs.past_key_values + elif "mems" in outputs: + past = outputs.mems + + if self.config.is_encoder_decoder and do_sample is False: + # TODO (PVP) still a bit hacky here - there might be a better solution + next_token_logits = self.adjust_logits_during_generation( + next_token_logits, cur_len=cur_len, max_length=max_length + ) + + scores = F.log_softmax(next_token_logits, dim=-1) # (batch_size * num_beams, vocab_size) + + scores = self.postprocess_next_token_scores( + scores=scores, + input_ids=input_ids, + no_repeat_ngram_size=no_repeat_ngram_size, + bad_words_ids=bad_words_ids, + cur_len=cur_len, + min_length=min_length, + max_length=max_length, + eos_token_id=eos_token_id, + repetition_penalty=repetition_penalty, + batch_size=batch_size, + num_beams=num_beams, + ) + + assert scores.shape == (batch_size * num_beams, vocab_size), "Shapes of scores: {} != {}".format( + scores.shape, (batch_size * num_beams, vocab_size) + ) + + if do_sample: + _scores = scores + beam_scores[:, None].expand_as(scores) # (batch_size * num_beams, vocab_size) + # Temperature + if temperature != 1.0: + _scores = _scores / temperature + # Top-p/top-k filtering + _scores = top_k_top_p_filtering( + _scores, top_k=top_k, top_p=top_p, min_tokens_to_keep=2 + ) # (batch_size * num_beams, vocab_size) + # re-organize to group the beam together to sample from all beam_idxs + _scores = _scores.contiguous().view( + batch_size, num_beams * vocab_size + ) # (batch_size, num_beams * vocab_size) + + # Sample 2 next tokens for each beam (so we have some spare tokens and match output of greedy beam search) + probs = F.softmax(_scores, dim=-1) + next_tokens = torch.multinomial(probs, num_samples=2 * num_beams) # (batch_size, num_beams * 2) + # Compute next scores + next_scores = torch.gather(_scores, -1, next_tokens) # (batch_size, num_beams * 2) + # sort the sampled vector to make sure that the first num_beams samples are the best + next_scores, next_scores_indices = torch.sort(next_scores, descending=True, dim=1) + next_tokens = torch.gather(next_tokens, -1, next_scores_indices) # (batch_size, num_beams * 2) + + else: + next_scores = scores + beam_scores[:, None].expand_as(scores) # (batch_size * num_beams, vocab_size) + + # re-organize to group the beam together (we are keeping top hypothesis accross beams) + next_scores = next_scores.view( + batch_size, num_beams * vocab_size + ) # (batch_size, num_beams * vocab_size) + + next_scores, next_tokens = torch.topk(next_scores, 2 * num_beams, dim=1, largest=True, sorted=True) + + assert next_scores.size() == next_tokens.size() == (batch_size, 2 * num_beams) + + # next batch beam content + next_batch_beam = [] + + # for each sentence + for batch_idx in range(batch_size): + + # if we are done with this sentence, add a pad token + if done[batch_idx]: + assert ( + len(generated_hyps[batch_idx]) >= num_beams + ), "Batch can only be done if at least {} beams have been generated".format(num_beams) + assert ( + eos_token_id is not None and pad_token_id is not None + ), "generated beams >= num_beams -> eos_token_id and pad_token have to be defined" + next_batch_beam.extend([(0, pad_token_id, 0)] * num_beams) # pad the batch + continue + + # next sentence beam content, this will get added to next_batch_beam + next_sent_beam = [] + + # next tokens for this sentence + for beam_token_rank, (beam_token_id, beam_token_score) in enumerate( + zip(next_tokens[batch_idx], next_scores[batch_idx]) + ): + # get beam and token IDs + beam_id = beam_token_id // vocab_size + token_id = beam_token_id % vocab_size + + effective_beam_id = batch_idx * num_beams + beam_id + # add to generated hypotheses if end of sentence + if (eos_token_id is not None) and (token_id.item() == eos_token_id): + # if beam_token does not belong to top num_beams tokens, it should not be added + is_beam_token_worse_than_top_num_beams = beam_token_rank >= num_beams + if is_beam_token_worse_than_top_num_beams: + continue + generated_hyps[batch_idx].add( + input_ids[effective_beam_id].clone(), + beam_token_score.item(), + ) + else: + # add next predicted token since it is not eos_token + next_sent_beam.append((beam_token_score, token_id, effective_beam_id)) + + # once the beam for next step is full, don't add more tokens to it. + if len(next_sent_beam) == num_beams: + break + + # Check if we are done so that we can save a pad step if all(done) + done[batch_idx] = done[batch_idx] or generated_hyps[batch_idx].is_done( + next_scores[batch_idx].max().item(), cur_len + ) + + # update next beam content + assert len(next_sent_beam) == num_beams, "Beam should always be full" + next_batch_beam.extend(next_sent_beam) + assert len(next_batch_beam) == num_beams * (batch_idx + 1), "We should have added num_beams each step" + + # stop when we are done with each sentence + if all(done): + break + + # sanity check / prepare next batch + assert len(next_batch_beam) == batch_size * num_beams + beam_scores = beam_scores.new([x[0] for x in next_batch_beam]) + beam_tokens = input_ids.new([x[1] for x in next_batch_beam]) + beam_idx = input_ids.new([x[2] for x in next_batch_beam]) + + # re-order batch and update current length + input_ids = input_ids[beam_idx, :] + input_ids = torch.cat([input_ids, beam_tokens.unsqueeze(1)], dim=-1) + cur_len = cur_len + 1 + + # re-order internal states + if past is not None: + past = self._reorder_cache(past, beam_idx) + + # extend attention_mask for new generated input if only decoder + # (huxu): move out since we trim attention_mask by ourselves. + # if self.config.is_encoder_decoder is False: + # attention_mask = torch.cat( + # [attention_mask, attention_mask.new_ones((attention_mask.shape[0], 1))], dim=-1 + # ) + + # finalize all open beam hypotheses and add to generated hypotheses + for batch_idx in range(batch_size): + if done[batch_idx]: + continue + + # test that beam scores match previously calculated scores if not eos and batch_idx not done + if eos_token_id is not None and all( + (token_id % vocab_size).item() != eos_token_id for token_id in next_tokens[batch_idx] + ): + assert torch.all( + next_scores[batch_idx, :num_beams] == beam_scores.view(batch_size, num_beams)[batch_idx] + ), "If batch_idx is not done, final next scores: {} have to equal to accumulated beam_scores: {}".format( + next_scores[:, :num_beams][batch_idx], + beam_scores.view(batch_size, num_beams)[batch_idx], + ) + + # need to add best num_beams hypotheses to generated hyps + for beam_id in range(num_beams): + effective_beam_id = batch_idx * num_beams + beam_id + final_score = beam_scores[effective_beam_id].item() + final_tokens = input_ids[effective_beam_id] + generated_hyps[batch_idx].add(final_tokens, final_score) + + # depending on whether greedy generation is wanted or not define different output_batch_size and output_num_return_sequences_per_batch + output_batch_size = batch_size if do_sample else batch_size * num_return_sequences + output_num_return_sequences_per_batch = 1 if do_sample else num_return_sequences + + # select the best hypotheses + sent_lengths = input_ids.new(output_batch_size) + best = [] + + # retrieve best hypotheses + for i, hypotheses in enumerate(generated_hyps): + sorted_hyps = sorted(hypotheses.beams, key=lambda x: x[0]) + for j in range(output_num_return_sequences_per_batch): + effective_batch_idx = output_num_return_sequences_per_batch * i + j + best_hyp = sorted_hyps.pop()[1] + sent_lengths[effective_batch_idx] = len(best_hyp) + best.append(best_hyp) + + # prepare for adding eos + sent_max_len = min(sent_lengths.max().item() + 1, max_length) + decoded = input_ids.new(output_batch_size, sent_max_len) + # shorter batches are padded if needed + if sent_lengths.min().item() != sent_lengths.max().item(): + assert pad_token_id is not None, "`pad_token_id` has to be defined" + decoded.fill_(pad_token_id) + + # fill with hypotheses and eos_token_id if the latter fits in + for i, hypo in enumerate(best): + decoded[i, : sent_lengths[i]] = hypo + if sent_lengths[i] < max_length: + decoded[i, sent_lengths[i]] = eos_token_id + + return decoded + + def _generate_no_beam_search( + self, + input_ids, + cur_len, + max_length, + min_length, + do_sample, + temperature, + top_k, + top_p, + repetition_penalty, + no_repeat_ngram_size, + bad_words_ids, + pad_token_id, + eos_token_id, + batch_size, + attention_mask, + use_cache, + model_kwargs, + ): + """Generate sequences for each example without beam search (num_beams == 1). + All returned sequence are generated independantly. + """ + # length of generated sentences / unfinished sentences + unfinished_sents = input_ids.new(batch_size).fill_(1) + sent_lengths = input_ids.new(batch_size).fill_(max_length) + + past = None + while cur_len < max_length: + model_inputs = self.prepare_inputs_for_generation( + input_ids, past=past, attention_mask=attention_mask, use_cache=use_cache, **model_kwargs + ) + + outputs = self(**model_inputs, return_dict=True) + next_token_logits = outputs.logits[:, -1, :] + scores = self.postprocess_next_token_scores( + scores=next_token_logits, + input_ids=input_ids, + no_repeat_ngram_size=no_repeat_ngram_size, + bad_words_ids=bad_words_ids, + cur_len=cur_len, + min_length=min_length, + max_length=max_length, + eos_token_id=eos_token_id, + repetition_penalty=repetition_penalty, + batch_size=batch_size, + num_beams=1, + ) + + # if model has past, then set the past variable to speed up decoding + if "past_key_values" in outputs: + past = outputs.past_key_values + elif "mems" in outputs: + past = outputs.mems + + if do_sample: + # Temperature (higher temperature => more likely to sample low probability tokens) + if temperature != 1.0: + scores = scores / temperature + # Top-p/top-k filtering + next_token_logscores = top_k_top_p_filtering(scores, top_k=top_k, top_p=top_p) + # Sample + probs = F.softmax(next_token_logscores, dim=-1) + next_token = torch.multinomial(probs, num_samples=1).squeeze(1) + else: + # Greedy decoding + next_token = torch.argmax(next_token_logits, dim=-1) + + # print(next_token_logits[0,next_token[0]], next_token_logits[0,eos_token_id]) + + # update generations and finished sentences + if eos_token_id is not None: + # pad finished sentences if eos_token_id exist + tokens_to_add = next_token * unfinished_sents + (pad_token_id) * (1 - unfinished_sents) + else: + tokens_to_add = next_token + + # add token and increase length by one + input_ids = torch.cat([input_ids, tokens_to_add.unsqueeze(-1)], dim=-1) + cur_len = cur_len + 1 + + if eos_token_id is not None: + eos_in_sents = tokens_to_add == eos_token_id + # if sentence is unfinished and the token to add is eos, sent_lengths is filled with current length + is_sents_unfinished_and_token_to_add_is_eos = unfinished_sents.mul(eos_in_sents.long()).bool() + sent_lengths.masked_fill_(is_sents_unfinished_and_token_to_add_is_eos, cur_len) + # unfinished_sents is set to zero if eos in sentence + unfinished_sents.mul_((~eos_in_sents).long()) + + # stop when there is a in each sentence, or if we exceed the maximul length + if unfinished_sents.max() == 0: + break + + + # extend attention_mask for new generated input if only decoder + # if self.config.is_encoder_decoder is False: + # attention_mask = torch.cat( + # [attention_mask, attention_mask.new_ones((attention_mask.shape[0], 1))], dim=-1 + # ) + + return input_ids diff --git a/data/fairseq/examples/MMPT/mmpt/models/transformermodel.py b/data/fairseq/examples/MMPT/mmpt/models/transformermodel.py new file mode 100644 index 0000000000000000000000000000000000000000..6acc419f09edbd5c9007c4d33d517d59b2f79b77 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/models/transformermodel.py @@ -0,0 +1,734 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright (c) Facebook, Inc. All Rights Reserved + +import torch + +from torch import nn + +try: + from transformers.modeling_bert import ( + BertPreTrainedModel, + BertModel, + BertEncoder, + BertPredictionHeadTransform, + ) +except ImportError: + pass + +from ..modules import VideoTokenMLP, MMBertEmbeddings + + +# --------------- fine-tuning models --------------- +class MMBertForJoint(BertPreTrainedModel): + """A BertModel with isolated attention mask to separate modality.""" + + def __init__(self, config): + super().__init__(config) + self.videomlp = VideoTokenMLP(config) + self.bert = MMBertModel(config) + self.init_weights() + + def forward( + self, + input_ids=None, + input_video_embeds=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + next_sentence_label=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + separate_forward_split=None, + ): + return_dict = ( + return_dict if return_dict is not None + else self.config.use_return_dict + ) + video_tokens = self.videomlp(input_video_embeds) + + outputs = self.bert( + input_ids, + video_tokens, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + separate_forward_split=separate_forward_split, + ) + + return outputs + + +class MMBertForTokenClassification(BertPreTrainedModel): + """A BertModel similar to MMJointUni, with extra wrapper layer + to be fine-tuned from other pretrained MMFusion model.""" + + def __init__(self, config): + super().__init__(config) + self.videomlp = VideoTokenMLP(config) + self.bert = MMBertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + # TODO(huxu): 779 is the number of classes for COIN: move to config? + self.classifier = nn.Linear(config.hidden_size, 779) + self.init_weights() + + def forward( + self, + input_ids=None, + input_video_embeds=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + next_sentence_label=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + separate_forward_split=None, + ): + return_dict = ( + return_dict if return_dict is not None + else self.config.use_return_dict + ) + + video_tokens = self.videomlp(input_video_embeds) + outputs = self.bert( + input_ids, + video_tokens, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + separate_forward_split=separate_forward_split, + ) + + return (self.classifier(outputs[0]),) + + +# ------------ pre-training models ---------------- + +class MMBertForEncoder(BertPreTrainedModel): + """A BertModel for Contrastive Learning.""" + def __init__(self, config): + super().__init__(config) + self.videomlp = VideoTokenMLP(config) + self.bert = MMBertModel(config) + self.init_weights() + + def forward( + self, + input_ids=None, + input_video_embeds=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + return_dict = ( + return_dict if return_dict is not None + else self.config.use_return_dict + ) + if input_video_embeds is not None: + video_tokens = self.videomlp(input_video_embeds) + else: + video_tokens = None + + outputs = self.bert( + input_ids, + video_tokens, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + return outputs + + +class MMBertForMFMMLM(BertPreTrainedModel): + """A BertModel with shared prediction head on MFM-MLM.""" + def __init__(self, config): + super().__init__(config) + self.videomlp = VideoTokenMLP(config) + self.bert = MMBertModel(config) + self.cls = MFMMLMHead(config) + self.hidden_size = config.hidden_size + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def forward( + self, + input_ids=None, + input_video_embeds=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + masked_frame_labels=None, + target_video_hidden_states=None, + non_masked_frame_mask=None, + masked_lm_labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + return_dict = ( + return_dict if return_dict is not None + else self.config.use_return_dict + ) + if input_video_embeds is not None: + video_tokens = self.videomlp(input_video_embeds) + else: + video_tokens = None + + if target_video_hidden_states is not None: + target_video_hidden_states = self.videomlp( + target_video_hidden_states) + + non_masked_frame_hidden_states = video_tokens.masked_select( + non_masked_frame_mask.unsqueeze(-1) + ).view(-1, self.hidden_size) + + outputs = self.bert( + input_ids, + video_tokens, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + mfm_scores, prediction_scores = None, None + if masked_frame_labels is not None and masked_lm_labels is not None: + # split the sequence. + text_offset = masked_frame_labels.size(1) + 1 # [CLS] + video_sequence_output = sequence_output[ + :, 1:text_offset + ] # remove [SEP] as not in video_label. + text_sequence_output = torch.cat( + [sequence_output[:, :1], sequence_output[:, text_offset:]], + dim=1 + ) + + hidden_size = video_sequence_output.size(-1) + selected_video_output = video_sequence_output.masked_select( + masked_frame_labels.unsqueeze(-1) + ).view(-1, hidden_size) + + # only compute select tokens to training to speed up. + hidden_size = text_sequence_output.size(-1) + # masked_lm_labels = masked_lm_labels.reshape(-1) + labels_mask = masked_lm_labels != -100 + + selected_text_output = text_sequence_output.masked_select( + labels_mask.unsqueeze(-1) + ).view(-1, hidden_size) + mfm_scores, prediction_scores = self.cls( + selected_video_output, + target_video_hidden_states, + non_masked_frame_hidden_states, + selected_text_output, + ) + + output = ( + mfm_scores, + prediction_scores, + ) + outputs + return output + + +class BertMFMMLMPredictionHead(nn.Module): + def __init__(self, config): + super().__init__() + self.transform = BertPredictionHeadTransform(config) + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear( + config.hidden_size, config.vocab_size, bias=False) + + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + # Need a link between the two variables so that the bias is correctly + # resized with `resize_token_embeddings` + self.decoder.bias = self.bias + + def forward( + self, + video_hidden_states=None, + target_video_hidden_states=None, + non_masked_frame_hidden_states=None, + text_hidden_states=None, + ): + video_logits, text_logits = None, None + if video_hidden_states is not None: + video_hidden_states = self.transform(video_hidden_states) + non_masked_frame_logits = torch.mm( + video_hidden_states, + non_masked_frame_hidden_states.transpose(1, 0) + ) + masked_frame_logits = torch.bmm( + video_hidden_states.unsqueeze(1), + target_video_hidden_states.unsqueeze(-1), + ).squeeze(-1) + video_logits = torch.cat( + [masked_frame_logits, non_masked_frame_logits], dim=1 + ) + + if text_hidden_states is not None: + text_hidden_states = self.transform(text_hidden_states) + text_logits = self.decoder(text_hidden_states) + return video_logits, text_logits + + +class MFMMLMHead(nn.Module): + def __init__(self, config): + super().__init__() + self.predictions = BertMFMMLMPredictionHead(config) + + def forward( + self, + video_hidden_states=None, + target_video_hidden_states=None, + non_masked_frame_hidden_states=None, + text_hidden_states=None, + ): + video_logits, text_logits = self.predictions( + video_hidden_states, + target_video_hidden_states, + non_masked_frame_hidden_states, + text_hidden_states, + ) + return video_logits, text_logits + + +class MMBertForMTM(MMBertForMFMMLM): + def __init__(self, config): + BertPreTrainedModel.__init__(self, config) + self.videomlp = VideoTokenMLP(config) + self.bert = MMBertModel(config) + self.cls = MTMHead(config) + self.hidden_size = config.hidden_size + self.init_weights() + + +class BertMTMPredictionHead(nn.Module): + def __init__(self, config): + super().__init__() + self.transform = BertPredictionHeadTransform(config) + self.decoder = nn.Linear( + config.hidden_size, config.vocab_size, bias=False) + + def forward( + self, + video_hidden_states=None, + target_video_hidden_states=None, + non_masked_frame_hidden_states=None, + text_hidden_states=None, + ): + non_masked_frame_hidden_states = non_masked_frame_hidden_states.transpose(1, 0) + video_logits, text_logits = None, None + if video_hidden_states is not None: + video_hidden_states = self.transform(video_hidden_states) + + masked_frame_logits = torch.bmm( + video_hidden_states.unsqueeze(1), + target_video_hidden_states.unsqueeze(-1), + ).squeeze(-1) + + non_masked_frame_logits = torch.mm( + video_hidden_states, + non_masked_frame_hidden_states + ) + video_on_vocab_logits = self.decoder(video_hidden_states) + video_logits = torch.cat([ + masked_frame_logits, + non_masked_frame_logits, + video_on_vocab_logits], dim=1) + + if text_hidden_states is not None: + text_hidden_states = self.transform(text_hidden_states) + # text first so label does not need to be shifted. + text_on_vocab_logits = self.decoder(text_hidden_states) + text_on_video_logits = torch.mm( + text_hidden_states, + non_masked_frame_hidden_states + ) + text_logits = torch.cat([ + text_on_vocab_logits, + text_on_video_logits + ], dim=1) + + return video_logits, text_logits + + +class MTMHead(nn.Module): + def __init__(self, config): + super().__init__() + self.predictions = BertMTMPredictionHead(config) + + def forward( + self, + video_hidden_states=None, + target_video_hidden_states=None, + non_masked_frame_hidden_states=None, + text_hidden_states=None, + ): + video_logits, text_logits = self.predictions( + video_hidden_states, + target_video_hidden_states, + non_masked_frame_hidden_states, + text_hidden_states, + ) + return video_logits, text_logits + + +class MMBertModel(BertModel): + """MMBertModel has MMBertEmbedding to support video tokens.""" + + def __init__(self, config, add_pooling_layer=True): + super().__init__(config) + # overwrite embedding + self.embeddings = MMBertEmbeddings(config) + self.encoder = MultiLayerAttentionMaskBertEncoder(config) + self.init_weights() + + def forward( + self, + input_ids=None, + input_video_embeds=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + separate_forward_split=None, + ): + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + return_dict = ( + return_dict if return_dict is not None + else self.config.use_return_dict + ) + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + "You cannot specify both input_ids " + "and inputs_embeds at the same time" + ) + elif input_ids is not None: + if input_video_embeds is not None: + input_shape = ( + input_ids.size(0), + input_ids.size(1) + input_video_embeds.size(1), + ) + else: + input_shape = ( + input_ids.size(0), + input_ids.size(1), + ) + elif inputs_embeds is not None: + if input_video_embeds is not None: + input_shape = ( + inputs_embeds.size(0), + inputs_embeds.size(1) + input_video_embeds.size(1), + ) + else: + input_shape = ( + input_ids.size(0), + input_ids.size(1), + ) + else: + raise ValueError( + "You have to specify either input_ids or inputs_embeds") + + device = input_ids.device if input_ids is not None \ + else inputs_embeds.device + + if attention_mask is None: + attention_mask = torch.ones(input_shape, device=device) + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions + # [batch_size, from_seq_length, to_seq_length] + # ourselves in which case + # we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = \ + self.get_extended_attention_mask( + attention_mask, input_shape, device) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to + # [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder and encoder_hidden_states is not None: + ( + encoder_batch_size, + encoder_sequence_length, + _, + ) = encoder_hidden_states.size() + encoder_hidden_shape = ( + encoder_batch_size, encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask + ) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or + # [num_hidden_layers x num_heads] + # and head_mask is converted to shape + # [num_hidden_layers x batch x num_heads x seq_length x seq_length] + + head_mask = self.get_head_mask( + head_mask, self.config.num_hidden_layers) + + embedding_output = self.embeddings( + input_ids, + input_video_embeds, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + ) + + if separate_forward_split is not None: + split_embedding_output = \ + embedding_output[:, :separate_forward_split] + split_extended_attention_mask = extended_attention_mask[ + :, :, :, :separate_forward_split, :separate_forward_split + ] + split_encoder_outputs = self.encoder( + split_embedding_output, + attention_mask=split_extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + assert ( + len(split_encoder_outputs) <= 2 + ), "we do not support merge on attention for now." + encoder_outputs = [] + encoder_outputs.append([split_encoder_outputs[0]]) + if len(split_encoder_outputs) == 2: + encoder_outputs.append([]) + for _all_hidden_states in split_encoder_outputs[1]: + encoder_outputs[-1].append([_all_hidden_states]) + + split_embedding_output = \ + embedding_output[:, separate_forward_split:] + split_extended_attention_mask = extended_attention_mask[ + :, :, :, separate_forward_split:, separate_forward_split: + ] + + split_encoder_outputs = self.encoder( + split_embedding_output, + attention_mask=split_extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + assert ( + len(split_encoder_outputs) <= 2 + ), "we do not support merge on attention for now." + encoder_outputs[0].append(split_encoder_outputs[0]) + encoder_outputs[0] = torch.cat(encoder_outputs[0], dim=1) + if len(split_encoder_outputs) == 2: + for layer_idx, _all_hidden_states in enumerate( + split_encoder_outputs[1] + ): + encoder_outputs[1][layer_idx].append(_all_hidden_states) + encoder_outputs[1][layer_idx] = torch.cat( + encoder_outputs[1][layer_idx], dim=1 + ) + encoder_outputs = tuple(encoder_outputs) + else: + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = encoder_outputs[0] + pooled_output = ( + self.pooler(sequence_output) if self.pooler is not None else None + ) + + return (sequence_output, pooled_output) + encoder_outputs[1:] + + def get_extended_attention_mask(self, attention_mask, input_shape, device): + """This is borrowed from `modeling_utils.py` with the support of + multi-layer attention masks. + The second dim is expected to be number of layers. + See `MMAttentionMaskProcessor`. + Makes broadcastable attention and causal masks so that future + and masked tokens are ignored. + + Arguments: + attention_mask (:obj:`torch.Tensor`): + Mask with ones indicating tokens to attend to, + zeros for tokens to ignore. + input_shape (:obj:`Tuple[int]`): + The shape of the input to the model. + device: (:obj:`torch.device`): + The device of the input to the model. + + Returns: + :obj:`torch.Tensor` The extended attention mask, \ + with a the same dtype as :obj:`attention_mask.dtype`. + """ + # We can provide a self-attention mask of dimensions + # [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable + # to all heads. + if attention_mask.dim() == 4: + extended_attention_mask = attention_mask[:, :, None, :, :] + extended_attention_mask = extended_attention_mask.to( + dtype=self.dtype + ) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) \ + * -10000.0 + return extended_attention_mask + else: + return super().get_extended_attention_mask( + attention_mask, input_shape, device + ) + + +class MultiLayerAttentionMaskBertEncoder(BertEncoder): + """extend BertEncoder with the capability of + multiple layers of attention mask.""" + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + output_attentions=False, + output_hidden_states=False, + return_dict=False, + ): + all_hidden_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + layer_head_mask = head_mask[i] if head_mask is not None else None + + layer_attention_mask = ( + attention_mask[:, i, :, :, :] + if attention_mask.dim() == 5 + else attention_mask + ) + + if getattr(self.config, "gradient_checkpointing", False): + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs, output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + layer_attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + layer_attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions, + ) + hidden_states = layer_outputs[0] + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + return tuple( + v + for v in [hidden_states, all_hidden_states, all_attentions] + if v is not None + ) diff --git a/data/fairseq/examples/MMPT/mmpt/modules/__init__.py b/data/fairseq/examples/MMPT/mmpt/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4c78594c21d7ccdcb2ab11c918dd211adce27c14 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/modules/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +from .mm import * + +try: + from .expmm import * +except ImportError: + pass diff --git a/data/fairseq/examples/MMPT/mmpt/modules/mm.py b/data/fairseq/examples/MMPT/mmpt/modules/mm.py new file mode 100644 index 0000000000000000000000000000000000000000..5d9777371a5e7afc235a8b4e6c164b2a35258eb6 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/modules/mm.py @@ -0,0 +1,145 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright (c) Facebook, Inc. All Rights Reserved + + +import torch + +from torch import nn + +try: + from transformers.modeling_bert import ( + BertEmbeddings, + ACT2FN, + ) +except ImportError: + pass + + +class VideoTokenMLP(nn.Module): + def __init__(self, config): + super().__init__() + input_dim = config.input_dim if hasattr(config, "input_dim") else 512 + self.linear1 = nn.Linear(input_dim, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size) + self.activation = ACT2FN[config.hidden_act] + self.linear2 = nn.Linear(config.hidden_size, config.hidden_size) + + def forward(self, hidden_states): + hidden_states = self.linear1(hidden_states) + hidden_states = self.activation(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + hidden_states = self.linear2(hidden_states) + return hidden_states + + +class MMBertEmbeddings(BertEmbeddings): + def __init__(self, config): + super().__init__(config) + self.max_video_len = config.max_video_len + if hasattr(config, "use_seg_emb") and config.use_seg_emb: + """the original VLM paper uses seg_embeddings for temporal space. + although not used it changed the randomness of initialization. + we keep it for reproducibility. + """ + self.seg_embeddings = nn.Embedding(256, config.hidden_size) + + def forward( + self, + input_ids, + input_video_embeds, + token_type_ids=None, + position_ids=None, + inputs_embeds=None, + ): + input_tensor = input_ids if input_ids is not None else inputs_embeds + if input_video_embeds is not None: + input_shape = ( + input_tensor.size(0), + input_tensor.size(1) + input_video_embeds.size(1), + ) + else: + input_shape = (input_tensor.size(0), input_tensor.size(1)) + + if position_ids is None: + """ + Auto skip position embeddings for text only case. + use cases: + (1) action localization and segmentation: + feed in len-1 dummy video token needs text part to + skip input_video_embeds.size(1) for the right + position_ids for video [SEP] and rest text tokens. + (2) MMFusionShare for two forward passings: + in `forward_text`: input_video_embeds is None. + need to skip video [SEP] token. + + # video_len + 1: [CLS] + video_embed + # self.max_video_len + 1: [SEP] for video. + # self.max_video_len + 2: [SEP] for video. + # self.max_video_len + input_ids.size(1): rest for text. + """ + if input_video_embeds is not None: + video_len = input_video_embeds.size(1) + starting_offset = self.max_video_len + 1 # video [SEP] + ending_offset = self.max_video_len + input_ids.size(1) + else: + video_len = 0 + starting_offset = self.max_video_len + 2 # first text token. + ending_offset = self.max_video_len + input_ids.size(1) + 1 + position_ids = torch.cat([ + self.position_ids[:, :video_len + 1], + self.position_ids[:, starting_offset:ending_offset] + ], dim=1) + + if token_type_ids is None: + token_type_ids = torch.zeros( + input_shape, dtype=torch.long, device=self.position_ids.device + ) + + """ + the format of input_ids is [CLS] [SEP] caption [SEP] padding. + the goal is to build [CLS] video tokens [SEP] caption [SEP] . + """ + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + if input_video_embeds is not None: + inputs_mm_embeds = torch.cat([ + inputs_embeds[:, :1], input_video_embeds, inputs_embeds[:, 1:] + ], dim=1) + else: + # text only for `MMFusionShare`. + inputs_mm_embeds = inputs_embeds + + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + embeddings = inputs_mm_embeds + position_embeddings + embeddings += token_type_embeddings + + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class AlignHead(nn.Module): + """this will load pre-trained weights for NSP, which is desirable.""" + + def __init__(self, config): + super().__init__() + self.seq_relationship = nn.Linear(config.hidden_size, 2) + + def forward(self, dropout_pooled_output): + logits = self.seq_relationship(dropout_pooled_output) + return logits diff --git a/data/fairseq/examples/MMPT/mmpt/modules/retri.py b/data/fairseq/examples/MMPT/mmpt/modules/retri.py new file mode 100644 index 0000000000000000000000000000000000000000..d1b288f8e52308eeaefcb4411cb3d23e543f37d0 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/modules/retri.py @@ -0,0 +1,429 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import os +import numpy as np +import pickle +import time + +try: + import faiss +except ImportError: + pass + +from collections import defaultdict + +from ..utils import get_local_rank, print_on_rank0 + + +class VectorRetriever(object): + """ + How2 Video Retriver. + Reference usage of FAISS: + https://github.com/fairinternal/fairseq-py/blob/paraphrase_pretraining/fairseq/data/multilingual_faiss_dataset.py + """ + + def __init__(self, hidden_size, cent, db_type, examples_per_cent_to_train): + if db_type == "flatl2": + quantizer = faiss.IndexFlatL2(hidden_size) # the other index + self.db = faiss.IndexIVFFlat( + quantizer, hidden_size, cent, faiss.METRIC_L2) + elif db_type == "pq": + self.db = faiss.index_factory( + hidden_size, f"IVF{cent}_HNSW32,PQ32" + ) + else: + raise ValueError("unknown type of db", db_type) + self.train_thres = cent * examples_per_cent_to_train + self.train_cache = [] + self.train_len = 0 + self.videoid_to_vectoridx = {} + self.vectoridx_to_videoid = None + self.make_direct_maps_done = False + + def make_direct_maps(self): + faiss.downcast_index(self.db).make_direct_map() + + def __len__(self): + return self.db.ntotal + + def save(self, out_dir): + faiss.write_index( + self.db, + os.path.join(out_dir, "faiss_idx") + ) + with open( + os.path.join( + out_dir, "videoid_to_vectoridx.pkl"), + "wb") as fw: + pickle.dump( + self.videoid_to_vectoridx, fw, + protocol=pickle.HIGHEST_PROTOCOL + ) + + def load(self, out_dir): + fn = os.path.join(out_dir, "faiss_idx") + self.db = faiss.read_index(fn) + with open( + os.path.join(out_dir, "videoid_to_vectoridx.pkl"), "rb") as fr: + self.videoid_to_vectoridx = pickle.load(fr) + + def add(self, hidden_states, video_ids, last=False): + assert len(hidden_states) == len(video_ids), "{}, {}".format( + str(len(hidden_states)), str(len(video_ids))) + assert len(hidden_states.shape) == 2 + assert hidden_states.dtype == np.float32 + + valid_idx = [] + for idx, video_id in enumerate(video_ids): + if video_id not in self.videoid_to_vectoridx: + valid_idx.append(idx) + self.videoid_to_vectoridx[video_id] = \ + len(self.videoid_to_vectoridx) + + hidden_states = hidden_states[valid_idx] + if not self.db.is_trained: + self.train_cache.append(hidden_states) + self.train_len += hidden_states.shape[0] + if self.train_len < self.train_thres: + return + self.finalize_training() + else: + self.db.add(hidden_states) + + def finalize_training(self): + hidden_states = np.concatenate(self.train_cache, axis=0) + del self.train_cache + local_rank = get_local_rank() + if local_rank == 0: + start = time.time() + print("training db on", self.train_thres, "/", self.train_len) + self.db.train(hidden_states[:self.train_thres]) + if local_rank == 0: + print("training db for", time.time() - start) + self.db.add(hidden_states) + + def search( + self, + query_hidden_states, + orig_dist, + ): + if len(self.videoid_to_vectoridx) != self.db.ntotal: + raise ValueError( + "cannot search: size mismatch in-between index and db", + len(self.videoid_to_vectoridx), + self.db.ntotal + ) + + if self.vectoridx_to_videoid is None: + self.vectoridx_to_videoid = { + self.videoid_to_vectoridx[videoid]: videoid + for videoid in self.videoid_to_vectoridx + } + assert len(self.vectoridx_to_videoid) \ + == len(self.videoid_to_vectoridx) + + # MultilingualFaissDataset uses the following; not sure the purpose. + # faiss.ParameterSpace().set_index_parameter(self.db, "nprobe", 10) + queried_dist, index = self.db.search(query_hidden_states, 1) + queried_dist, index = queried_dist[:, 0], index[:, 0] + + outputs = np.array( + [self.vectoridx_to_videoid[_index] + if _index != -1 else (-1, -1, -1) for _index in index], + dtype=np.int32) + outputs[queried_dist <= orig_dist] = -1 + return outputs + + def search_by_video_ids( + self, + video_ids, + retri_factor + ): + if len(self.videoid_to_vectoridx) != self.db.ntotal: + raise ValueError( + len(self.videoid_to_vectoridx), + self.db.ntotal + ) + + if not self.make_direct_maps_done: + self.make_direct_maps() + + if self.vectoridx_to_videoid is None: + self.vectoridx_to_videoid = { + self.videoid_to_vectoridx[videoid]: videoid + for videoid in self.videoid_to_vectoridx + } + assert len(self.vectoridx_to_videoid) \ + == len(self.videoid_to_vectoridx) + + query_hidden_states = [] + vector_ids = [] + for video_id in video_ids: + vector_id = self.videoid_to_vectoridx[video_id] + vector_ids.append(vector_id) + query_hidden_state = self.db.reconstruct(vector_id) + query_hidden_states.append(query_hidden_state) + query_hidden_states = np.stack(query_hidden_states) + + # MultilingualFaissDataset uses the following; not sure the reason. + # faiss.ParameterSpace().set_index_parameter(self.db, "nprobe", 10) + _, index = self.db.search(query_hidden_states, retri_factor) + outputs = [] + for sample_idx, sample in enumerate(index): + # the first video_id is always the video itself. + cands = [video_ids[sample_idx]] + for vector_idx in sample: + if vector_idx >= 0 \ + and vector_ids[sample_idx] != vector_idx: + cands.append( + self.vectoridx_to_videoid[vector_idx] + ) + outputs.append(cands) + return outputs + + +class VectorRetrieverDM(VectorRetriever): + """ + with direct map. + How2 Video Retriver. + Reference usage of FAISS: + https://github.com/fairinternal/fairseq-py/blob/paraphrase_pretraining/fairseq/data/multilingual_faiss_dataset.py + """ + + def __init__( + self, + hidden_size, + cent, + db_type, + examples_per_cent_to_train + ): + super().__init__( + hidden_size, cent, db_type, examples_per_cent_to_train) + self.make_direct_maps_done = False + + def make_direct_maps(self): + faiss.downcast_index(self.db).make_direct_map() + self.make_direct_maps_done = True + + def search( + self, + query_hidden_states, + orig_dist, + ): + if len(self.videoid_to_vectoridx) != self.db.ntotal: + raise ValueError( + len(self.videoid_to_vectoridx), + self.db.ntotal + ) + + if not self.make_direct_maps_done: + self.make_direct_maps() + if self.vectoridx_to_videoid is None: + self.vectoridx_to_videoid = { + self.videoid_to_vectoridx[videoid]: videoid + for videoid in self.videoid_to_vectoridx + } + assert len(self.vectoridx_to_videoid) \ + == len(self.videoid_to_vectoridx) + + # MultilingualFaissDataset uses the following; not sure the reason. + # faiss.ParameterSpace().set_index_parameter(self.db, "nprobe", 10) + queried_dist, index = self.db.search(query_hidden_states, 1) + outputs = [] + for sample_idx, sample in enumerate(index): + # and queried_dist[sample_idx] < thres \ + if sample >= 0 \ + and queried_dist[sample_idx] < orig_dist[sample_idx]: + outputs.append(self.vectoridx_to_videoid[sample]) + else: + outputs.append(None) + return outputs + + def search_by_video_ids( + self, + video_ids, + retri_factor=8 + ): + if len(self.videoid_to_vectoridx) != self.db.ntotal: + raise ValueError( + len(self.videoid_to_vectoridx), + self.db.ntotal + ) + + if not self.make_direct_maps_done: + self.make_direct_maps() + if self.vectoridx_to_videoid is None: + self.vectoridx_to_videoid = { + self.videoid_to_vectoridx[videoid]: videoid + for videoid in self.videoid_to_vectoridx + } + assert len(self.vectoridx_to_videoid) \ + == len(self.videoid_to_vectoridx) + + query_hidden_states = [] + vector_ids = [] + for video_id in video_ids: + vector_id = self.videoid_to_vectoridx[video_id] + vector_ids.append(vector_id) + query_hidden_state = self.db.reconstruct(vector_id) + query_hidden_states.append(query_hidden_state) + query_hidden_states = np.stack(query_hidden_states) + + # MultilingualFaissDataset uses the following; not sure the reason. + # faiss.ParameterSpace().set_index_parameter(self.db, "nprobe", 10) + _, index = self.db.search(query_hidden_states, retri_factor) + outputs = [] + for sample_idx, sample in enumerate(index): + # the first video_id is always the video itself. + cands = [video_ids[sample_idx]] + for vector_idx in sample: + if vector_idx >= 0 \ + and vector_ids[sample_idx] != vector_idx: + cands.append( + self.vectoridx_to_videoid[vector_idx] + ) + outputs.append(cands) + return outputs + + +class MMVectorRetriever(VectorRetrieverDM): + """ + multimodal vector retriver: + text retrieve video or video retrieve text. + """ + + def __init__(self, hidden_size, cent, db_type, examples_per_cent_to_train): + super().__init__( + hidden_size, cent, db_type, examples_per_cent_to_train) + video_db = self.db + super().__init__( + hidden_size, cent, db_type, examples_per_cent_to_train) + text_db = self.db + self.db = {"video": video_db, "text": text_db} + self.video_to_videoid = defaultdict(list) + + def __len__(self): + assert self.db["video"].ntotal == self.db["text"].ntotal + return self.db["video"].ntotal + + def make_direct_maps(self): + faiss.downcast_index(self.db["video"]).make_direct_map() + faiss.downcast_index(self.db["text"]).make_direct_map() + + def save(self, out_dir): + faiss.write_index( + self.db["video"], + os.path.join(out_dir, "video_faiss_idx") + ) + faiss.write_index( + self.db["text"], + os.path.join(out_dir, "text_faiss_idx") + ) + + with open( + os.path.join( + out_dir, "videoid_to_vectoridx.pkl"), + "wb") as fw: + pickle.dump( + self.videoid_to_vectoridx, fw, + protocol=pickle.HIGHEST_PROTOCOL + ) + + def load(self, out_dir): + fn = os.path.join(out_dir, "video_faiss_idx") + video_db = faiss.read_index(fn) + fn = os.path.join(out_dir, "text_faiss_idx") + text_db = faiss.read_index(fn) + self.db = {"video": video_db, "text": text_db} + with open( + os.path.join(out_dir, "videoid_to_vectoridx.pkl"), "rb") as fr: + self.videoid_to_vectoridx = pickle.load(fr) + self.video_to_videoid = defaultdict(list) + + def add(self, hidden_states, video_ids): + """hidden_states is a pair `(video, text)`""" + assert len(hidden_states) == len(video_ids), "{}, {}".format( + str(len(hidden_states)), str(len(video_ids))) + assert len(hidden_states.shape) == 3 + assert len(self.video_to_videoid) == 0 + + valid_idx = [] + for idx, video_id in enumerate(video_ids): + if video_id not in self.videoid_to_vectoridx: + valid_idx.append(idx) + self.videoid_to_vectoridx[video_id] = \ + len(self.videoid_to_vectoridx) + + batch_size = hidden_states.shape[0] + hidden_states = hidden_states[valid_idx] + + hidden_states = np.transpose(hidden_states, (1, 0, 2)).copy() + if not self.db["video"].is_trained: + self.train_cache.append(hidden_states) + train_len = batch_size * len(self.train_cache) + if train_len < self.train_thres: + return + + hidden_states = np.concatenate(self.train_cache, axis=1) + del self.train_cache + self.db["video"].train(hidden_states[0, :self.train_thres]) + self.db["text"].train(hidden_states[1, :self.train_thres]) + self.db["video"].add(hidden_states[0]) + self.db["text"].add(hidden_states[1]) + + def get_clips_by_video_id(self, video_id): + if not self.video_to_videoid: + for video_id, video_clip, text_clip in self.videoid_to_vectoridx: + self.video_to_videoid[video_id].append( + (video_id, video_clip, text_clip)) + return self.video_to_videoid[video_id] + + def search( + self, + video_ids, + target_modality, + retri_factor=8 + ): + if len(self.videoid_to_vectoridx) != len(self): + raise ValueError( + len(self.videoid_to_vectoridx), + len(self) + ) + + if not self.make_direct_maps_done: + self.make_direct_maps() + if self.vectoridx_to_videoid is None: + self.vectoridx_to_videoid = { + self.videoid_to_vectoridx[videoid]: videoid + for videoid in self.videoid_to_vectoridx + } + assert len(self.vectoridx_to_videoid) \ + == len(self.videoid_to_vectoridx) + + src_modality = "text" if target_modality == "video" else "video" + + query_hidden_states = [] + vector_ids = [] + for video_id in video_ids: + vector_id = self.videoid_to_vectoridx[video_id] + vector_ids.append(vector_id) + query_hidden_state = self.db[src_modality].reconstruct(vector_id) + query_hidden_states.append(query_hidden_state) + query_hidden_states = np.stack(query_hidden_states) + + # MultilingualFaissDataset uses the following; not sure the reason. + # faiss.ParameterSpace().set_index_parameter(self.db, "nprobe", 10) + _, index = self.db[target_modality].search( + query_hidden_states, retri_factor) + outputs = [] + for sample_idx, sample in enumerate(index): + cands = [] + for vector_idx in sample: + if vector_idx >= 0: + cands.append( + self.vectoridx_to_videoid[vector_idx] + ) + outputs.append(cands) + return outputs diff --git a/data/fairseq/examples/MMPT/mmpt/modules/vectorpool.py b/data/fairseq/examples/MMPT/mmpt/modules/vectorpool.py new file mode 100644 index 0000000000000000000000000000000000000000..d2b23d2da888c714d7605d508c558fbad34e709d --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/modules/vectorpool.py @@ -0,0 +1,246 @@ +# Copyright (c) Facebook, Inc. All Rights Reserved + +import torch +import os +import numpy as np +import pickle + +from . import retri +from ..utils import get_local_rank + + +class VectorPool(object): + """ + Base class of retrieval space. + """ + + def __init__(self, config): + from transformers import AutoConfig + self.hidden_size = AutoConfig.from_pretrained( + config.dataset.bert_name).hidden_size + self.retriever_cls = getattr(retri, config.retriever_cls) + + def __call__(self, sample, **kwargs): + raise NotImplementedError + + def build_retriver( + self, + retriever_cls=None, + hidden_size=None, + centroids=512, + db_type="flatl2", + examples_per_cent_to_train=48 + ): + + """merge results from multiple gpus and return a retriver..""" + self.retriver = retriever_cls( + hidden_size, centroids, db_type, examples_per_cent_to_train) + return self.retriver + + def __repr__(self): + if hasattr(self, "retriver"): + retriver_name = str(len(self.retriver)) + else: + retriver_name = "no retriver field yet" + return self.__class__.__name__ \ + + "(" + retriver_name + ")" + + +class VideoVectorPool(VectorPool): + """ + average clips of a video as video representation. + """ + def __init__(self, config): + super().__init__(config) + self.build_retriver(self.retriever_cls, self.hidden_size) + + def __call__(self, sample, subsampling, **kwargs): + hidden_states = ( + sample["pooled_video"] + sample["pooled_text"]) / 2. + hidden_states = hidden_states.view( + -1, subsampling, + hidden_states.size(-1)) + hidden_states = torch.mean(hidden_states, dim=1) + hidden_states = hidden_states.cpu().detach().numpy() + video_ids = [] + for offset_idx, video_id in enumerate(sample["video_id"]): + if isinstance(video_id, tuple) and len(video_id) == 3: + # a sharded video_id. + video_id = video_id[0] + video_ids.append(video_id) + assert len(video_ids) == len(hidden_states) + self.retriver.add( + hidden_states.astype("float32"), + video_ids + ) + + +class DistributedVectorPool(VectorPool): + """ + support sync of multiple gpus/nodes. + """ + def __init__(self, config): + super().__init__(config) + self.out_dir = os.path.join( + config.fairseq.checkpoint.save_dir, + "retri") + os.makedirs(self.out_dir, exist_ok=True) + self.hidden_states = [] + self.video_ids = [] + + def build_retriver( + self, + retriever_cls=None, + hidden_size=None, + centroids=4096, + db_type="flatl2", + examples_per_cent_to_train=48 + ): + if retriever_cls is None: + retriever_cls = self.retriever_cls + if hidden_size is None: + hidden_size = self.hidden_size + """merge results from multiple gpus and return a retriver..""" + if torch.distributed.is_initialized(): + self.save() + # sync saving. + torch.distributed.barrier() + world_size = torch.distributed.get_world_size() + else: + world_size = 1 + self.retriver = retriever_cls( + hidden_size, centroids, db_type, examples_per_cent_to_train) + # each gpu process has its own retriever. + for local_rank in range(world_size): + if get_local_rank() == 0: + print("load local_rank", local_rank) + hidden_states, video_ids = self.load(local_rank) + hidden_states = hidden_states.astype("float32") + self.retriver.add(hidden_states, video_ids) + return self.retriver + + def load(self, local_rank): + hidden_states = np.load( + os.path.join( + self.out_dir, + "hidden_state" + str(local_rank) + ".npy" + ) + ) + + with open( + os.path.join( + self.out_dir, "video_id" + str(local_rank) + ".pkl"), + "rb") as fr: + video_ids = pickle.load(fr) + return hidden_states, video_ids + + def save(self): + hidden_states = np.vstack(self.hidden_states) + assert len(hidden_states) == len(self.video_ids), "{}, {}".format( + len(hidden_states), + len(self.video_ids) + ) + local_rank = torch.distributed.get_rank() \ + if torch.distributed.is_initialized() else 0 + + np.save( + os.path.join( + self.out_dir, + "hidden_state" + str(local_rank) + ".npy"), + hidden_states) + + with open( + os.path.join( + self.out_dir, + "video_id" + str(local_rank) + ".pkl"), + "wb") as fw: + pickle.dump( + self.video_ids, + fw, + protocol=pickle.HIGHEST_PROTOCOL + ) + + +class DistributedVideoVectorPool(DistributedVectorPool): + """ + average clips of a video as video representation. + """ + def __call__(self, sample, subsampling, **kwargs): + hidden_states = ( + sample["pooled_video"] + sample["pooled_text"]) / 2. + hidden_states = hidden_states.view( + -1, subsampling, + hidden_states.size(-1)) + hidden_states = torch.mean(hidden_states, dim=1) + hidden_states = hidden_states.cpu().detach().numpy() + video_ids = [] + for offset_idx, video_id in enumerate(sample["video_id"]): + if isinstance(video_id, tuple) and len(video_id) == 3: + # a sharded video_id. + video_id = video_id[0] + video_ids.append(video_id) + assert len(video_ids) == len(hidden_states) + self.hidden_states.append(hidden_states) + self.video_ids.extend(video_ids) + + +# ------------ the following are deprecated -------------- + +class TextClipVectorPool(VectorPool): + def __init__(self, config): + from transformers import AutoConfig + hidden_size = AutoConfig.from_pretrained( + config.dataset.bert_name).hidden_size + retriever_cls = getattr(retri, config.retriever_cls) + self.build_retriver(retriever_cls, hidden_size) + + def __call__(self, sample, **kwargs): + clip_meta = sample["clip_meta"].cpu() + assert torch.all(torch.le(clip_meta[:, 4], clip_meta[:, 5])) + text_meta = [tuple(item.tolist()) for item in clip_meta[:, 3:]] + + if hasattr(self, "retriver"): + # build_retriver is called. + self.retriver.add( + sample["pooled_text"].cpu().numpy().astype("float32"), + text_meta + ) + else: + raise NotImplementedError + + +class MMClipVectorPool(VectorPool): + """ + Multimodal Clip-level vector pool. + """ + def __init__(self, out_dir): + """use hidden_states to store `(video, text)`.""" + """use video_ids to store `(video_id, start, end)`.""" + super().__init__(out_dir) + + def __call__(self, sample, **kwargs): + pooled_video = sample["pooled_video"].cpu().unsqueeze(1).numpy() + pooled_text = sample["pooled_text"].cpu().unsqueeze(1).numpy() + + self.hidden_states.append( + np.concatenate([pooled_video, pooled_text], axis=1) + ) + + video_starts = sample["video_start"].cpu() + video_ends = sample["video_end"].cpu() + assert torch.all(torch.le(video_starts, video_ends)) + + text_starts = sample["text_start"].cpu() + text_ends = sample["text_end"].cpu() + assert torch.all(torch.le(text_starts, text_ends)) + subsample_size = sample["pooled_video"].size(0) // len(sample["video_id"]) + video_ids = [video_id for video_id in sample["video_id"] + for _ in range(subsample_size) + ] + for video_id, video_start, video_end, text_start, text_end in zip( + video_ids, video_starts, video_ends, text_starts, text_ends): + self.video_ids.append(( + video_id, + (int(video_start), int(video_end)), + (int(text_start), int(text_end)) + )) diff --git a/data/fairseq/examples/MMPT/mmpt/processors/__init__.py b/data/fairseq/examples/MMPT/mmpt/processors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..434d1d92b95846b0cd87625834c5d9bed279a44e --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/processors/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +from .processor import * + +from .how2processor import * +from .how2retriprocessor import * + +from .dsprocessor import * + +try: + from .rawvideoprocessor import * + from .codecprocessor import * + from .webvidprocessor import * + from .expprocessor import * + from .exphow2processor import * + from .exphow2retriprocessor import * + from .expcodecprocessor import * + from .expfeatureencoder import * + from .expdsprocessor import * +except ImportError: + pass diff --git a/data/fairseq/examples/MMPT/mmpt/processors/dedupprocessor.py b/data/fairseq/examples/MMPT/mmpt/processors/dedupprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..8a1ad402cd890516457c188910ec265a47fc6e8e --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/processors/dedupprocessor.py @@ -0,0 +1,242 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import random +import json +import pickle +from tqdm import tqdm +import os +import numpy as np + + +class CaptionDedupProcessor(object): + """remove overlapping of caption sentences(clip). + Some statistics: + caption: + {'t_clip_len': 246.6448431320854, + 'video_len': 281.09174795676245, + 'clip_tps': 0.8841283727427481, + 'video_tps': 0.7821156477732097, + 'min_clip_len': 0.0, + 'max_clip_len': 398.3, + 'mean_clip_len': 3.196580003006861, + 'num_clip': 77.15897706301081} + + raw_caption: + {'t_clip_len': 238.95908778424115, + 'video_len': 267.5914859862507, + 'clip_tps': 2.4941363624267963, + 'video_tps': 2.258989769647173, + 'min_clip_len': 0.0, + 'max_clip_len': 398.3, + 'mean_clip_len': 3.0537954186814265, + 'num_clip': 78.24986779481756} + """ + + def __init__(self, pkl_file): + with open(pkl_file, "rb") as fd: + self.data = pickle.load(fd) + self.stat = { + "t_clip_len": [], + "video_len": [], + "clip_tps": [], + "video_tps": [], + "clip_len": [], + } + + def __call__(self): + for idx, video_id in enumerate(tqdm(self.data)): + caption = json.loads(self.data[video_id]) + caption = self._dedup(caption) + if idx < 4096: # for the first 4096 examples, compute the statistics. + self.save_stat(video_id, caption) + self.data[video_id] = json.dumps(caption) + self.print_stat() + + def single(self, video_id): + caption = json.loads(self.data[video_id]) + for clip_idx, (start, end, text) in enumerate( + zip(caption["start"], caption["end"], caption["text"]) + ): + print(start, end, text) + print("@" * 100) + caption = self._dedup(caption) + for clip_idx, (start, end, text) in enumerate( + zip(caption["start"], caption["end"], caption["text"]) + ): + print(start, end, text) + print("#" * 100) + self.save_stat(video_id, caption) + self.print_stat() + + def finalize(self, tgt_fn): + with open(tgt_fn, "wb") as fw: + pickle.dump(self.data, fw, pickle.HIGHEST_PROTOCOL) + + def save_stat(self, video_id, caption): + video_fn = os.path.join( + "data/feat/feat_how2_s3d", video_id + ".npy" + ) + if os.path.isfile(video_fn): + with open(video_fn, "rb", 1) as fr: # 24 is the buffer size. buffered + version = np.lib.format.read_magic(fr) + shape, fortran, dtype = np.lib.format._read_array_header(fr, version) + video_len = shape[0] + + t_clip_len = 0.0 + t_tokens = 0 + for idx, (start, end, text) in enumerate( + zip(caption["start"], caption["end"], caption["text"]) + ): + clip_len = ( + (end - max(caption["end"][idx - 1], start)) + if idx > 0 + else end - start + ) + t_clip_len += clip_len + t_tokens += len(text.split(" ")) + self.stat["clip_len"].append(clip_len) + self.stat["t_clip_len"].append(t_clip_len) + self.stat["video_len"].append(video_len) + self.stat["clip_tps"].append(t_tokens / t_clip_len) + self.stat["video_tps"].append(t_tokens / video_len) + + def print_stat(self): + result = { + "t_clip_len": np.mean(self.stat["t_clip_len"]), + "video_len": np.mean(self.stat["video_len"]), + "clip_tps": np.mean(self.stat["clip_tps"]), + "video_tps": np.mean(self.stat["video_tps"]), + "min_clip_len": min(self.stat["clip_len"]), + "max_clip_len": max(self.stat["clip_len"]), + "mean_clip_len": np.mean(self.stat["clip_len"]), + "num_clip": len(self.stat["clip_len"]) / len(self.stat["video_tps"]), + } + print(result) + + def _dedup(self, caption): + def random_merge(end_idx, start, end, text, starts, ends, texts): + if random.random() > 0.5: + # print(clip_idx, "[PARTIAL INTO PREV]", end_idx) + # overlapped part goes to the end of previous. + ends[-1] = max(ends[-1], start) # ? + rest_text = text[end_idx:].strip() + if rest_text: + starts.append(max(ends[-1], start)) + ends.append(max(end, starts[-1])) + texts.append(rest_text) + else: # goes to the beginning of the current. + # strip the previous. + left_text = texts[-1][:-end_idx].strip() + if left_text: + # print(clip_idx, "[PREV PARTIAL INTO CUR]", end_idx) + ends[-1] = min(ends[-1], start) + texts[-1] = left_text + else: + # print(clip_idx, "[PREV LEFT NOTHING ALL INTO CUR]", end_idx) + starts.pop(-1) + ends.pop(-1) + texts.pop(-1) + starts.append(start) + ends.append(end) + texts.append(text) + + starts, ends, texts = [], [], [] + for clip_idx, (start, end, text) in enumerate( + zip(caption["start"], caption["end"], caption["text"]) + ): + if not isinstance(text, str): + continue + text = text.replace("\n", " ").strip() + if len(text) == 0: + continue + starts.append(start) + ends.append(end) + texts.append(text) + break + + for clip_idx, (start, end, text) in enumerate( + zip( + caption["start"][clip_idx + 1:], + caption["end"][clip_idx + 1:], + caption["text"][clip_idx + 1:], + ) + ): + if not isinstance(text, str): + continue + text = text.replace("\n", " ").strip() + if len(text) == 0: + continue + + # print(clip_idx, texts[-5:]) + # print(clip_idx, start, end, text) + if texts[-1].endswith(text): # subset of prev caption -> merge + # print(clip_idx, "[MERGE INTO PREV]") + ends[-1] = max(ends[-1], end) + elif text.startswith(texts[-1]): # superset of prev caption -> merge + # print(clip_idx, "[PREV MERGE INTO CUR]") + texts[-1] = text + starts[-1] = min(starts[-1], start) + ends[-1] = max(ends[-1], end) + else: # overlapping or non-overlapping. + for end_idx in range(1, len(text) + 1): + if texts[-1].endswith(text[:end_idx]): + random_merge(end_idx, start, end, text, starts, ends, texts) + break + else: + starts.append(start) + ends.append(end) + texts.append(text) + + assert (ends[-1] + 0.001) >= starts[-1] and len( + texts[-1] + ) > 0, "{} {} {} <- {} {} {}, {} {} {}".format( + str(starts[-1]), + str(ends[-1]), + texts[-1], + caption["start"][clip_idx - 1], + caption["end"][clip_idx - 1], + caption["text"][clip_idx - 1], + str(start), + str(end), + text, + ) + + return {"start": starts, "end": ends, "text": texts} + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="dedup how2 caption") + parser.add_argument('--how2dir', default="data/how2") + args = parser.parse_args() + + raw_caption_json = os.path.join(args.how2dir, "raw_caption.json") + raw_caption_pickle = os.path.join(args.how2dir, "raw_caption.pkl") + raw_caption_dedup_pickle = os.path.join(args.how2dir, "raw_caption_dedup.pkl") + + def convert_to_pickle(src_fn, tgt_fn): + with open(src_fn) as fd: + captions = json.load(fd) + + for video_id in captions: + captions[video_id] = json.dumps(captions[video_id]) + + with open(tgt_fn, "wb") as fw: + pickle.dump(captions, fw, pickle.HIGHEST_PROTOCOL) + + if not os.path.isfile(raw_caption_pickle): + convert_to_pickle(raw_caption_json, raw_caption_pickle) + + deduper = CaptionDedupProcessor(raw_caption_pickle) + deduper() + deduper.finalize(raw_caption_dedup_pickle) + + """ + # demo + deduper = CaptionDedupProcessor("data/how2/raw_caption.pkl") + deduper.single("HfIeQ9pzL5U") + """ diff --git a/data/fairseq/examples/MMPT/mmpt/processors/dsprocessor.py b/data/fairseq/examples/MMPT/mmpt/processors/dsprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..ecebf0eea5c57b7846a2bd46ddffbdbf55bd83ab --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/processors/dsprocessor.py @@ -0,0 +1,848 @@ +# Copyright (c) Facebook, Inc. All Rights Reserved + +""" +Processors for all downstream (ds) tasks. +""" + +import json +import os +import pickle +import random +import math +import numpy as np +import torch + +from collections import defaultdict + +from .processor import ( + MetaProcessor, + VideoProcessor, + TextProcessor, + Aligner, + MMAttentionMask2DProcessor, +) + +from .how2processor import TextGenerationProcessor + + +# ------------- A General Aligner for all downstream tasks----------------- + + +class DSAligner(Aligner): + """ + Downstream (DS) aligner shared by all datasets. + """ + + def __call__(self, video_id, video_feature, text_feature, wps=0.7): + # random sample a starting sec for video. + video_start = 0 + video_end = min(len(video_feature), self.max_video_len) + # the whole sequence is a single clip. + video_clips = {"start": [video_start], "end": [video_end]} + + text_feature = { + "cap": [text_feature], + "start": [video_start], + "end": [len(text_feature) / wps], + } + text_clip_indexs = [0] + + vfeats, vmasks = self._build_video_seq( + video_feature, video_clips + ) + caps, cmasks = self._build_text_seq( + text_feature, text_clip_indexs + ) + + return { + "caps": caps, + "cmasks": cmasks, + "vfeats": vfeats, + "vmasks": vmasks, + "video_id": video_id, + } + + +class NLGTextProcessor(TextProcessor): + """ + Also return the original text as ref. + """ + def __call__(self, text_id): + return super().__call__(text_id), text_id + + +class DSNLGAligner(DSAligner): + """extend with the capability of 2d mask for generation.""" + def __init__(self, config): + super().__init__(config) + self.attnmasker = MMAttentionMask2DProcessor() + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained( + self.bert_name, use_fast=self.use_fast, + bos_token="[CLS]", eos_token="[SEP]" + ) + self.tokenizer = tokenizer + self.bos_token_id = tokenizer.bos_token_id + self.eos_token_id = tokenizer.eos_token_id + self.textgen = TextGenerationProcessor(tokenizer) + + def __call__(self, video_id, video_feature, text_feature): + output = super().__call__(video_id, video_feature, text_feature[0]) + if self.split == "test": + # output.update({"ref": text_feature[1]}) + output.update({"ref": self.tokenizer.decode( + output["caps"], skip_special_tokens=True)}) + text_label = output["caps"] + cmasks = torch.BoolTensor([1] * text_label.size(0)) + caps = torch.LongTensor([ + self.cls_token_id, + self.sep_token_id, + self.bos_token_id]) + else: + caps, text_label = self.textgen(output["caps"]) + cmasks = output["cmasks"] + + attention_mask = self.attnmasker( + output["vmasks"], cmasks, "textgen") + + output.update({ + "caps": caps, + "cmasks": cmasks, + "text_label": text_label, + "attention_mask": attention_mask, + }) + return output + + +# -------------------- MSRVTT ------------------------ + + +class MSRVTTMetaProcessor(MetaProcessor): + """MSRVTT dataset. + reference: `howto100m/msrvtt_dataloader.py` + """ + + def __init__(self, config): + super().__init__(config) + import pandas as pd + data = pd.read_csv(self._get_split_path(config)) + # TODO: add a text1ka flag. + if config.split == "train" \ + and config.full_test_path is not None \ + and config.jsfusion_path is not None: + # add testing videos from full_test_path not used by jfusion. + additional_data = pd.read_csv(config.full_test_path) + jsfusion_data = pd.read_csv(config.jsfusion_path) + + for video_id in additional_data["video_id"]: + if video_id not in jsfusion_data["video_id"].values: + data = data.append( + {"video_id": video_id}, ignore_index=True) + + if config.dup is not None and config.split == "train": + data = data.append([data] * (config.dup - 1), ignore_index=True) + self.data = data + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + """slightly modify with if condition to combine train/test.""" + vid, sentence = None, None + vid = self.data["video_id"].values[idx] + if "sentence" in self.data: # for testing. + sentence = self.data["sentence"].values[idx] + else: # for training. + sentence = vid + return vid, sentence + + +class MSRVTTTextProcessor(TextProcessor): + """MSRVTT dataset. + reference: `msrvtt_dataloader.py` `MSRVTT_TrainDataLoader`. + TODO (huxu): add max_words. + """ + + def __init__(self, config): + super().__init__(config) + self.sentences = None + if config.json_path is not None and config.split == "train": + with open(config.json_path) as fd: + self.data = json.load(fd) + self.sentences = defaultdict(list) + for s in self.data["sentences"]: + self.sentences[s["video_id"]].append(s["caption"]) + + def __call__(self, text_id): + if self.sentences is not None: + rind = random.randint(0, len(self.sentences[text_id]) - 1) + sentence = self.sentences[text_id][rind] + else: + sentence = text_id + caption = self.tokenizer(sentence, add_special_tokens=False) + return caption["input_ids"] + + +class MSRVTTNLGTextProcessor(MSRVTTTextProcessor): + """TODO: change dsaligner and merge to avoid any NLG text processor.""" + def __call__(self, text_id): + if self.sentences is not None: + rind = random.randint(0, len(self.sentences[text_id]) - 1) + sentence = self.sentences[text_id][rind] + else: + sentence = text_id + caption = self.tokenizer(sentence, add_special_tokens=False) + return caption["input_ids"], sentence + + +class MSRVTTQAMetaProcessor(MetaProcessor): + """MSRVTT-QA: retrieval-based multi-choice QA from JSFusion dataset. + For simplicity, we use the train retrieval model. + reference: `https://github.com/yj-yu/lsmdc` + """ + + def __init__(self, config): + super().__init__(config) + import pandas as pd + csv_data = pd.read_csv(self._get_split_path(config), sep="\t") + data = [] + for video_id, a1, a2, a3, a4, a5, answer in zip( + csv_data["vid_key"].values, + csv_data["a1"].values, + csv_data["a2"].values, + csv_data["a3"].values, + csv_data["a4"].values, + csv_data["a5"].values, + csv_data["answer"].values): + video_id = video_id.replace("msr", "video") + data.append((video_id, (answer, [a1, a2, a3, a4, a5]))) + self.data = data + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + return self.data[idx] + + +class MSRVTTQATextProcessor(TextProcessor): + """MSRVTT-QA dataset. + text_ans is of format `(answer, [a1, a2, a3, a4, a5])`. + """ + + def __call__(self, text_ans): + for ans_idx, ans in enumerate(text_ans[1]): + if isinstance(ans, str): + text_ans[1][ans_idx] = self.tokenizer(ans, add_special_tokens=False)["input_ids"] + return text_ans + + +class MSRVTTQAAligner(DSAligner): + """MSRVTT dataset. + similar to sample in how2. + we call __call__ multiple times. + """ + + def __call__(self, video_id, video_feature, text_feature, wps=0.7): + caps = [] + cmasks = [] + answer = text_feature[0] + for ans_idx, _text_feature in enumerate(text_feature[1]): + output = super().__call__( + video_id, video_feature, _text_feature, wps) + caps.append(output["caps"]) + cmasks.append(output["cmasks"]) + output.update({ + "caps": torch.stack(caps), + "cmasks": torch.stack(cmasks), + "answers": torch.LongTensor([answer]), + }) + return output + + +# -------------------- Youcook ----------------------- + + +class YoucookMetaProcessor(MetaProcessor): + """Youcook dataset. + reference: `howto100m/youcook_dataloader.py` + note that the data can be different as the + (1) some videos already in Howto100m are removed. + (2) stop words are removed from caption + TODO (huxu): make a flag to load the original caption. + (see youcookii_annotations_trainval.json). + + The max_video_len can be 264 and text can be 64 tokens. + In reality we may not need that long. see projects/task/youcook.yaml + """ + + def __init__(self, config): + super().__init__(config) + vfeat_dir = config.vfeat_dir + print(self._get_split_path(config)) + with open(self._get_split_path(config), "rb") as fd: + data = pickle.load(fd) + all_valid_video_ids = set( + [os.path.splitext(fn)[0] for fn in os.listdir(vfeat_dir)] + ) + recs = [] + video_ids = set() + valid_video_ids = set() + for rec in data: # filter videos not available. + udl_idx = rec["id"].rindex("_") + video_id = rec["id"][:udl_idx] + video_ids.add(video_id) + if video_id in all_valid_video_ids: + valid_video_ids.add(video_id) + recs.append(rec) + print("total video_ids in .pkl", len(video_ids)) + print("valid video_ids in .pkl", len(valid_video_ids)) + print("please verify {train,val}_list.txt") + data = recs + self.data = data + + with open(config.trainval_annotation) as fd: + self.youcook_annotation = json.load(fd)["database"] + if config.use_annotation_text is True: + print("using text in annotation.") + self.use_annotation_caption = True + else: + self.use_annotation_caption = False + + def __getitem__(self, idx): + def _get_video_and_caption(rec): + vid = rec["id"] + udl_idx = vid.rindex("_") + video_id, clip_id = vid[:udl_idx], int(vid[udl_idx + 1:]) + clip = self.youcook_annotation[video_id]["annotations"][clip_id] + start, end = clip["segment"] + if self.use_annotation_caption: + caption = clip["sentence"] + else: + caption = rec["caption"] + return (video_id, start, end), caption + + rec = self.data[idx] + video_info, text_info = _get_video_and_caption(rec) + return video_info, text_info + + +class YoucookVideoProcessor(VideoProcessor): + """video_fn is a tuple of (video_id, start, end) now.""" + + def __call__(self, video_fn): + video_id, start, end = video_fn + feat = np.load(os.path.join(self.vfeat_dir, video_id + ".npy")) + return feat[start:end] + + +class YoucookNLGMetaProcessor(MetaProcessor): + """NLG uses the original split: + `train_list.txt` and `val_list.txt` + """ + + def __init__(self, config): + super().__init__(config) + vfeat_dir = config.vfeat_dir + print(self._get_split_path(config)) + with open(self._get_split_path(config)) as fd: + video_ids = [ + line.strip().split("/")[1] for line in fd.readlines()] + print("total video_ids in train/val_list.txt", len(video_ids)) + + all_valid_video_ids = set( + [os.path.splitext(fn)[0] for fn in os.listdir(vfeat_dir)] + ) + video_ids = [ + video_id for video_id in video_ids + if video_id in all_valid_video_ids] + + print("valid video_ids in train/val_list.txt", len(video_ids)) + with open(config.trainval_annotation) as fd: + self.youcook_annotation = json.load(fd)["database"] + + data = [] + for video_id in video_ids: + for clip in self.youcook_annotation[video_id]["annotations"]: + start, end = clip["segment"] + caption = clip["sentence"] + data.append(((video_id, start, end), caption)) + self.data = data + + def __getitem__(self, idx): + return self.data[idx] + + +# --------------------- CrossTask ------------------------- + +class CrossTaskMetaProcessor(MetaProcessor): + def __init__(self, config): + super().__init__(config) + np.random.seed(0) # deterministic random split. + task_vids = self._get_vids( + config.train_csv_path, + config.vfeat_dir, + config.annotation_path) + + val_vids = self._get_vids( + config.val_csv_path, + config.vfeat_dir, + config.annotation_path) + + # filter out those task and vids appear in val_vids. + task_vids = { + task: [ + vid for vid in vids + if task not in val_vids or vid not in val_vids[task]] + for task, vids in task_vids.items()} + + primary_info = self._read_task_info(config.primary_path) + test_tasks = set(primary_info['steps'].keys()) + + # if args.use_related: + related_info = self._read_task_info(config.related_path) + task_steps = {**primary_info['steps'], **related_info['steps']} + n_steps = {**primary_info['n_steps'], **related_info['n_steps']} + # else: + # task_steps = primary_info['steps'] + # n_steps = primary_info['n_steps'] + all_tasks = set(n_steps.keys()) + # filter and keep task in primary or related. + task_vids = { + task: vids for task, vids in task_vids.items() + if task in all_tasks} + # vocab-by-step matrix (A) and vocab (M) + # (huxu): we do not use BoW. + # A, M = self._get_A(task_steps, share="words") + + train_vids, test_vids = self._random_split( + task_vids, test_tasks, config.n_train) + print("train_num_videos", sum(len(vids) for vids in train_vids.values())) + print("test_num_videos", sum(len(vids) for vids in test_vids.values())) + # added by huxu to automatically determine the split. + split_map = { + "train": train_vids, + "valid": test_vids, + "test": test_vids + } + task_vids = split_map[config.split] + + self.vids = [] + for task, vids in task_vids.items(): + self.vids.extend([(task, vid) for vid in vids]) + self.task_steps = task_steps + self.n_steps = n_steps + + def __getitem__(self, idx): + task, vid = self.vids[idx] + n_steps = self.n_steps[task] + steps = self.task_steps[task] + assert len(steps) == n_steps + return (task, vid, steps, n_steps), (task, vid, steps, n_steps) + + def __len__(self): + return len(self.vids) + + def _random_split(self, task_vids, test_tasks, n_train): + train_vids = {} + test_vids = {} + for task, vids in task_vids.items(): + if task in test_tasks and len(vids) > n_train: + train_vids[task] = np.random.choice( + vids, n_train, replace=False).tolist() + test_vids[task] = [ + vid for vid in vids if vid not in train_vids[task]] + else: + train_vids[task] = vids + return train_vids, test_vids + + def _get_vids(self, path, vfeat_dir, annotation_path): + """refactored from + https://github.com/DmZhukov/CrossTask/blob/master/data.py + changes: add `vfeat_dir` to check if the video is available. + add `annotation_path` to check if the video is available. + """ + + task_vids = {} + with open(path, 'r') as f: + for line in f: + task, vid, url = line.strip().split(',') + # double check the video is available. + if not os.path.exists( + os.path.join(vfeat_dir, vid + ".npy")): + continue + # double check the annotation is available. + if not os.path.exists(os.path.join( + annotation_path, + task + "_" + vid + ".csv")): + continue + if task not in task_vids: + task_vids[task] = [] + task_vids[task].append(vid) + return task_vids + + def _read_task_info(self, path): + titles = {} + urls = {} + n_steps = {} + steps = {} + with open(path, 'r') as f: + idx = f.readline() + while idx != '': + idx = idx.strip() + titles[idx] = f.readline().strip() + urls[idx] = f.readline().strip() + n_steps[idx] = int(f.readline().strip()) + steps[idx] = f.readline().strip().split(',') + next(f) + idx = f.readline() + return { + 'title': titles, + 'url': urls, + 'n_steps': n_steps, + 'steps': steps + } + + def _get_A(self, task_steps, share="words"): + raise ValueError("running get_A is not allowed for BERT.") + """Step-to-component matrices.""" + if share == 'words': + # share words + task_step_comps = { + task: [step.split(' ') for step in steps] + for task, steps in task_steps.items()} + elif share == 'task_words': + # share words within same task + task_step_comps = { + task: [[task+'_'+tok for tok in step.split(' ')] for step in steps] + for task, steps in task_steps.items()} + elif share == 'steps': + # share whole step descriptions + task_step_comps = { + task: [[step] for step in steps] for task, steps in task_steps.items()} + else: + # no sharing + task_step_comps = { + task: [[task+'_'+step] for step in steps] + for task, steps in task_steps.items()} + # BERT tokenizer here? + vocab = [] + for task, steps in task_step_comps.items(): + for step in steps: + vocab.extend(step) + vocab = {comp: m for m, comp in enumerate(set(vocab))} + M = len(vocab) + A = {} + for task, steps in task_step_comps.items(): + K = len(steps) + a = torch.zeros(M, K) + for k, step in enumerate(steps): + a[[vocab[comp] for comp in step], k] = 1 + a /= a.sum(dim=0) + A[task] = a + return A, M + + +class CrossTaskVideoProcessor(VideoProcessor): + def __call__(self, video_fn): + task, vid, steps, n_steps = video_fn + video_fn = os.path.join(self.vfeat_dir, vid + ".npy") + feat = np.load(video_fn) + return feat + + +class CrossTaskTextProcessor(TextProcessor): + def __call__(self, text_id): + task, vid, steps, n_steps = text_id + step_ids = [] + for step_str in steps: + step_ids.append( + self.tokenizer(step_str, add_special_tokens=False)["input_ids"] + ) + return step_ids + + +class CrossTaskAligner(Aligner): + """ + TODO: it's not clear yet the formulation of the task; finish this later. + """ + def __init__(self, config): + super().__init__(config) + self.annotation_path = config.annotation_path + self.sliding_window = config.sliding_window + self.sliding_window_size = config.sliding_window_size + + def __call__(self, video_id, video_feature, text_feature): + task, vid, steps, n_steps = video_id + annot_path = os.path.join( + self.annotation_path, task + '_' + vid + '.csv') + video_len = len(video_feature) + + labels = torch.from_numpy(self._read_assignment( + video_len, n_steps, annot_path)).float() + + vfeats, vmasks, targets = [], [], [] + # sliding window on video features and targets. + for window_start in range(0, video_len, self.sliding_window): + video_start = 0 + video_end = min(video_len - window_start, self.sliding_window_size) + video_clip = {"start": [video_start], "end": [video_end]} + + vfeat, vmask = self._build_video_seq( + video_feature[window_start: window_start + video_end], + video_clip + ) + + target = labels[window_start: window_start + video_end] + assert len(vfeat) >= len(target), "{},{}".format(len(vfeat), len(target)) + # TODO: randomly drop all zero targets for training ? + # if self.split == "train" and target.sum() == 0: + # continue + vfeats.append(vfeat) + vmasks.append(vmask) + targets.append(target) + + if (video_len - window_start) <= self.sliding_window_size: + break + + vfeats = torch.stack(vfeats) + vmasks = torch.stack(vmasks) + targets = torch.cat(targets, dim=0) + + caps, cmasks = [], [] + for step in text_feature: + step_text_feature = {"start": [0], "end": [1], "cap": [step]} + step_text_clip_index = [0] + cap, cmask = self._build_text_seq( + step_text_feature, step_text_clip_index + ) + caps.append(cap) + cmasks.append(cmask) + caps = torch.stack(caps) + cmasks = torch.stack(cmasks) + + return { + "caps": caps, + "cmasks": cmasks, + "vfeats": vfeats, # X for original code. + "vmasks": vmasks, + "targets": targets, + "video_id": vid, + "task": task, + "video_len": video_len # for later checking. + } + + def _read_assignment(self, T, K, path): + """ + refactored from https://github.com/DmZhukov/CrossTask/blob/master/data.py + Howto interpret contraints on loss that is going to be minimized: + lambd is a big number; + self.lambd * C is a big number for all valid position (csv stores invalids) + + def forward(self, O, Y, C): + return (Y*(self.lambd * C - self.lsm(O))).mean(dim=0).sum() + + This will load the csv file and fill-in the step col from start to end rows. + """ + + Y = np.zeros([T, K], dtype=np.uint8) + with open(path, 'r') as f: + for line in f: + step, start, end = line.strip().split(',') + start = int(math.floor(float(start))) + end = int(math.ceil(float(end))) + step = int(step) - 1 + Y[start:end, step] = 1 + return Y + + +# --------------------- COIN ------------------------- + +class MetaTextBinarizer(Aligner): + def __call__(self, text_feature): + text_feature = { + "cap": [text_feature], + "start": [0.], + "end": [100.], + } + text_clip_indexs = [0] + + caps, cmasks = self._build_text_seq( + text_feature, text_clip_indexs + ) + return {"caps": caps, "cmasks": cmasks} + + +class COINActionSegmentationMetaProcessor(MetaProcessor): + split_map = { + "train": "training", + "valid": "testing", + "test": "testing", + } + + def __init__(self, config): + super().__init__(config) + with open(self._get_split_path(config)) as fr: + database = json.load(fr)["database"] + id2label = {} + data = [] + # filter the data by split. + for video_id, rec in database.items(): + # always use testing to determine label_set + if rec["subset"] == "testing": + for segment in rec["annotation"]: + id2label[int(segment["id"])] = segment["label"] + # text_labels is used for ZS setting + self.text_labels = ["none"] * len(id2label) + for label_id in id2label: + self.text_labels[label_id-1] = id2label[label_id] + + id2label[0] = "O" + print("num of labels", len(id2label)) + + for video_id, rec in database.items(): + if not os.path.isfile(os.path.join(config.vfeat_dir, video_id + ".npy")): + continue + if rec["subset"] == COINActionSegmentationMetaProcessor.split_map[self.split]: + starts, ends, labels = [], [], [] + for segment in rec["annotation"]: + start, end = segment["segment"] + label = int(segment["id"]) + starts.append(start) + ends.append(end) + labels.append(label) + data.append( + (video_id, {"start": starts, "end": ends, "label": labels})) + self.data = data + + def meta_text_labels(self, config): + from transformers import default_data_collator + from ..utils import get_local_rank + + text_processor = TextProcessor(config) + binarizer = MetaTextBinarizer(config) + # TODO: add prompts to .yaml. + text_labels = [label for label in self.text_labels] + + if get_local_rank() == 0: + print(text_labels) + + outputs = [] + for text_label in text_labels: + text_feature = text_processor(text_label) + outputs.append(binarizer(text_feature)) + return default_data_collator(outputs) + + def __getitem__(self, idx): + return self.data[idx] + + +class COINActionSegmentationTextProcessor(TextProcessor): + def __call__(self, text_label): + return text_label + + +class COINActionSegmentationAligner(Aligner): + def __init__(self, config): + super().__init__(config) + self.sliding_window = config.sliding_window + self.sliding_window_size = config.sliding_window_size + + def __call__(self, video_id, video_feature, text_feature): + starts, ends, label_ids = text_feature["start"], text_feature["end"], text_feature["label"] + # sliding window. + video_len = len(video_feature) + + vfeats, vmasks, targets = [], [], [] + # sliding window on video features and targets. + for window_start in range(0, video_len, self.sliding_window): + video_start = 0 + video_end = min(video_len - window_start, self.sliding_window_size) + video_clip = {"start": [video_start], "end": [video_end]} + vfeat, vmask = self._build_video_seq( + video_feature[window_start: window_start + video_end], + video_clip + ) + # covers video length only. + target = torch.full_like(vmask, -100, dtype=torch.long) + target[vmask] = 0 + for start, end, label_id in zip(starts, ends, label_ids): + if (window_start < end) and (start < (window_start + video_end)): + start_offset = max(0, math.floor(start) - window_start) + end_offset = min(video_end, math.ceil(end) - window_start) + target[start_offset:end_offset] = label_id + vfeats.append(vfeat) + vmasks.append(vmask) + targets.append(target) + if (video_len - window_start) <= self.sliding_window_size: + break + + vfeats = torch.stack(vfeats) + vmasks = torch.stack(vmasks) + targets = torch.stack(targets) + video_targets = torch.full((video_len,), 0) + for start, end, label_id in zip(starts, ends, label_ids): + start_offset = max(0, math.floor(start)) + end_offset = min(video_len, math.ceil(end)) + video_targets[start_offset:end_offset] = label_id + + caps = torch.LongTensor( + [[self.cls_token_id, self.sep_token_id, + self.pad_token_id, self.sep_token_id]], + ).repeat(vfeats.size(0), 1) + cmasks = torch.BoolTensor( + [[0, 1, 0, 1]] # pad are valid for attention. + ).repeat(vfeats.size(0), 1) + return { + "caps": caps, + "cmasks": cmasks, + "vfeats": vfeats, # X for original code. + "vmasks": vmasks, + "targets": targets, + "video_id": video_id, + "video_len": video_len, # for later checking. + "video_targets": video_targets + } + + +class DiDeMoMetaProcessor(MetaProcessor): + """reference: https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/eval.py + https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/data_processing.py + """ + def __init__(self, config): + super().__init__(config) + + assert "test" in self._get_split_path(config), "DiDeMo only supports zero-shot testing for now." + + with open(self._get_split_path(config)) as data_file: + json_data = json.load(data_file) + + data = [] + for record in json_data: + data.append((record["video"], record["description"])) + self.data = data + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + return self.data[idx] + + +class DiDeMoTextProcessor(TextProcessor): + """reference: https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/eval.py + https://github.com/LisaAnne/LocalizingMoments/blob/master/utils/data_processing.py + """ + + def __call__(self, text): + return self.tokenizer(text, add_special_tokens=False)["input_ids"] + + +class DiDeMoAligner(DSAligner): + """ + check video length. + """ + + def __call__(self, video_id, video_feature, text_feature): + # print(video_feature.shape[0]) + return super().__call__(video_id, video_feature, text_feature) diff --git a/data/fairseq/examples/MMPT/mmpt/processors/how2processor.py b/data/fairseq/examples/MMPT/mmpt/processors/how2processor.py new file mode 100644 index 0000000000000000000000000000000000000000..bed2168b1df28babc7a12e81b6bc31d36d73bc99 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/processors/how2processor.py @@ -0,0 +1,887 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright (c) Facebook, Inc. All Rights Reserved + + +import torch +import math +import pickle +import random +import os +import numpy as np + +from collections import deque +from typing import Optional, Tuple, List +from .processor import ( + Processor, + MetaProcessor, + TextProcessor, + Aligner, + MMAttentionMask2DProcessor +) + +from ..utils import ShardedTensor + + +class How2MetaProcessor(MetaProcessor): + def __init__(self, config): + super().__init__(config) + path = self._get_split_path(config) + with open(path) as fd: + self.data = [line.strip() for line in fd] + + def __getitem__(self, idx): + video_id = self.data[idx] + return video_id, video_id + + +class ShardedHow2MetaProcessor(How2MetaProcessor): + def __init__(self, config): + super().__init__(config) + self.split = str(config.split) + self.vfeat_dir = config.vfeat_dir + self._init_shard() + + def _init_shard(self): + if self.split == "train": + meta_fn = os.path.join(self.vfeat_dir, "train" + "_meta.pkl") + with open(meta_fn, "rb") as fr: + meta = pickle.load(fr) + elif self.split == "valid": + meta_fn = os.path.join(self.vfeat_dir, "val" + "_meta.pkl") + with open(meta_fn, "rb") as fr: + meta = pickle.load(fr) + elif self.split == "test": + print("use how2 val as test.") + meta_fn = os.path.join(self.vfeat_dir, "val" + "_meta.pkl") + with open(meta_fn, "rb") as fr: + meta = pickle.load(fr) + else: + raise ValueError("unsupported for MetaProcessor:", self.split) + video_id_to_shard = {} + for shard_id in meta: + for video_idx, video_id in enumerate(meta[shard_id]): + video_id_to_shard[video_id] = (shard_id, video_idx) + self.video_id_to_shard = video_id_to_shard + + def __getitem__(self, idx): + video_id, video_id = super().__getitem__(idx) + shard_id, shard_idx = self.video_id_to_shard[video_id] + meta = (video_id, idx, shard_id, shard_idx) + return meta, meta + + +class ShardedVideoProcessor(Processor): + """ + mmaped shards of numpy video features. + """ + + def __init__(self, config): + self.split = str(config.split) + self.vfeat_dir = config.vfeat_dir + + def __call__(self, video_id): + _, _, shard_id, video_idx = video_id + if self.split == "train": + shard = ShardedTensor.load( + os.path.join(self.vfeat_dir, "train" + "_" + str(shard_id)), + "r" + ) + elif self.split == "valid": + shard = ShardedTensor.load( + os.path.join(self.vfeat_dir, "val" + "_" + str(shard_id)), + "r" + ) + elif self.split == "test": + shard = ShardedTensor.load( + os.path.join(self.vfeat_dir, "val" + "_" + str(shard_id)), + "r" + ) + else: + raise ValueError("unknown split", self.split) + feat = shard[video_idx] + return feat + + +class ShardedTextProcessor(Processor): + def __init__(self, config): + self.tfeat_dir = str(config.tfeat_dir) + self.split = str(config.split) + + def __call__(self, video_id): + _, _, shard_id, shard_idx = video_id + if self.split == "train": + target_path = self.tfeat_dir + "train" + "_" + str(shard_id) + elif self.split == "valid": + target_path = self.tfeat_dir + "val" + "_" + str(shard_id) + elif self.split == "test": + target_path = self.tfeat_dir + "val" + "_" + str(shard_id) + else: + raise ValueError("unknown split", self.split) + + startend = ShardedTensor.load( + target_path + ".startends", "r")[shard_idx] + cap_ids = ShardedTensor.load( + target_path + ".caps_ids", "r")[shard_idx] + cap = [] + for clip_idx in range(len(cap_ids)): + clip = cap_ids[clip_idx] + cap.append(clip[clip != -1].tolist()) + start, end = startend[:, 0].tolist(), startend[:, 1].tolist() + return {"start": start, "end": end, "cap": cap} + + +class FixedLenAligner(Aligner): + """ + In the model we assume text is on the left (closer to BERT formulation) + and video is on the right. + We fix the total length of text + video. + max_video_len is in number of secs. + max_text_len is in number of tokens. + + special tokens formats: + we use the format [CLS] [SEP] text tokens [SEP] [PAD] ... + [CLS] will be splitted out into: + [CLS] video tokens [SEP] text tokens [SEP] [PAD] ... + token_type_ids will be generated by the model (for now). + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence | + so each sequence owns a [SEP] token for no-ops. + """ + + def __init__(self, config): + super().__init__(config) + self.text_clip_sampler = TextClipSamplingProcessor( + self.max_len - self.max_video_len - 3 + ) + """ + decide subsampling: + `config.subsampling` will change batch_size in trainer. + `config.clip_per_video` (used by RetriTask) doesn't + change batch_size in trainer. + """ + subsampling = config.subsampling \ + if config.subsampling is not None else None + if config.clip_per_video is not None: + subsampling = config.clip_per_video + self.subsampling = subsampling + + def _get_text_maxlen(self): + # use max text len + return self.text_clip_sampler.max_text_len + + def __call__(self, video_id, video_feature, text_feature): + from transformers import default_data_collator + video_idx = video_id[1] + if self.subsampling is not None and self.subsampling >= 1: + batch = [] + for _ in range(self.subsampling): + centerclip_idx = random.randint( + 0, len(text_feature["start"]) - 1) + batch.append( + self.sampling( + video_idx, + video_feature, + text_feature, + centerclip_idx, + self._get_text_maxlen() + )) + batch = self.batch_post_processing(batch, video_feature) + batch = default_data_collator(batch) + else: + raise ValueError( + "dataset.subsampling must be >= 1 for efficient video loading.") + batch = self.sampling(video_idx, video_feature, text_feature) + batch = self.batch_post_processing(batch, video_feature) + + batch["video_id"] = video_id if isinstance(video_id, str) \ + else video_id[0] + # e2e: make sure frame ids is into tensor. + assert torch.is_tensor(batch["vfeats"]) + return batch + + def sampling( + self, + video_idx, + video_feature, + text_feature, + centerclip_idx=None, + sampled_max_text_len=None, + ): + text_clip_indexs = self.text_clip_sampler( + text_feature, centerclip_idx, + sampled_max_text_len + ) + if isinstance(video_feature, np.ndarray): + video_len = len(video_feature) + else: + video_len = math.ceil(text_feature["end"][-1]) + + video_end = min( + math.ceil(text_feature["end"][text_clip_indexs[-1]]), + video_len + ) + video_start = max( + min( + math.floor(text_feature["start"][text_clip_indexs[0]]), + video_end), + 0 + ) + + video_clips = {"start": [video_start], "end": [video_end]} + + # tensorize. + vfeats, vmasks = self._build_video_seq( + video_feature, video_clips + ) + caps, cmasks = self._build_text_seq( + text_feature, text_clip_indexs + ) + + text_start = text_clip_indexs[0] + text_end = text_clip_indexs[-1] + 1 + + return { + "caps": caps, + "cmasks": cmasks, + "vfeats": vfeats, + "vmasks": vmasks, + "video_start": video_start, + "video_end": video_end, + "text_start": text_start, + "text_end": text_end, + } + + +class VariedLenAligner(FixedLenAligner): + def __init__(self, config): + super().__init__(config) + self.sampled_min_len = config.sampled_min_len + self.sampled_max_len = config.sampled_max_len + + def _get_text_maxlen(self): + return random.randint(self.sampled_min_len, self.sampled_max_len) + + +class StartClipAligner(VariedLenAligner): + def sampling( + self, + video_idx, + video_feature, + text_feature, + centerclip_idx=None, + sampled_max_text_len=None, + ): + return super().sampling( + video_idx, video_feature, text_feature, 0) + + +class OverlappedAligner(VariedLenAligner): + """video clip and text clip has overlappings + but may not be the same start/end.""" + def __init__(self, config): + super().__init__(config) + self.sampled_video_min_len = config.sampled_video_min_len + self.sampled_video_max_len = config.sampled_video_max_len + + self.video_clip_sampler = VideoClipSamplingProcessor() + + def _get_video_maxlen(self): + return random.randint( + self.sampled_video_min_len, self.sampled_video_max_len) + + def sampling( + self, + video_idx, + video_feature, + text_feature, + centerclip_idx=None, + sampled_max_text_len=None, + ): + text_clip_indexs = self.text_clip_sampler( + text_feature, centerclip_idx, + sampled_max_text_len + ) + if isinstance(video_feature, np.ndarray): + video_len = len(video_feature) + else: + video_len = math.ceil(text_feature["end"][-1]) + low = math.floor(text_feature["start"][text_clip_indexs[0]]) + high = math.ceil(text_feature["end"][text_clip_indexs[-1]]) + if low < high: + center = random.randint(low, high) + else: + center = int((low + high) // 2) + center = max(0, min(video_feature.shape[0] - 1, center)) + + assert 0 <= center < video_feature.shape[0] + + video_clips = self.video_clip_sampler( + video_len, self._get_video_maxlen(), center + ) + video_start = video_clips["start"][0] + video_end = video_clips["end"][0] + + # tensorize. + vfeats, vmasks = self._build_video_seq( + video_feature, video_clips + ) + caps, cmasks = self._build_text_seq( + text_feature, text_clip_indexs + ) + + text_start = text_clip_indexs[0] + text_end = text_clip_indexs[-1] + 1 + + return { + "caps": caps, + "cmasks": cmasks, + "vfeats": vfeats, + "vmasks": vmasks, + "video_start": video_start, + "video_end": video_end, + "text_start": text_start, + "text_end": text_end, + } + + +class MFMMLMAligner(FixedLenAligner): + """ + `FixedLenAligner` with Masked Language Model and Masked Frame Model. + """ + + def __init__(self, config): + super().__init__(config) + keep_prob = config.keep_prob if config.keep_prob is not None else 1.0 + self.text_clip_sampler = TextClipSamplingProcessor( + self.max_len - self.max_video_len - 3, keep_prob + ) + self.sampled_min_len = config.sampled_min_len + self.sampled_max_len = config.sampled_max_len + self.masked_token_sampler = TextMaskingProcessor(config) + self.mm_type = config.mm_type \ + if config.mm_type is not None else "full" + self.attnmasker = MMAttentionMask2DProcessor() \ + if self.mm_type == "textgen" else None + self.masked_frame_sampler = FrameMaskingProcessor(config) + self.lazy_vfeat_mask = ( + False if config.lazy_vfeat_mask is None else config.lazy_vfeat_mask + ) + self.mm_prob = config.mm_prob if config.mm_prob is not None else 0. + + def __call__(self, video_id, video_feature, text_feature): + from transformers import default_data_collator + if self.subsampling is not None and self.subsampling > 1: + batch = [] + for _ in range(self.subsampling): + centerclip_idx = random.randint( + 0, len(text_feature["start"]) - 1) + sampled_max_text_len = random.randint( + self.sampled_min_len, self.sampled_max_len + ) + batch.append( + self.sampling( + video_id, + video_feature, + text_feature, + centerclip_idx, + sampled_max_text_len, + ) + ) + batch = self.batch_post_processing(batch, video_feature) + batch = default_data_collator(batch) + else: + batch = self.sampling(video_id, video_feature, text_feature) + batch = self.batch_post_processing(batch, video_feature) + batch["video_id"] = video_id if isinstance(video_id, str) \ + else video_id[0] + return batch + + def sampling( + self, + video_id, + video_feature, + text_feature, + centerclip_idx=None, + sampled_max_text_len=None, + ): + output = FixedLenAligner.sampling(self, + video_id, video_feature, text_feature, + centerclip_idx, sampled_max_text_len) + + masking_text, masking_video = None, None + if random.random() < self.mm_prob: + if random.random() > 0.5: + masking_text, masking_video = self.mm_type, "no" + else: + masking_text, masking_video = "no", "full" + video_feats = output["vfeats"] if not self.lazy_vfeat_mask else None + video_label = self.masked_frame_sampler( + output["vmasks"], masking_video, vfeats=video_feats) + caps, text_label = self.masked_token_sampler( + output["caps"], masking_text) + + output.update({ + "caps": caps, + "video_label": video_label, + "text_label": text_label, + }) + + if self.attnmasker is not None: + attention_mask = self.attnmasker( + output["vmasks"], output["cmasks"], masking_text) + output.update({ + "attention_mask": attention_mask + }) + return output + + +class FrameMaskingProcessor(Processor): + def __init__(self, config): + self.mfm_probability = 0.15 + if config.mfm_probability is not None: + self.mfm_probability = config.mfm_probability + + def __call__(self, vmasks, modality_masking=None, vfeats=None): + """ + We perform lazy masking to save data transfer time. + It only generates video_labels by default and MFM model + will do actualy masking. + Return: `video_label` is a binary mask. + """ + video_label = vmasks.clone() + if modality_masking is not None: + if modality_masking == "full": + probability_matrix = torch.full(video_label.shape, 1.) + elif modality_masking == "no": + probability_matrix = torch.full(video_label.shape, 0.) + elif modality_masking == "inverse": + probability_matrix = torch.full( + video_label.shape, 1. - self.mfm_probability) + else: + raise ValueError("unknown modality masking.", modality_masking) + else: + probability_matrix = torch.full( + video_label.shape, self.mfm_probability) + masked_indices = torch.bernoulli(probability_matrix).bool() + # We only compute loss on masked tokens + video_label[~masked_indices] = 0 + if vfeats is not None: + vfeats[video_label, :] = 0.0 + return video_label + + +class TextGenerationProcessor(Processor): + def __init__(self, tokenizer): + self.bos_token_id = tokenizer.bos_token_id + self.pad_token_id = tokenizer.pad_token_id + + def __call__(self, inputs): + labels = inputs.clone() + # [CLS] [SEP] for video + labels[:2] = -100 + # keep [SEP] for text. + pad_mask = labels == self.pad_token_id + labels[pad_mask] = -100 + inputs[2:] = torch.cat([ + torch.LongTensor([self.bos_token_id]), + inputs[2:-1]]) + inputs[pad_mask] = self.pad_token_id + assert len(inputs) == len(labels) + return inputs, labels + + +class TextMaskingProcessor(Processor): + def __init__(self, config): + """this function is borrowed from + `transformers/data/data_collator.DataCollatorForLanguageModeling`""" + self.mlm_probability = 0.15 + if config.mlm_probability is not None: + self.mlm_probability = config.mlm_probability + self.bert_name = config.bert_name + # [CLS] is used as bos_token and [SEP] is used as eos_token. + # https://huggingface.co/transformers/master/model_doc/bertgeneration.html + from transformers import AutoTokenizer + self.tokenizer = AutoTokenizer.from_pretrained( + self.bert_name, bos_token="[CLS]", eos_token="[SEP]") + self.textgen = TextGenerationProcessor(self.tokenizer) + + def __call__( + self, inputs: torch.Tensor, + modality_masking=None, + special_tokens_mask: Optional[torch.Tensor] = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + expand modality_masking into + None: traditional bert masking. + "no": no masking. + "full": all [MASK] token for generation. + "gen": autoregressive generation. + """ + """ + Prepare masked tokens inputs/labels for masked language modeling: + 80% MASK, 10% random, 10% original. + """ + labels = inputs.clone() + # We sample a few tokens in each sequence for MLM training + # (with probability `self.mlm_probability`) + if modality_masking is not None: + if modality_masking == "full": + probability_matrix = torch.full(labels.shape, 1.) + elif modality_masking == "no": + probability_matrix = torch.full(labels.shape, 0.) + elif modality_masking.startswith("textgen"): + # [CLS] [SEP] ... + inputs, labels = self.textgen(inputs) + if "mask" not in modality_masking: + return inputs, labels + inputs = self.mask_input(inputs, special_tokens_mask) + return inputs, labels + elif modality_masking == "mask": + inputs = self.mask_input(inputs, special_tokens_mask) + labels = torch.full(inputs.shape, -100) + return inputs, labels + elif modality_masking == "inverse": + probability_matrix = torch.full(labels.shape, 1. - self.mlm_probability) + else: + raise ValueError("unknown modality masking.", modality_masking) + else: + probability_matrix = torch.full(labels.shape, self.mlm_probability) + + if special_tokens_mask is None: + special_tokens_mask = self.get_special_tokens_mask( + labels.tolist(), already_has_special_tokens=True + ) + special_tokens_mask = torch.tensor( + special_tokens_mask, dtype=torch.bool) + else: + special_tokens_mask = special_tokens_mask.bool() + + probability_matrix.masked_fill_(special_tokens_mask, value=0.0) + masked_indices = torch.bernoulli(probability_matrix).bool() + labels[~masked_indices] = -100 # We only compute loss on masked tokens + + # 80% of the time, + # we replace masked input tokens with tokenizer.mask_token ([MASK]) + indices_replaced = ( + torch.bernoulli( + torch.full(labels.shape, 0.8)).bool() & masked_indices + ) + inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids( + self.tokenizer.mask_token + ) + + # 10% of the time, we replace masked input tokens with random word + indices_random = ( + torch.bernoulli(torch.full(labels.shape, 0.5)).bool() + & masked_indices + & ~indices_replaced + ) + random_words = torch.randint( + len(self.tokenizer), labels.shape, dtype=torch.long + ) + inputs[indices_random] = random_words[indices_random] + + # The rest of the time (10% of the time) we keep the masked input + # tokens unchanged + return inputs, labels + + def mask_input(self, inputs, special_tokens_mask=None): + # the following is new with masked autoregressive. + probability_matrix = torch.full( + inputs.shape, self.mlm_probability) + if special_tokens_mask is None: + special_tokens_mask = self.get_special_tokens_mask( + inputs.tolist(), already_has_special_tokens=True + ) + special_tokens_mask = torch.tensor( + special_tokens_mask, dtype=torch.bool) + else: + special_tokens_mask = special_tokens_mask.bool() + probability_matrix.masked_fill_(special_tokens_mask, value=0.0) + masked_indices = torch.bernoulli(probability_matrix).bool() + indices_replaced = ( + torch.bernoulli( + torch.full(inputs.shape, 0.8)).bool() & masked_indices + ) + inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids( + self.tokenizer.mask_token + ) + + # 10% of the time, we replace masked input tokens with random word + indices_random = ( + torch.bernoulli(torch.full(inputs.shape, 0.5)).bool() + & masked_indices + & ~indices_replaced + ) + random_words = torch.randint( + len(self.tokenizer), inputs.shape, dtype=torch.long + ) + inputs[indices_random] = random_words[indices_random] + return inputs + + def get_special_tokens_mask( + self, token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None, + already_has_special_tokens: bool = False + ) -> List[int]: + """ + Note: the version from transformers do not consider pad + as special tokens. + """ + + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError( + "You should not supply a second sequence if" + "the provided sequence of " + "ids is already formated with special tokens " + "for the model." + ) + return list(map(lambda x: 1 if x in [ + self.tokenizer.sep_token_id, + self.tokenizer.cls_token_id, + self.tokenizer.pad_token_id] else 0, token_ids_0)) + + if token_ids_1 is not None: + return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1] + + +class TextClipSamplingProcessor(Processor): + def __init__(self, max_text_len, keep_prob=1.0): + self.max_text_len = max_text_len + self.max_video_len = 256 # always hold. + self.keep_prob = keep_prob + + def __call__( + self, + text_feature, + centerclip_idx=None, + sampled_max_text_len=None, + sampled_max_video_len=None, + ): + # Let's use all caps for now and see if 256 can cover all of them. + if sampled_max_text_len is not None: + max_text_len = sampled_max_text_len + else: + max_text_len = self.max_text_len + if sampled_max_video_len is not None: + max_video_len = sampled_max_video_len + else: + max_video_len = self.max_video_len + + t_num_clips = len(text_feature["start"]) + + if centerclip_idx is None: + centerclip_idx = random.randint(0, t_num_clips - 1) + + start_idx, end_idx = centerclip_idx, centerclip_idx + 1 + text_clip_indexs = deque() + text_clip_indexs.append(start_idx) + text_len = len(text_feature["cap"][start_idx]) + + video_len = max( + 0, + text_feature["end"][start_idx] + - text_feature["start"][start_idx], + ) + + while ( + (start_idx > 0 or end_idx < t_num_clips) + and text_len < max_text_len + and video_len < max_video_len + ): + if random.random() > 0.5 and end_idx < t_num_clips: + # skip the next one? + if random.random() > self.keep_prob and (end_idx + 1) < t_num_clips: + end_idx = end_idx + 1 + text_clip_indexs.append(end_idx) + text_len += len(text_feature["cap"][end_idx]) + end_idx += 1 + elif start_idx > 0: + if random.random() > self.keep_prob and (start_idx - 1) > 0: + start_idx = start_idx - 1 + start_idx -= 1 + text_clip_indexs.insert(0, start_idx) + text_len += len(text_feature["cap"][start_idx]) + else: + if end_idx < t_num_clips: + if random.random() > self.keep_prob and (end_idx + 1) < t_num_clips: + end_idx = end_idx + 1 + text_clip_indexs.append(end_idx) + text_len += len(text_feature["cap"][end_idx]) + end_idx += 1 + else: + return text_clip_indexs + video_len = max( + 0, + text_feature["end"][text_clip_indexs[-1]] + - text_feature["start"][text_clip_indexs[0]], + ) + return text_clip_indexs + + +class VideoClipSamplingProcessor(Processor): + def __call__(self, video_len, max_video_len, center): + """ + `video_len`: length of the video. + `max_video_len`: maximum video tokens allowd in a sequence. + `center`: initial starting index. + """ + assert center >= 0 and center < video_len + t_clip_len = 0 + start, end = center, center + while (start > 0 or end < video_len) and t_clip_len < max_video_len: + # decide the direction to grow. + if start <= 0: + end += 1 + elif end >= video_len: + start -= 1 + elif random.random() > 0.5: + end += 1 + else: + start -= 1 + t_clip_len += 1 + return {"start": [start], "end": [end]} + + +class How2MILNCEAligner(FixedLenAligner): + """reference: `antoine77340/MIL-NCE_HowTo100M/video_loader.py`""" + + def __init__(self, config): + super().__init__(config) + self.num_candidates = 4 + self.min_time = 5.0 + self.num_sec = 3.2 + # self.num_sec = self.num_frames / float(self.fps) num_frames=16 / fps = 5 + # self.num_frames = 16 + + def sampling( + self, + video_id, + video_feature, + text_feature, + centerclip_idx=None, # will be ignored. + sampled_max_text_len=None # will be ignored. + ): + text, start, end = self._get_text(text_feature) + video = self._get_video(video_feature, start, end) + + vfeats = torch.zeros((self.max_video_len, video_feature.shape[1])) + vmasks = torch.zeros((self.max_video_len,), dtype=torch.bool) + vfeats[: video.shape[0]] = torch.from_numpy(np.array(video)) + vmasks[: video.shape[0]] = 1 + + caps, cmasks = [], [] + for words in text: + cap, cmask = self._build_text_seq(text_feature, words) + caps.append(cap) + cmasks.append(cmask) + caps = torch.stack(caps) + cmasks = torch.stack(cmasks) + # video of shape: (video_len) + # text of shape (num_candidates, max_text_len) + + return { + "caps": caps, + "cmasks": cmasks, + "vfeats": vfeats, + "vmasks": vmasks, + # "video_id": video_id, + } + + def _get_video(self, video_feature, start, end): + start_seek = random.randint(start, int(max(start, end - self.num_sec))) + # duration = self.num_sec + 0.1 + return video_feature[start_seek : int(start_seek + self.num_sec)] + + def _get_text(self, cap): + ind = random.randint(0, len(cap["start"]) - 1) + if self.num_candidates == 1: + words = [ind] + else: + words = [] + cap_start = self._find_nearest_candidates(cap, ind) + for i in range(self.num_candidates): + words.append([max(0, min(len(cap["cap"]) - 1, cap_start + i))]) + + start, end = cap["start"][ind], cap["end"][ind] + # TODO: May need to be improved for edge cases. + # expand the min time. + if end - start < self.min_time: + diff = self.min_time - end + start + start = max(0, start - diff / 2) + end = start + self.min_time + return words, int(start), int(end) + + def _find_nearest_candidates(self, caption, ind): + """find the range of the clips.""" + start, end = ind, ind + #diff = caption["end"][end] - caption["start"][start] + n_candidate = 1 + while n_candidate < self.num_candidates: + # the first clip + if start == 0: + return 0 + # we add () in the following condition to fix the bug. + elif end == (len(caption["start"]) - 1): + return start - (self.num_candidates - n_candidate) + elif (caption["end"][end] - caption["start"][start - 1]) < ( + caption["end"][end + 1] - caption["start"][start] + ): + start -= 1 + else: + end += 1 + n_candidate += 1 + return start + + +class PKLJSONStrTextProcessor(TextProcessor): + """`caption.json` from howto100m are preprocessed as a + dict `[video_id, json_str]`. + Json parsing tokenization are conducted on-the-fly and cached into dict. + """ + + def __init__(self, config, max_clip_text_len=96): + print("[Warning] PKLJSONStrTextProcessor is slow for num_workers > 0.") + self.caption_pkl_path = str(config.caption_pkl_path) + with open(self.caption_pkl_path, "rb") as fd: + self.data = pickle.load(fd) + self.max_clip_text_len = max_clip_text_len + from transformers import AutoTokenizer + self.tokenizer = AutoTokenizer.from_pretrained( + str(config.bert_name), use_fast=config.use_fast + ) + + def __call__(self, video_id): + caption = self.data[video_id] + if isinstance(caption, str): + import json + caption = json.loads(caption) + cap = [] + for clip_idx, text_clip in enumerate(caption["text"]): + clip_ids = [] + if isinstance(text_clip, str): + clip_ids = self.tokenizer( + text_clip[: self.max_clip_text_len], + add_special_tokens=False + )["input_ids"] + cap.append(clip_ids) + caption["cap"] = cap + caption.pop("text") # save space. + self.data[video_id] = caption + return caption diff --git a/data/fairseq/examples/MMPT/mmpt/processors/how2retriprocessor.py b/data/fairseq/examples/MMPT/mmpt/processors/how2retriprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..b5a7730ec0bbe91d9997564214fffb10d0aef519 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/processors/how2retriprocessor.py @@ -0,0 +1,100 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from .how2processor import ( + ShardedHow2MetaProcessor, + ShardedVideoProcessor, + ShardedTextProcessor, + VariedLenAligner, + OverlappedAligner +) + + +class ShardedHow2VideoRetriMetaProcessor(ShardedHow2MetaProcessor): + def __init__(self, config): + super().__init__(config) + self.num_video_per_batch = config.num_video_per_batch + self.cands = [ + self.data[batch_offset:batch_offset + self.num_video_per_batch] + for batch_offset in + range(0, (len(self.data) // (8 * self.num_video_per_batch)) * 8 * self.num_video_per_batch, self.num_video_per_batch)] + + def __len__(self): + return len(self.cands) + + def set_candidates(self, cands): + # no changes on num of batches. + print(len(self.cands), "->", len(cands)) + # assert len(self.cands) == len(cands) + self.cands = cands + + def __getitem__(self, idx): + video_ids = self.cands[idx] + assert isinstance(video_ids, list) + sharded_video_idxs = [] + for video_id in video_ids: + shard_id, video_idx = self.video_id_to_shard[video_id] + sharded_video_idxs.append((video_id, -1, shard_id, video_idx)) + return sharded_video_idxs, sharded_video_idxs + + +class ShardedVideoRetriVideoProcessor(ShardedVideoProcessor): + """In retrival case the video_id + is a list of tuples: `(shard_id, video_idx)` .""" + + def __call__(self, sharded_video_idxs): + assert isinstance(sharded_video_idxs, list) + cand_feats = [] + for shared_video_idx in sharded_video_idxs: + feat = super().__call__(shared_video_idx) + cand_feats.append(feat) + return cand_feats + + +class ShardedVideoRetriTextProcessor(ShardedTextProcessor): + """In retrival case the video_id + is a list of tuples: `(shard_id, video_idx)` .""" + + def __call__(self, sharded_video_idxs): + assert isinstance(sharded_video_idxs, list) + cand_caps = [] + for shared_video_idx in sharded_video_idxs: + caps = super().__call__(shared_video_idx) + cand_caps.append(caps) + return cand_caps + + +class VideoRetriAligner(VariedLenAligner): + # Retritask will trim dim-0. + def __call__(self, sharded_video_idxs, video_features, text_features): + from transformers import default_data_collator + batch, video_ids = [], [] + for video_id, video_feature, text_feature in \ + zip(sharded_video_idxs, video_features, text_features): + sub_batch = super().__call__(video_id, video_feature, text_feature) + batch.append(sub_batch) + if isinstance(video_id, tuple): + video_id = video_id[0] + video_ids.append(video_id) + batch = default_data_collator(batch) + batch["video_id"] = video_ids + return batch + + +class VideoRetriOverlappedAligner(OverlappedAligner): + # Retritask will trim dim-0. + def __call__(self, sharded_video_idxs, video_features, text_features): + from transformers import default_data_collator + batch, video_ids = [], [] + for video_id, video_feature, text_feature in \ + zip(sharded_video_idxs, video_features, text_features): + sub_batch = super().__call__(video_id, video_feature, text_feature) + batch.append(sub_batch) + if isinstance(video_id, tuple): + video_id = video_id[0] + video_ids.append(video_id) + batch = default_data_collator(batch) + batch["video_id"] = video_ids + return batch diff --git a/data/fairseq/examples/MMPT/mmpt/processors/models/s3dg.py b/data/fairseq/examples/MMPT/mmpt/processors/models/s3dg.py new file mode 100644 index 0000000000000000000000000000000000000000..6c7a691e33551807ef258a54282884a409408691 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/processors/models/s3dg.py @@ -0,0 +1,336 @@ +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Contains a PyTorch definition for Gated Separable 3D network (S3D-G) +with a text module for computing joint text-video embedding from raw text +and video input. The following code will enable you to load the HowTo100M +pretrained S3D Text-Video model from: + A. Miech, J.-B. Alayrac, L. Smaira, I. Laptev, J. Sivic and A. Zisserman, + End-to-End Learning of Visual Representations from Uncurated Instructional Videos. + https://arxiv.org/abs/1912.06430. + +S3D-G was proposed by: + S. Xie, C. Sun, J. Huang, Z. Tu and K. Murphy, + Rethinking Spatiotemporal Feature Learning For Video Understanding. + https://arxiv.org/abs/1712.04851. + Tensorflow code: https://github.com/tensorflow/models/blob/master/research/slim/nets/s3dg.py + +The S3D architecture was slightly modified with a space to depth trick for TPU +optimization. +""" + +import torch as th +import torch.nn.functional as F +import torch.nn as nn +import os +import numpy as np +import re + + +class InceptionBlock(nn.Module): + def __init__( + self, + input_dim, + num_outputs_0_0a, + num_outputs_1_0a, + num_outputs_1_0b, + num_outputs_2_0a, + num_outputs_2_0b, + num_outputs_3_0b, + gating=True, + ): + super(InceptionBlock, self).__init__() + self.conv_b0 = STConv3D(input_dim, num_outputs_0_0a, [1, 1, 1]) + self.conv_b1_a = STConv3D(input_dim, num_outputs_1_0a, [1, 1, 1]) + self.conv_b1_b = STConv3D( + num_outputs_1_0a, num_outputs_1_0b, [3, 3, 3], padding=1, separable=True + ) + self.conv_b2_a = STConv3D(input_dim, num_outputs_2_0a, [1, 1, 1]) + self.conv_b2_b = STConv3D( + num_outputs_2_0a, num_outputs_2_0b, [3, 3, 3], padding=1, separable=True + ) + self.maxpool_b3 = th.nn.MaxPool3d((3, 3, 3), stride=1, padding=1) + self.conv_b3_b = STConv3D(input_dim, num_outputs_3_0b, [1, 1, 1]) + self.gating = gating + self.output_dim = ( + num_outputs_0_0a + num_outputs_1_0b + num_outputs_2_0b + num_outputs_3_0b + ) + if gating: + self.gating_b0 = SelfGating(num_outputs_0_0a) + self.gating_b1 = SelfGating(num_outputs_1_0b) + self.gating_b2 = SelfGating(num_outputs_2_0b) + self.gating_b3 = SelfGating(num_outputs_3_0b) + + def forward(self, input): + """Inception block + """ + b0 = self.conv_b0(input) + b1 = self.conv_b1_a(input) + b1 = self.conv_b1_b(b1) + b2 = self.conv_b2_a(input) + b2 = self.conv_b2_b(b2) + b3 = self.maxpool_b3(input) + b3 = self.conv_b3_b(b3) + if self.gating: + b0 = self.gating_b0(b0) + b1 = self.gating_b1(b1) + b2 = self.gating_b2(b2) + b3 = self.gating_b3(b3) + return th.cat((b0, b1, b2, b3), dim=1) + + +class SelfGating(nn.Module): + def __init__(self, input_dim): + super(SelfGating, self).__init__() + self.fc = nn.Linear(input_dim, input_dim) + + def forward(self, input_tensor): + """Feature gating as used in S3D-G. + """ + spatiotemporal_average = th.mean(input_tensor, dim=[2, 3, 4]) + weights = self.fc(spatiotemporal_average) + weights = th.sigmoid(weights) + return weights[:, :, None, None, None] * input_tensor + + +class STConv3D(nn.Module): + def __init__( + self, input_dim, output_dim, kernel_size, stride=1, padding=0, separable=False + ): + super(STConv3D, self).__init__() + self.separable = separable + self.relu = nn.ReLU(inplace=True) + assert len(kernel_size) == 3 + if separable and kernel_size[0] != 1: + spatial_kernel_size = [1, kernel_size[1], kernel_size[2]] + temporal_kernel_size = [kernel_size[0], 1, 1] + if isinstance(stride, list) and len(stride) == 3: + spatial_stride = [1, stride[1], stride[2]] + temporal_stride = [stride[0], 1, 1] + else: + spatial_stride = [1, stride, stride] + temporal_stride = [stride, 1, 1] + if isinstance(padding, list) and len(padding) == 3: + spatial_padding = [0, padding[1], padding[2]] + temporal_padding = [padding[0], 0, 0] + else: + spatial_padding = [0, padding, padding] + temporal_padding = [padding, 0, 0] + if separable: + self.conv1 = nn.Conv3d( + input_dim, + output_dim, + kernel_size=spatial_kernel_size, + stride=spatial_stride, + padding=spatial_padding, + bias=False, + ) + self.bn1 = nn.BatchNorm3d(output_dim) + self.conv2 = nn.Conv3d( + output_dim, + output_dim, + kernel_size=temporal_kernel_size, + stride=temporal_stride, + padding=temporal_padding, + bias=False, + ) + self.bn2 = nn.BatchNorm3d(output_dim) + else: + self.conv1 = nn.Conv3d( + input_dim, + output_dim, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=False, + ) + self.bn1 = nn.BatchNorm3d(output_dim) + + def forward(self, input): + out = self.relu(self.bn1(self.conv1(input))) + if self.separable: + out = self.relu(self.bn2(self.conv2(out))) + return out + + +class MaxPool3dTFPadding(th.nn.Module): + def __init__(self, kernel_size, stride=None, padding="SAME"): + super(MaxPool3dTFPadding, self).__init__() + if padding == "SAME": + padding_shape = self._get_padding_shape(kernel_size, stride) + self.padding_shape = padding_shape + self.pad = th.nn.ConstantPad3d(padding_shape, 0) + self.pool = th.nn.MaxPool3d(kernel_size, stride, ceil_mode=True) + + def _get_padding_shape(self, filter_shape, stride): + def _pad_top_bottom(filter_dim, stride_val): + pad_along = max(filter_dim - stride_val, 0) + pad_top = pad_along // 2 + pad_bottom = pad_along - pad_top + return pad_top, pad_bottom + + padding_shape = [] + for filter_dim, stride_val in zip(filter_shape, stride): + pad_top, pad_bottom = _pad_top_bottom(filter_dim, stride_val) + padding_shape.append(pad_top) + padding_shape.append(pad_bottom) + depth_top = padding_shape.pop(0) + depth_bottom = padding_shape.pop(0) + padding_shape.append(depth_top) + padding_shape.append(depth_bottom) + return tuple(padding_shape) + + def forward(self, inp): + inp = self.pad(inp) + out = self.pool(inp) + return out + + +class Sentence_Embedding(nn.Module): + def __init__( + self, + embd_dim, + num_embeddings=66250, + word_embedding_dim=300, + token_to_word_path="dict.npy", + max_words=16, + output_dim=2048, + ): + super(Sentence_Embedding, self).__init__() + self.word_embd = nn.Embedding(num_embeddings, word_embedding_dim) + self.fc1 = nn.Linear(word_embedding_dim, output_dim) + self.fc2 = nn.Linear(output_dim, embd_dim) + self.word_to_token = {} + self.max_words = max_words + token_to_word = np.load(token_to_word_path) + for i, t in enumerate(token_to_word): + self.word_to_token[t] = i + 1 + + def _zero_pad_tensor_token(self, tensor, size): + if len(tensor) >= size: + return tensor[:size] + else: + zero = th.zeros(size - len(tensor)).long() + return th.cat((tensor, zero), dim=0) + + def _split_text(self, sentence): + w = re.findall(r"[\w']+", str(sentence)) + return w + + def _words_to_token(self, words): + words = [ + self.word_to_token[word] for word in words if word in self.word_to_token + ] + if words: + we = self._zero_pad_tensor_token(th.LongTensor(words), self.max_words) + return we + else: + return th.zeros(self.max_words).long() + + def _words_to_ids(self, x): + split_x = [self._words_to_token(self._split_text(sent.lower())) for sent in x] + return th.stack(split_x, dim=0) + + def forward(self, x): + x = self._words_to_ids(x) + x = self.word_embd(x) + x = F.relu(self.fc1(x)) + x = th.max(x, dim=1)[0] + x = self.fc2(x) + return {'text_embedding': x} + + +class S3D(nn.Module): + def __init__(self, dict_path, num_classes=512, gating=True, space_to_depth=True): + super(S3D, self).__init__() + self.num_classes = num_classes + self.gating = gating + self.space_to_depth = space_to_depth + if space_to_depth: + self.conv1 = STConv3D( + 24, 64, [2, 4, 4], stride=1, padding=(1, 2, 2), separable=False + ) + else: + self.conv1 = STConv3D( + 3, 64, [3, 7, 7], stride=2, padding=(1, 3, 3), separable=False + ) + self.conv_2b = STConv3D(64, 64, [1, 1, 1], separable=False) + self.conv_2c = STConv3D(64, 192, [3, 3, 3], padding=1, separable=True) + self.gating = SelfGating(192) + self.maxpool_2a = MaxPool3dTFPadding( + kernel_size=(1, 3, 3), stride=(1, 2, 2), padding="SAME" + ) + self.maxpool_3a = MaxPool3dTFPadding( + kernel_size=(1, 3, 3), stride=(1, 2, 2), padding="SAME" + ) + self.mixed_3b = InceptionBlock(192, 64, 96, 128, 16, 32, 32) + self.mixed_3c = InceptionBlock( + self.mixed_3b.output_dim, 128, 128, 192, 32, 96, 64 + ) + self.maxpool_4a = MaxPool3dTFPadding( + kernel_size=(3, 3, 3), stride=(2, 2, 2), padding="SAME" + ) + self.mixed_4b = InceptionBlock( + self.mixed_3c.output_dim, 192, 96, 208, 16, 48, 64 + ) + self.mixed_4c = InceptionBlock( + self.mixed_4b.output_dim, 160, 112, 224, 24, 64, 64 + ) + self.mixed_4d = InceptionBlock( + self.mixed_4c.output_dim, 128, 128, 256, 24, 64, 64 + ) + self.mixed_4e = InceptionBlock( + self.mixed_4d.output_dim, 112, 144, 288, 32, 64, 64 + ) + self.mixed_4f = InceptionBlock( + self.mixed_4e.output_dim, 256, 160, 320, 32, 128, 128 + ) + self.maxpool_5a = self.maxPool3d_5a_2x2 = MaxPool3dTFPadding( + kernel_size=(2, 2, 2), stride=(2, 2, 2), padding="SAME" + ) + self.mixed_5b = InceptionBlock( + self.mixed_4f.output_dim, 256, 160, 320, 32, 128, 128 + ) + self.mixed_5c = InceptionBlock( + self.mixed_5b.output_dim, 384, 192, 384, 48, 128, 128 + ) + self.fc = nn.Linear(self.mixed_5c.output_dim, num_classes) + self.text_module = Sentence_Embedding(num_classes, + token_to_word_path=dict_path) + + def _space_to_depth(self, input): + """3D space to depth trick for TPU optimization. + """ + B, C, T, H, W = input.shape + input = input.view(B, C, T // 2, 2, H // 2, 2, W // 2, 2) + input = input.permute(0, 3, 5, 7, 1, 2, 4, 6) + input = input.contiguous().view(B, 8 * C, T // 2, H // 2, W // 2) + return input + + def forward(self, inputs): + """Defines the S3DG base architecture.""" + if self.space_to_depth: + inputs = self._space_to_depth(inputs) + net = self.conv1(inputs) + if self.space_to_depth: + # we need to replicate 'SAME' tensorflow padding + net = net[:, :, 1:, 1:, 1:] + net = self.maxpool_2a(net) + net = self.conv_2b(net) + net = self.conv_2c(net) + if self.gating: + net = self.gating(net) + net = self.maxpool_3a(net) + net = self.mixed_3b(net) + net = self.mixed_3c(net) + net = self.maxpool_4a(net) + net = self.mixed_4b(net) + net = self.mixed_4c(net) + net = self.mixed_4d(net) + net = self.mixed_4e(net) + net = self.mixed_4f(net) + net = self.maxpool_5a(net) + net = self.mixed_5b(net) + net = self.mixed_5c(net) + net = th.mean(net, dim=[2, 3, 4]) + return {'video_embedding': self.fc(net), 'mixed_5c': net} diff --git a/data/fairseq/examples/MMPT/mmpt/processors/processor.py b/data/fairseq/examples/MMPT/mmpt/processors/processor.py new file mode 100644 index 0000000000000000000000000000000000000000..98edb051f16efef81fba98b0b2f6befbad09f2d4 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/processors/processor.py @@ -0,0 +1,274 @@ +# Copyright (c) Facebook, Inc. All Rights Reserved + +import numpy as np +import os +import torch + + +class Processor(object): + """ + A generic processor for video (codec, feature etc.) and text. + """ + + def __call__(self, **kwargs): + raise NotImplementedError + + +class MetaProcessor(Processor): + """ + A meta processor is expected to load the metadata of a dataset: + (e.g., video_ids, or captions). + You must implement the `__getitem__` (meta datasets are rather diverse.). + """ + + def __init__(self, config): + self.split = config.split + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + raise NotImplementedError + + def _get_split_path(self, config): + splits = { + "train": config.train_path, + "valid": config.val_path, + "test": config.test_path, + } + if config.split is not None: + return splits[config.split] + return config.train_path + + +class TextProcessor(Processor): + """ + A generic Text processor: rename this as `withTokenizer`. + tokenize a string of text on-the-fly. + Warning: mostly used for end tasks. + (on-the-fly tokenization is slow for how2.) + TODO(huxu): move this class as a subclass. + """ + + def __init__(self, config): + self.bert_name = str(config.bert_name) + self.use_fast = config.use_fast + from transformers import AutoTokenizer + self.tokenizer = AutoTokenizer.from_pretrained( + self.bert_name, use_fast=self.use_fast + ) + + def __call__(self, text_id): + caption = self.tokenizer(text_id, add_special_tokens=False) + return caption["input_ids"] + + +class VideoProcessor(Processor): + """ + A generic video processor: load a numpy video tokens by default. + """ + + def __init__(self, config): + self.vfeat_dir = config.vfeat_dir + + def __call__(self, video_fn): + if isinstance(video_fn, tuple): + video_fn = video_fn[0] + assert isinstance(video_fn, str) + video_fn = os.path.join(self.vfeat_dir, video_fn + ".npy") + feat = np.load(video_fn) + return feat + + +class Aligner(object): + """ + An alignprocessor align video and text and output a dict of tensors (for a model). + """ + def __init__(self, config): + """__init__ needs to be light weight for more workers/threads.""" + self.split = config.split + self.max_video_len = config.max_video_len + self.max_len = config.max_len + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained( + str(config.bert_name), use_fast=config.use_fast + ) + self.cls_token_id = tokenizer.cls_token_id + self.sep_token_id = tokenizer.sep_token_id + self.pad_token_id = tokenizer.pad_token_id + self.mask_token_id = tokenizer.mask_token_id + + def __call__(self, video_id, video_feature, text_feature): + raise NotImplementedError + + def _build_video_seq(self, video_feature, video_clips=None): + """ + `video_feature`: available video tokens. + `video_clips`: video clip sequence to build. + """ + if not isinstance(video_feature, np.ndarray): + raise ValueError( + "unsupported type of video_feature", type(video_feature) + ) + + if video_clips is None: + # this is borrowed from DSAligner + video_start = 0 + video_end = min(len(video_feature), self.max_video_len) + # the whole sequence is a single clip. + video_clips = {"start": [video_start], "end": [video_end]} + + vfeats = np.zeros( + (self.max_video_len, video_feature.shape[1]), dtype=np.float32 + ) + vmasks = torch.zeros((self.max_video_len,), dtype=torch.bool) + video_len = 0 + for start, end in zip(video_clips["start"], video_clips["end"]): + clip_len = min(self.max_video_len - video_len, (end - start)) + if clip_len > 0: + vfeats[video_len: video_len + clip_len] = video_feature[ + start: start + clip_len + ] + vmasks[video_len: video_len + clip_len] = 1 + video_len += clip_len + vfeats = torch.from_numpy(vfeats) + + return vfeats, vmasks + + def _build_text_seq(self, text_feature, text_clip_indexs=None): + """ + `text_feature`: all available clips. + `text_clip_indexes`: clip sequence to build. + """ + if text_clip_indexs is None: + text_clip_indexs = [0] + + full_caps = [] + if isinstance(text_feature, dict): + for clip_idx in text_clip_indexs: + full_caps.extend(text_feature["cap"][clip_idx]) + else: + full_caps = text_feature + max_text_len = self.max_len - self.max_video_len - 3 + full_caps = full_caps[:max_text_len] + full_caps = ( + [self.cls_token_id, self.sep_token_id] + full_caps + [self.sep_token_id] + ) + text_pad_len = self.max_len - len(full_caps) - self.max_video_len + padded_full_caps = full_caps + [self.pad_token_id] * text_pad_len + caps = torch.LongTensor(padded_full_caps) + cmasks = torch.zeros((len(padded_full_caps),), dtype=torch.bool) + cmasks[: len(full_caps)] = 1 + + return caps, cmasks + + def batch_post_processing(self, batch, video_feature): + return batch + + +class MMAttentionMask2DProcessor(Processor): + """text generation requires 2d mask + that is harder to generate by GPU at this stage.""" + + def __call__(self, vmask, cmask, mtype): + if mtype == "textgen": + return self._build_textgeneration_mask(vmask, cmask) + elif mtype == "videogen": + return self._build_videogeneration_mask(vmask, cmask) + else: + return self._build_mm_mask(vmask, cmask) + + def _build_mm_mask(self, vmask, cmask): + mask_1d = torch.cat([cmask[:1], vmask, cmask[1:]], dim=0) + return mask_1d[None, :].repeat(mask_1d.size(0), 1) + + def _build_videogeneration_mask(self, vmask, cmask): + # cls_mask is only about text otherwise it will leak generation. + cls_text_mask = torch.cat([ + # [CLS] + torch.ones( + (1,), dtype=torch.bool, device=cmask.device), + # video tokens and [SEP] for video. + torch.zeros( + (vmask.size(0) + 1,), dtype=torch.bool, device=cmask.device), + cmask[2:] + ], dim=0) + + # concat horizontially. + video_len = int(vmask.sum()) + video_masks = torch.cat([ + # [CLS] + torch.ones( + (video_len, 1), dtype=torch.bool, device=cmask.device + ), + torch.tril( + torch.ones( + (video_len, video_len), + dtype=torch.bool, device=cmask.device)), + # video_padding + torch.zeros( + (video_len, vmask.size(0) - video_len), + dtype=torch.bool, device=cmask.device + ), + # [SEP] for video (unused). + torch.zeros( + (video_len, 1), dtype=torch.bool, device=cmask.device + ), + cmask[2:].unsqueeze(0).repeat(video_len, 1) + ], dim=1) + + text_masks = cls_text_mask[None, :].repeat( + cmask.size(0) - 2, 1) + video_padding_masks = cls_text_mask[None, :].repeat( + vmask.size(0) - video_len, 1) + + return torch.cat([ + cls_text_mask[None, :], + video_masks, + video_padding_masks, + torch.cat([cmask[:1], vmask, cmask[1:]], dim=0)[None,:], + text_masks + ], dim=0) + + def _build_textgeneration_mask(self, vmask, cmask): + # cls_mask is only about video otherwise it will leak generation. + cls_video_mask = torch.cat([ + # [CLS] + torch.ones( + (1,), dtype=torch.bool, device=cmask.device), + vmask, + # [SEP] + torch.ones((1,), dtype=torch.bool, device=cmask.device), + torch.zeros( + (cmask.size(0)-2,), dtype=torch.bool, device=cmask.device) + ], dim=0) + + # concat horizontially. + text_len = int(cmask[2:].sum()) + text_masks = torch.cat([ + # [CLS] + torch.ones( + (text_len, 1), dtype=torch.bool, device=cmask.device + ), + vmask.unsqueeze(0).repeat(text_len, 1), + # [SEP] for video. + torch.ones( + (text_len, 1), dtype=torch.bool, device=cmask.device + ), + torch.tril( + torch.ones( + (text_len, text_len), + dtype=torch.bool, device=cmask.device)), + # padding. + torch.zeros( + (text_len, cmask.size(0) - text_len - 2), + dtype=torch.bool, device=cmask.device + ) + ], dim=1) + + cls_video_masks = cls_video_mask[None, :].repeat( + vmask.size(0) + 2, 1) + text_padding_masks = cls_video_mask[None, :].repeat( + cmask.size(0) - text_len - 2, 1) + return torch.cat([ + cls_video_masks, text_masks, text_padding_masks], dim=0) diff --git a/data/fairseq/examples/MMPT/mmpt/tasks/__init__.py b/data/fairseq/examples/MMPT/mmpt/tasks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e2e9323a530672ef9daecd793ef645a3c1d0f3e6 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/tasks/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +from .task import * +from .vlmtask import * +from .retritask import * + +try: + from .fairseqmmtask import * +except ImportError: + pass + +try: + from .milncetask import * +except ImportError: + pass + +try: + from .expretritask import * +except ImportError: + pass diff --git a/data/fairseq/examples/MMPT/mmpt/tasks/fairseqmmtask.py b/data/fairseq/examples/MMPT/mmpt/tasks/fairseqmmtask.py new file mode 100644 index 0000000000000000000000000000000000000000..f6b6115a39b2342be5513edd53016187ab91eb01 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/tasks/fairseqmmtask.py @@ -0,0 +1,104 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +""" +make a general fairseq task for MM pretraining. +""" + +import random + +from fairseq.tasks import LegacyFairseqTask, register_task + +from .task import Task +from .retritask import RetriTask +from ..datasets import FairseqMMDataset +from .. import utils + + +@register_task("mmtask") +class FairseqMMTask(LegacyFairseqTask): + @staticmethod + def add_args(parser): + # Add some command-line arguments for specifying where the data is + # located and the maximum supported input length. + parser.add_argument( + "taskconfig", + metavar="FILE", + help=("taskconfig to load all configurations" "outside fairseq parser."), + ) + + @classmethod + def setup_task(cls, args, **kwargs): + return FairseqMMTask(args) + + def __init__(self, args): + super().__init__(args) + config = utils.load_config(args) + self.mmtask = Task.config_task(config) + self.mmtask.build_dataset() + self.mmtask.build_model() + self.mmtask.build_loss() + + def load_dataset(self, split, **kwargs): + split_map = { + "train": self.mmtask.train_data, + "valid": self.mmtask.val_data, + "test": self.mmtask.test_data, + } + if split not in split_map: + raise ValueError("unknown split type.") + if split_map[split] is not None: + self.datasets[split] = FairseqMMDataset(split_map[split]) + + def get_batch_iterator( + self, + dataset, + max_tokens=None, + max_sentences=None, + max_positions=None, + ignore_invalid_inputs=False, + required_batch_size_multiple=1, + seed=1, + num_shards=1, + shard_id=0, + num_workers=0, + epoch=1, + data_buffer_size=0, + disable_iterator_cache=False, + skip_remainder_batch=False, + grouped_shuffling=False, + update_epoch_batch_itr=False, + ): + random.seed(epoch) + if dataset.mmdataset.split == "train" and isinstance(self.mmtask, RetriTask): + if epoch >= self.mmtask.config.retri_epoch: + if not hasattr(self.mmtask, "retri_dataloader"): + self.mmtask.build_dataloader() + self.mmtask.retrive_candidates(epoch) + + return super().get_batch_iterator( + dataset, + max_tokens, + max_sentences, + max_positions, + ignore_invalid_inputs, + required_batch_size_multiple, + seed, + num_shards, + shard_id, + num_workers, + epoch, + data_buffer_size, + disable_iterator_cache, + grouped_shuffling, + update_epoch_batch_itr, + ) + + @property + def source_dictionary(self): + return None + + @property + def target_dictionary(self): + return None diff --git a/data/fairseq/examples/MMPT/mmpt/tasks/milncetask.py b/data/fairseq/examples/MMPT/mmpt/tasks/milncetask.py new file mode 100644 index 0000000000000000000000000000000000000000..61b6ab0597f9f3a78bbcf1474613630b20c5a874 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/tasks/milncetask.py @@ -0,0 +1,27 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import torch + +from .task import Task + + +class MILNCETask(Task): + def reshape_subsample(self, sample): + if ( + hasattr(self.config.dataset, "subsampling") + and self.config.dataset.subsampling is not None + and self.config.dataset.subsampling > 1 + ): + for key in sample: + if torch.is_tensor(sample[key]): + tensor = self.flat_subsample(sample[key]) + if key in ["caps", "cmasks"]: + size = tensor.size() + batch_size = size[0] * size[1] + expanded_size = (batch_size,) + size[2:] + tensor = tensor.view(expanded_size) + sample[key] = tensor + return sample diff --git a/data/fairseq/examples/MMPT/mmpt/tasks/retritask.py b/data/fairseq/examples/MMPT/mmpt/tasks/retritask.py new file mode 100644 index 0000000000000000000000000000000000000000..b43f20fddb31f29b210b1c726e5f6ccaad04bcf0 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/tasks/retritask.py @@ -0,0 +1,253 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import os +import torch +import pickle +import random + +from tqdm import tqdm +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler + +from ..processors import ( + ShardedHow2MetaProcessor, + ShardedVideoProcessor, + ShardedTextProcessor, + VariedLenAligner, +) + +from ..datasets import MMDataset +from .task import Task +from ..modules import vectorpool +from ..evaluators.predictor import Predictor +from ..utils import set_seed, get_local_rank, get_world_size + + +class RetriTask(Task): + """abstract class for task with retrival.""" + + def reshape_subsample(self, sample): + for key in sample: + if torch.is_tensor(sample[key]): + sample[key] = self.flat_subsample(sample[key]) + return sample + + def flat_subsample(self, tensor): + if tensor.size(0) == 1: + tensor = tensor.squeeze(0) + return tensor + + def build_dataloader(self): + """called by `get_batch_iterator` in fairseqmmtask. """ + # TODO: hard-code dataloader for retri for now and configurable in .yaml. + # reuse the `train.lst`. + self.config.dataset.split = "train" + meta_processor = ShardedHow2MetaProcessor(self.config.dataset) + video_processor = ShardedVideoProcessor(self.config.dataset) + text_processor = ShardedTextProcessor(self.config.dataset) + + aligner = VariedLenAligner(self.config.dataset) + aligner.subsampling = self.config.dataset.clip_per_video + + self.retri_data = MMDataset( + meta_processor, video_processor, text_processor, aligner + ) + + retri_sampler = DistributedSampler(self.retri_data) + infer_scale = 16 + batch_size = self.config.dataset.num_video_per_batch \ + * infer_scale + + self.retri_dataloader = DataLoader( + self.retri_data, + collate_fn=self.retri_data.collater, + batch_size=batch_size, + shuffle=False, + sampler=retri_sampler, + num_workers=self.config.fairseq.dataset.num_workers + ) + return self.retri_dataloader + + def retrive_candidates(self, epoch, dataloader=None): + if get_local_rank() == 0: + print("running retrieval model.") + out_dir = os.path.join( + self.config.fairseq.checkpoint.save_dir, "retri") + os.makedirs(out_dir, exist_ok=True) + + if not os.path.isfile( + os.path.join( + out_dir, "batched_e" + str(epoch) + "_videos0.pkl") + ): + if dataloader is None: + dataloader = self.retri_dataloader + + self.model.eval() + self.model.is_train = False + + assert self.retri_data.meta_processor.data == \ + self.train_data.meta_processor.data # video_ids not mutated. + + self._retri_predict(epoch, dataloader) + + self.model.train() + self.model.is_train = True + + torch.distributed.barrier() + output = self._retri_sync(epoch, out_dir) + torch.distributed.barrier() + self.train_data.meta_processor.set_candidates(output) + return output + + +class VideoRetriTask(RetriTask): + """RetriTask on video level.""" + + def reshape_subsample(self, sample): + if ( + hasattr(self.config.dataset, "clip_per_video") + and self.config.dataset.clip_per_video is not None + and self.config.dataset.clip_per_video > 1 + ): + for key in sample: + if torch.is_tensor(sample[key]): + sample[key] = self.flat_subsample(sample[key]) + return sample + + def flat_subsample(self, tensor): + if tensor.size(0) == 1: + tensor = tensor.squeeze(0) + return Task.flat_subsample(self, tensor) + + def _retri_predict(self, epoch, dataloader): + set_seed(epoch) + # save for retrival. + predictor = VideoPredictor(self.config) + predictor.predict_loop( + self.model, dataloader) + set_seed(epoch) # get the same text clips. + # retrival. + retri_predictor = VideoRetriPredictor( + self.config) + retri_predictor.predict_loop( + self.model, predictor.vecpool.retriver, epoch) + del predictor + del retri_predictor + + def _retri_sync(self, epoch, out_dir): + # gpu do the same merge. + batched_videos = [] + for local_rank in range(get_world_size()): + fn = os.path.join( + out_dir, + "batched_e" + str(epoch) + "_videos" + str(local_rank) + ".pkl") + with open(fn, "rb") as fr: + batched_videos.extend(pickle.load(fr)) + print( + "[INFO] batched_videos", + len(batched_videos), len(batched_videos[0])) + return batched_videos + + +class VideoPredictor(Predictor): + def __init__(self, config): + vectorpool_cls = getattr(vectorpool, config.vectorpool_cls) + self.vecpool = vectorpool_cls(config) + + def predict_loop( + self, + model, + dataloader, + early_stop=-1, + ): + with torch.no_grad(): + if get_local_rank() == 0: + dataloader = tqdm(dataloader) + for batch_idx, batch in enumerate(dataloader): + if batch_idx == early_stop: + break + self(batch, model) + return self.finalize() + + def __call__(self, sample, model, **kwargs): + param = next(model.parameters()) + dtype = param.dtype + device = param.device + subsample = sample["vfeats"].size(1) + sample = self.to_ctx(sample, device, dtype) + for key in sample: + if torch.is_tensor(sample[key]): + size = sample[key].size() + if len(size) >= 2: + batch_size = size[0] * size[1] + expanded_size = ( + (batch_size,) + size[2:] if len(size) > 2 + else (batch_size,) + ) + sample[key] = sample[key].view(expanded_size) + + outputs = model(**sample) + sample.update(outputs) + self.vecpool(sample, subsample) + + def finalize(self): + print("[INFO]", self.vecpool) + if not self.vecpool.retriver.db.is_trained: + self.vecpool.retriver.finalize_training() + return self.vecpool.retriver + + +class VideoRetriPredictor(Predictor): + """ + Online Retrieval Predictor for Clips (used by RetriTask). + TODO: merge this with VisPredictor? + """ + + def __init__(self, config): + self.pred_dir = os.path.join( + config.fairseq.checkpoint.save_dir, + "retri") + self.num_cands = config.num_cands + self.num_video_per_batch = config.dataset.num_video_per_batch + + def predict_loop( + self, + model, + retriver, + epoch, + early_stop=-1 + ): + # a fake loop that only try to recover video vector + # from video_id. + batched_videos = [] + # obtain available video_ids. + video_ids = list(retriver.videoid_to_vectoridx.keys()) + + dataloader = random.sample( + video_ids, + len(video_ids) // self.num_video_per_batch + ) + + if get_local_rank() == 0: + dataloader = tqdm(dataloader) + for batch_idx, batch in enumerate(dataloader): + # batch is one video id. + if batch_idx == early_stop: + break + video_ids = retriver.search_by_video_ids( + [batch], self.num_cands)[0] + if len(video_ids) > self.num_video_per_batch: + # we moved the center to make cluster robust. + video_ids = random.sample(video_ids, self.num_video_per_batch) + batched_videos.append(video_ids) + return self.finalize(batched_videos, epoch) + + def finalize(self, batched_videos, epoch): + fn = os.path.join( + self.pred_dir, + "batched_e" + str(epoch) + "_videos" + str(get_local_rank()) + ".pkl") + with open(fn, "wb") as fw: + pickle.dump(batched_videos, fw, pickle.HIGHEST_PROTOCOL) + return batched_videos diff --git a/data/fairseq/examples/MMPT/mmpt/tasks/task.py b/data/fairseq/examples/MMPT/mmpt/tasks/task.py new file mode 100644 index 0000000000000000000000000000000000000000..8bb50f24df49878d5b430f9df5ba1ffb1cf30e32 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/tasks/task.py @@ -0,0 +1,184 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import torch + +from .. import tasks +from .. import models +from .. import losses +from ..datasets import MMDataset +from .. import processors + + +class Task(object): + """ + A task refers to one generic training task (e.g., training one model). + """ + + @classmethod + def config_task(cls, config): + """ + determine whether to load a hard-coded task or config from a generic one. + via if a task string is available in config. + """ + if config.task is not None: + # TODO (huxu): expand the search scope. + task_cls = getattr(tasks, config.task) + return task_cls(config) + else: + return Task(config) + + def __init__(self, config): + self.config = config + self.train_data = None + self.val_data = None + self.test_data = None + + self.model = None + self.loss_fn = None + self.eval_fn = None + + def build_dataset(self): + """TODO (huxu): move processor breakdown to MMDataset.""" + """fill-in `self.train_data`, `self.val_data` and `self.test_data`.""" + + meta_processor_cls = getattr( + processors, self.config.dataset.meta_processor) + video_processor_cls = getattr( + processors, self.config.dataset.video_processor) + text_processor_cls = getattr( + processors, self.config.dataset.text_processor) + aligner_cls = getattr( + processors, self.config.dataset.aligner) + + if self.config.dataset.train_path is not None: + self.config.dataset.split = "train" + # may be used by meta processor. + # meta_processor controls different dataset. + meta_processor = meta_processor_cls(self.config.dataset) + video_processor = video_processor_cls(self.config.dataset) + text_processor = text_processor_cls(self.config.dataset) + aligner = aligner_cls(self.config.dataset) + self.train_data = MMDataset( + meta_processor, video_processor, text_processor, aligner + ) + print("train_len", len(self.train_data)) + output = self.train_data[0] + self.train_data.print_example(output) + if self.config.dataset.val_path is not None: + self.config.dataset.split = "valid" + # may be used by meta processor. + meta_processor = meta_processor_cls(self.config.dataset) + video_processor = video_processor_cls(self.config.dataset) + text_processor = text_processor_cls(self.config.dataset) + aligner = aligner_cls(self.config.dataset) + self.val_data = MMDataset( + meta_processor, video_processor, text_processor, aligner + ) + print("val_len", len(self.val_data)) + output = self.val_data[0] + self.val_data.print_example(output) + + if self.config.dataset.split == "test": + # the following is run via lauching fairseq-validate. + meta_processor = meta_processor_cls(self.config.dataset) + video_processor = video_processor_cls(self.config.dataset) + text_processor = text_processor_cls(self.config.dataset) + + self.test_data = MMDataset( + meta_processor, video_processor, text_processor, aligner + ) + print("test_len", len(self.test_data)) + output = self.test_data[0] + self.test_data.print_example(output) + + def build_model(self, checkpoint=None): + if self.model is None: + model_cls = getattr(models, self.config.model.model_cls) + self.model = model_cls(self.config) + if checkpoint is not None: + self.load_checkpoint(checkpoint) + return self.model + + def load_checkpoint(self, checkpoint): + if self.model is None: + raise ValueError("model is not initialized.") + state_dict = torch.load(checkpoint) + state_dict = self._trim_state_dict(state_dict) + self.model.load_state_dict(state_dict, strict=False) + # if it's a fp16 model, turn it back. + if next(self.model.parameters()).dtype == torch.float16: + self.model = self.model.float() + return self.model + + def _trim_state_dict(self, state_dict): + from collections import OrderedDict + + if "state_dict" in state_dict: + state_dict = state_dict["state_dict"] + if "model" in state_dict: # fairseq checkpoint format. + state_dict = state_dict["model"] + ret_state_dict = OrderedDict() + for ( + key, + value, + ) in state_dict.items(): + # remove fairseq wrapper since this is a task. + if key.startswith("mmmodel"): + key = key[len("mmmodel."):] + ret_state_dict[key] = value + return ret_state_dict + + def build_loss(self): + if self.loss_fn is None and self.config.loss is not None: + loss_cls = getattr(losses, self.config.loss.loss_cls) + self.loss_fn = loss_cls() + return self.loss_fn + + def flat_subsample(self, tensor): + size = tensor.size() + if len(size) >= 2: + batch_size = size[0] * size[1] + expanded_size = ( + (batch_size,) + size[2:] if len(size) > 2 + else (batch_size,) + ) + tensor = tensor.view(expanded_size) + return tensor + + def reshape_subsample(self, sample): + if ( + hasattr(self.config.dataset, "subsampling") + and self.config.dataset.subsampling is not None + and self.config.dataset.subsampling > 1 + ): + for key in sample: + if torch.is_tensor(sample[key]): + sample[key] = self.flat_subsample(sample[key]) + return sample + + def __call__(self, model, sample): + loss = None + loss_scalar = float("inf") + + sample = self.reshape_subsample(sample) + outputs = self.model(**sample) + sample.update(outputs) + if self.loss_fn is not None: + loss = self.loss_fn(**sample) + loss_scalar = loss.item() + + batch_size = sample["caps"].size(0) + sample_size = 1 + return { + "loss": loss, + "loss_scalar": loss_scalar, + "max_len": self.config.dataset.max_len, + "batch_size": batch_size, + "sample_size": sample_size, + } + + def build_dataloader(self): + """only used for trainer that lacks building loaders.""" + raise NotImplementedError diff --git a/data/fairseq/examples/MMPT/mmpt/tasks/vlmtask.py b/data/fairseq/examples/MMPT/mmpt/tasks/vlmtask.py new file mode 100644 index 0000000000000000000000000000000000000000..57dc4c91705fdb1292f2f2accbb42acb993eb6aa --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt/tasks/vlmtask.py @@ -0,0 +1,27 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import torch + +from .task import Task + + +class VLMTask(Task): + """A VLM task for reproducibility. + the collator split subsamples into two sub-batches. + This has should have no logic changes. + but changed the randomness in frame masking. + """ + + def flat_subsample(self, tensor): + size = tensor.size() + if len(size) >= 2: + batch_size = size[0] * (size[1] // 2) + expanded_size = ( + (batch_size, 2) + size[2:] if len(size) > 2 + else (batch_size, 2) + ) + tensor = tensor.view(expanded_size) + tensor = torch.cat([tensor[:, 0], tensor[:, 1]], dim=0) + return tensor diff --git a/data/fairseq/examples/MMPT/mmpt_cli/localjob.py b/data/fairseq/examples/MMPT/mmpt_cli/localjob.py new file mode 100644 index 0000000000000000000000000000000000000000..2675d3511a9ca700185d2d7b853dc56ad70c638c --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt_cli/localjob.py @@ -0,0 +1,117 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import os + +from mmpt.utils import recursive_config + + +class BaseJob(object): + def __init__(self, yaml_file, dryrun=False): + self.yaml_file = yaml_file + self.config = recursive_config(yaml_file) + self.dryrun = dryrun + + def submit(self, **kwargs): + raise NotImplementedError + + def _normalize_cmd(self, cmd_list): + cmd_list = list(cmd_list) + yaml_index = cmd_list.index("[yaml]") + cmd_list[yaml_index] = self.yaml_file + return cmd_list + + +class LocalJob(BaseJob): + + CMD_CONFIG = { + "local_single": [ + "fairseq-train", "[yaml]", "--user-dir", "mmpt", + "--task", "mmtask", "--arch", "mmarch", + "--criterion", "mmloss", + ], + "local_small": [ + "fairseq-train", "[yaml]", "--user-dir", "mmpt", + "--task", "mmtask", "--arch", "mmarch", + "--criterion", "mmloss", + "--distributed-world-size", "2" + ], + "local_big": [ + "fairseq-train", "[yaml]", "--user-dir", "mmpt", + "--task", "mmtask", "--arch", "mmarch", + "--criterion", "mmloss", + "--distributed-world-size", "8" + ], + "local_predict": ["python", "mmpt_cli/predict.py", "[yaml]"], + } + + def __init__(self, yaml_file, job_type=None, dryrun=False): + super().__init__(yaml_file, dryrun) + if job_type is None: + self.job_type = "local_single" + if self.config.task_type is not None: + self.job_type = self.config.task_type + else: + self.job_type = job_type + if self.job_type in ["local_single", "local_small"]: + if self.config.fairseq.dataset.batch_size > 32: + print("decreasing batch_size to 32 for local testing?") + + def submit(self): + cmd_list = self._normalize_cmd(LocalJob.CMD_CONFIG[self.job_type]) + if "predict" not in self.job_type: + # append fairseq args. + from mmpt.utils import load_config + + config = load_config(config_file=self.yaml_file) + for field in config.fairseq: + for key in config.fairseq[field]: + if key in ["fp16", "reset_optimizer", "reset_dataloader", "reset_meters"]: # a list of binary flag. + param = ["--" + key.replace("_", "-")] + else: + if key == "lr": + value = str(config.fairseq[field][key][0]) + elif key == "adam_betas": + value = "'"+str(config.fairseq[field][key])+"'" + else: + value = str(config.fairseq[field][key]) + param = [ + "--" + key.replace("_", "-"), + value + ] + cmd_list.extend(param) + + print("launching", " ".join(cmd_list)) + if not self.dryrun: + os.system(" ".join(cmd_list)) + return JobStatus("12345678") + + +class JobStatus(object): + def __init__(self, job_id): + self.job_id = job_id + + def __repr__(self): + return self.job_id + + def __str__(self): + return self.job_id + + def done(self): + return False + + def running(self): + return False + + def result(self): + if self.done(): + return "{} is done.".format(self.job_id) + else: + return "{} is running.".format(self.job_id) + + def stderr(self): + return self.result() + + def stdout(self): + return self.result() diff --git a/data/fairseq/examples/MMPT/mmpt_cli/predict.py b/data/fairseq/examples/MMPT/mmpt_cli/predict.py new file mode 100644 index 0000000000000000000000000000000000000000..4071e196d211f7b11170db2e7e35b716d3deeb69 --- /dev/null +++ b/data/fairseq/examples/MMPT/mmpt_cli/predict.py @@ -0,0 +1,113 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import os +import glob +import argparse +import pprint +import omegaconf + +from omegaconf import OmegaConf +from torch.utils.data import DataLoader + +from mmpt.utils import load_config, set_seed +from mmpt.evaluators import Evaluator +from mmpt.evaluators import predictor as predictor_path +from mmpt.tasks import Task +from mmpt import processors +from mmpt.datasets import MMDataset + + +def get_dataloader(config): + meta_processor_cls = getattr(processors, config.dataset.meta_processor) + video_processor_cls = getattr(processors, config.dataset.video_processor) + text_processor_cls = getattr(processors, config.dataset.text_processor) + aligner_cls = getattr(processors, config.dataset.aligner) + + meta_processor = meta_processor_cls(config.dataset) + video_processor = video_processor_cls(config.dataset) + text_processor = text_processor_cls(config.dataset) + aligner = aligner_cls(config.dataset) + + test_data = MMDataset( + meta_processor, + video_processor, + text_processor, + aligner, + ) + print("test_len", len(test_data)) + output = test_data[0] + test_data.print_example(output) + + test_dataloader = DataLoader( + test_data, + batch_size=config.fairseq.dataset.batch_size, + shuffle=False, + num_workers=6, + collate_fn=test_data.collater, + ) + return test_dataloader + + +def main(args): + config = load_config(args) + + if isinstance(config, omegaconf.dictconfig.DictConfig): + print(OmegaConf.to_yaml(config)) + else: + pp = pprint.PrettyPrinter(indent=4) + pp.print(config) + + mmtask = Task.config_task(config) + mmtask.build_model() + + test_dataloader = get_dataloader(config) + checkpoint_search_path = os.path.dirname(config.eval.save_path) + results = [] + + prefix = os.path.basename(args.taskconfig) + if prefix.startswith("test"): + # loop all checkpoint for datasets without validation set. + if "best" not in config.fairseq.common_eval.path: + print("eval each epoch.") + for checkpoint in glob.glob(checkpoint_search_path + "/checkpoint*"): + model = mmtask.load_checkpoint(checkpoint) + ckpt = os.path.basename(checkpoint) + evaluator = Evaluator(config) + output = evaluator.evaluate( + model, test_dataloader, ckpt + "_merged") + results.append((checkpoint, output)) + # use the one specified by the config lastly. + model = mmtask.load_checkpoint(config.fairseq.common_eval.path) + evaluator = Evaluator(config) + output = evaluator.evaluate(model, test_dataloader) + results.append((config.fairseq.common_eval.path, output)) + + best_result = None + best_metric = 0. + for checkpoint, result in results: + print(checkpoint) + evaluator.metric.print_computed_metrics(result) + best_score = evaluator.metric.best_metric(result) + if best_score > best_metric: + best_result = (checkpoint, result) + best_metric = best_score + print("best results:") + print(best_result[0]) + evaluator.metric.print_computed_metrics(best_result[1]) + + elif prefix.startswith("vis"): + model = mmtask.load_checkpoint(config.fairseq.common_eval.path) + predictor_cls = getattr(predictor_path, config.predictor) + predictor = predictor_cls(config) + predictor.predict_loop(model, test_dataloader, mmtask, None) + else: + raise ValueError("unknown prefix of the config file", args.taskconfig) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("taskconfig", type=str) + args = parser.parse_args() + main(args) diff --git a/data/fairseq/examples/mr_hubert/README.md b/data/fairseq/examples/mr_hubert/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e72c09c047958363dedae8fa49106f2209b098ba --- /dev/null +++ b/data/fairseq/examples/mr_hubert/README.md @@ -0,0 +1,187 @@ +# MR-HuBERT + +## Pre-trained models + +### Main models +Model | Pretraining Data | Model | Paper Reference +|---|---|---|--- +MR-HuBERT Base (~97M) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/mono_base/mrhubert_mono_base.pt) | mono\_base +MR-HuBERT Base (~321M) | [Libri-Light](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/mono_large/mrhubert_mono_large.pt) | mono\_large +Multilingual MR-HuBERT Base (~97M) | [Voxpopuli](https://github.com/facebookresearch/voxpopuli) 100k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/multi_base/multi_base.pt) | multi\_base +Multilingual MR-HuBERT Large (~321M) | [Voxpopuli](https://github.com/facebookresearch/voxpopuli) 100k hr | [download 400k steps](https://dl.fbaipublicfiles.com/mrhubert/multi_large/multi_large_400k.pt) or [download 600k steps](https://dl.fbaipublicfiles.com/mrhubert/multi_large/multi_large_600k.pt) | Not in the paper + + +### Abalation models +Model | Pretraining Data | Model | Paper Reference +|---|---|---|--- +MR-HuBERT Base (2-4-6 lyrs) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b1-a/b1-a.pt) | (B.1)-a +MR-HuBERT Base (5-2-5 lyrs) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b1-b/b1-b.pt) | (B.1)-b +MR-HuBERT Base (6-4-2 lyrs) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b1-c/b1-c.pt) | (B.1)-c +MR-HuBERT Base (3res 3-2-2-2-3 lyrs) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b2-a/b2-a.pt) | (B.2)-a +MR-HuBERT Base (3res 2-2-4-2-2 lyrs) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b2-b/b2-b.pt) | (B.2)-b +MR-HuBERT Base (3res 2-2-2-2-2 lyrs) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b2-c/b2-c.pt) | (B.2)-c +MR-HuBERT Base (Simple sampling) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b3-a/b3-a.pt) | (B.3)-a +MR-HuBERT Base (Single target) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b4-a/b4-a.pt) | (B.4)-a +MR-HuBERT Base (Simple Sampling + single target) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b4-b/b4-b.pt) | (B.4)-b +MR-HuBERT Base (Mono-resolution 20ms) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b5-a/b5-a.pt) | (B.5)-a +MR-HuBERT Base (3-3-3 lyrs) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b6-a/b6-a.pt) | (B.6)-a +MR-HuBERT Base (Mono-resolution 20ms, 3-3-3 lyrs) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b6-b/b6-b.pt) | (B.6)-b +MR-HuBERT Base (HuBERT 20ms&40ms units) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b7-a/b7-a.pt) | (B.7)-a +MR-HuBERT Base (Encodec 50Hz unit) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b7-b/b7-b.pt) | (B.7)-b +MR-HuBERT Base (Encodec 50Hz units and 25Hz units) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b7-c/b7-c.pt) | (B.7)-c +MR-HuBERT Base (Encodec 50Hz units stream 0&1 ) | [Librispeech](http://www.openslr.org/12) 960 hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b7-d/b7-d.pt) | (B.7)-d +MR-HuBERT Large (no audio norm) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b8-a/b8-a.pt) | (B.8)-a +MR-HuBERT Large (check paper ) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b8-b/b8-b.pt) | (B.8)-b +MR-HuBERT Large (check paper ) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b8-c/b8-c.pt) | (B.8)-c +MR-HuBERT Large (check paper ) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b8-d/b8-d.pt) | (B.8)-d +MR-HuBERT Large (check paper ) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b8-e/b8-e.pt) | (B.8)-e +MR-HuBERT Large (check paper ) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b8-f/b8-f.pt) | (B.8)-f +MR-HuBERT Large (check paper ) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b8-g/b8-g.pt) | (B.8)-g +MR-HuBERT Large (check paper ) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b8-h/b8-h.pt) | (B.8)-h +MR-HuBERT Large (check paper ) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b8-i/b8-i.pt) | (B.8)-i +MR-HuBERT Large (check paper ) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/b8-j/b8-j.pt) | (B.8)-j +Multilingual MR-HuBERT Large (Simple sampling) | [Voxpopuli](https://github.com/facebookresearch/voxpopuli) 100k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/multi_large_simple/multi_large_simple.pt) | Not in paper +MR-HuBERT xLarge (from HuBERT-base label) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/mono_xlarge/v1.pt) | Not in paper +MR-HuBERT xLarge (from HuBERT-large label) | [LibriLight](https://github.com/facebookresearch/libri-light) 60k hr | [download](https://dl.fbaipublicfiles.com/mrhubert/mono_xlarge/v2.pt) | Not in paper + +## Load a model +``` +ckpt_path = "/path/to/the/checkpoint.pt" +models, cfg, task = fairseq.checkpoint_utils.load_model_ensemble_and_task([ckpt_path]) +model = models[0] +``` + +## Train a new model + +### Data preparation + +Follow the steps in `./simple_kmeans` to create: +- `{train,valid}.tsv` waveform list files with length information +``` +/path/to/your/audio/files +file1.wav\t160000 +file2.wav\t154600 +... +filen.wav\t54362 +``` +- `{train,valid}.km` frame-aligned pseudo label files (the order is the same as wavefiles in the tsv file). +``` +44 44 44 48 48 962 962 962 962 962 962 962 962 967 967 967 967 967 967 967 967 370 852 370 ... 18 18 745 745 +44 44 44 48 48 962 962 962 147 147 147 147 147 147 147 147 147 147 147 147 176 176 271 271 ... 27 27 745 745 +... +44 44 44 48 962 962 962 962 962 962 377 377 377 77 77 852 696 694 433 578 578 82 740 622 ... 27 27 745 745 +``` +- `dict.km.txt` a dummy dictionary (first column is id, the second is dummy one) +``` +0 1 +1 1 +2 1 +... +999 1 +``` + +The `label_rate` is the same as the feature frame rate used for clustering, +which is 100Hz for MFCC features and 50Hz for HuBERT features by default. + +### Pre-train a MR-HuBERT model + +Suppose `{train,valid}.tsv` are saved at `/path/to/data`, `{train,valid}.km` +are saved at `/path/to/labels`, and the label rate is 100Hz. + +To train a base model (12 layer transformer), run: +```sh +$ python fairseq_cli/hydra_train.py \ + --config-dir /path/to/fairseq-py/examples/mr_hubert/config/pretrain \ + --config-name mrhubert_base_librispeech \ + task.data=/path/to/data task.label_dir=/path/to/labels \ + task.labels='["km"]' model.label_rate=100 \ + task.label_rate_ratios='[1, 2]' \ +``` + +Please see sample pre-training scripts `train.sh` for an example script. + +### Fine-tune a MR-HuBERT model with a CTC loss + +Suppose `{train,valid}.tsv` are saved at `/path/to/data`, and their +corresponding character transcripts `{train,valid}.ltr` are saved at +`/path/to/trans`. A typical ltr file is with the same order of tsv waveform files as +``` +HOW | ARE | YOU +... +THANK | YOU +``` + +To fine-tune a pre-trained MR-HuBERT model at `/path/to/checkpoint`, run +```sh +$ python fairseq_cli/hydra_train.py \ + --config-dir /path/to/fairseq-py/examples/mr_hubert/config/finetune \ + --config-name base_10h \ + task.data=/path/to/data task.label_dir=/path/to/trans \ + model.w2v_path=/path/to/checkpoint +``` + +Please see sample fine-tuning scripts `finetune.sh` for an example script. + +### Decode a MR-HuBERT model + +Suppose the `test.tsv` and `test.ltr` are the waveform list and transcripts of +the split to be decoded, saved at `/path/to/data`, and the fine-tuned model is +saved at `/path/to/checkpoint`. + + +We support three decoding modes: +- Viterbi decoding: greedy decoding without a language model +- KenLM decoding: decoding with an arpa-format KenLM n-gram language model +- Fairseq-LM deocding: decoding with a Fairseq neural language model (not fully tested) + + +#### Viterbi decoding + +`task.normalize` needs to be consistent with the value used during fine-tuning. +Decoding results will be saved at +`/path/to/experiment/directory/decode/viterbi/test`. + +```sh +$ python examples/speech_recognition/new/infer.py \ + --config-dir /path/to/fairseq-py/examples/mr_hubert/config/decode \ + --config-name infer \ + task.data=/path/to/data \ + task.normalize=[true|false] \ + decoding.exp_dir=/path/to/experiment/directory \ + common_eval.path=/path/to/checkpoint + dataset.gen_subset=test \ +``` + +#### KenLM / Fairseq-LM decoding + +Suppose the pronunciation lexicon and the n-gram LM are saved at +`/path/to/lexicon` and `/path/to/arpa`, respectively. Decoding results will be +saved at `/path/to/experiment/directory/decode/kenlm/test`. + +```sh +$ python examples/speech_recognition/new/infer.py \ + --config-dir /path/to/fairseq-py/examples/mr_hubert/config/decode \ + --config-name infer_lm \ + task.data=/path/to/data \ + task.normalize=[true|false] \ + decoding.exp_dir=/path/to/experiment/directory \ + common_eval.path=/path/to/checkpoint + dataset.gen_subset=test \ + decoding.decoder.lexicon=/path/to/lexicon \ + decoding.decoder.lmpath=/path/to/arpa +``` + +The command above uses the default decoding hyperparameter, which can be found +in `examples/speech_recognition/hydra/decoder.py`. These parameters can be +configured from the command line. For example, to search with a beam size of +500, we can append the command above with `decoding.decoder.beam=500`. +Important parameters include: +- decoding.decoder.beam +- decoding.decoder.beamthreshold +- decoding.decoder.lmweight +- decoding.decoder.wordscore +- decoding.decoder.silweight + +To decode with a Fairseq LM, you may check the usage examples in wav2vec2 or hubert examples. + +Please see sample decoding scripts `decode.sh` for an example script. diff --git a/data/fairseq/examples/mr_hubert/config/decode/infer.yaml b/data/fairseq/examples/mr_hubert/config/decode/infer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eff39802e7a398c804e4c7eacdb2eee6ab33db81 --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/decode/infer.yaml @@ -0,0 +1,30 @@ +# @package _group_ + +defaults: + - model: null + +hydra: + run: + dir: ${common_eval.results_path}/viterbi + sweep: + dir: ${common_eval.results_path} + subdir: viterbi + +task: + _name: multires_hubert_pretraining + single_target: true + fine_tuning: true + label_rate_ratios: ??? + data: ??? + normalize: false + +decoding: + type: viterbi + unique_wer_file: true +common_eval: + results_path: ??? + path: ??? + post_process: letter +dataset: + max_tokens: 1100000 + gen_subset: ??? diff --git a/data/fairseq/examples/mr_hubert/config/decode/infer_lm.yaml b/data/fairseq/examples/mr_hubert/config/decode/infer_lm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..535b95077560dd1c64a9fa2d6ae43641068afc59 --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/decode/infer_lm.yaml @@ -0,0 +1,37 @@ +# @package _group_ + +defaults: + - model: null + +hydra: + run: + dir: ${common_eval.results_path}/beam${decoding.beam}_th${decoding.beamthreshold}_lmw${decoding.lmweight}_wrd${decoding.wordscore}_sil${decoding.silweight} + sweep: + dir: ${common_eval.results_path} + subdir: beam${decoding.beam}_th${decoding.beamthreshold}_lmw${decoding.lmweight}_wrd${decoding.wordscore}_sil${decoding.silweight} + +task: + _name: multires_hubert_pretraining + single_target: true + fine_tuning: true + data: ??? + label_rate_ratios: ??? + normalize: ??? + +decoding: + type: kenlm + lexicon: ??? + lmpath: ??? + beamthreshold: 100 + beam: 500 + lmweight: 1.5 + wordscore: -1 + silweight: 0 + unique_wer_file: true +common_eval: + results_path: ??? + path: ??? + post_process: letter +dataset: + max_tokens: 1100000 + gen_subset: ??? diff --git a/data/fairseq/examples/mr_hubert/config/decode/run/submitit_slurm.yaml b/data/fairseq/examples/mr_hubert/config/decode/run/submitit_slurm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b8065832ecacf9dd4fe4e99c87941e00fb3ef7f --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/decode/run/submitit_slurm.yaml @@ -0,0 +1,17 @@ +# @package _global_ +hydra: + launcher: + cpus_per_task: ${distributed_training.distributed_world_size} + gpus_per_node: ${distributed_training.distributed_world_size} + tasks_per_node: ${hydra.launcher.gpus_per_node} + nodes: 1 + mem_gb: 200 + timeout_min: 4320 + max_num_timeout: 50 + name: ${hydra.job.config_name} + submitit_folder: ${hydra.sweep.dir}/submitit + +distributed_training: + distributed_world_size: 1 + distributed_no_spawn: true + distributed_port: 29761 diff --git a/data/fairseq/examples/mr_hubert/config/decode/run/submitit_slurm_8gpu.yaml b/data/fairseq/examples/mr_hubert/config/decode/run/submitit_slurm_8gpu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f669f376312dbfe4611cc08f4996a314155fb87 --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/decode/run/submitit_slurm_8gpu.yaml @@ -0,0 +1,17 @@ +# @package _global_ +hydra: + launcher: + cpus_per_task: ${distributed_training.distributed_world_size} + gpus_per_node: ${distributed_training.distributed_world_size} + tasks_per_node: ${hydra.launcher.gpus_per_node} + nodes: 1 + mem_gb: 200 + timeout_min: 4320 + max_num_timeout: 50 + name: ${hydra.job.config_name} + submitit_folder: ${hydra.sweep.dir}/submitit + +distributed_training: + distributed_world_size: 8 + distributed_no_spawn: true + distributed_port: 29761 diff --git a/data/fairseq/examples/mr_hubert/config/finetune/base_100h.yaml b/data/fairseq/examples/mr_hubert/config/finetune/base_100h.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c52a118cb819d8504eb44501fed90a3e8f68bc9f --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/finetune/base_100h.yaml @@ -0,0 +1,97 @@ +# @package _group_ + +common: + fp16: true + log_format: json + log_interval: 200 + tensorboard_logdir: tblog + seed: 1337 + +checkpoint: + no_epoch_checkpoints: true + best_checkpoint_metric: wer + +distributed_training: + ddp_backend: c10d + find_unused_parameters: true + distributed_world_size: 8 + distributed_port: 29671 + nprocs_per_node: 8 + +task: + _name: multires_hubert_pretraining + data: ??? + fine_tuning: true + label_dir: ??? + label_rate_ratios: ??? + normalize: false # must be consistent with pre-training + labels: ["ltr"] + single_target: true + +dataset: + num_workers: 0 + max_tokens: 3200000 + validate_after_updates: ${model.freeze_finetune_updates} + validate_interval: 5 + train_subset: train_100h + valid_subset: dev_other + +criterion: + _name: ctc + zero_infinity: true + +optimization: + max_update: 80000 + lr: [3e-5] + sentence_avg: true + update_freq: [1] + +optimizer: + _name: adam + adam_betas: (0.9,0.98) + adam_eps: 1e-08 + +lr_scheduler: + _name: tri_stage + phase_ratio: [0.1, 0.4, 0.5] + final_lr_scale: 0.05 + +model: + _name: multires_hubert_ctc + multires_hubert_path: ??? + apply_mask: true + mask_selection: static + mask_length: 10 + mask_other: 0 + mask_prob: 0.75 + mask_channel_selection: static + mask_channel_length: 64 + mask_channel_other: 0 + mask_channel_prob: 0.5 + layerdrop: 0.1 + dropout: 0.0 + activation_dropout: 0.1 + attention_dropout: 0.0 + feature_grad_mult: 0.0 + freeze_finetune_updates: 10000 + +hydra: + job: + config: + override_dirname: + kv_sep: '-' + item_sep: '__' + exclude_keys: + - run + - task.data + - task.label_dir + - model.multires_hubert_path + - dataset.train_subset + - dataset.valid_subset + - criterion.wer_kenlm_model + - criterion.wer_lexicon + run: + dir: ??? + sweep: + dir: ??? + subdir: ${hydra.job.config_name}__${hydra.job.override_dirname} diff --git a/data/fairseq/examples/mr_hubert/config/finetune/base_100h_large.yaml b/data/fairseq/examples/mr_hubert/config/finetune/base_100h_large.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d0c0da3dbc336495640f7c1fc35c9b72814021f --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/finetune/base_100h_large.yaml @@ -0,0 +1,97 @@ +# @package _group_ + +common: + fp16: true + log_format: json + log_interval: 200 + tensorboard_logdir: tblog + seed: 1337 + +checkpoint: + no_epoch_checkpoints: true + best_checkpoint_metric: wer + +distributed_training: + ddp_backend: c10d + find_unused_parameters: true + distributed_world_size: 8 + distributed_port: 29671 + nprocs_per_node: 8 + +task: + _name: multires_hubert_pretraining + data: ??? + fine_tuning: true + label_dir: ??? + label_rate_ratios: ??? + normalize: true # must be consistent with pre-training + labels: ["ltr"] + single_target: true + +dataset: + num_workers: 0 + max_tokens: 1600000 + validate_after_updates: ${model.freeze_finetune_updates} + validate_interval: 5 + train_subset: train_100h + valid_subset: dev_other + +criterion: + _name: ctc + zero_infinity: true + +optimization: + max_update: 80000 + lr: [3e-5] + sentence_avg: true + update_freq: [2] + +optimizer: + _name: adam + adam_betas: (0.9,0.98) + adam_eps: 1e-08 + +lr_scheduler: + _name: tri_stage + phase_ratio: [0.1, 0.4, 0.5] + final_lr_scale: 0.05 + +model: + _name: multires_hubert_ctc + multires_hubert_path: ??? + apply_mask: true + mask_selection: static + mask_length: 10 + mask_other: 0 + mask_prob: 0.75 + mask_channel_selection: static + mask_channel_length: 64 + mask_channel_other: 0 + mask_channel_prob: 0.5 + layerdrop: 0.1 + dropout: 0.0 + activation_dropout: 0.1 + attention_dropout: 0.0 + feature_grad_mult: 0.0 + freeze_finetune_updates: 10000 + +hydra: + job: + config: + override_dirname: + kv_sep: '-' + item_sep: '__' + exclude_keys: + - run + - task.data + - task.label_dir + - model.multires_hubert_path + - dataset.train_subset + - dataset.valid_subset + - criterion.wer_kenlm_model + - criterion.wer_lexicon + run: + dir: ??? + sweep: + dir: ??? + subdir: ${hydra.job.config_name}__${hydra.job.override_dirname} diff --git a/data/fairseq/examples/mr_hubert/config/finetune/base_10h.yaml b/data/fairseq/examples/mr_hubert/config/finetune/base_10h.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25123e44816d13fd5d53e77bcd6e9e6fa0369030 --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/finetune/base_10h.yaml @@ -0,0 +1,101 @@ +# @package _group_ + +common: + fp16: true + log_format: json + log_interval: 200 + tensorboard_logdir: tblog + seed: 1337 + +checkpoint: + save_interval: 5 + keep_interval_updates: 1 + no_epoch_checkpoints: true + best_checkpoint_metric: wer + +distributed_training: + ddp_backend: c10d + find_unused_parameters: true + distributed_world_size: 8 + distributed_port: 29671 + nprocs_per_node: 8 + +task: + _name: multires_hubert_pretraining + data: ??? + fine_tuning: true + label_dir: ??? + label_rate_ratios: ??? + normalize: false # must be consistent with pre-training + labels: ["ltr"] + single_target: true + +dataset: + num_workers: 0 + max_tokens: 3200000 + validate_after_updates: ${model.freeze_finetune_updates} + validate_interval: 5 + train_subset: train_10h + valid_subset: dev + +criterion: + _name: ctc + zero_infinity: true + +optimization: + max_update: 25000 + lr: [2e-5] + sentence_avg: true + update_freq: [1] + +optimizer: + _name: adam + adam_betas: (0.9,0.98) + adam_eps: 1e-08 + +lr_scheduler: + _name: tri_stage + warmup_steps: 8000 + hold_steps: 0 + decay_steps: 72000 + final_lr_scale: 0.05 + +model: + _name: multires_hubert_ctc + multires_hubert_path: ??? + apply_mask: true + mask_selection: static + mask_length: 10 + mask_other: 0 + mask_prob: 0.75 + mask_channel_selection: static + mask_channel_length: 64 + mask_channel_other: 0 + mask_channel_prob: 0.5 + layerdrop: 0.1 + dropout: 0.0 + activation_dropout: 0.1 + attention_dropout: 0.0 + feature_grad_mult: 0.0 + freeze_finetune_updates: 10000 + +hydra: + job: + config: + override_dirname: + kv_sep: '-' + item_sep: '__' + exclude_keys: + - run + - task.data + - task.label_dir + - model.multires_hubert_path + - dataset.train_subset + - dataset.valid_subset + - criterion.wer_kenlm_model + - criterion.wer_lexicon + run: + dir: ??? + sweep: + dir: ??? + subdir: ${hydra.job.config_name}__${hydra.job.override_dirname} diff --git a/data/fairseq/examples/mr_hubert/config/finetune/base_10h_large.yaml b/data/fairseq/examples/mr_hubert/config/finetune/base_10h_large.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65448c7722c7cc059fcb7e153c86d1583fff10bd --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/finetune/base_10h_large.yaml @@ -0,0 +1,101 @@ +# @package _group_ + +common: + fp16: true + log_format: json + log_interval: 200 + tensorboard_logdir: tblog + seed: 1337 + +checkpoint: + save_interval: 5 + keep_interval_updates: 1 + no_epoch_checkpoints: true + best_checkpoint_metric: wer + +distributed_training: + ddp_backend: c10d + find_unused_parameters: true + distributed_world_size: 8 + distributed_port: 29671 + nprocs_per_node: 8 + +task: + _name: multires_hubert_pretraining + data: ??? + fine_tuning: true + label_dir: ??? + label_rate_ratios: ??? + normalize: true # must be consistent with pre-training + labels: ["ltr"] + single_target: true + +dataset: + num_workers: 0 + max_tokens: 3200000 + validate_after_updates: ${model.freeze_finetune_updates} + validate_interval: 5 + train_subset: train_10h + valid_subset: dev + +criterion: + _name: ctc + zero_infinity: true + +optimization: + max_update: 25000 + lr: [2e-5] + sentence_avg: true + update_freq: [1] + +optimizer: + _name: adam + adam_betas: (0.9,0.98) + adam_eps: 1e-08 + +lr_scheduler: + _name: tri_stage + warmup_steps: 8000 + hold_steps: 0 + decay_steps: 72000 + final_lr_scale: 0.05 + +model: + _name: multires_hubert_ctc + multires_hubert_path: ??? + apply_mask: true + mask_selection: static + mask_length: 10 + mask_other: 0 + mask_prob: 0.75 + mask_channel_selection: static + mask_channel_length: 64 + mask_channel_other: 0 + mask_channel_prob: 0.5 + layerdrop: 0.1 + dropout: 0.0 + activation_dropout: 0.1 + attention_dropout: 0.0 + feature_grad_mult: 0.0 + freeze_finetune_updates: 10000 + +hydra: + job: + config: + override_dirname: + kv_sep: '-' + item_sep: '__' + exclude_keys: + - run + - task.data + - task.label_dir + - model.multires_hubert_path + - dataset.train_subset + - dataset.valid_subset + - criterion.wer_kenlm_model + - criterion.wer_lexicon + run: + dir: ??? + sweep: + dir: ??? + subdir: ${hydra.job.config_name}__${hydra.job.override_dirname} diff --git a/data/fairseq/examples/mr_hubert/config/finetune/base_1h.yaml b/data/fairseq/examples/mr_hubert/config/finetune/base_1h.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7459c3fc4c4ccdd96ae6648d1f95d7e2bb59ab1f --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/finetune/base_1h.yaml @@ -0,0 +1,100 @@ +# @package _group_ + +common: + fp16: true + log_format: json + log_interval: 200 + tensorboard_logdir: tblog + seed: 1337 + +checkpoint: + save_interval: 50 + keep_interval_updates: 1 + save_interval_updates: 1000 + no_epoch_checkpoints: true + best_checkpoint_metric: wer + +distributed_training: + ddp_backend: c10d + find_unused_parameters: true + distributed_world_size: 8 + distributed_port: 29671 + nprocs_per_node: 8 + +task: + _name: multires_hubert_pretraining + data: ??? + fine_tuning: true + label_dir: ??? + label_rate_ratios: ??? + normalize: false # must be consistent with pre-training + labels: ["ltr"] + single_target: true + +dataset: + num_workers: 0 + max_tokens: 3200000 + validate_after_updates: ${model.freeze_finetune_updates} + validate_interval: 1000 + train_subset: train_1h + valid_subset: dev_other + +criterion: + _name: ctc + zero_infinity: true + +optimization: + max_update: 13000 + lr: [5e-5] + sentence_avg: true + update_freq: [4] + +optimizer: + _name: adam + adam_betas: (0.9,0.98) + adam_eps: 1e-08 + +lr_scheduler: + _name: tri_stage + phase_ratio: [0.1, 0.4, 0.5] + final_lr_scale: 0.05 + +model: + _name: multires_hubert_ctc + multires_hubert_path: ??? + apply_mask: true + mask_selection: static + mask_length: 10 + mask_other: 0 + mask_prob: 0.75 + mask_channel_selection: static + mask_channel_length: 64 + mask_channel_other: 0 + mask_channel_prob: 0.5 + layerdrop: 0.1 + dropout: 0.0 + activation_dropout: 0.1 + attention_dropout: 0.0 + feature_grad_mult: 0.0 + freeze_finetune_updates: 10000 + +hydra: + job: + config: + override_dirname: + kv_sep: '-' + item_sep: '__' + exclude_keys: + - run + - task.data + - task.label_dir + - model.multires_hubert_path + - dataset.train_subset + - dataset.valid_subset + - criterion.wer_kenlm_model + - criterion.wer_lexicon + run: + dir: ??? + sweep: + dir: ??? + subdir: ${hydra.job.config_name}__${hydra.job.override_dirname} diff --git a/data/fairseq/examples/mr_hubert/config/finetune/base_1h_large.yaml b/data/fairseq/examples/mr_hubert/config/finetune/base_1h_large.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34ef4dc19de87b20a53af49b527b5fc0002625d2 --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/finetune/base_1h_large.yaml @@ -0,0 +1,99 @@ +# @package _group_ + +common: + fp16: true + log_format: json + log_interval: 200 + tensorboard_logdir: tblog + seed: 1337 + +checkpoint: + save_interval: 1000 + keep_interval_updates: 1 + no_epoch_checkpoints: true + best_checkpoint_metric: wer + +distributed_training: + ddp_backend: c10d + find_unused_parameters: true + distributed_world_size: 8 + distributed_port: 29671 + nprocs_per_node: 8 + +task: + _name: multires_hubert_pretraining + data: ??? + fine_tuning: true + label_dir: ??? + label_rate_ratios: ??? + normalize: true # must be consistent with pre-training + labels: ["ltr"] + single_target: true + +dataset: + num_workers: 0 + max_tokens: 1280000 + validate_after_updates: ${model.freeze_finetune_updates} + validate_interval: 5 + train_subset: train_10h + valid_subset: dev + +criterion: + _name: ctc + zero_infinity: true + +optimization: + max_update: 25000 + lr: [3e-4] + sentence_avg: true + update_freq: [5] + +optimizer: + _name: adam + adam_betas: (0.9,0.98) + adam_eps: 1e-08 + +lr_scheduler: + _name: tri_stage + phase_ratio: [0.1, 0.4, 0.5] + final_lr_scale: 0.05 + +model: + _name: multires_hubert_ctc + multires_hubert_path: ??? + apply_mask: true + mask_selection: static + mask_length: 10 + mask_other: 0 + mask_prob: 0.75 + mask_channel_selection: static + mask_channel_length: 64 + mask_channel_other: 0 + mask_channel_prob: 0.5 + layerdrop: 0.1 + dropout: 0.0 + activation_dropout: 0.1 + attention_dropout: 0.0 + feature_grad_mult: 0.0 + freeze_finetune_updates: 10000 + +hydra: + job: + config: + override_dirname: + kv_sep: '-' + item_sep: '__' + exclude_keys: + - run + - task.data + - task.label_dir + - model.multires_hubert_path + - dataset.train_subset + - dataset.valid_subset + - criterion.wer_kenlm_model + - criterion.wer_lexicon + run: + dir: ??? + sweep: + dir: ??? + subdir: ${hydra.job.config_name}__${hydra.job.override_dirname} diff --git a/data/fairseq/examples/mr_hubert/config/pretrain/mrhubert_base_librispeech.yaml b/data/fairseq/examples/mr_hubert/config/pretrain/mrhubert_base_librispeech.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16a35d340a6994210da6d73280369b4635cb9c61 --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/pretrain/mrhubert_base_librispeech.yaml @@ -0,0 +1,103 @@ +# @package _group_ + +common: + fp16: true + log_format: json + log_interval: 200 + seed: 1337 + tensorboard_logdir: tblog + min_loss_scale: 1e-8 + +checkpoint: + save_interval_updates: 25000 + keep_interval_updates: 1 + no_epoch_checkpoints: true + +distributed_training: + ddp_backend: no_c10d + distributed_backend: 'nccl' + distributed_world_size: 32 + distributed_port: 29671 + nprocs_per_node: 8 + find_unused_parameters: true + +task: + _name: multires_hubert_pretraining + data: ??? + label_dir: ??? + labels: ??? + label_rate: ${model.label_rate} + label_rate_ratios: ??? + sample_rate: 16000 + max_sample_size: 250000 + min_sample_size: 32000 + pad_audio: false + random_crop: true + normalize: false # must be consistent with extractor + # max_keep_size: 300000 + # max_keep_size: 50000 + + +dataset: + num_workers: 0 + max_tokens: 1000000 + skip_invalid_size_inputs_valid_test: true + validate_interval: 5 + validate_interval_updates: 10000 + +criterion: + _name: hubert + pred_masked_weight: 1.0 + pred_nomask_weight: 0.0 + loss_weights: [10,] + +optimization: + max_update: 400000 + lr: [0.0005] + clip_norm: 10.0 + +optimizer: + _name: adam + adam_betas: (0.9,0.98) + adam_eps: 1e-06 + weight_decay: 0.01 + +lr_scheduler: + _name: polynomial_decay + warmup_updates: 32000 + +model: + _name: multires_hubert + label_rate: ??? + label_rate_ratios: ${task.label_rate_ratios} + skip_masked: false + skip_nomask: false + mask_prob: 0.80 + extractor_mode: default + conv_feature_layers: '[(512,10,5)] + [(512,3,2)] * 4 + [(512,2,2)] * 2' + final_dim: 256 + encoder_layers: 4 + encoder_layerdrop: 0.05 + dropout_input: 0.1 + dropout_features: 0.1 + dropout: 0.1 + attention_dropout: 0.1 + feature_grad_mult: 0.1 + untie_final_proj: true + activation_dropout: 0.0 + conv_adapator_kernal: 1 + use_single_target: true + +hydra: + job: + config: + override_dirname: + kv_sep: '-' + item_sep: '/' + exclude_keys: + - run + - task.data + - task.label_dir + - common.min_loss_scale + - common.log_interval + - optimization.clip_norm diff --git a/data/fairseq/examples/mr_hubert/config/pretrain/mrhubert_large_librilight.yaml b/data/fairseq/examples/mr_hubert/config/pretrain/mrhubert_large_librilight.yaml new file mode 100644 index 0000000000000000000000000000000000000000..423f3b25c22bfa5a18216ddd317546eedb785544 --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/pretrain/mrhubert_large_librilight.yaml @@ -0,0 +1,107 @@ +# @package _group_ + +common: + memory_efficient_fp16: true + log_format: json + log_interval: 200 + seed: 1337 + tensorboard_logdir: tblog + +checkpoint: + save_interval_updates: 25000 + keep_interval_updates: 1 + no_epoch_checkpoints: true + + +distributed_training: + ddp_backend: no_c10d + distributed_backend: 'nccl' + distributed_world_size: 128 + distributed_port: 29671 + nprocs_per_node: 8 + find_unused_parameters: true + +task: + _name: multires_hubert_pretraining + data: ??? + label_dir: ??? + labels: ??? + label_rate: ${model.label_rate} + label_rate_ratios: ??? + sample_rate: 16000 + max_sample_size: 250000 + min_sample_size: 32000 + pad_audio: false + random_crop: true + normalize: true # must be consistent with extractor + # max_keep_size: 50000 + +dataset: + num_workers: 0 + max_tokens: 300000 + skip_invalid_size_inputs_valid_test: true + validate_interval: 5 + validate_interval_updates: 10000 + +criterion: + _name: hubert + pred_masked_weight: 1.0 + pred_nomask_weight: 0.0 + loss_weights: [10,] + +optimization: + max_update: 400000 + lr: [0.0015] + clip_norm: 1.0 + update_freq: [3] + +optimizer: + _name: adam + adam_betas: (0.9,0.98) + adam_eps: 1e-06 + weight_decay: 0.01 + +lr_scheduler: + _name: polynomial_decay + warmup_updates: 32000 + +model: + _name: multires_hubert + label_rate: ??? + label_rate_ratios: ${task.label_rate_ratios} + encoder_layers: 8 + encoder_embed_dim: 1024 + encoder_ffn_embed_dim: 4096 + encoder_attention_heads: 16 + final_dim: 768 + skip_masked: false + skip_nomask: false + mask_prob: 0.80 + extractor_mode: layer_norm + conv_feature_layers: '[(512,10,5)] + [(512,3,2)] * 4 + [(512,2,2)] * 2' + encoder_layerdrop: 0.0 + dropout_input: 0.0 + dropout_features: 0.0 + dropout: 0.0 + attention_dropout: 0.0 + layer_norm_first: true + feature_grad_mult: 1.0 + untie_final_proj: true + activation_dropout: 0.0 + conv_adapator_kernal: 1 + use_single_target: true + +hydra: + job: + config: + override_dirname: + kv_sep: '-' + item_sep: '__' + exclude_keys: + - run + - task.data + run: + dir: /checkpoint/wnhsu/w2v/hubert_final/hydra_pt + sweep: + dir: /checkpoint/wnhsu/w2v/hubert_final/hydra_pt + subdir: ${hydra.job.config_name}__${hydra.job.override_dirname} diff --git a/data/fairseq/examples/mr_hubert/config/pretrain/run/submitit_reg.yaml b/data/fairseq/examples/mr_hubert/config/pretrain/run/submitit_reg.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46c979cd2835fe026b0a532a54533904d1001e54 --- /dev/null +++ b/data/fairseq/examples/mr_hubert/config/pretrain/run/submitit_reg.yaml @@ -0,0 +1,20 @@ +# @package _global_ + +hydra: + launcher: + cpus_per_task: 8 + gpus_per_node: 8 + tasks_per_node: ${hydra.launcher.gpus_per_node} + nodes: 4 + comment: null + mem_gb: 384 + timeout_min: 4320 + max_num_timeout: 100 + constraint: volta32gb + name: ${hydra.job.config_name}/${hydra.job.override_dirname} + submitit_folder: ${hydra.sweep.dir}/submitit/%j + +distributed_training: + distributed_world_size: 32 + distributed_port: 29671 + nprocs_per_node: 8 diff --git a/data/fairseq/examples/mr_hubert/decode.sh b/data/fairseq/examples/mr_hubert/decode.sh new file mode 100644 index 0000000000000000000000000000000000000000..1ff423a84c638bb8f0a89073a93380b57246423f --- /dev/null +++ b/data/fairseq/examples/mr_hubert/decode.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +FAIRSEQ= # Setup your fairseq directory + +config_dir=${FAIRSEQ}/examples/mr_hubert/config +config_name=mr_hubert_base_librispeech + + +# Prepared Data Directory + +data_dir=librispeech +# -- data_dir +# -- test.tsv +# -- test.ltr +# -- dict.ltr.txt + + +exp_dir=exp # Target experiments directory (where you have your pre-trained model with checkpoint_best.pt) +ratios="[1, 2]" # Default label rate ratios + +_opts= + +# If use slurm, uncomment this line and modify the job submission at +# _opts="${_opts} hydra/launcher=submitit_slurm +hydra.launcher.partition=${your_slurm_partition} +run=submitit_reg" + +# If want to set additional experiment tag, uncomment this line +# _opts="${_opts} hydra.sweep.subdir=${your_experiment_tag}" + +# If use un-normalized audio, uncomment this line +# _opts="${_opts} task.normalize=false" + + + +PYTHONPATH=${FAIRSEQ} +python examples/speech_recognition/new/infer.py \ + --config-dir ${config_dir} \ + --config-name infer_multires \ + ${_opts} \ + task.data=${data_dir} \ + task.label_rate_ratios='${ratios}' \ + common_eval.results_path=${exp_dir} \ + common_eval.path=${exp_dir}/checkpoint_best.pt \ + dataset.max_tokens=2000000 \ + dataset.gen_subset=test \ + dataset.skip_invalid_size_inputs_valid_test=true + diff --git a/data/fairseq/examples/mr_hubert/finetune.sh b/data/fairseq/examples/mr_hubert/finetune.sh new file mode 100644 index 0000000000000000000000000000000000000000..31ba645560346fd50aacca1fcc14dfe7cd5c3449 --- /dev/null +++ b/data/fairseq/examples/mr_hubert/finetune.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +FAIRSEQ= # Setup your fairseq directory + +config_dir=${FAIRSEQ}/examples/mr_hubert/config +config_name=mr_hubert_base_librispeech + +# override configs if need +max_tokens=3200000 +max_sample_size=1000000 +max_update=50000 + + +# Prepared Data Directory + +data_dir=librispeech +# -- data_dir +# -- train.tsv +# -- train.ltr +# -- valid.tsv +# -- valid.ltr +# -- dict.ltr.txt + + +exp_dir=exp # Target experiments directory +ratios="[1, 2]" # Default label rate ratios +hubert_path=/path/of/your/hubert.pt + +_opts= + +# If use slurm, uncomment this line and modify the job submission at +# _opts="${_opts} hydra/launcher=submitit_slurm +hydra.launcher.partition=${your_slurm_partition} +run=submitit_reg" + +# If want to set additional experiment tag, uncomment this line +# _opts="${_opts} hydra.sweep.subdir=${your_experiment_tag}" + + +python ${FAIRSEQ}/fairseq_cli/hydra_train.py \ + -m --config-dir ${config_dir} --config-name ${config_name} ${_opts} \ + task.data=${data_dir} +task.max_sample_size=${max_sample_size} \ + task.label_dir=${data_dir} \ + task.label_rate_ratios='${ratios}' \ + dataset.max_tokens=${max_tokens} \ + optimization.max_update=${max_update} \ + model.multires_hubert_path=${hubert_path} \ + hydra.sweep.dir=${exp_dir} & diff --git a/data/fairseq/examples/mr_hubert/train.sh b/data/fairseq/examples/mr_hubert/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..da561eb171416f3c321f5dac681cdc9603bebe83 --- /dev/null +++ b/data/fairseq/examples/mr_hubert/train.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +FAIRSEQ= # Setup your fairseq directory + +config_dir=${FAIRSEQ}/examples/mr_hubert/config +config_name=mr_hubert_base_librispeech + +# Prepared Data Directory +data_dir=librispeech +# -- data_dir +# -- train.tsv +# -- valid.tsv + +label_dir=labels +# -- label_dir +# -- train.km +# -- valid.km +# -- dict.km.txt + + +exp_dir=exp # Target experiments directory +ratios="[1, 2]" # Default label rate ratios +label_rate=50 # Base label rate + + +_opts= + +# If use slurm, uncomment this line and modify the job submission at +# _opts="${_opts} hydra/launcher=submitit_slurm +hydra.launcher.partition=${your_slurm_partition} +run=submitit_reg" + +# If want to set additional experiment tag, uncomment this line +# _opts="${_opts} hydra.sweep.subdir=${your_experiment_tag}" + + +python ${FAIRSEQ}/fairseq_cli/hydra_train.py \ + -m --config-dir ${config_dir} --config-name ${config_name} ${_opts} \ + task.data=${data_dir} \ + task.label_dir=${label_dir} \ + task.labels='["km"]' \ + model.label_rate=${label_rate} \ + task.label_rate_ratios='${ratios}' \ + hydra.sweep.dir=${exp_dir} & + + + diff --git a/data/fairseq/examples/multilingual/ML50_langs.txt b/data/fairseq/examples/multilingual/ML50_langs.txt new file mode 100644 index 0000000000000000000000000000000000000000..558abbc785072629de8000e343fc02a32c0afb97 --- /dev/null +++ b/data/fairseq/examples/multilingual/ML50_langs.txt @@ -0,0 +1,52 @@ +ar_AR +cs_CZ +de_DE +en_XX +es_XX +et_EE +fi_FI +fr_XX +gu_IN +hi_IN +it_IT +ja_XX +kk_KZ +ko_KR +lt_LT +lv_LV +my_MM +ne_NP +nl_XX +ro_RO +ru_RU +si_LK +tr_TR +vi_VN +zh_CN +af_ZA +az_AZ +bn_IN +fa_IR +he_IL +hr_HR +id_ID +ka_GE +km_KH +mk_MK +ml_IN +mn_MN +mr_IN +pl_PL +ps_AF +pt_XX +sv_SE +sw_KE +ta_IN +te_IN +th_TH +tl_XX +uk_UA +ur_PK +xh_ZA +gl_ES +sl_SI \ No newline at end of file diff --git a/data/fairseq/examples/multilingual/README.md b/data/fairseq/examples/multilingual/README.md new file mode 100644 index 0000000000000000000000000000000000000000..46ff9c351b1030e0729f89f246e0cd86444c1633 --- /dev/null +++ b/data/fairseq/examples/multilingual/README.md @@ -0,0 +1,158 @@ +# Multilingual Translation + +[[Multilingual Translation with Extensible Multilingual Pretraining and Finetuning, https://arxiv.org/abs/2008.00401]](https://arxiv.org/abs/2008.00401) + +## Introduction + +This work is for training multilingual translation models with multiple bitext datasets. This multilingual translation framework supports (see [[training section]](#Training) and [[finetuning section]](#Finetuning) for examples) + +* temperature based sampling over unbalancing datasets of different translation directions + - --sampling-method' with + choices=['uniform', 'temperature', 'concat'] + - --sampling-temperature +* configurable to automatically add source and/or target language tokens to source/target sentences using data which are prepared in the same way as bilignual training + - --encoder-langtok with choices=['src', 'tgt', None] to specify whether to add source or target language tokens to the source sentences + - --decoder-langtok (binary option) to specify whether to add target language tokens to the target sentences or not +* finetuning mBART pretrained models for multilingual translation + - --finetune-from-model to specify the path from which to load the pretrained model + +## Preprocessing data +Multilingual training requires a joint BPE vocab. Please follow [mBART's preprocessing steps](https://github.com/pytorch/fairseq/tree/main/examples/mbart#bpe-data) to reuse our pretrained sentence-piece model. + +You can also train a joint BPE model on your own dataset and then follow the steps in [[link]](https://github.com/pytorch/fairseq/tree/main/examples/translation#multilingual-translation). + +## Training + + +```bash +lang_pairs= +path_2_data= +lang_list= + +fairseq-train $path_2_data \ + --encoder-normalize-before --decoder-normalize-before \ + --arch transformer --layernorm-embedding \ + --task translation_multi_simple_epoch \ + --sampling-method "temperature" \ + --sampling-temperature 1.5 \ + --encoder-langtok "src" \ + --decoder-langtok \ + --lang-dict "$lang_list" \ + --lang-pairs "$lang_pairs" \ + --criterion label_smoothed_cross_entropy --label-smoothing 0.2 \ + --optimizer adam --adam-eps 1e-06 --adam-betas '(0.9, 0.98)' \ + --lr-scheduler inverse_sqrt --lr 3e-05 --warmup-updates 2500 --max-update 40000 \ + --dropout 0.3 --attention-dropout 0.1 --weight-decay 0.0 \ + --max-tokens 1024 --update-freq 2 \ + --save-interval 1 --save-interval-updates 5000 --keep-interval-updates 10 --no-epoch-checkpoints \ + --seed 222 --log-format simple --log-interval 2 +``` + +## Finetuning +We can also finetune multilingual models from a monolingual pretrained models, e.g. [mMBART](https://github.com/pytorch/fairseq/tree/main/examples/mbart). +```bash +lang_pairs= +path_2_data= +lang_list= +pretrained_model= + +fairseq-train $path_2_data \ + --finetune-from-model $pretrained_model \ + --encoder-normalize-before --decoder-normalize-before \ + --arch transformer --layernorm-embedding \ + --task translation_multi_simple_epoch \ + --sampling-method "temperature" \ + --sampling-temperature 1.5 \ + --encoder-langtok "src" \ + --decoder-langtok \ + --lang-dict "$lang_list" \ + --lang-pairs "$lang_pairs" \ + --criterion label_smoothed_cross_entropy --label-smoothing 0.2 \ + --optimizer adam --adam-eps 1e-06 --adam-betas '(0.9, 0.98)' \ + --lr-scheduler inverse_sqrt --lr 3e-05 --warmup-updates 2500 --max-update 40000 \ + --dropout 0.3 --attention-dropout 0.1 --weight-decay 0.0 \ + --max-tokens 1024 --update-freq 2 \ + --save-interval 1 --save-interval-updates 5000 --keep-interval-updates 10 --no-epoch-checkpoints \ + --seed 222 --log-format simple --log-interval 2 +``` +## Generate +The following command uses the multilingual task (translation_multi_simple_epoch) to generate translation from $source_lang to $target_lang on the test dataset. During generaton, the source language tokens are added to source sentences and the target language tokens are added as the starting token to decode target sentences. Options --lang-dict and --lang-pairs are needed to tell the generation process the ordered list of languages and translation directions that the trained model are awared of; they will need to be consistent with the training. + +```bash +model= +source_lang= +target_lang= + +fairseq-generate $path_2_data \ + --path $model \ + --task translation_multi_simple_epoch \ + --gen-subset test \ + --source-lang $source_lang \ + --target-lang $target_lang + --sacrebleu --remove-bpe 'sentencepiece'\ + --batch-size 32 \ + --encoder-langtok "src" \ + --decoder-langtok \ + --lang-dict "$lang_list" \ + --lang-pairs "$lang_pairs" > ${source_lang}_${target_lang}.txt +``` +Fairseq will generate translation into a file {source_lang}_${target_lang}.txt with sacreblue at the end. + +You can also use costomized tokenizer to compare the performance with the literature. For example, you get a tokenizer [here](https://github.com/rsennrich/wmt16-scripts) and do the following: +```bash +TOKENIZER= +TOK_CMD=<"$TOKENIZER $target_lang" or cat for sacrebleu> + +cat {source_lang}_${target_lang}.txt | grep -P "^H" |sort -V |cut -f 3- |$TOK_CMD > ${source_lang}_${target_lang}.hyp +cat {source_lang}_${target_lang}.txt | grep -P "^T" |sort -V |cut -f 2- |$TOK_CMD > ${source_lang}_${target_lang}.ref +sacrebleu -tok 'none' -s 'none' ${source_lang}_${target_lang}.ref < ${source_lang}_${target_lang}.hyp +``` + +# mBART50 models + +* [mMBART 50 pretrained model](https://dl.fbaipublicfiles.com/fairseq/models/mbart50/mbart50.pretrained.tar.gz). +* [mMBART 50 finetuned many-to-one](https://dl.fbaipublicfiles.com/fairseq/models/mbart50/mbart50.ft.n1.tar.gz). +* [mMBART 50 finetuned one-to-many](https://dl.fbaipublicfiles.com/fairseq/models/mbart50/mbart50.ft.1n.tar.gz). +* [mMBART 50 finetuned many-to-many](https://dl.fbaipublicfiles.com/fairseq/models/mbart50/mbart50.ft.nn.tar.gz). + +Please download and extract from the above tarballs. Each tarball contains +* The fairseq model checkpoint: model.pt +* The list of supported languages: ML50_langs.txt +* Sentence piece model: sentence.bpe.model +* Fairseq dictionary of each language: dict.{lang}.txt (please replace lang with a language specified in ML50_langs.txt) + +To use the trained models, +* use the tool [binarize.py](./data_scripts/binarize.py) to binarize your data using sentence.bpe.model and dict.{lang}.txt, and copy the dictionaries to your data path +* then run the generation command: +```bash +path_2_data= +model=/model.pt +lang_list=/ML50_langs.txt +source_lang= +target_lang= + +fairseq-generate $path_2_data \ + --path $model \ + --task translation_multi_simple_epoch \ + --gen-subset test \ + --source-lang $source_lang \ + --target-lang $target_lang + --sacrebleu --remove-bpe 'sentencepiece'\ + --batch-size 32 \ + --encoder-langtok "src" \ + --decoder-langtok \ + --lang-dict "$lang_list" +``` + +## Citation + +```bibtex +@article{tang2020multilingual, + title={Multilingual Translation with Extensible Multilingual Pretraining and Finetuning}, + author={Yuqing Tang and Chau Tran and Xian Li and Peng-Jen Chen and Naman Goyal and Vishrav Chaudhary and Jiatao Gu and Angela Fan}, + year={2020}, + eprint={2008.00401}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +``` diff --git a/data/fairseq/examples/multilingual/data_scripts/README.md b/data/fairseq/examples/multilingual/data_scripts/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cc610c0c9e936a5ae4659ceda691c6db6d387296 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/README.md @@ -0,0 +1,24 @@ + +# Install dependency +```bash +pip install -r requirement.txt +``` + +# Download the data set +```bash +export WORKDIR_ROOT= + +``` +The downloaded data will be at $WORKDIR_ROOT/ML50 + +# preprocess the data +Install SPM [here](https://github.com/google/sentencepiece) +```bash +export WORKDIR_ROOT= +export SPM_PATH= +``` +* $WORKDIR_ROOT/ML50/raw: extracted raw data +* $WORKDIR_ROOT/ML50/dedup: dedup data +* $WORKDIR_ROOT/ML50/clean: data with valid and test sentences removed from the dedup data + + diff --git a/data/fairseq/examples/multilingual/data_scripts/binarize.py b/data/fairseq/examples/multilingual/data_scripts/binarize.py new file mode 100644 index 0000000000000000000000000000000000000000..ee54c6aabf021ca526743f8f1f67b91889e1e335 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/binarize.py @@ -0,0 +1,200 @@ +import shutil +import os, sys +from subprocess import check_call, check_output +import glob +import argparse +import shutil +import pathlib +import itertools + +def call_output(cmd): + print(f"Executing: {cmd}") + ret = check_output(cmd, shell=True) + print(ret) + return ret + +def call(cmd): + print(cmd) + check_call(cmd, shell=True) + + +WORKDIR_ROOT = os.environ.get('WORKDIR_ROOT', None) + +if WORKDIR_ROOT is None or not WORKDIR_ROOT.strip(): + print('please specify your working directory root in OS environment variable WORKDIR_ROOT. Exitting..."') + sys.exit(-1) + +SPM_PATH = os.environ.get('SPM_PATH', None) + +if SPM_PATH is None or not SPM_PATH.strip(): + print("Please install sentence piecence from https://github.com/google/sentencepiece and set SPM_PATH pointing to the installed spm_encode.py. Exitting...") + sys.exit(-1) + + +SPM_MODEL = f'{WORKDIR_ROOT}/sentence.bpe.model' +SPM_VOCAB = f'{WORKDIR_ROOT}/dict_250k.txt' + +SPM_ENCODE = f'{SPM_PATH}' + +if not os.path.exists(SPM_MODEL): + call(f"wget https://dl.fbaipublicfiles.com/fairseq/models/mbart50/sentence.bpe.model -O {SPM_MODEL}") + + +if not os.path.exists(SPM_VOCAB): + call(f"wget https://dl.fbaipublicfiles.com/fairseq/models/mbart50/dict_250k.txt -O {SPM_VOCAB}") + + + +def get_data_size(raw): + cmd = f'wc -l {raw}' + ret = call_output(cmd) + return int(ret.split()[0]) + +def encode_spm(model, direction, prefix='', splits=['train', 'test', 'valid'], pairs_per_shard=None): + src, tgt = direction.split('-') + + for split in splits: + src_raw, tgt_raw = f'{RAW_DIR}/{split}{prefix}.{direction}.{src}', f'{RAW_DIR}/{split}{prefix}.{direction}.{tgt}' + if os.path.exists(src_raw) and os.path.exists(tgt_raw): + cmd = f"""python {SPM_ENCODE} \ + --model {model}\ + --output_format=piece \ + --inputs {src_raw} {tgt_raw} \ + --outputs {BPE_DIR}/{direction}{prefix}/{split}.bpe.{src} {BPE_DIR}/{direction}{prefix}/{split}.bpe.{tgt} """ + print(cmd) + call(cmd) + + +def binarize_( + bpe_dir, + databin_dir, + direction, spm_vocab=SPM_VOCAB, + splits=['train', 'test', 'valid'], +): + src, tgt = direction.split('-') + + try: + shutil.rmtree(f'{databin_dir}', ignore_errors=True) + os.mkdir(f'{databin_dir}') + except OSError as error: + print(error) + cmds = [ + "fairseq-preprocess", + f"--source-lang {src} --target-lang {tgt}", + f"--destdir {databin_dir}/", + f"--workers 8", + ] + if isinstance(spm_vocab, tuple): + src_vocab, tgt_vocab = spm_vocab + cmds.extend( + [ + f"--srcdict {src_vocab}", + f"--tgtdict {tgt_vocab}", + ] + ) + else: + cmds.extend( + [ + f"--joined-dictionary", + f"--srcdict {spm_vocab}", + ] + ) + input_options = [] + if 'train' in splits and glob.glob(f"{bpe_dir}/train.bpe*"): + input_options.append( + f"--trainpref {bpe_dir}/train.bpe", + ) + if 'valid' in splits and glob.glob(f"{bpe_dir}/valid.bpe*"): + input_options.append(f"--validpref {bpe_dir}/valid.bpe") + if 'test' in splits and glob.glob(f"{bpe_dir}/test.bpe*"): + input_options.append(f"--testpref {bpe_dir}/test.bpe") + if len(input_options) > 0: + cmd = " ".join(cmds + input_options) + print(cmd) + call(cmd) + + +def binarize( + databin_dir, + direction, spm_vocab=SPM_VOCAB, prefix='', + splits=['train', 'test', 'valid'], + pairs_per_shard=None, +): + def move_databin_files(from_folder, to_folder): + for bin_file in glob.glob(f"{from_folder}/*.bin") \ + + glob.glob(f"{from_folder}/*.idx") \ + + glob.glob(f"{from_folder}/dict*"): + try: + shutil.move(bin_file, to_folder) + except OSError as error: + print(error) + bpe_databin_dir = f"{BPE_DIR}/{direction}{prefix}_databin" + bpe_dir = f"{BPE_DIR}/{direction}{prefix}" + if pairs_per_shard is None: + binarize_(bpe_dir, bpe_databin_dir, direction, spm_vocab=spm_vocab, splits=splits) + move_databin_files(bpe_databin_dir, databin_dir) + else: + # binarize valid and test which will not be sharded + binarize_( + bpe_dir, bpe_databin_dir, direction, + spm_vocab=spm_vocab, splits=[s for s in splits if s != "train"]) + for shard_bpe_dir in glob.glob(f"{bpe_dir}/shard*"): + path_strs = os.path.split(shard_bpe_dir) + shard_str = path_strs[-1] + shard_folder = f"{bpe_databin_dir}/{shard_str}" + databin_shard_folder = f"{databin_dir}/{shard_str}" + print(f'working from {shard_folder} to {databin_shard_folder}') + os.makedirs(databin_shard_folder, exist_ok=True) + binarize_( + shard_bpe_dir, shard_folder, direction, + spm_vocab=spm_vocab, splits=["train"]) + + for test_data in glob.glob(f"{bpe_databin_dir}/valid.*") + glob.glob(f"{bpe_databin_dir}/test.*"): + filename = os.path.split(test_data)[-1] + try: + os.symlink(test_data, f"{databin_shard_folder}/{filename}") + except OSError as error: + print(error) + move_databin_files(shard_folder, databin_shard_folder) + + +def load_langs(path): + with open(path) as fr: + langs = [l.strip() for l in fr] + return langs + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--data_root", default=f"{WORKDIR_ROOT}/ML50") + parser.add_argument("--raw-folder", default='raw') + parser.add_argument("--bpe-folder", default='bpe') + parser.add_argument("--databin-folder", default='databin') + + args = parser.parse_args() + + DATA_PATH = args.data_root #'/private/home/yuqtang/public_data/ML50' + RAW_DIR = f'{DATA_PATH}/{args.raw_folder}' + BPE_DIR = f'{DATA_PATH}/{args.bpe_folder}' + DATABIN_DIR = f'{DATA_PATH}/{args.databin_folder}' + os.makedirs(BPE_DIR, exist_ok=True) + + raw_files = itertools.chain( + glob.glob(f'{RAW_DIR}/train*'), + glob.glob(f'{RAW_DIR}/valid*'), + glob.glob(f'{RAW_DIR}/test*'), + ) + + directions = [os.path.split(file_path)[-1].split('.')[1] for file_path in raw_files] + + for direction in directions: + prefix = "" + splits = ['train', 'valid', 'test'] + try: + shutil.rmtree(f'{BPE_DIR}/{direction}{prefix}', ignore_errors=True) + os.mkdir(f'{BPE_DIR}/{direction}{prefix}') + os.makedirs(DATABIN_DIR, exist_ok=True) + except OSError as error: + print(error) + spm_model, spm_vocab = SPM_MODEL, SPM_VOCAB + encode_spm(spm_model, direction=direction, splits=splits) + binarize(DATABIN_DIR, direction, spm_vocab=spm_vocab, splits=splits) diff --git a/data/fairseq/examples/multilingual/data_scripts/check_iswlt_test_data.py b/data/fairseq/examples/multilingual/data_scripts/check_iswlt_test_data.py new file mode 100644 index 0000000000000000000000000000000000000000..f8e2eb0f15699f1b458a8445d0c1dd6229a21f77 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/check_iswlt_test_data.py @@ -0,0 +1,67 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import os, sys +import subprocess +import re +from subprocess import check_call, check_output + +WORKDIR_ROOT = os.environ.get('WORKDIR_ROOT', None) + +if WORKDIR_ROOT is None or not WORKDIR_ROOT.strip(): + print('please specify your working directory root in OS environment variable WORKDIR_ROOT. Exitting..."') + sys.exit(-1) + + +BLEU_REGEX = re.compile("^BLEU\\S* = (\\S+) ") +def run_eval_bleu(cmd): + output = check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode("utf-8").strip() + print(output) + bleu = -1.0 + for line in output.strip().split('\n'): + m = BLEU_REGEX.search(line) + if m is not None: + bleu = m.groups()[0] + bleu = float(bleu) + break + return bleu + +def check_data_test_bleu(raw_folder, data_lang_pairs): + not_matchings = [] + for sacrebleu_set, src_tgts in data_lang_pairs: + for src_tgt in src_tgts: + print(f'checking test bleus for: {src_tgt} at {sacrebleu_set}') + src, tgt = src_tgt.split('-') + ssrc, stgt = src[:2], tgt[:2] + if os.path.exists(f'{raw_folder}/test.{tgt}-{src}.{src}'): + # reversed direction may have different test set + test_src = f'{raw_folder}/test.{tgt}-{src}.{src}' + else: + test_src = f'{raw_folder}/test.{src}-{tgt}.{src}' + cmd1 = f'cat {test_src} | sacrebleu -t "{sacrebleu_set}" -l {stgt}-{ssrc}; [ $? -eq 0 ] || echo ""' + test_tgt = f'{raw_folder}/test.{src}-{tgt}.{tgt}' + cmd2 = f'cat {test_tgt} | sacrebleu -t "{sacrebleu_set}" -l {ssrc}-{stgt}; [ $? -eq 0 ] || echo ""' + bleu1 = run_eval_bleu(cmd1) + if bleu1 != 100.0: + not_matchings.append(f'{sacrebleu_set}:{src_tgt} source side not matching: {test_src}') + bleu2 = run_eval_bleu(cmd2) + if bleu2 != 100.0: + not_matchings.append(f'{sacrebleu_set}:{src_tgt} target side not matching: {test_tgt}') + return not_matchings + +if __name__ == "__main__": + to_data_path = f'{WORKDIR_ROOT}/iwsltv2' + not_matching = check_data_test_bleu( + f'{to_data_path}/raw', + [ + ('iwslt17', ['en_XX-ar_AR', 'en_XX-ko_KR', 'ar_AR-en_XX', 'ko_KR-en_XX']), + ('iwslt17', ['en_XX-it_IT', 'en_XX-nl_XX', 'it_IT-en_XX', 'nl_XX-en_XX']), + ('iwslt17/tst2015', ['en_XX-vi_VN', "vi_VN-en_XX"]), + ] + ) + if len(not_matching) > 0: + print('the following datasets do not have matching test datasets:\n\t', '\n\t'.join(not_matching)) + diff --git a/data/fairseq/examples/multilingual/data_scripts/check_self_overlaps.py b/data/fairseq/examples/multilingual/data_scripts/check_self_overlaps.py new file mode 100644 index 0000000000000000000000000000000000000000..07b338dcfd2d7f10317608274631d0edd93ba889 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/check_self_overlaps.py @@ -0,0 +1,103 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import os +import glob +import argparse +from utils.dedup import deup +import sys + +WORKDIR_ROOT = os.environ.get('WORKDIR_ROOT', None) + +if WORKDIR_ROOT is None or not WORKDIR_ROOT.strip(): + print('please specify your working directory root in OS environment variable WORKDIR_ROOT. Exitting..."') + sys.exit(-1) + +def get_directions(folder): + raw_files = glob.glob(f'{folder}/train*') + directions = [os.path.split(file_path)[-1].split('.')[1] for file_path in raw_files] + return directions + +def diff_list(lhs, rhs): + return set(lhs).difference(set(rhs)) + +def check_diff( + from_src_file, from_tgt_file, + to_src_file, to_tgt_file, +): + seen_in_from = set() + seen_src_in_from = set() + seen_tgt_in_from = set() + from_count = 0 + with open(from_src_file, encoding='utf-8') as fsrc, \ + open(from_tgt_file, encoding='utf-8') as ftgt: + for s, t in zip(fsrc, ftgt): + seen_in_from.add((s, t)) + seen_src_in_from.add(s) + seen_tgt_in_from.add(t) + from_count += 1 + common = 0 + common_src = 0 + common_tgt = 0 + to_count = 0 + seen = set() + + with open(to_src_file, encoding='utf-8') as fsrc, \ + open(to_tgt_file, encoding='utf-8') as ftgt: + for s, t in zip(fsrc, ftgt): + to_count += 1 + if (s, t) not in seen: + if (s, t) in seen_in_from: + common += 1 + if s in seen_src_in_from: + common_src += 1 + seen_src_in_from.remove(s) + if t in seen_tgt_in_from: + common_tgt += 1 + seen_tgt_in_from.remove(t) + seen.add((s, t)) + return common, common_src, common_tgt, from_count, to_count + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--folder", type=str, required=True, + help="the data folder ") + parser.add_argument("--split", type=str, default='test', + help="split (valid, test) to check against training data") + parser.add_argument('--directions', type=str, default=None, required=False) + + args = parser.parse_args() + + if args.directions is None: + directions = set(get_directions(args.folder)) + directions = sorted(directions) + else: + directions = args.directions.split(',') + directions = sorted(set(directions)) + + results = [] + print(f'checking where {args.split} split data are in training') + print(f'direction\tcommon_count\tsrc common\ttgt common\tfrom_size\tto_size') + + for direction in directions: + src, tgt = direction.split('-') + from_src_file = f'{args.folder}/{args.split}.{src}-{tgt}.{src}' + from_tgt_file = f'{args.folder}/{args.split}.{src}-{tgt}.{tgt}' + if not os.path.exists(from_src_file): + # some test/valid data might in reverse directinos: + from_src_file = f'{args.folder}/{args.split}.{tgt}-{src}.{src}' + from_tgt_file = f'{args.folder}/{args.split}.{tgt}-{src}.{tgt}' + to_src_file = f'{args.folder}/train.{src}-{tgt}.{src}' + to_tgt_file = f'{args.folder}/train.{src}-{tgt}.{tgt}' + if not os.path.exists(to_src_file) or not os.path.exists(from_src_file): + continue + r = check_diff(from_src_file, from_tgt_file, to_src_file, to_tgt_file) + results.append(r) + print(f'{direction}\t', '\t'.join(map(str, r))) + + +if __name__ == "__main__": + main() diff --git a/data/fairseq/examples/multilingual/data_scripts/check_valid_test_overlaps.py b/data/fairseq/examples/multilingual/data_scripts/check_valid_test_overlaps.py new file mode 100644 index 0000000000000000000000000000000000000000..40fa9aecdf9108e095feb3661236453c0f7ed7c4 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/check_valid_test_overlaps.py @@ -0,0 +1,124 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import os +import argparse +import pandas as pd +import sys + + +WORKDIR_ROOT = os.environ.get('WORKDIR_ROOT', None) + +if WORKDIR_ROOT is None or not WORKDIR_ROOT.strip(): + print('please specify your working directory root in OS environment variable WORKDIR_ROOT. Exitting..."') + sys.exit(-1) + +def load_langs(path): + with open(path) as fr: + langs = [l.strip() for l in fr] + return langs + + + +def load_sentences(raw_data, split, direction): + src, tgt = direction.split('-') + src_path = f"{raw_data}/{split}.{direction}.{src}" + tgt_path = f"{raw_data}/{split}.{direction}.{tgt}" + if os.path.exists(src_path) and os.path.exists(tgt_path): + return [(src, open(src_path).read().splitlines()), (tgt, open(tgt_path).read().splitlines())] + else: + return [] + +def swap_direction(d): + src, tgt = d.split('-') + return f'{tgt}-{src}' + +def get_all_test_data(raw_data, directions, split='test'): + test_data = [ + x + for dd in directions + for d in [dd, swap_direction(dd)] + for x in load_sentences(raw_data, split, d) + ] + # all_test_data = {s for _, d in test_data for s in d} + all_test_data = {} + for lang, d in test_data: + for s in d: + s = s.strip() + lgs = all_test_data.get(s, set()) + lgs.add(lang) + all_test_data[s] = lgs + return all_test_data, test_data + + +def check_train_sentences(src_path, tgt_path, direction, all_test_data, mess_up_train={}): + # src, tgt = direction.split('-') + print(f'check training data for {direction} in {src_path} and {tgt_path}') + size = 0 + overlapped_size_counted_dup = 0 + if not os.path.exists(tgt_path) or not os.path.exists(src_path): + return mess_up_train, size, overlapped_size_counted_dup + + with open(src_path) as f, open(tgt_path) as g: + for src_line, tgt_line in zip(f, g): + s = src_line.strip() + t = tgt_line.strip() + size += 1 + if s in all_test_data: + langs = mess_up_train.get(s, set()) + langs.add(direction) + mess_up_train[s] = langs + overlapped_size_counted_dup += 1 + if t in all_test_data: + langs = mess_up_train.get(t, set()) + langs.add(direction) + mess_up_train[t] = langs + overlapped_size_counted_dup += 1 + print(f'{direction}: size={size}, overlapped={overlapped_size_counted_dup}') + return mess_up_train, size, overlapped_size_counted_dup + +def check_train_all(raw_data, directions, all_test_data): + mess_up_train = {} + data_sizes = {} + # raw_data = '~chau/data-bin/MineBART/multilingual_mined_100M/en_XX/et_EE-en_XX/all.{en_XX, et_EE}' + print(f'checking training data againsts # {len(all_test_data)} sentences') + print(f'example test data: ', [s for i, s in enumerate(all_test_data.keys()) if i < 10]) + for direction in directions: + src, tgt = direction.split('-') + path = f'{raw_data}/en_XX/{direction}/all' + src_path = f'{path}.{src}' + tgt_path = f'{path}.{tgt}' + print(f'checking {src_path} {tgt_path}') + _, size, overlapped_size_counted_dup = check_train_sentences(src_path, tgt_path, direction, all_test_data, mess_up_train) + data_sizes[direction] = (size, overlapped_size_counted_dup) + return mess_up_train, data_sizes + + + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--folder", type=str, required=True, + help="the data folder ") + parser.add_argument("--test-data", type=str, required=True, + help="the test data folder ") + parser.add_argument('--directions', type=str, default=None, required=False) + + args = parser.parse_args() + directions = args.directions.split(',') + directions = sorted(set(directions)) + + results = [] + # print(f'checking where {args.split} split data are in training') + # print(f'direction\tcommon_count\tsrc common\ttgt common\tfrom_size\tto_size') + raw_data = args.folder + all_test_data, test_data = get_all_test_data(args.test_data, directions, split='test') + mess_up_train, data_sizes = check_train_all(raw_data, directions, all_test_data) + print(data_sizes) + + +if __name__ == "__main__": + main() diff --git a/data/fairseq/examples/multilingual/data_scripts/dedup_all.py b/data/fairseq/examples/multilingual/data_scripts/dedup_all.py new file mode 100644 index 0000000000000000000000000000000000000000..ef39c05ee606aaeda1d9e94970932d2241a8b281 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/dedup_all.py @@ -0,0 +1,52 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + + +import os +import glob +import argparse +from utils.dedup import deup + +import sys +WORKDIR_ROOT = os.environ.get('WORKDIR_ROOT', None) + +if WORKDIR_ROOT is None or not WORKDIR_ROOT.strip(): + print('please specify your working directory root in OS environment variable WORKDIR_ROOT. Exitting..."') + sys.exit(-1) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--from-folder", type=str, required=True, + help="the data folder to be dedup") + parser.add_argument("--to-folder", type=str, required=True, + help="the data folder to save deduped data") + parser.add_argument('--directions', type=str, default=None, required=False) + + args = parser.parse_args() + + if args.directions is None: + raw_files = glob.glob(f'{args.from_folder}/train*') + + directions = [os.path.split(file_path)[-1].split('.')[1] for file_path in raw_files] + else: + directions = args.directions.split(',') + directions = sorted(set(directions)) + + for direction in directions: + src, tgt = direction.split('-') + src_file = f'{args.from_folder}/train.{src}-{tgt}.{src}' + tgt_file = f'{args.from_folder}/train.{src}-{tgt}.{tgt}' + src_file_out = f'{args.to_folder}/train.{src}-{tgt}.{src}' + tgt_file_out = f'{args.to_folder}/train.{src}-{tgt}.{tgt}' + assert src_file != src_file_out + assert tgt_file != tgt_file_out + print(f'deduping {src_file}, {tgt_file}') + deup(src_file, tgt_file, src_file_out, tgt_file_out) + + +if __name__ == "__main__": + main() diff --git a/data/fairseq/examples/multilingual/data_scripts/download_ML50_v1.sh b/data/fairseq/examples/multilingual/data_scripts/download_ML50_v1.sh new file mode 100644 index 0000000000000000000000000000000000000000..99fbc75920836a4b4bbdbd6b523749843288e450 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/download_ML50_v1.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +if [ -z $WORKDIR_ROOT ] ; +then + echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." + exit +fi + +# first run download_wmt20.sh; it will install a few useful tools for other scripts +# TODO: need to print out instructions on downloading a few files which requires manually authentication from the websites +bash ./download_wmt20.sh + +python ./download_wmt19_and_before.py +bash ./download_wat19_my.sh +python ./download_ted_and_extract.py +bash ./download_lotus.sh +bash ./download_iitb.sh +bash ./download_af_xh.sh + + +# IWSLT downloading URLs have changed in between; TODO: fix them: +bash ./download_iwslt_and_extract.sh + +# TODO: globalvoices URLs changed; need to be fixed +bash ./download_flores_data.sh diff --git a/data/fairseq/examples/multilingual/data_scripts/download_af_xh.sh b/data/fairseq/examples/multilingual/data_scripts/download_af_xh.sh new file mode 100644 index 0000000000000000000000000000000000000000..a78fbbbbccb6f6ae005a1f03b97f083a2d958ebe --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/download_af_xh.sh @@ -0,0 +1,164 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +# set -x -e + +if [ -z $WORKDIR_ROOT ] ; +then + echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." + exit +fi + + +# put intermediate files +TMP_DIR=$WORKDIR_ROOT/temp/af_xhv2 +# output {train,valid,test} files to dest +DEST=${WORKDIR_ROOT}/ML50/raw + + + +ROOT=${WORKDIR_ROOT} +UTILS=$PWD/utils +TMX2CORPUS="${UTILS}/tmx2corpus" +TMX_TOOL="python ${TMX2CORPUS}/tmx2corpus.py" + +mkdir -p $TMP_DIR +mkdir -p $DEST +mkdir -p $UTILS + +function download_opus(){ + src=$1 + tgt=$2 + subset=$3 + ulr=$4 + + mkdir extract_$subset.$src-$tgt + pushd extract_$subset.$src-$tgt + if [ ! -f "$subset.$src-$tgt.tmx.gz" ]; then + wget $url -O "$subset.$src-$tgt.tmx.gz" + gzip -d "$subset.$src-$tgt.tmx.gz" + f=$subset.$src-$tgt.tmx + $TMX_TOOL $f + mv bitext.$src ../$subset.$src-$tgt.$src + mv bitext.$tgt ../$subset.$src-$tgt.$tgt + fi + popd +} + +function concat_subsets(){ + src=$1 + tgt=$2 + subsets=$3 + src_train=raw_train.$src-$tgt.$src + tgt_train=raw_train.$src-$tgt.$tgt + > $src_train + > $tgt_train + for subset in $subsets; do + cat $subset.$src-$tgt.$src >> $src_train + cat $subset.$src-$tgt.$tgt >> $tgt_train + done +} + + + +function get_seeded_random() +{ + seed="$1" + openssl enc -aes-256-ctr -pass pass:"$seed" -nosalt \ + /dev/null +} + +function split_train_valid(){ + src=$1 + tgt=$2 + raw_src_train=raw_train.$src-$tgt.$src + raw_tgt_train=raw_train.$src-$tgt.$tgt + + shuf --random-source=<(get_seeded_random 43) $raw_src_train > shuffled.$src-$tgt.$src + shuf --random-source=<(get_seeded_random 43) $raw_tgt_train > shuffled.$src-$tgt.$tgt + + head -n 1500 shuffled.$src-$tgt.$src > valid.$src-$tgt.$src + head -n 1500 shuffled.$src-$tgt.$tgt > valid.$src-$tgt.$tgt + + tail +1501 shuffled.$src-$tgt.$src > train.$src-$tgt.$src + tail +1501 shuffled.$src-$tgt.$tgt > train.$src-$tgt.$tgt +} + +function copy2dst(){ + lsrc=$1 + ltgt=$2 + src=${lsrc:0:2} + tgt=${ltgt:0:2} + + + cp valid.$src-$tgt.$src $DEST/valid.$lsrc-$ltgt.$lsrc + cp valid.$src-$tgt.$tgt $DEST/valid.$lsrc-$ltgt.$ltgt + + cp train.$src-$tgt.$src $DEST/train.$lsrc-$ltgt.$lsrc + cp train.$src-$tgt.$tgt $DEST/train.$lsrc-$ltgt.$ltgt +} + + + + +#for xh-en +declare -A xh_en_urls +xh_en_urls=( + [Tatoeba]=https://object.pouta.csc.fi/OPUS-Tatoeba/v20190709/tmx/en-xh.tmx.gz + [wikimedia]=https://object.pouta.csc.fi/OPUS-wikimedia/v20190628/tmx/en-xh.tmx.gz + [memat]=https://object.pouta.csc.fi/OPUS-memat/v1/tmx/en-xh.tmx.gz + [uedin]=https://object.pouta.csc.fi/OPUS-bible-uedin/v1/tmx/en-xh.tmx.gz + [GNOME]=https://object.pouta.csc.fi/OPUS-GNOME/v1/tmx/en-xh.tmx.gz + [XhosaNavy]=https://object.pouta.csc.fi/OPUS-XhosaNavy/v1/tmx/en-xh.tmx.gz + [KDE4]=https://object.pouta.csc.fi/OPUS-KDE4/v2/tmx/en-xh.tmx.gz + [Ubuntu]=https://object.pouta.csc.fi/OPUS-Ubuntu/v14.10/tmx/en-xh.tmx.gz +) + +mkdir $TMP_DIR/xh-en +pushd $TMP_DIR/xh-en +for k in "${!xh_en_urls[@]}" +do + name=$k + url=${xh_en_urls[$k]} + echo "$name: $url" + download_opus xh en $name $ulr +done +concat_subsets xh en "${!xh_en_urls[@]}" +split_train_valid xh en +copy2dst xh_ZA en_XX +popd + + +## +#for af-en +declare -A af_en_urls +af_en_urls=( + [Tatoeba]=https://object.pouta.csc.fi/OPUS-Tatoeba/v20190709/tmx/af-en.tmx.gz + [uedin]=https://object.pouta.csc.fi/OPUS-bible-uedin/v1/tmx/af-en.tmx.gz + [GNOME]=https://object.pouta.csc.fi/OPUS-GNOME/v1/tmx/af-en.tmx.gz + [QED]=https://object.pouta.csc.fi/OPUS-QED/v2.0a/tmx/af-en.tmx.gz + [KDE4]=https://object.pouta.csc.fi/OPUS-KDE4/v2/tmx/af-en.tmx.gz + [OpenSubtitles]=https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/tmx/af-en.tmx.gz + [SPC]=https://object.pouta.csc.fi/OPUS-SPC/v1/tmx/af-en.tmx.gz + [Ubuntu]=https://object.pouta.csc.fi/OPUS-Ubuntu/v14.10/tmx/af-en.tmx.gz +) + +mkdir $TMP_DIR/af-en +pushd $TMP_DIR/af-en +for k in "${!af_en_urls[@]}" +do + name=$k + url=${af_en_urls[$k]} + echo "$name: $url" + download_opus af en $name $ulr +done +concat_subsets af en "${!af_en_urls[@]}" +split_train_valid af en +copy2dst af_ZA en_XX +popd + + diff --git a/data/fairseq/examples/multilingual/data_scripts/download_flores_data.sh b/data/fairseq/examples/multilingual/data_scripts/download_flores_data.sh new file mode 100644 index 0000000000000000000000000000000000000000..e6175ce0c38b06a1ebddaeca808f71b47f77f500 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/download_flores_data.sh @@ -0,0 +1,246 @@ +#!/bin/bash + +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + +if [ -z $WORKDIR_ROOT ] ; +then + echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." + exit +fi + + +set -e +set -o pipefail + +SRC=en +SI_TGT=si +NE_TGT=ne + +DESTDIR=${WORKDIR_ROOT}/ML50/raw/ + +ROOT=${WORKDIR_ROOT}/tmp +mkdir -p $ROOT +DATA=$ROOT/data +NE_ROOT=$DATA/all-clean-ne +SI_ROOT=$DATA/all-clean-si + +mkdir -p $DATA $NE_ROOT $SI_ROOT + +SI_OPUS_DATASETS=( + "$SI_ROOT/GNOME.en-si" + "$SI_ROOT/Ubuntu.en-si" + "$SI_ROOT/KDE4.en-si" + "$SI_ROOT/OpenSubtitles.en-si" +) + +SI_OPUS_URLS=( + "https://object.pouta.csc.fi/OPUS-GNOME/v1/moses/en-si.txt.zip" + "https://object.pouta.csc.fi/OPUS-Ubuntu/v14.10/moses/en-si.txt.zip" + "https://object.pouta.csc.fi/OPUS-KDE4/v2/moses/en-si.txt.zip" + "https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/moses/en-si.txt.zip" +) + +NE_OPUS_DATASETS=( + "$NE_ROOT/GNOME.en-ne" + "$NE_ROOT/Ubuntu.en-ne" + "$NE_ROOT/KDE4.en-ne" +) + +NE_OPUS_URLS=( + "https://object.pouta.csc.fi/OPUS-GNOME/v1/moses/en-ne.txt.zip" + "https://object.pouta.csc.fi/OPUS-Ubuntu/v14.10/moses/en-ne.txt.zip" + "https://object.pouta.csc.fi/OPUS-KDE4/v2/moses/en-ne.txt.zip" +) + +REMOVE_FILE_PATHS=() + +# Download data +download_data() { + CORPORA=$1 + URL=$2 + + if [ -f $CORPORA ]; then + echo "$CORPORA already exists, skipping download" + else + echo "Downloading $URL" + wget $URL -O $CORPORA --no-check-certificate || rm -f $CORPORA + if [ -f $CORPORA ]; then + echo "$URL successfully downloaded." + else + echo "$URL not successfully downloaded." + rm -f $CORPORA + exit -1 + fi + fi +} + +# Example: download_opus_data $LANG_ROOT $TGT +download_opus_data() { + LANG_ROOT=$1 + TGT=$2 + + if [ "$TGT" = "si" ]; then + URLS=("${SI_OPUS_URLS[@]}") + DATASETS=("${SI_OPUS_DATASETS[@]}") + else + URLS=("${NE_OPUS_URLS[@]}") + DATASETS=("${NE_OPUS_DATASETS[@]}") + fi + + # Download and extract data + for ((i=0;i<${#URLS[@]};++i)); do + URL=${URLS[i]} + CORPORA=${DATASETS[i]} + + download_data $CORPORA $URL + unzip -o $CORPORA -d $LANG_ROOT + REMOVE_FILE_PATHS+=( $CORPORA $CORPORA.xml $CORPORA.ids $LANG_ROOT/README $LANG_ROOT/LICENSE ) + done + + cat ${DATASETS[0]}.$SRC ${DATASETS[1]}.$SRC ${DATASETS[2]}.$SRC > $LANG_ROOT/GNOMEKDEUbuntu.$SRC-$TGT.$SRC + cat ${DATASETS[0]}.$TGT ${DATASETS[1]}.$TGT ${DATASETS[2]}.$TGT > $LANG_ROOT/GNOMEKDEUbuntu.$SRC-$TGT.$TGT + + REMOVE_FILE_PATHS+=( ${DATASETS[0]}.$SRC ${DATASETS[1]}.$SRC ${DATASETS[2]}.$SRC ) + REMOVE_FILE_PATHS+=( ${DATASETS[0]}.$TGT ${DATASETS[1]}.$TGT ${DATASETS[2]}.$TGT ) +} + +download_opus_data $SI_ROOT $SI_TGT +cp ${SI_OPUS_DATASETS[3]}.$SRC $SI_ROOT/OpenSubtitles2018.$SRC-$SI_TGT.$SRC +cp ${SI_OPUS_DATASETS[3]}.$SI_TGT $SI_ROOT/OpenSubtitles2018.$SRC-$SI_TGT.$SI_TGT +REMOVE_FILE_PATHS+=( ${SI_OPUS_DATASETS[3]}.$SRC ${SI_OPUS_DATASETS[3]}.$SI_TGT ) + +download_opus_data $NE_ROOT $NE_TGT + + +# Download and extract Global Voices data +GLOBAL_VOICES="$NE_ROOT/globalvoices.2018q4.ne-en" +GLOBAL_VOICES_URL="http://www.casmacat.eu/corpus/global-voices/globalvoices.ne-en.xliff.gz" + +download_data $GLOBAL_VOICES.gz $GLOBAL_VOICES_URL +gunzip -Nf $GLOBAL_VOICES.gz + +sed -ne 's?.*\(.*\).*?\1?p' $GLOBAL_VOICES > $GLOBAL_VOICES.$NE_TGT +sed -ne 's?.*]*>\(.*\).*?\1?p' $GLOBAL_VOICES > $GLOBAL_VOICES.$SRC + +REMOVE_FILE_PATHS+=( $GLOBAL_VOICES ) + +# Download and extract the bible dataset +BIBLE_TOOLS=bible-corpus-tools +XML_BIBLES=XML_Bibles +XML_BIBLES_DUP=XML_Bibles_dup + +if [ ! -e $BIBLE_TOOLS ]; then + echo "Cloning bible-corpus-tools repository..." + git clone https://github.com/christos-c/bible-corpus-tools.git +fi + +mkdir -p $BIBLE_TOOLS/bin $XML_BIBLES $XML_BIBLES_DUP +javac -cp "$BIBLE_TOOLS/lib/*" -d $BIBLE_TOOLS/bin $BIBLE_TOOLS/src/bible/readers/*.java $BIBLE_TOOLS/src/bible/*.java + +download_data bible.tar.gz "https://github.com/christos-c/bible-corpus/archive/v1.2.1.tar.gz" +tar xvzf bible.tar.gz + +cp bible-corpus-1.2.1/bibles/{Greek.xml,English.xml,Nepali.xml} $XML_BIBLES/ +cp bible-corpus-1.2.1/bibles/{Greek.xml,English-WEB.xml,Nepali.xml} $XML_BIBLES_DUP/ + +java -cp $BIBLE_TOOLS/lib/*:$BIBLE_TOOLS/bin bible.CreateMLBooks $XML_BIBLES +java -cp $BIBLE_TOOLS/lib/*:$BIBLE_TOOLS/bin bible.CreateMLBooks $XML_BIBLES_DUP +java -cp $BIBLE_TOOLS/lib/*:$BIBLE_TOOLS/bin bible.CreateVerseAlignedBooks $XML_BIBLES +java -cp $BIBLE_TOOLS/lib/*:$BIBLE_TOOLS/bin bible.CreateVerseAlignedBooks $XML_BIBLES_DUP + +cat $XML_BIBLES/aligned/*/English.txt > $NE_ROOT/bible.$SRC-$NE_TGT.$SRC +cat $XML_BIBLES/aligned/*/Nepali.txt > $NE_ROOT/bible.$SRC-$NE_TGT.$NE_TGT +cat $XML_BIBLES_DUP/aligned/*/English-WEB.txt > $NE_ROOT/bible_dup.$SRC-$NE_TGT.$SRC +cat $XML_BIBLES_DUP/aligned/*/Nepali.txt > $NE_ROOT/bible_dup.$SRC-$NE_TGT.$NE_TGT +REMOVE_FILE_PATHS+=( bible-corpus-1.2.1 bible.tar.gz $BIBLE_TOOLS $XML_BIBLES $XML_BIBLES_DUP ) + +# Download and extract the Penn Treebank dataset +NE_TAGGED=$ROOT/new_submissions_parallel_corpus_project_Nepal +NE_TAGGED_URL="http://www.cle.org.pk/Downloads/ling_resources/parallelcorpus/NepaliTaggedCorpus.zip" +EN_TAGGED_PATCH_URL="https://dl.fbaipublicfiles.com/fairseq/data/nepali-penn-treebank.en.patch" +NE_TAGGED_PATCH_URL="https://dl.fbaipublicfiles.com/fairseq/data/nepali-penn-treebank.ne.patch" +MOSES=mosesdecoder +MOSES_TOK=$MOSES/scripts/tokenizer +EN_PATCH_REGEX="{s:\\\/:\/:g;s/\*\T\*\-\n+//g;s/\-LCB\-/\{/g;s/\-RCB\-/\}/g; s/\-LSB\-/\[/g; s/\-RSB\-/\]/g;s/\-LRB\-/\(/g; s/\-RRB\-/\)/g; s/\'\'/\"/g; s/\`\`/\"/g; s/\ +\'s\ +/\'s /g; s/\ +\'re\ +/\'re /g; s/\"\ +/\"/g; s/\ +\"/\"/g; s/\ n't([\ \.\"])/n't\1/g; s/\r+(.)/\1/g;}" +NE_PATCH_REGEX="{s:\p{Cf}::g;s:\\\/:\/:g;s/\*\T\*\-\n+//g;s/\-LCB\-/\{/g;s/\-RCB\-/\}/g; s/\-LSB\-/\[/g; s/\-RSB\-/\]/g;s/\-LRB\-/\(/g; s/\-RRB\-/\)/g; s/\'\'/\"/g; s/\`\`/\"/g; s/\ +\'s\ +/\'s /g; s/\ +\'re\ +/\'re /g; s/\"\ +/\"/g; s/\ +\"/\"/g; s/\ n't([\ \.\"])/n't\1/g; s/\r+(.)/\1/g;}" + +download_data $DATA/nepali-penn-treebank.$SRC.patch $EN_TAGGED_PATCH_URL +download_data $DATA/nepali-penn-treebank.$NE_TGT.patch $NE_TAGGED_PATCH_URL +download_data original.zip $NE_TAGGED_URL +unzip -o original.zip -d $ROOT + +cat $NE_TAGGED/00.txt $NE_TAGGED/01.txt $NE_TAGGED/02.txt > $NE_TAGGED/nepali-penn-treebank.$SRC +cat $NE_TAGGED/00ne_revised.txt $NE_TAGGED/01ne_revised.txt $NE_TAGGED/02ne_revised.txt > $NE_TAGGED/nepali-penn-treebank.$NE_TGT + +patch $NE_TAGGED/nepali-penn-treebank.$SRC -i $DATA/nepali-penn-treebank.$SRC.patch -o $NE_TAGGED/nepali-penn-treebank-patched.$SRC +patch $NE_TAGGED/nepali-penn-treebank.$NE_TGT -i $DATA/nepali-penn-treebank.$NE_TGT.patch -o $NE_TAGGED/nepali-penn-treebank-patched.$NE_TGT + +if [ ! -e $MOSES ]; then + echo "Cloning moses repository..." + git clone https://github.com/moses-smt/mosesdecoder.git +fi + +cat $NE_TAGGED/nepali-penn-treebank-patched.$SRC | \ + perl -anpe "$EN_PATCH_REGEX" | \ + $MOSES_TOK/tokenizer.perl -l $SRC | \ + $MOSES_TOK/detokenizer.perl -l $SRC > $NE_ROOT/nepali-penn-treebank.$SRC + +cat $NE_TAGGED/nepali-penn-treebank-patched.$NE_TGT | \ + perl -CIO -anpe "$NE_PATCH_REGEX" | \ + $MOSES_TOK/detokenizer.perl -l $SRC > $NE_ROOT/nepali-penn-treebank.$NE_TGT + + +# Download nepali dictionary data +NE_DICT=$NE_ROOT/dictionaries +download_data $NE_DICT "http://www.seas.upenn.edu/~nlp/resources/TACL-data-release/dictionaries.tar.gz" +tar xvzf $NE_DICT +cp dictionaries/dict.ne $NE_ROOT/dictionary.$NE_TGT-$SRC +REMOVE_FILE_PATHS+=( $NE_DICT dictionaries ) + +REMOVE_FILE_PATHS+=( $MOSES $NE_TAGGED original.zip $DATA/nepali-penn-treebank.$SRC.patch $DATA/nepali-penn-treebank.$NE_TGT.patch ) + + +# Remove the temporary files +for ((i=0;i<${#REMOVE_FILE_PATHS[@]};++i)); do + rm -rf ${REMOVE_FILE_PATHS[i]} +done + +# Copy the training data +si=si_LK +ne=ne_NP +en=en_XX +cat $SI_ROOT/GNOMEKDEUbuntu.en-si.si $SI_ROOT/OpenSubtitles2018.en-si.si > $DESTDIR/train.$si-$en.$si +cat $SI_ROOT/GNOMEKDEUbuntu.en-si.en $SI_ROOT/OpenSubtitles2018.en-si.en > $DESTDIR/train.$si-$en.$en + +cat $NE_ROOT/bible_dup.en-ne.ne $NE_ROOT/bible.en-ne.ne $NE_ROOT/globalvoices.2018q4.ne-en.ne $NE_ROOT/GNOMEKDEUbuntu.en-ne.ne $NE_ROOT/nepali-penn-treebank.ne > $DESTDIR/train.$ne-$en.$ne +cat $NE_ROOT/bible_dup.en-ne.en $NE_ROOT/bible.en-ne.en $NE_ROOT/globalvoices.2018q4.ne-en.en $NE_ROOT/GNOMEKDEUbuntu.en-ne.en $NE_ROOT/nepali-penn-treebank.en > $DESTDIR/train.$ne-$en.$en + + +#Download the test sets +wget https://github.com/facebookresearch/flores/raw/master/data/wikipedia_en_ne_si_test_sets.tgz +tar -xvzf wikipedia_en_ne_si_test_sets.tgz + +cp wikipedia_en_ne_si_test_sets/wikipedia.dev.ne-en.ne $DESTDIR/valid.$ne-$en.$ne +cp wikipedia_en_ne_si_test_sets/wikipedia.dev.ne-en.en $DESTDIR/valid.$ne-$en.$en + +cp wikipedia_en_ne_si_test_sets/wikipedia.dev.si-en.si $DESTDIR/valid.$si-$en.$si +cp wikipedia_en_ne_si_test_sets/wikipedia.dev.si-en.en $DESTDIR/valid.$si-$en.$en + +cp wikipedia_en_ne_si_test_sets/wikipedia.devtest.ne-en.ne $DESTDIR/devtest.$ne-$en.$ne +cp wikipedia_en_ne_si_test_sets/wikipedia.devtest.ne-en.en $DESTDIR/devtest.$ne-$en.$en + +cp wikipedia_en_ne_si_test_sets/wikipedia.devtest.si-en.si $DESTDIR/devtest.$si-$en.$si +cp wikipedia_en_ne_si_test_sets/wikipedia.devtest.si-en.en $DESTDIR/devtest.$si-$en.$en + +cp wikipedia_en_ne_si_test_sets/wikipedia.test.ne-en.ne $DESTDIR/test.$ne-$en.$ne +cp wikipedia_en_ne_si_test_sets/wikipedia.test.ne-en.en $DESTDIR/test.$ne-$en.$en + +cp wikipedia_en_ne_si_test_sets/wikipedia.test.si-en.si $DESTDIR/test.$si-$en.$si +cp wikipedia_en_ne_si_test_sets/wikipedia.test.si-en.en $DESTDIR/test.$si-$en.$en + +rm -rf wikipedia_en_ne_si_test_sets.tgz wikipedia_en_ne_si_test_sets diff --git a/data/fairseq/examples/multilingual/data_scripts/download_iitb.sh b/data/fairseq/examples/multilingual/data_scripts/download_iitb.sh new file mode 100644 index 0000000000000000000000000000000000000000..a884e20839e2a41a57405cb6af362e37bd16ab6f --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/download_iitb.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + + +if [ -z $WORKDIR_ROOT ] ; +then + echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." + exit +fi + +IITB=$WORKDIR_ROOT/IITB +mkdir -p $IITB +pushd $IITB + +wget http://www.cfilt.iitb.ac.in/~moses/iitb_en_hi_parallel/iitb_corpus_download/parallel.tgz +tar -xvzf parallel.tgz + +wget http://www.cfilt.iitb.ac.in/~moses/iitb_en_hi_parallel/iitb_corpus_download/dev_test.tgz +tar -xvzf dev_test.tgz + +DESTDIR=${WORKDIR_ROOT}/ML50/raw/ + +cp parallel/IITB.en-hi.en $DESTDIR/train.hi_IN-en_XX.en_XX +cp parallel/IITB.en-hi.hi $DESTDIR/train.hi_IN-en_XX.hi_IN + +cp dev_test/dev.en $DESTDIR/valid.hi_IN-en_XX.en_XX +cp dev_test/dev.hi $DESTDIR/valid.hi_IN-en_XX.hi_IN + +cp dev_test/test.en $DESTDIR/test.hi_IN-en_XX.en_XX +cp dev_test/test.hi $DESTDIR/test.hi_IN-en_XX.hi_IN +popd \ No newline at end of file diff --git a/data/fairseq/examples/multilingual/data_scripts/download_iwslt_and_extract.sh b/data/fairseq/examples/multilingual/data_scripts/download_iwslt_and_extract.sh new file mode 100644 index 0000000000000000000000000000000000000000..ca3591b3db1715f136773d62e4b9b9ede97d436c --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/download_iwslt_and_extract.sh @@ -0,0 +1,225 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +#echo 'Cloning Moses github repository (for tokenization scripts)...' +#git clone https://github.com/moses-smt/mosesdecoder.git + +if [ -z $WORKDIR_ROOT ] ; +then + echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." + exit +fi + + + +data_root=${WORKDIR_ROOT}/iwsltv2 +DESTDIR=${WORKDIR_ROOT}/ML50/raw + + +langs="ar_AR it_IT nl_XX ko_KR vi_VN" +echo "data_root: $data_root" + +download_path=${data_root}/downloads +raw=${DESTDIR} +tmp=${data_root}/tmp +orig=${data_root}/orig + +mkdir -p $download_path $orig $raw $tmp +####################### +download_iwslt(){ + iwslt_key=$1 + src=$2 + tgt=$3 + save_prefix=$4 + pushd ${download_path} + if [[ ! -f ${save_prefix}$src-$tgt.tgz ]]; then + wget https://wit3.fbk.eu/archive/${iwslt_key}/texts/$src/$tgt/$src-$tgt.tgz -O ${save_prefix}$src-$tgt.tgz + [ $? -eq 0 ] && return 0 + fi + popd +} + +extract_iwslt(){ + src=$1 + tgt=$2 + prefix=$3 + pushd $orig + tar zxvf ${download_path}/${prefix}$src-${tgt}.tgz + popd +} + +generate_train(){ + lsrc=$1 + ltgt=$2 + src=${lsrc:0:2} + tgt=${ltgt:0:2} + for ll in $lsrc $ltgt; do + l=${ll:0:2} + f="$orig/*/train.tags.$src-$tgt.$l" + f_raw=$raw/train.$lsrc-$ltgt.$ll + cat $f \ + | grep -v '' \ + | grep -v '' \ + | grep -v '' \ + | grep -v '' \ + | grep -v '' \ + | sed -e 's///g' \ + | sed -e 's/<\/title>//g' \ + | sed -e 's/<description>//g' \ + | sed -e 's/<\/description>//g' \ + | sed 's/^\s*//g' \ + | sed 's/\s*$//g' \ + > $f_raw + [ $? -eq 0 ] && echo "extracted $f to $f_raw" + done + return 0 +} + +convert_valid_test(){ + src=$1 + tgt=$2 + for l in $src $tgt; do + echo "lang: ${l}" + for o in `ls $orig/*/IWSLT*.TED*.$src-$tgt.$l.xml`; do + fname=${o##*/} + f=$tmp/${fname%.*} + echo "$o => $f" + grep '<seg id' $o \ + | sed -e 's/<seg id="[0-9]*">\s*//g' \ + | sed -e 's/\s*<\/seg>\s*//g' \ + | sed -e "s/\’/\'/g" \ + > $f + echo "" + done + done +} + +generate_subset(){ + lsrc=$1 + ltgt=$2 + src=${lsrc:0:2} + tgt=${ltgt:0:2} + subset=$3 + prefix=$4 + for ll in $lsrc $ltgt; do + l=${ll:0:2} + f=$tmp/$prefix.${src}-${tgt}.$l + if [[ -f $f ]]; then + cp $f $raw/$subset.${lsrc}-$ltgt.${ll} + fi + done +} +################# + +echo "downloading iwslt training and dev data" +# using multilingual for it, nl +download_iwslt "2017-01-trnmted" DeEnItNlRo DeEnItNlRo +download_iwslt "2017-01-trnted" ar en +download_iwslt "2017-01-trnted" en ar +download_iwslt "2017-01-trnted" ko en +download_iwslt "2017-01-trnted" en ko +download_iwslt "2015-01" vi en +download_iwslt "2015-01" en vi + +echo "donwloading iwslt test data" +download_iwslt "2017-01-mted-test" it en "test." +download_iwslt "2017-01-mted-test" en it "test." +download_iwslt "2017-01-mted-test" nl en "test." +download_iwslt "2017-01-mted-test" en nl "test." + +download_iwslt "2017-01-ted-test" ar en "test." +download_iwslt "2017-01-ted-test" en ar "test." +download_iwslt "2017-01-ted-test" ko en "test." +download_iwslt "2017-01-ted-test" en ko "test." +download_iwslt "2015-01-test" vi en "test." +download_iwslt "2015-01-test" en vi "test." + +echo "extract training data tar balls" +extract_iwslt DeEnItNlRo DeEnItNlRo +extract_iwslt ar en +extract_iwslt en ar +extract_iwslt ko en +extract_iwslt en ko +extract_iwslt vi en +extract_iwslt en vi + + +echo "extracting iwslt test data" +for lang in $langs; do + l=${lang:0:2} + extract_iwslt $l en "test." + extract_iwslt en $l "test." +done + +echo "convert dev and test data" +for lang in $langs; do + s_lang=${lang:0:2} + convert_valid_test $s_lang en + convert_valid_test en $s_lang +done + + + +echo "creating training data into $raw" +for lang in $langs; do + generate_train $lang en_XX + generate_train en_XX $lang +done + +echo "creating iwslt dev data into raw" +generate_subset en_XX vi_VN valid "IWSLT15.TED.tst2013" +generate_subset vi_VN en_XX valid "IWSLT15.TED.tst2013" + +generate_subset en_XX ar_AR valid "IWSLT17.TED.tst2016" +generate_subset ar_AR en_XX valid "IWSLT17.TED.tst2016" +generate_subset en_XX ko_KR valid "IWSLT17.TED.tst2016" +generate_subset ko_KR en_XX valid "IWSLT17.TED.tst2016" + + +generate_subset en_XX it_IT valid "IWSLT17.TED.tst2010" +generate_subset it_IT en_XX valid "IWSLT17.TED.tst2010" +generate_subset en_XX nl_XX valid "IWSLT17.TED.tst2010" +generate_subset nl_XX en_XX valid "IWSLT17.TED.tst2010" + +echo "creating iswslt test data into raw" +generate_subset en_XX vi_VN test "IWSLT15.TED.tst2015" +generate_subset vi_VN en_XX test "IWSLT15.TED.tst2015" + +generate_subset en_XX ar_AR test "IWSLT17.TED.tst2017" +generate_subset ar_AR en_XX test "IWSLT17.TED.tst2017" +generate_subset en_XX ko_KR test "IWSLT17.TED.tst2017" +generate_subset ko_KR en_XX test "IWSLT17.TED.tst2017" + +generate_subset en_XX it_IT test "IWSLT17.TED.tst2017.mltlng" +generate_subset it_IT en_XX test "IWSLT17.TED.tst2017.mltlng" +generate_subset en_XX nl_XX test "IWSLT17.TED.tst2017.mltlng" +generate_subset nl_XX en_XX test "IWSLT17.TED.tst2017.mltlng" + +# normalze iwslt directions into x-en +pushd $raw +for lang in $langs; do + for split in test valid; do + x_en_f1=$split.$lang-en_XX.en_XX + x_en_f2=$split.$lang-en_XX.${lang} + + en_x_f1=$split.en_XX-$lang.en_XX + en_x_f2=$split.en_XX-$lang.${lang} + + if [ -f $en_x_f1 ] && [ ! -f $x_en_f1 ]; then + echo "cp $en_x_f1 $x_en_f1" + cp $en_x_f1 $x_en_f1 + fi + if [ -f $x_en_f2 ] && [ ! -f $x_en_f2 ]; then + echo "cp $en_x_f2 $x_en_f2" + cp $en_x_f2 $x_en_f2 + fi + done +done +popd \ No newline at end of file diff --git a/data/fairseq/examples/multilingual/data_scripts/download_lotus.sh b/data/fairseq/examples/multilingual/data_scripts/download_lotus.sh new file mode 100644 index 0000000000000000000000000000000000000000..c08c701314a8e575637deff78381ab02c2ef6728 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/download_lotus.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + + +if [ -z $WORKDIR_ROOT ] ; +then + echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." + exit +fi + + +SRCDIR=$WORKDIR_ROOT/indic_languages_corpus +DESTDIR=${WORKDIR_ROOT}/ML50/raw/ +mkdir -p $SRCDIR +mkdir -p $DESTDIR + +cd $SRCDIR +wget http://lotus.kuee.kyoto-u.ac.jp/WAT/indic-multilingual/indic_languages_corpus.tar.gz +tar -xvzf indic_languages_corpus.tar.gz + +SRC_EXTRACT_DIR=$SRCDIR/indic_languages_corpus/bilingual + +cp $SRC_EXTRACT_DIR/ml-en/train.ml $DESTDIR/train.ml_IN-en_XX.ml_IN +cp $SRC_EXTRACT_DIR/ml-en/train.en $DESTDIR/train.ml_IN-en_XX.en_XX +cp $SRC_EXTRACT_DIR/ml-en/dev.ml $DESTDIR/valid.ml_IN-en_XX.ml_IN +cp $SRC_EXTRACT_DIR/ml-en/dev.en $DESTDIR/valid.ml_IN-en_XX.en_XX +cp $SRC_EXTRACT_DIR/ml-en/test.ml $DESTDIR/test.ml_IN-en_XX.ml_IN +cp $SRC_EXTRACT_DIR/ml-en/test.en $DESTDIR/test.ml_IN-en_XX.en_XX + +cp $SRC_EXTRACT_DIR/ur-en/train.ur $DESTDIR/train.ur_PK-en_XX.ur_PK +cp $SRC_EXTRACT_DIR/ur-en/train.en $DESTDIR/train.ur_PK-en_XX.en_XX +cp $SRC_EXTRACT_DIR/ur-en/dev.ur $DESTDIR/valid.ur_PK-en_XX.ur_PK +cp $SRC_EXTRACT_DIR/ur-en/dev.en $DESTDIR/valid.ur_PK-en_XX.en_XX +cp $SRC_EXTRACT_DIR/ur-en/test.ur $DESTDIR/test.ur_PK-en_XX.ur_PK +cp $SRC_EXTRACT_DIR/ur-en/test.en $DESTDIR/test.ur_PK-en_XX.en_XX + +cp $SRC_EXTRACT_DIR/te-en/train.te $DESTDIR/train.te_IN-en_XX.te_IN +cp $SRC_EXTRACT_DIR/te-en/train.en $DESTDIR/train.te_IN-en_XX.en_XX +cp $SRC_EXTRACT_DIR/te-en/dev.te $DESTDIR/valid.te_IN-en_XX.te_IN +cp $SRC_EXTRACT_DIR/te-en/dev.en $DESTDIR/valid.te_IN-en_XX.en_XX +cp $SRC_EXTRACT_DIR/te-en/test.te $DESTDIR/test.te_IN-en_XX.te_IN +cp $SRC_EXTRACT_DIR/te-en/test.en $DESTDIR/test.te_IN-en_XX.en_XX diff --git a/data/fairseq/examples/multilingual/data_scripts/download_ted_and_extract.py b/data/fairseq/examples/multilingual/data_scripts/download_ted_and_extract.py new file mode 100644 index 0000000000000000000000000000000000000000..eb756680fa7dc31a14ba45c216776a6d60c16b60 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/download_ted_and_extract.py @@ -0,0 +1,338 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import itertools +import os +import csv +from collections import defaultdict +from six.moves import zip +import io +import wget +import sys + +from subprocess import check_call, check_output + +# scripts and data locations +CWD = os.getcwd() +UTILS = f"{CWD}/utils" + +MOSES = f"{UTILS}/mosesdecoder" + +WORKDIR_ROOT = os.environ.get('WORKDIR_ROOT', None) + +if WORKDIR_ROOT is None or not WORKDIR_ROOT.strip(): + print('please specify your working directory root in OS environment variable WORKDIR_ROOT. Exitting..."') + sys.exit(-1) + + +# please donwload mosesdecoder here: +detok_cmd = f'{MOSES}/scripts/tokenizer/detokenizer.perl' + + +def call(cmd): + print(f"Executing: {cmd}") + check_call(cmd, shell=True) + +class MultiLingualAlignedCorpusReader(object): + """A class to read TED talk dataset + """ + + def __init__(self, corpus_path, delimiter='\t', + target_token=True, bilingual=True, corpus_type='file', + lang_dict={'source': ['fr'], 'target': ['en']}, + eval_lang_dict=None, zero_shot=False, + detok=True, + ): + + self.empty_line_flag = 'NULL' + self.corpus_path = corpus_path + self.delimiter = delimiter + self.bilingual = bilingual + self.lang_dict = lang_dict + self.lang_set = set() + self.target_token = target_token + self.zero_shot = zero_shot + self.eval_lang_dict = eval_lang_dict + self.corpus_type = corpus_type + self.detok = detok + + for list_ in self.lang_dict.values(): + for lang in list_: + self.lang_set.add(lang) + + self.data = dict() + self.data['train'] = self.read_aligned_corpus(split_type='train') + self.data['test'] = self.read_aligned_corpus(split_type='test') + self.data['dev'] = self.read_aligned_corpus(split_type='dev') + + def read_data(self, file_loc_): + data_list = list() + with io.open(file_loc_, 'r', encoding='utf8') as fp: + for line in fp: + try: + text = line.strip() + except IndexError: + text = self.empty_line_flag + data_list.append(text) + return data_list + + def filter_text(self, dict_): + if self.target_token: + field_index = 1 + else: + field_index = 0 + data_dict = defaultdict(list) + list1 = dict_['source'] + list2 = dict_['target'] + for sent1, sent2 in zip(list1, list2): + try: + src_sent = ' '.join(sent1.split()[field_index: ]) + except IndexError: + src_sent = 'NULL' + + if src_sent.find(self.empty_line_flag) != -1 or len(src_sent) == 0: + continue + + elif sent2.find(self.empty_line_flag) != -1 or len(sent2) == 0: + continue + + else: + data_dict['source'].append(sent1) + data_dict['target'].append(sent2) + return data_dict + + def read_file(self, split_type, data_type): + return self.data[split_type][data_type] + + def save_file(self, path_, split_type, data_type, lang): + tok_file = tok_file_name(path_, lang) + with io.open(tok_file, 'w', encoding='utf8') as fp: + for line in self.data[split_type][data_type]: + fp.write(line + '\n') + if self.detok: + de_tok(tok_file, lang) + + def add_target_token(self, list_, lang_id): + new_list = list() + token = '__' + lang_id + '__' + for sent in list_: + new_list.append(token + ' ' + sent) + return new_list + + def read_from_single_file(self, path_, s_lang, t_lang): + data_dict = defaultdict(list) + with io.open(path_, 'r', encoding='utf8') as fp: + reader = csv.DictReader(fp, delimiter='\t', quoting=csv.QUOTE_NONE) + for row in reader: + data_dict['source'].append(row[s_lang]) + data_dict['target'].append(row[t_lang]) + + if self.target_token: + text = self.add_target_token(data_dict['source'], t_lang) + data_dict['source'] = text + + return data_dict['source'], data_dict['target'] + + def read_aligned_corpus(self, split_type='train'): + data_dict = defaultdict(list) + iterable = [] + s_list = [] + t_list = [] + + if self.zero_shot: + if split_type == "train": + iterable = zip(self.lang_dict['source'], self.lang_dict['target']) + else: + iterable = zip(self.eval_lang_dict['source'], self.eval_lang_dict['target']) + + elif self.bilingual: + iterable = itertools.product(self.lang_dict['source'], self.lang_dict['target']) + + for s_lang, t_lang in iterable: + if s_lang == t_lang: + continue + if self.corpus_type == 'file': + split_type_file_path = os.path.join(self.corpus_path, + "all_talks_{}.tsv".format(split_type)) + s_list, t_list = self.read_from_single_file(split_type_file_path, + s_lang=s_lang, + t_lang=t_lang) + data_dict['source'] += s_list + data_dict['target'] += t_list + new_data_dict = self.filter_text(data_dict) + return new_data_dict + + +def read_langs(corpus_path): + split_type_file_path = os.path.join(corpus_path, 'extracted', + "all_talks_dev.tsv") + with io.open(split_type_file_path, 'r', encoding='utf8') as fp: + reader = csv.DictReader(fp, delimiter='\t', quoting=csv.QUOTE_NONE) + header = next(reader) + return [k for k in header.keys() if k != 'talk_name'] + +def extra_english(corpus_path, split): + split_type_file_path = os.path.join(corpus_path, + f"all_talks_{split}.tsv") + output_split_type_file_path = os.path.join(corpus_path, + f"all_talks_{split}.en") + with io.open(split_type_file_path, 'r', encoding='utf8') as fp, io.open(output_split_type_file_path, 'w', encoding='utf8') as fw: + reader = csv.DictReader(fp, delimiter='\t', quoting=csv.QUOTE_NONE) + for row in reader: + line = row['en'] + fw.write(line + '\n') + de_tok(output_split_type_file_path, 'en') + + + +def tok_file_name(filename, lang): + seps = filename.split('.') + seps.insert(-1, 'tok') + tok_file = '.'.join(seps) + return tok_file + +def de_tok(tok_file, lang): + # seps = tok_file.split('.') + # seps.insert(-1, 'detok') + # de_tok_file = '.'.join(seps) + de_tok_file = tok_file.replace('.tok.', '.') + cmd = 'perl {detok_cmd} -l {lang} < {tok_file} > {de_tok_file}'.format( + detok_cmd=detok_cmd, tok_file=tok_file, + de_tok_file=de_tok_file, lang=lang[:2]) + call(cmd) + +def extra_bitex( + ted_data_path, + lsrc_lang, + ltrg_lang, + target_token, + output_data_path, +): + def get_ted_lang(lang): + long_langs = ['pt-br', 'zh-cn', 'zh-tw', 'fr-ca'] + if lang[:5] in long_langs: + return lang[:5] + elif lang[:4] =='calv': + return lang[:5] + elif lang in ['pt_BR', 'zh_CN', 'zh_TW', 'fr_CA']: + return lang.lower().replace('_', '-') + return lang[:2] + src_lang = get_ted_lang(lsrc_lang) + trg_lang = get_ted_lang(ltrg_lang) + train_lang_dict={'source': [src_lang], 'target': [trg_lang]} + eval_lang_dict = {'source': [src_lang], 'target': [trg_lang]} + + obj = MultiLingualAlignedCorpusReader(corpus_path=ted_data_path, + lang_dict=train_lang_dict, + target_token=target_token, + corpus_type='file', + eval_lang_dict=eval_lang_dict, + zero_shot=False, + bilingual=True) + + os.makedirs(output_data_path, exist_ok=True) + lsrc_lang = lsrc_lang.replace('-', '_') + ltrg_lang = ltrg_lang.replace('-', '_') + obj.save_file(output_data_path + f"/train.{lsrc_lang}-{ltrg_lang}.{lsrc_lang}", + split_type='train', data_type='source', lang=src_lang) + obj.save_file(output_data_path + f"/train.{lsrc_lang}-{ltrg_lang}.{ltrg_lang}", + split_type='train', data_type='target', lang=trg_lang) + + obj.save_file(output_data_path + f"/test.{lsrc_lang}-{ltrg_lang}.{lsrc_lang}", + split_type='test', data_type='source', lang=src_lang) + obj.save_file(output_data_path + f"/test.{lsrc_lang}-{ltrg_lang}.{ltrg_lang}", + split_type='test', data_type='target', lang=trg_lang) + + obj.save_file(output_data_path + f"/valid.{lsrc_lang}-{ltrg_lang}.{lsrc_lang}", + split_type='dev', data_type='source', lang=src_lang) + obj.save_file(output_data_path + f"/valid.{lsrc_lang}-{ltrg_lang}.{ltrg_lang}", + split_type='dev', data_type='target', lang=trg_lang) + + +def bar_custom(current, total, width=80): + print("Downloading: %d%% [%d / %d] Ks" % (current / total * 100, current / 1000, total / 1000), end='\r') + + +def download_and_extract(download_to, extract_to): + url = 'http://phontron.com/data/ted_talks.tar.gz' + filename = f"{download_to}/ted_talks.tar.gz" + if os.path.exists(filename): + print(f'{filename} has already been downloaded so skip') + else: + filename = wget.download(url, filename, bar=bar_custom) + if os.path.exists(f'{extract_to}/all_talks_train.tsv'): + print(f'Already extracted so skip') + else: + extract_cmd = f'tar xzfv "{filename}" -C "{extract_to}"' + call(extract_cmd) + + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--ted_data_path', type=str, default=WORKDIR_ROOT, required=False) + parser.add_argument( + '--direction-list', + type=str, + # default=None, + #for ML50 + default=( + "bn_IN-en_XX,he_IL-en_XX,fa_IR-en_XX,id_ID-en_XX,sv_SE-en_XX,pt_XX-en_XX,ka_GE-en_XX,ka_GE-en_XX,th_TH-en_XX," + "mr_IN-en_XX,hr_HR-en_XX,uk_UA-en_XX,az_AZ-en_XX,mk_MK-en_XX,gl_ES-en_XX,sl_SI-en_XX,mn_MN-en_XX," + #non-english directions + # "fr_XX-de_DE," # replaced with wmt20 + # "ja_XX-ko_KR,es_XX-pt_XX,ru_RU-sv_SE,hi_IN-bn_IN,id_ID-ar_AR,cs_CZ-pl_PL,ar_AR-tr_TR" + ), + required=False) + parser.add_argument('--target-token', action='store_true', default=False) + parser.add_argument('--extract-all-english', action='store_true', default=False) + + args = parser.parse_args() + + import sys + import json + + # TED Talks data directory + ted_data_path = args.ted_data_path + + download_to = f'{ted_data_path}/downloads' + extract_to = f'{ted_data_path}/extracted' + + #DESTDIR=${WORKDIR_ROOT}/ML50/raw/ + output_path = f'{ted_data_path}/ML50/raw' + os.makedirs(download_to, exist_ok=True) + os.makedirs(extract_to, exist_ok=True) + os.makedirs(output_path, exist_ok=True) + download_and_extract(download_to, extract_to) + + + if args.extract_all_english: + for split in ['train', 'dev', 'test']: + extra_english(ted_data_path, split) + exit(0) + if args.direction_list is not None: + directions = args.direction_list.strip().split(',') + directions = [tuple(d.strip().split('-', 1)) for d in directions if d] + else: + langs = read_langs(ted_data_path) + # directions = [ + # '{}.{}'.format(src, tgt) + # for src in langs + # for tgt in langs + # if src < tgt + # ] + directions = [('en', tgt) for tgt in langs if tgt != 'en'] + print(f'num directions={len(directions)}: {directions}') + + for src_lang, trg_lang in directions: + print('--working on {}-{}'.format(src_lang, trg_lang)) + extra_bitex( + extract_to, + src_lang, + trg_lang, + target_token=args.target_token, + output_data_path=output_path + ) diff --git a/data/fairseq/examples/multilingual/data_scripts/download_wat19_my.sh b/data/fairseq/examples/multilingual/data_scripts/download_wat19_my.sh new file mode 100644 index 0000000000000000000000000000000000000000..c1e2d47287a29af4576e7a63641e8152ecb63c44 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/download_wat19_my.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + + +if [ -z $WORKDIR_ROOT ] ; +then + echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." + exit +fi + + +SRCDIR=$WORKDIR_ROOT/indic_languages_corpus +DESTDIR=$WORKDIR_ROOT/ML50/raw +mkdir -p $SRCDIR +mkdir -p $DESTDIR + +WAT_MY_EN=wat2020.my-en.zip +cd $SRCDIR +# please refer to http://lotus.kuee.kyoto-u.ac.jp/WAT/my-en-data/ for latest URL if the following url expired +#- The data used for WAT2020 are identical to those used in WAT2019. +wget http://lotus.kuee.kyoto-u.ac.jp/WAT/my-en-data/$WAT_MY_EN +unzip $WAT_MY_EN + + +SRC_EXTRACT_DIR=$SRCDIR/wat2020.my-en/alt + +cp $SRC_EXTRACT_DIR/train.alt.en $DESTDIR/train.my_MM-en_XX.en_XX +cp $SRC_EXTRACT_DIR/train.alt.my $DESTDIR/train.my_MM-en_XX.my_MM +cp $SRC_EXTRACT_DIR/dev.alt.en $DESTDIR/valid.my_MM-en_XX.en_XX +cp $SRC_EXTRACT_DIR/dev.alt.my $DESTDIR/valid.my_MM-en_XX.my_MM +cp $SRC_EXTRACT_DIR/test.alt.en $DESTDIR/test.my_MM-en_XX.en_XX +cp $SRC_EXTRACT_DIR/test.alt.my $DESTDIR/test.my_MM-en_XX.my_MM diff --git a/data/fairseq/examples/multilingual/data_scripts/download_wmt19_and_before.py b/data/fairseq/examples/multilingual/data_scripts/download_wmt19_and_before.py new file mode 100644 index 0000000000000000000000000000000000000000..3465731eb3e55047c44d1b336a97e99cb3a89a53 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/download_wmt19_and_before.py @@ -0,0 +1,899 @@ +from typing import NamedTuple, List +from urllib.parse import urlparse +import os, sys +import subprocess +from subprocess import check_call, check_output +import glob +import wget +import re +import multiprocessing as mp +from functools import partial +import pathlib +from collections import OrderedDict + +WORKDIR_ROOT = os.environ.get('WORKDIR_ROOT', None) + +if WORKDIR_ROOT is None or not WORKDIR_ROOT.strip(): + print('please specify your working directory root in OS environment variable WORKDIR_ROOT. Exitting..."') + sys.exit(-1) + +# scripts and data locations +CWD = os.getcwd() +UTILS = f"{CWD}/utils" + +MOSES = f"{UTILS}/mosesdecoder" +SGM_TOOL = f'{MOSES}/scripts/ems/support/input-from-sgm.perl' + +TMX2CORPUS = f"{UTILS}/tmx2corpus" +TMX_TOOL = f'python {TMX2CORPUS}/tmx2corpus.py' + +to_data_path = f'{WORKDIR_ROOT}/wmt' +download_to = f'{to_data_path}/downloads' +manually_downloads = f'{to_data_path}/downloads' +extract_to = f'{to_data_path}/extracted' +#DESTDIR=${WORKDIR_ROOT}/ML50/raw/ +raw_data = f'{WORKDIR_ROOT}/ML50/raw' +#### + +class DLDataset(NamedTuple): + name: str + train_urls: List[str] + valid_urls: List[str] + test_urls: List[str] + train_files_patterns: List[str] = [] + valid_files_patterns: List[str] = [] + test_files_patterns: List[str] = [] + + + +def bar_custom(current, total, width=80): + print("Downloading: %d%% [%d / %d] Ks" % (current / total * 100, current / 1000, total / 1000), end='\r') + +def get_downloaded_file(dl_folder, url): + if isinstance(url, tuple): + url, f = url + else: + url_f = urlparse(url) + # f = os.path.split(url_f.path)[-1] + f = '_'.join(url_f.path.split('/')[1:]) + return url, f"{dl_folder}/{f}" + +def download_parts_and_combine(dl_folder, urls, filename): + parts = [] + for url_record in urls: + url, part_file = get_downloaded_file(dl_folder, url_record) + if os.path.exists(part_file): + print(f'{part_file} has already been downloaded so skip') + else: + part_file = wget.download(url, part_file, bar=bar_custom) + parts.append(part_file) + + def get_combine_cmd(parts): + #default as tar.gz.?? + return f'cat {" ".join(parts)} > {filename}' + + combine_cmd = get_combine_cmd(parts) + call(combine_cmd, debug=True) + return filename + +def download_a_url(dl_folder, url): + url, filename = get_downloaded_file(dl_folder, url) + if os.path.exists(filename): + print(f'{filename} has already been downloaded so skip') + return filename + + print(f'downloading {url} to {filename}') + if isinstance(url, list) or isinstance(url, tuple): + download_parts_and_combine(dl_folder, url, filename) + else: + wget.download(url, filename, bar=bar_custom) + print(f'dowloaded: {filename}') + return filename + +def download_files(dl_folder, urls, completed_urls={}): + for url_record in urls: + url, _ = get_downloaded_file(dl_folder, url_record) + filename = download_a_url(dl_folder, url_record) + completed_urls[str(url)] = filename + return completed_urls + +def check_need_manual_downalod(dl_folder, to_manually_download_urls): + to_be_manually_dowloaded = [] + manually_completed_urls = {} + for url_record, instruction in to_manually_download_urls: + url, filename = get_downloaded_file(dl_folder, url_record) + if not os.path.exists(filename): + print(f'{url} need to be download manually, please download it manually following {instruction}; and copy it to {filename}') + to_be_manually_dowloaded.append((url, filename)) + else: + manually_completed_urls[url] = filename + # if len(to_be_manually_dowloaded) > 0: + # raise ValueError('Missing files that need to be downloaded manually; stop the process now.') + return to_be_manually_dowloaded + +def download_dataset(to_folder, dl_dataset, completed_urls={}): + download_files(to_folder, dl_dataset.train_urls, completed_urls) + download_files(to_folder, dl_dataset.valid_urls, completed_urls) + download_files(to_folder, dl_dataset.test_urls, completed_urls) + print('completed downloading') + return completed_urls + +def call(cmd, debug=False): + if debug: + print(cmd) + check_call(cmd, shell=True) + + +def get_extract_name(file_path): + path = os.path.split(file_path) + return path[-1] + '_extract' #.split('.')[0] + +def extract_file(downloaded_file, extract_folder, get_extract_name=get_extract_name, debug=False): + extract_name = get_extract_name(downloaded_file) + extract_to = f'{extract_folder}/{extract_name}' + os.makedirs(extract_to, exist_ok=True) + if os.path.exists(f'{extract_to}/DONE'): + print(f'{downloaded_file} has already been extracted to {extract_to} so skip') + return extract_to + def get_extract_cmd(filename): + if filename.endswith('.tgz') or filename.endswith('tar.gz'): + return f'tar xzfv {filename} -C {extract_to}' + elif filename.endswith('.gz.tar'): + return f'tar xfv {filename} -C {extract_to}; (cd {extract_to}; gzip -d *.gz; [ $? -eq 0 ] || gzip -d */*.gz)' + elif filename.endswith('.tar'): + return f'tar xfv {filename} -C {extract_to}' + elif filename.endswith('.gz'): + return f'cp {filename} {extract_to}; (cd {extract_to}; gzip -d *.gz)' + elif filename.endswith('.zip'): + return f'unzip {filename} -d {extract_to}' + extract_cmd = get_extract_cmd(downloaded_file) + print(f'extracting {downloaded_file}') + if isinstance(extract_cmd, list): + for c in extract_cmd: + call(c, debug=debug) + else: + call(extract_cmd, debug=debug) + call(f'echo DONE > {extract_to}/DONE') + return extract_to + + +def extract_all_files( + completed_urls, extract_folder, + get_extract_name=get_extract_name, + completed_extraction={}, + debug=False): + extracted_folders = OrderedDict() + for url, downloaded_file in set(completed_urls.items()): + if downloaded_file in completed_extraction: + print(f'{downloaded_file} is already extracted; so skip') + continue + folder = extract_file(downloaded_file, extract_folder, get_extract_name, debug) + extracted_folders[url] = folder + return extracted_folders + + +def my_glob(folder): + for p in [f'{folder}/*', f'{folder}/*/*', f'{folder}/*/*/*']: + for f in glob.glob(p): + yield f + + +def sgm2raw(sgm, debug): + to_file = sgm[0:len(sgm) - len('.sgm')] + if os.path.exists(to_file): + debug and print(f'{sgm} already converted to {to_file}; so skip') + return to_file + cmd = f'{SGM_TOOL} < {sgm} > {to_file}' + call(cmd, debug) + return to_file + +def tmx2raw(tmx, debug): + to_file = tmx[0:len(tmx) - len('.tmx')] + to_folder = os.path.join(*os.path.split(tmx)[:-1]) + if os.path.exists(f'{to_folder}/bitext.en'): + debug and print(f'{tmx} already extracted to {to_file}; so skip') + return to_file + cmd = f'(cd {to_folder}; {TMX_TOOL} {tmx})' + call(cmd, debug) + return to_file + +CZENG16_REGEX = re.compile(r'.*?data.plaintext-format/0[0-9]train$') +WMT19_WIKITITLES_REGEX = re.compile(r'.*?wikititles-v1.(\w\w)-en.tsv.gz') +TSV_REGEX = re.compile(r'.*?(\w\w)-(\w\w).tsv$') + + + +def cut_wikitles(wiki_file, debug): + # different languages have different file names: + if wiki_file.endswith('wiki/fi-en/titles.fi-en'): + to_file1 = f'{wiki_file}.fi' + to_file2 = f'{wiki_file}.en' + BACKSLASH = '\\' + cmd1 = f"cat {wiki_file} | sed 's/|||/{BACKSLASH}t/g' |cut -f1 |awk '{{$1=$1}};1' > {to_file1}" + cmd2 = f"cat {wiki_file} | sed 's/|||/{BACKSLASH}t/g' |cut -f2 |awk '{{$1=$1}};1' > {to_file2}" +# elif WMT19_WIKITITLES_REGEX.match(wiki_file): +# src = WMT19_WIKITITLES_REGEX.match(wiki_file).groups()[0] +# to_file1 = f'{wiki_file}.{src}' +# to_file2 = f'{wiki_file}.en' +# cmd1 = f"cat {wiki_file} | cut -f1 |awk '{{$1=$1}};1' > {to_file1}" +# cmd2 = f"cat {wiki_file} | cut -f2 |awk '{{$1=$1}};1' > {to_file2}" + else: + return None + if os.path.exists(to_file1) and os.path.exists(to_file2): + debug and print(f'{wiki_file} already processed to {to_file1} and {to_file2}; so skip') + return wiki_file + + call(cmd1, debug=debug) + call(cmd2, debug=debug) + return wiki_file + +def cut_tsv(file, debug): + m = TSV_REGEX.match(file) + if m is None: + raise ValueError(f'{file} is not matching tsv pattern') + src = m.groups()[0] + tgt = m.groups()[1] + + to_file1 = f'{file}.{src}' + to_file2 = f'{file}.{tgt}' + cmd1 = f"cat {file} | cut -f1 |awk '{{$1=$1}};1' > {to_file1}" + cmd2 = f"cat {file} | cut -f2 |awk '{{$1=$1}};1' > {to_file2}" + if os.path.exists(to_file1) and os.path.exists(to_file2): + debug and print(f'{file} already processed to {to_file1} and {to_file2}; so skip') + return file + + call(cmd1, debug=debug) + call(cmd2, debug=debug) + return file + + +def convert_file_if_needed(file, debug): + if file.endswith('.sgm'): + return sgm2raw(file, debug) + elif file.endswith('.tmx'): + return tmx2raw(file, debug) + elif file.endswith('wiki/fi-en/titles.fi-en'): + return cut_wikitles(file, debug) +# elif WMT19_WIKITITLES_REGEX.match(file): +# return cut_wikitles(file, debug) + elif file.endswith('.tsv'): + return cut_tsv(file, debug) + elif CZENG16_REGEX.match(file): + return convert2czeng17(file, debug) + else: + return file + + +def convert_files_if_needed(extracted_foldrs, my_glob=my_glob, debug=False): + return { + url: list(sorted(set(convert_file_if_needed(f, debug)) for f in sorted(set(my_glob(folder))))) + for url, folder in extracted_foldrs.items() + } + +def match_patt(file_path, file_pattern, src, tgt, lang): + return file_pattern.format(src=src, tgt=tgt, lang=lang) in file_path + +def match_patts(file_path, file_patterns, src, tgt, lang): + for file_pattern in file_patterns: + params = { k: v for k, v in [('src', src), ('tgt', tgt), ('lang', lang)] if k in file_pattern} + matching = file_pattern.format(**params) + + if isinstance(file_pattern, tuple): + pattern, directions = file_pattern + if f'{src}-{tgt}' in directions and matching in file_path: + return True + else: + if matching in file_path: + return True + return False + +def extracted_glob(extracted_folder, file_patterns, src, tgt, lang): + def get_matching_pattern(file_pattern): + params = { + k: v + for k, v in [('src', src), ('tgt', tgt), ('lang', lang)] + if '{' + k + '}' in file_pattern + } + file_pattern = re.sub(r'{src:(.*?)}', r'\1' if lang == src else '', file_pattern) + file_pattern = re.sub(r'{tgt:(.*?)}', r'\1' if lang == tgt else '', file_pattern) + file_pattern = file_pattern.format(**params) + return file_pattern + for file_pattern in file_patterns: + if isinstance(file_pattern, tuple): + file_pattern, lang_pairs = file_pattern + if f'{src}-{tgt}' not in lang_pairs: + continue +# print('working on pattern: ', file_pattern, lang_pairs ) + matching_pattern = get_matching_pattern(file_pattern) + if matching_pattern is None: + continue + glob_patterns = f'{extracted_folder}/{matching_pattern}' +# print('glob_patterns: ', glob_patterns) + for f in glob.glob(glob_patterns): + yield f + +# for debug usage +def all_extracted_files(split, src, tgt, extracted_folders, split_urls): + def get_url(url): + if isinstance(url, tuple): + url, downloaded_file = url + return url + return [ + f + for url in split_urls + for f in my_glob(extracted_folders[str(get_url(url))]) + ] + +def concat_files(split, src, tgt, extracted_folders, split_urls, path_patterns, to_folder, debug=False): +# if debug: +# print('extracted files to be filtered by patterns: ', +# '\n\t'.join(sorted(all_extracted_files(split, src, tgt, extracted_folders, split_urls)))) + for lang in [src, tgt]: + to_file = f'{to_folder}/{split}.{src}-{tgt}.{lang}' + s_src, s_tgt, s_lang = src.split('_')[0], tgt.split('_')[0], lang.split('_')[0] + files = [] + for url in split_urls: + if isinstance(url, tuple): + url, downloaded_file = url + if str(url) not in extracted_folders: + print(f'warning: {url} not in extracted files') + for extracted_file in set( + extracted_glob( + extracted_folders[str(url)], path_patterns, + s_src, s_tgt, s_lang)): + files.append(extracted_file) + if len(files) == 0: + print('warning: ', f'No files found for split {to_file}') + continue + files = sorted(set(files)) + print(f'concating {len(files)} files into {to_file}') + cmd = ['cat'] + [f'"{f}"' for f in files] + [f'>{to_file}'] + cmd = " ".join(cmd) + call(cmd, debug=debug) + +UTILS = os.path.join(pathlib.Path(__file__).parent, 'utils') +LID_MODEL = f'{download_to}/lid.176.bin' +LID_MULTI = f'{UTILS}/fasttext_multi_filter.py' + +def lid_filter(split, src, tgt, from_folder, to_folder, debug=False): + if not os.path.exists(LID_MODEL): + call(f'wget -nc https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin -O {LID_MODEL}') + from_prefix = f'{from_folder}/{split}.{src}-{tgt}' + to_prefix = f'{to_folder}/{split}.{src}-{tgt}' + if os.path.exists(f'{from_prefix}.{src}') and os.path.exists(f'{from_prefix}.{tgt}'): + s_src, s_tgt = src.split('_')[0], tgt.split('_')[0] + cmd = ( + f'python {LID_MULTI} --model {LID_MODEL} --inputs {from_prefix}.{src} {from_prefix}.{tgt} ' + f'--langs {s_src} {s_tgt} --outputs {to_prefix}.{src} {to_prefix}.{tgt}' + ) + print(f'filtering {from_prefix}') + call(cmd, debug=debug) + +def concat_into_splits(dl_dataset, src, tgt, extracted_folders, to_folder, debug): + to_folder_tmp = f"{to_folder}_tmp" + os.makedirs(to_folder_tmp, exist_ok=True) + concat_files('train', src, tgt, + extracted_folders, + split_urls=dl_dataset.train_urls, + path_patterns=dl_dataset.train_files_patterns, + to_folder=to_folder_tmp, debug=debug) + lid_filter('train', src, tgt, to_folder_tmp, to_folder, debug) + + concat_files('valid', src, tgt, + extracted_folders, + split_urls=dl_dataset.valid_urls, + path_patterns=dl_dataset.valid_files_patterns, + to_folder=to_folder, debug=debug) + concat_files('test', src, tgt, + extracted_folders, + split_urls=dl_dataset.test_urls, + path_patterns=dl_dataset.test_files_patterns, + to_folder=to_folder, debug=debug) + + +def download_multi(dl_folder, extract_folder, urls, num_processes=8, debug=False): + pool = mp.Pool(processes=num_processes) + download_f = partial(download_a_url, dl_folder) + downloaded_files = pool.imap_unordered(download_f, urls) + pool.close() + pool.join() + +BLEU_REGEX = re.compile("^BLEU\\S* = (\\S+) ") +def run_eval_bleu(cmd): + output = check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode("utf-8").strip() + print(output) + bleu = -1.0 + for line in output.strip().split('\n'): + m = BLEU_REGEX.search(line) + if m is not None: + bleu = m.groups()[0] + bleu = float(bleu) + break + return bleu + +def check_wmt_test_bleu(raw_folder, wmt_lang_pairs): + not_matchings = [] + for wmt, src_tgts in wmt_lang_pairs: + for src_tgt in src_tgts: + print(f'checking test bleus for: {src_tgt} at {wmt}') + src, tgt = src_tgt.split('-') + ssrc, stgt = src[:2], tgt[:2] + if os.path.exists(f'{raw_folder}/test.{tgt}-{src}.{src}'): + # reversed direction may have different test set + test_src = f'{raw_folder}/test.{tgt}-{src}.{src}' + else: + test_src = f'{raw_folder}/test.{src}-{tgt}.{src}' + cmd1 = f'cat {test_src} | sacrebleu -t "{wmt}" -l {stgt}-{ssrc}; [ $? -eq 0 ] || echo ""' + test_tgt = f'{raw_folder}/test.{src}-{tgt}.{tgt}' + cmd2 = f'cat {test_tgt} | sacrebleu -t "{wmt}" -l {ssrc}-{stgt}; [ $? -eq 0 ] || echo ""' + bleu1 = run_eval_bleu(cmd1) + if bleu1 != 100.0: + not_matchings.append(f'{wmt}:{src_tgt} source side not matching: {test_src}') + bleu2 = run_eval_bleu(cmd2) + if bleu2 != 100.0: + not_matchings.append(f'{wmt}:{src_tgt} target side not matching: {test_tgt}') + return not_matchings + +def download_and_extract( + to_folder, lang_pairs, dl_dataset, + to_manually_download_urls, + completed_urls={}, completed_extraction={}, + debug=False): + + dl_folder = f'{to_folder}/downloads' + extract_folder = f'{to_folder}/extracted' + raw_folder = f'{to_folder}/raw' + lid_filtered = f'{to_folder}/lid_filtered' + + os.makedirs(extract_folder, exist_ok=True) + os.makedirs(raw_folder, exist_ok=True) + os.makedirs(lid_filtered, exist_ok=True) + + + to_be_manually_dowloaded = check_need_manual_downalod(dl_folder, to_manually_download_urls) + + completed_urls = download_dataset( + dl_folder, dl_dataset, completed_urls) + if debug: + print('completed urls: ', completed_urls) + + + extracted_folders = extract_all_files( + completed_urls, + extract_folder=extract_folder, + completed_extraction=completed_extraction, + debug=debug) + if debug: + print('download files have been extracted to folders: ', extracted_folders) + + converted_files = convert_files_if_needed(extracted_folders, debug=False) + for src_tgt in lang_pairs: + print(f'working on {dl_dataset.name}: {src_tgt}') + src, tgt = src_tgt.split('-') + concat_into_splits(dl_dataset, + src=src, tgt=tgt, + extracted_folders=extracted_folders, + to_folder=raw_folder, debug=debug) + print('completed data into: ', raw_folder) + +def download_czang16(download_to, username=None): + wgets = [ + f'wget --user={username} --password=czeng -P {download_to} http://ufallab.ms.mff.cuni.cz/~bojar/czeng16-data/data-plaintext-format.{i}.tar' + for i in range(10)] + cmds = [] + for i, cmd in enumerate(wgets): + filename = f'{download_to}/data-plaintext-format.{i}.tar' + if os.path.exists(filename): + print(f'{filename} has already been downloaded; so skip') + continue + cmds.append(cmd) + if cmds and username is None: + raise ValueError('No czeng username is given; please register at http://ufal.mff.cuni.cz/czeng/czeng16 to obtain username to download') + for cmd in cmds: + call(cmd) + print('done with downloading czeng1.6') + +def download_czeng17_script(download_to, extract_folder, debug=False): + url = 'http://ufal.mff.cuni.cz/czeng/download.php?f=convert_czeng16_to_17.pl.zip' + filename = f'{download_to}/convert_czeng16_to_17.pl.zip' + extract_to = f'{extract_folder}/{get_extract_name(filename)}' + script_path = f'{extract_to}/convert_czeng16_to_17.pl' + + if not os.path.exists(script_path): + wget.download(url, filename, bar=bar_custom) + extract_to = extract_file(f'{download_to}/convert_czeng16_to_17.pl.zip', extract_folder, get_extract_name=get_extract_name, debug=debug) + return script_path + +czeng17_script_path = "" +def convert2czeng17(file, debug): + en_file = f'{file}.en' + cs_file = f'{file}.cs' + + if not os.path.exists(en_file) or not os.path.exists(cs_file): + cs_cmd = f'cat {file} | perl {czeng17_script_path} | cut -f3 > {cs_file}' + en_cmd = f'cat {file} | perl {czeng17_script_path} | cut -f4 > {en_file}' + call(cs_cmd, debug) + call(en_cmd, debug) + else: + print(f'already extracted: {en_file} and {cs_file}') + return file + +def extract_czeng17(extract_folder, debug=False): + url = 'http://ufal.mff.cuni.cz/czeng/download.php?f=convert_czeng16_to_17.pl.zip' + filename = f'{download_to}/convert_czeng16_to_17.pl.zip' + extract_to = f'{extract_folder}/{get_extract_name(filename)}' + script_path = f'{extract_to}/convert_czeng16_to_17.pl' + + if not os.path.exists(script_path): + wget.download(url, filename, bar=bar_custom) + extract_to = extract_file(f'{download_to}/convert_czeng16_to_17.pl.zip', extract_folder, get_extract_name=get_extract_name, debug=debug) + return script_path + +######### +# definitions of wmt data sources +# for es-en +# Punctuation in the official test sets will be encoded with ASCII characters (not complex Unicode characters) as much as possible. You may want to normalize your system's output before submission. You are able able to use a rawer version of the test sets that does not have this normalization. +# script to normalize punctuation: http://www.statmt.org/wmt11/normalize-punctuation.perl +wmt13_es_en = DLDataset( + name='wmt13_es-en', + train_urls=[ + 'http://www.statmt.org/wmt13/training-parallel-europarl-v7.tgz', + 'http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz', + 'http://www.statmt.org/wmt13/training-parallel-un.tgz', + 'http://www.statmt.org/wmt13/training-parallel-nc-v8.tgz', + ], + valid_urls=[ + ('http://www.statmt.org/wmt13/dev.tgz', 'wmt13_dev.tgz') + ], + test_urls=[ + ('http://www.statmt.org/wmt13/test.tgz', 'wmt13_test.tgz') + ], + train_files_patterns=[ + ('*/europarl-v7.{src}-{tgt}.{lang}', ['es-en']), + ('*commoncrawl.{src}-{tgt}.{lang}', ['es-en']), + ('*/news-commentary-v8.{src}-{tgt}.{lang}', ['es-en']), + ('un/*undoc.2000.{src}-{tgt}.{lang}', ['es-en']), + ] , + valid_files_patterns=[ + ('dev/newstest2012.{lang}', ['es-en']) + ], + test_files_patterns=[ + ('test/newstest*.{lang}', ['es-en']) + ], +) + +wmt14_de_fr_en = DLDataset( + name='wmt14_de_fr_en', + train_urls=[ + 'http://www.statmt.org/wmt13/training-parallel-europarl-v7.tgz', + 'http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz', + 'http://www.statmt.org/wmt13/training-parallel-un.tgz', + 'http://www.statmt.org/wmt14/training-parallel-nc-v9.tgz', + ('http://www.statmt.org/wmt10/training-giga-fren.tar', 'training-giga-fren.gz.tar'), #it is actuall a gz.tar + ], + valid_urls=[ + ('http://www.statmt.org/wmt14/dev.tgz', 'wmt14_dev.tgz'), + ], + test_urls=[ + ('http://www.statmt.org/wmt14/test-full.tgz', 'wmt14_test_full.tgz'), # cleaned test sets + ], + train_files_patterns=[ + ('*/europarl-v7.{src}-{tgt}.{lang}', ['fr-en', 'de-en']), + ('*commoncrawl.{src}-{tgt}.{lang}', ['fr-en', 'de-en']), + ('*/*news-commentary-v9.{src}-{tgt}.{lang}', ['fr-en', 'de-en']), + ('un/undoc.2000.{src}-{tgt}.{lang}', ['fr-en']), + ('*giga-{src}{tgt}*{lang}', ['fr-en']) + ], + valid_files_patterns=[ + ('dev/newstest2013.{lang}', ['fr-en', 'de-en']) + ], + test_files_patterns=[ + ('test-full/newstest*{src}{tgt}-{src:src}{tgt:ref}.{lang}', ['en-de', 'de-en', 'fr-en', 'en-fr']), + ], +) + +# pip install git+https://github.com/amake/tmx2corpus.git +wmt16_ro_en = DLDataset( + name='wmt16_ro-en', + train_urls=[ + ('http://data.statmt.org/wmt16/translation-task/training-parallel-ep-v8.tgz', 'wmt16_training-parallel-ep-v8.tgz'), + ('http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz', 'en-ro.tmx.gz'), + ], + valid_urls=[ + ('http://data.statmt.org/wmt16/translation-task/dev-romanian-updated.tgz', 'wmt16_dev.tgz') + ], + test_urls=[ + ('http://data.statmt.org/wmt16/translation-task/test.tgz', 'wmt16_test.tgz') + ], + train_files_patterns=[ + ('*/*europarl-v8.{src}-{tgt}.{lang}', ['ro-en']), + ('bitext.{lang}', ['ro-en']) #setimes from tmux + ] , + valid_files_patterns=[ + ('dev/newsdev2016*{src}{tgt}*.{lang}', ['ro-en', 'ro-en']) + ], + test_files_patterns=[ + ('test/newstest*{src}{tgt}*.{lang}', ['ro-en', 'en-ro']) + ], +) + +cwmt_wmt_instruction = 'cwmt download instruction at: http://nlp.nju.edu.cn/cwmt-wmt' +wmt17_fi_lv_tr_zh_en_manual_downloads = [ + # fake urls to have unique keys for the data + ( ('http://nlp.nju.edu.cn/cwmt-wmt/CASIA2015.zip', 'CASIA2015.zip'), cwmt_wmt_instruction), + ( ('http://nlp.nju.edu.cn/cwmt-wmt/CASICT2011.zip', 'CASICT2011.zip'), cwmt_wmt_instruction), + ( ('http://nlp.nju.edu.cn/cwmt-wmt/CASICT2015.zip', 'CASICT2015.zip'), cwmt_wmt_instruction), + ( ('http://nlp.nju.edu.cn/cwmt-wmt/Datum2015.zip', 'Datum2015.zip'), cwmt_wmt_instruction), + ( ('http://nlp.nju.edu.cn/cwmt-wmt/Datum2017.zip', 'Datum2017.zip'), cwmt_wmt_instruction), + ( ('http://nlp.nju.edu.cn/cwmt-wmt/NEU2017.zip', 'NEU2017.zip'), cwmt_wmt_instruction), +] +wmt17_fi_lv_tr_zh_en = DLDataset( + name='wmt17_fi_lv_tr_zh_en', + train_urls=[ + ('http://data.statmt.org/wmt17/translation-task/training-parallel-ep-v8.tgz', 'wmt17_training-parallel-ep-v8.tgz'), + 'http://data.statmt.org/wmt17/translation-task/training-parallel-nc-v12.tgz', + 'http://www.statmt.org/wmt15/wiki-titles.tgz', + ('http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-tr.tmx.gz', 'en-tr.tmx.gz'), + ('http://data.statmt.org/wmt17/translation-task/rapid2016.tgz', 'wmt17_rapid2016.tgz'), + 'http://data.statmt.org/wmt17/translation-task/leta.v1.tgz', + 'http://data.statmt.org/wmt17/translation-task/dcep.lv-en.v1.tgz', + 'http://data.statmt.org/wmt17/translation-task/books.lv-en.v1.tgz', + (('https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-zh.tar.gz.00', + 'https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-zh.tar.gz.01',), 'UNv1.0.en-zh.tar.gz'), + #manually download files: + ('http://nlp.nju.edu.cn/cwmt-wmt/CASIA2015.zip', 'CASIA2015.zip'), + ('http://nlp.nju.edu.cn/cwmt-wmt/CASICT2011.zip', 'CASICT2011.zip'), + ('http://nlp.nju.edu.cn/cwmt-wmt/CASICT2015.zip', 'CASICT2015.zip'), + ('http://nlp.nju.edu.cn/cwmt-wmt/Datum2015.zip', 'Datum2015.zip'), + ('http://nlp.nju.edu.cn/cwmt-wmt/Datum2017.zip', 'Datum2017.zip'), + ('http://nlp.nju.edu.cn/cwmt-wmt/NEU2017.zip', 'NEU2017.zip'), + ], + valid_urls=[ + ('http://data.statmt.org/wmt17/translation-task/dev.tgz', 'wmt17_dev.tgz'), + ], + test_urls=[ + #NEW: Improved translations for zh test sets + ('http://data.statmt.org/wmt17/translation-task/test-update-1.tgz', 'wmt17_test_zh_en.tgz'), + ('http://data.statmt.org/wmt17/translation-task/test.tgz', 'wmt17_test_others.tgz') + ], + train_files_patterns=[ + ('casict*/cas*{src:ch}{tgt:en}.txt', ['zh-en', 'zh-en'] ), + ('casia*/cas*{src:ch}{tgt:en}.txt', ['zh-en', 'zh-en'] ), + ('dataum*/Book*{src:cn}{tgt:en}.txt', ['zh-en', 'zh-en']), + ('neu*/NEU*{src:cn}{tgt:en}.txt', ['zh-en', 'zh-en'] ), + ('*/*UNv1.0.en-zh.{src:zh}{tgt:en}', ['zh-en']), + ('training/*news-commentary-v12.{src}-{tgt}.{lang}', ['zh-en', ]), + + ('*/*europarl-v8.{src}-{tgt}.{lang}', ['fi-en', 'lv-en']), + ('wiki/fi-en/titles.{src}-{tgt}.{lang}', ['fi-en', ]), + ('rapid2016.{tgt}-{src}.{lang}', ['fi-en', 'lv-en']), + ('*/leta.{lang}', ['lv-en']), + ('*/dcep.{lang}', ['lv-en']), + ('*/farewell.{lang}', ['lv-en']), + ('bitext.{lang}', ['tr-en']), + ] , + valid_files_patterns=[ + ('dev/newsdev2017*{src}{tgt}-{src:src}{tgt:ref}.{lang}', + [ + 'fi-en', 'lv-en', 'tr-en', 'zh-en', + 'en-fi', 'en-lv', 'en-tr', 'en-zh' + ]), + ('dev/newstest2016*{src}{tgt}-{src:src}{tgt:ref}.{lang}', + [ + 'fi-en', 'tr-en', + 'en-fi', 'en-tr', + ]), + ], + test_files_patterns=[ + ('test/newstest2017-{src}{tgt}-{src:src}{tgt:ref}.{lang}', + [ + 'fi-en', 'lv-en', 'tr-en', + 'en-fi', 'en-lv', 'en-tr', + ]), + ('newstest2017-{src}{tgt}-{src:src}{tgt:ref}.{lang}', + [ + 'zh-en', + 'en-zh' + ]), + ], +) + +czeng_instruction = 'download instruction at: http://ufal.mff.cuni.cz/czeng/czeng16' +#alternative: use the prepared data but detokenize it? +wmt18_cs_et_en_manual_downloads = [ +#for cs, need to register and download; Register and download CzEng 1.6. +#Better results can be obtained by using a subset of sentences, released under a new version name CzEng 1.7. + # ((f'http://ufallab.ms.mff.cuni.cz/~bojar/czeng16-data/data-plaintext-format.{i}.tar', + # f'data-plaintext-format.{i}.tar'), czeng_instruction) + # for i in range(10) +] + +wmt18_cs_et_en = DLDataset( + name='wmt18_cs_et_en', + train_urls=[ + 'http://www.statmt.org/wmt13/training-parallel-europarl-v7.tgz', + 'http://data.statmt.org/wmt18/translation-task/training-parallel-ep-v8.tgz', + 'https://s3.amazonaws.com/web-language-models/paracrawl/release1/paracrawl-release1.en-cs.zipporah0-dedup-clean.tgz', + 'https://s3.amazonaws.com/web-language-models/paracrawl/release1/paracrawl-release1.en-et.zipporah0-dedup-clean.tgz', + 'http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz', + 'http://data.statmt.org/wmt18/translation-task/training-parallel-nc-v13.tgz', + ('http://data.statmt.org/wmt18/translation-task/rapid2016.tgz', 'wmt18_rapid2016.tgz'), + # (tuple( + # (f'http://ufallab.ms.mff.cuni.cz/~bojar/czeng16-data/data-plaintext-format.{i}.tar', + # f'data-plaintext-format.{i}.tar') + # for i in range(10) + # ), + # 'czeng16_data_plaintext.gz.tar'), + ], + valid_urls=[ + ('http://data.statmt.org/wmt18/translation-task/dev.tgz', 'wmt18_dev.tgz'), + ], + test_urls=[ + ('http://data.statmt.org/wmt18/translation-task/test.tgz', 'wmt18_test.tgz'), + ], + train_files_patterns=[ + # ('*/*europarl-v7.{src}-{tgt}.{lang}', ['cs-en']), + ('*/*europarl-v8.{src}-{tgt}.{lang}', ['et-en']), + # ('*paracrawl-release1.{tgt}-{src}.zipporah0-dedup-clean.{lang}', ['cs-en', 'et-en']), + ('*paracrawl-release1.{tgt}-{src}.zipporah0-dedup-clean.{lang}', ['et-en']), + # ('*commoncrawl.{src}-{tgt}.{lang}', ['cs-en']), + # ('*/news-commentary-v13.{src}-{tgt}.{lang}', ['cs-en']), + # ('data.plaintext-format/*train.{lang}', ['cs-en']), + ('rapid2016.{tgt}-{src}.{lang}', ['et-en']), + ] , + valid_files_patterns=[ + ('dev/newsdev2018*{src}{tgt}-{src:src}{tgt:ref}.{lang}', ['et-en']), + # ('dev/newstest2017*{src}{tgt}-{src:src}{tgt:ref}.{lang}', ['cs-en']) + ], + test_files_patterns=[ + ('test/newstest2018-{src}{tgt}-{src:src}{tgt:ref}.{lang}', + # ['cs-en', 'et-en']), + ['et-en']), + ] +) + +ru_en_yandex_instruction = 'Yandex Corpus download instruction at: https://translate.yandex.ru/corpus?lang=en' +wmt19_ru_gu_kk_lt_manual_downloads = [ + (('https://translate.yandex.ru/corpus?lang=en', 'wmt19_1mcorpus.zip'), ru_en_yandex_instruction) +] +wmt19_ru_gu_kk_lt = DLDataset( + name='wmt19_ru_gu_kk_lt', + train_urls=[ + 'http://www.statmt.org/europarl/v9/training/europarl-v9.lt-en.tsv.gz', + 'https://s3.amazonaws.com/web-language-models/paracrawl/release3/en-lt.bicleaner07.tmx.gz', + 'https://s3.amazonaws.com/web-language-models/paracrawl/release1/paracrawl-release1.en-ru.zipporah0-dedup-clean.tgz', + 'http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz', + 'http://data.statmt.org/news-commentary/v14/training/news-commentary-v14-wmt19.en-kk.tsv.gz', + 'http://data.statmt.org/news-commentary/v14/training/news-commentary-v14.en-ru.tsv.gz', + 'http://data.statmt.org/wikititles/v1/wikititles-v1.kk-en.tsv.gz', + 'http://data.statmt.org/wikititles/v1/wikititles-v1.ru-en.tsv.gz', + 'http://data.statmt.org/wikititles/v1/wikititles-v1.kk-en.tsv.gz', + 'http://data.statmt.org/wikititles/v1/wikititles-v1.lt-en.tsv.gz', + 'http://data.statmt.org/wikititles/v1/wikititles-v1.gu-en.tsv.gz', + (('https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.00', + 'https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.01', + 'https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.02',), + 'wmt19_UNv1.0.en-ru.tar.gz'), + 'https://tilde-model.s3-eu-west-1.amazonaws.com/rapid2016.en-lt.tmx.zip', + ('https://translate.yandex.ru/corpus?lang=en', 'wmt19_1mcorpus.zip'), + ], + valid_urls=[ + ('http://data.statmt.org/wmt19/translation-task/dev.tgz', 'wmt19_dev.tgz'), + ], + test_urls=[ + ('http://data.statmt.org/wmt19/translation-task/test.tgz', 'wmt19_test.tgz'), + ], + train_files_patterns=[ + ('*europarl-v9.{src}-{tgt}.tsv.{lang}', ['lt-en']), + #paracrawl + ('*paracrawl-release1.{tgt}-{src}.zipporah0-dedup-clean.{lang}', ['ru-en']), + ('bitext.{lang}', ['lt-en',]), + ('*commoncrawl.{src}-{tgt}.{lang}', ['ru-en',]), + ('*news-commentary-v14-wmt19.{tgt}-{src}.tsv.{lang}', ['kk-en', ]), + ('*news-commentary-v14.{tgt}-{src}.tsv.{lang}', ['ru-en']), + #yandex + ('corpus.{tgt}_{src}.1m.{lang}', ['ru-en']), + ('wikititles_v1_wikititles-v1.{src}-{tgt}.tsv.{lang}', ['ru-en', 'kk-en', 'lt-en', 'gu-en']), + ('*/UNv1.0.{tgt}-{src}.{lang}', ['ru-en']), + #rapid + ('bitext.{lang}', ['lt-en']) + ], + valid_files_patterns=[ + ('dev/newsdev2019*{src}{tgt}-{src:src}{tgt:ref}.{lang}', ['gu-en', 'kk-en', 'lt-en']), + ('dev/newstest2018*{src}{tgt}-{src:src}{tgt:ref}.{lang}', ['ru-en']), + ], + test_files_patterns=[ + ('sgm/newstest2019-{src}{tgt}-{src:src}{tgt:ref}.{lang}', + ['ru-en', 'gu-en', 'kk-en', 'lt-en', 'en-ru', 'en-gu', 'en-kk', 'en-lt']), + ] +) + + +######### + +if __name__ == "__main__": + # speed up the downloads with multiple processing + dl_folder = f'{to_data_path}/downloads' + extract_folder = f'{to_data_path}/extracted' + + urls = [ + url + for dataset in [wmt13_es_en, wmt14_de_fr_en, wmt16_ro_en, wmt18_cs_et_en, wmt19_ru_gu_kk_lt] + for urls in [dataset.train_urls, dataset.valid_urls, dataset.test_urls] + for url in urls + ] + urls = set(urls) + download_multi(dl_folder, extract_folder, urls, num_processes=8, debug=True) + + # check manually downlaods + to_manually_download_urls = ( + wmt17_fi_lv_tr_zh_en_manual_downloads + wmt18_cs_et_en_manual_downloads + wmt19_ru_gu_kk_lt_manual_downloads + ) + to_be_manually_dowloaded = check_need_manual_downalod(dl_folder, to_manually_download_urls) + if len(to_be_manually_dowloaded) > 0: + print('Missing files that need to be downloaded manually; stop the process now.') + exit(-1) + + completed_urls = {} + completed_extraction = {} + def work_on_wmt(directions, wmt_data): + download_and_extract( + to_data_path, + directions, + wmt_data, + to_manually_download_urls=to_manually_download_urls, + completed_urls=completed_urls, completed_extraction=completed_extraction, debug=True) + + work_on_wmt( + ['es_XX-en_XX'], + wmt13_es_en,) + work_on_wmt( + [ + 'fr_XX-en_XX', 'en_XX-fr_XX', + # 'en_XX-de_DE', 'de_DE-en_XX', + ], + wmt14_de_fr_en,) + work_on_wmt( + ['ro_RO-en_XX', 'en_XX-ro_XX'], + wmt16_ro_en,) + work_on_wmt( + [ + # 'zh_CN-en_XX', + 'lv_LV-en_XX', 'fi_FI-en_XX', 'tr_TR-en_XX', + #in case the reversed directions have different train/valid/test data + # 'en_XX-zh_CN', + 'en_XX-lv_LV', 'en_XX-fi_FI', 'en_XX-tr_TR', + ], + wmt17_fi_lv_tr_zh_en, ) + # czeng17_script_path = download_czeng17_script(download_to, extract_to, debug=False) + # cz_username = None + work_on_wmt( + [ + # 'cs_CZ-en_XX', + 'et_EE-en_XX'], + wmt18_cs_et_en,) + work_on_wmt( + [ + # 'ru_RU-en_XX', 'en_XX-ru_RU', + 'gu_IN-en_XX', 'kk_KZ-en_XX', 'lt_LT-en_XX', + #in case the reversed directions have different train/valid/test data + 'en_XX-gu_IN', 'en_XX-kk_KZ', 'en_XX-lt_LT' + ], + wmt19_ru_gu_kk_lt,) + + not_matching = check_wmt_test_bleu( + f'{to_data_path}/raw', + [ + ('wmt13', ['es_XX-en_XX']), + ('wmt14/full', ['fr_XX-en_XX',]), + ('wmt16', ['ro_RO-en_XX',]), + # ('wmt17/improved', ['zh_CN-en_XX']), + ('wmt17', [ 'lv_LV-en_XX', 'fi_FI-en_XX', 'tr_TR-en_XX']), + ('wmt18', ['cs_CZ-en_XX', 'et_EE-en_XX']), + ('wmt19', ['gu_IN-en_XX', 'kk_KZ-en_XX', 'lt_LT-en_XX']), + #'ru_RU-en_XX', + ] + ) + if len(not_matching) > 0: + print('the following datasets do not have matching test datasets:\n\t', '\n\t'.join(not_matching)) + diff --git a/data/fairseq/examples/multilingual/data_scripts/download_wmt20.sh b/data/fairseq/examples/multilingual/data_scripts/download_wmt20.sh new file mode 100644 index 0000000000000000000000000000000000000000..31cd5c76b75081331ae03c5ea70ea7ddebaa06e1 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/download_wmt20.sh @@ -0,0 +1,547 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +if [ -z $WORKDIR_ROOT ] ; +then + echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." + exit +fi + + + +set -x -e + +# TODO update the workdir and dest dir name +# put fasttext model +WORKDIR=$WORKDIR_ROOT +# put intermediate files +TMP_DIR=$WORKDIR_ROOT/tmp/tmp_wmt20_lowres_download +# output {train,valid,test} files to dest +DEST=$WORKDIR_ROOT/ML50/raw + +UTILS=$PWD/utils + +# per dataset locations +COMMONCRAWL_DIR=$TMP_DIR/commoncrawl +YANDEX_CORPUS=$WORKDIR_ROOT/wmt20/official/ru/yandex/1mcorpus.zip +# unzipped +CZENG_CORPUS=$WORKDIR_ROOT/wmt20/official/cs/czeng/czeng20-train +CCMT_DIR=$WORKDIR_ROOT/wmt20/official/zh/ccmt/parallel + +download_and_select() { + SUBFOLDER=$1 + URL=$2 + UNCOMPRESS_CMD=$3 + LANG=$4 + INPUT_FILEPATH=$5 + if [[ $# -gt 5 ]]; then + LANG_COL=$6 + EN_COL=$7 + fi + + mkdir -p $SUBFOLDER + cd $SUBFOLDER + wget -nc --content-disposition $URL + $UNCOMPRESS_CMD + + if [[ $# -gt 5 ]]; then + cut -f$LANG_COL $INPUT_FILEPATH > $INPUT_FILEPATH.$LANG + cut -f$EN_COL $INPUT_FILEPATH > $INPUT_FILEPATH.en + fi + cd .. + + ln -sf $SUBFOLDER/$INPUT_FILEPATH.$LANG $SUBFOLDER.$LANG + ln -sf $SUBFOLDER/$INPUT_FILEPATH.en $SUBFOLDER.en +} + +prepare_lid() { + pip install fasttext + + # TODO specify global workdir + MODEL=$WORKDIR/fasttext/lid.176.bin + LID_MULTI=$UTILS/fasttext_multi_filter.py + + if [ ! -f "$MODEL" ]; then + echo "downloading fasttext lid model..." + mkdir -p $WORKDIR/fasttext + wget -nc https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin -O $MODEL + fi +} + +prepare_moses() { + pushd $UTILS + echo 'Cloning Moses github repository (for tokenization scripts)...' + git clone https://github.com/moses-smt/mosesdecoder.git + popd +} + +lid_filter() { + # TODO specify global workdir + MODEL=$WORKDIR/fasttext/lid.176.bin + LID_MULTI=$UTILS/fasttext_multi_filter.py + + prepare_lid + + SRC=$1 + SRC_FILE=$2 + SRC_OUTPUT=$3 + TGT=$4 + TGT_FILE=$5 + TGT_OUTPUT=$6 + python $LID_MULTI --model $MODEL --inputs $SRC_FILE $TGT_FILE --langs $SRC $TGT --outputs $SRC_OUTPUT $TGT_OUTPUT +} + +prepare_ja_ted() { + mkdir -p ted + cd ted + + wget -nc https://wit3.fbk.eu/archive/2017-01-trnted//texts/en/ja/en-ja.tgz + tar -zxvf en-ja.tgz + cat en-ja/train.tags.en-ja.en | grep -v -P "^[ ]*\<" | sed 's/^[ \t]*//g' | sed 's/[ \t]*$//g' > en-ja/train.en-ja.en + cat en-ja/train.tags.en-ja.ja | grep -v -P "^[ ]*\<" | sed 's/^[ \t]*//g' | sed 's/[ \t]*$//g' > en-ja/train.en-ja.ja + + cd .. + ln -sf ted/en-ja/train.en-ja.ja ted.ja + ln -sf ted/en-ja/train.en-ja.en ted.en +} + +prepare_ja() { + OUTPUT_DIR=$TMP_DIR/ja + mkdir -p $OUTPUT_DIR + cd $OUTPUT_DIR + + download_and_select paracrawl "http://www.kecl.ntt.co.jp/icl/lirg/jparacrawl/release/2.0/bitext/en-ja.tar.gz" "tar -zxvf en-ja.tar.gz" ja en-ja/en-ja.bicleaner05.txt 4 3 & + download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.en-ja.tsv.gz" "gunzip -f news-commentary-v15.en-ja.tsv.gz" ja news-commentary-v15.en-ja.tsv 2 1 & + download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ja-en.tsv.gz" "gunzip -f wikititles-v2.ja-en.tsv.gz" ja wikititles-v2.ja-en.tsv 1 2 & + download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-ja.langid.tsv.gz" "gunzip -f WikiMatrix.v1.en-ja.langid.tsv.gz" ja WikiMatrix.v1.en-ja.langid.tsv 3 2 & + download_and_select subtitle "https://nlp.stanford.edu/projects/jesc/data/split.tar.gz" "tar -zxvf split.tar.gz" ja split/train 2 1 & + download_and_select kftt "http://www.phontron.com/kftt/download/kftt-data-1.0.tar.gz" "tar -zxvf kftt-data-1.0.tar.gz" ja kftt-data-1.0/data/orig/kyoto-train & + + prepare_ja_ted & + + # ted data needs to + + wait + + # remove previous results + rm -f all.?? + find ./ -maxdepth 1 -name "*.ja" | sort -V | xargs cat > all.ja + find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en + lid_filter ja all.ja $DEST/train.ja_XX-en_XX.ja_XX en all.en $DEST/train.ja_XX-en_XX.en_XX +} + +prepare_ta() { + OUTPUT_DIR=$TMP_DIR/ta + mkdir -p $OUTPUT_DIR + cd $OUTPUT_DIR + + download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ta-en.tsv.gz" "gunzip -f wikititles-v2.ta-en.tsv.gz" ta wikititles-v2.ta-en.tsv 1 2 & + download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-ta.langid.tsv.gz" "gunzip -f WikiMatrix.v1.en-ta.langid.tsv.gz" ta WikiMatrix.v1.en-ta.langid.tsv 3 2 & + download_and_select pmindia "http://data.statmt.org/pmindia/v1/parallel/pmindia.v1.ta-en.tsv" "" ta pmindia.v1.ta-en.tsv 2 1 & + download_and_select tanzil "https://object.pouta.csc.fi/OPUS-Tanzil/v1/moses/en-ta.txt.zip" "unzip en-ta.txt.zip" ta Tanzil.en-ta & + download_and_select pib "http://preon.iiit.ac.in/~jerin/resources/datasets/pib-v0.tar" "tar -xvf pib-v0.tar" ta pib/en-ta/train & + download_and_select mkb "http://preon.iiit.ac.in/~jerin/resources/datasets/mkb-v0.tar" "tar -xvf mkb-v0.tar" ta mkb/en-ta/mkb & + download_and_select ufal "http://ufal.mff.cuni.cz/~ramasamy/parallel/data/v2/en-ta-parallel-v2.tar.gz" "tar -zxvf en-ta-parallel-v2.tar.gz" ta en-ta-parallel-v2/corpus.bcn.train & + + wait + + # need special handling for nlpc + mkdir -p nlpc + cd nlpc + wget -nc https://raw.githubusercontent.com/nlpc-uom/English-Tamil-Parallel-Corpus/master/En-Ta%20Corpus/En-Ta%20English.txt + wget -nc https://github.com/nlpc-uom/English-Tamil-Parallel-Corpus/raw/master/En-Ta%20Corpus/En-Ta%20Tamil.txt + tail -n +4 "En-Ta English.txt" > en-ta.en + tail -n +4 "En-Ta Tamil.txt" > en-ta.ta + cd .. + ln -sf nlpc/en-ta.en nlpc.en + ln -sf nlpc/en-ta.ta nlpc.ta + + # remove previous results + rm -f all.?? + find ./ -maxdepth 1 -name "*.ta" | sort -V | xargs cat > all.ta + find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en + lid_filter ta all.ta $DEST/train.ta_IN-en_XX.ta_IN en all.en $DEST/train.ta_IN-en_XX.en_XX +} + +prepare_iu() { + OUTPUT_DIR=$TMP_DIR/iu + mkdir -p $OUTPUT_DIR + cd $OUTPUT_DIR + + download_and_select nh "https://nrc-digital-repository.canada.ca/eng/view/dataset/?id=c7e34fa7-7629-43c2-bd6d-19b32bf64f60" "tar -zxvf Nunavut-Hansard-Inuktitut-English-Parallel-Corpus-3.0.1.tgz" iu Nunavut-Hansard-Inuktitut-English-Parallel-Corpus-3.0/NunavutHansard > /dev/null & + download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.iu-en.tsv.gz" "gunzip -f wikititles-v2.iu-en.tsv.gz" iu wikititles-v2.iu-en.tsv 1 2 & + + wait + + # remove previous results + rm -f all.?? + find ./ -maxdepth 1 -name "*.iu" | sort -V | xargs cat | nh/Nunavut-Hansard-Inuktitut-English-Parallel-Corpus-3.0/scripts/normalize-iu-spelling.pl > all.iu + find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en + paste all.iu all.en | awk -F $'\t' '$1!=""&&$2!=""' > all.iuen + cut -f1 all.iuen > $DEST/train.iu_CA-en_XX.iu_CA + cut -f2 all.iuen > $DEST/train.iu_CA-en_XX.en_XX +} + +prepare_km() { + OUTPUT_DIR=$TMP_DIR/km + mkdir -p $OUTPUT_DIR + cd $OUTPUT_DIR + + download_and_select paracrawl "http://data.statmt.org/wmt20/translation-task/ps-km/wmt20-sent.en-km.xz" "unxz wmt20-sent.en-km.zx" km wmt20-sent.en-km 2 1 & + + # km-parallel has multiple sets, concat all of them together + mkdir -p opus + cd opus + wget -nc "http://data.statmt.org/wmt20/translation-task/ps-km/km-parallel.tgz" + tar -zxvf km-parallel.tgz + find ./km-parallel -maxdepth 1 -name "*.km" | sort -V | xargs cat > opus.km + find ./km-parallel -maxdepth 1 -name "*.en" | sort -V | xargs cat > opus.en + cd .. + ln -sf opus/opus.km . + ln -sf opus/opus.en . + + wait + + # remove previous results + rm -f all.?? + find ./ -maxdepth 1 -name "*.km" | sort -V | xargs cat > all.km + find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en + lid_filter km all.km $DEST/train.km_KH-en_XX.km_KH en all.en $DEST/train.km_KH-en_XX.en_XX +} + +prepare_ps() { + OUTPUT_DIR=$TMP_DIR/ps + mkdir -p $OUTPUT_DIR + cd $OUTPUT_DIR + + download_and_select paracrawl "http://data.statmt.org/wmt20/translation-task/ps-km/wmt20-sent.en-ps.xz" "unxz wmt20-sent.en-ps.xz" ps wmt20-sent.en-ps 2 1 & + download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ps-en.tsv.gz" "gunzip -f wikititles-v2.ps-en.tsv.gz" ps wikititles-v2.ps-en.tsv 1 2 & + # ps-parallel has multiple sets, concat all of them together + mkdir -p opus + cd opus + wget -nc "http://data.statmt.org/wmt20/translation-task/ps-km/ps-parallel.tgz" + tar -zxvf ps-parallel.tgz + find ./ps-parallel -maxdepth 1 -name "*.ps" | sort -V | xargs cat > opus.ps + find ./ps-parallel -maxdepth 1 -name "*.en" | sort -V | xargs cat > opus.en + cd .. + ln -sf opus/opus.ps opus.ps + ln -sf opus/opus.en opus.en + + wait + + # remove previous results + rm -f all.?? + find ./ -maxdepth 1 -name "*.ps" | sort -V | xargs cat > all.ps + find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en + lid_filter ps all.ps $DEST/train.ps_AF-en_XX.ps_AF en all.en $DEST/train.ps_AF-en_XX.en_XX +} + +download_commoncrawl() { + mkdir -p $COMMONCRAWL_DIR + cd $COMMONCRAWL_DIR + + wget -nc "http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz" + tar -zxvf training-parallel-commoncrawl.tgz +} +link_commoncrawl() { + LANG=$1 + ln -sf $COMMONCRAWL_DIR/commoncrawl.$LANG-en.en commoncrawl.en + ln -sf $COMMONCRAWL_DIR/commoncrawl.$LANG-en.$LANG commoncrawl.$LANG +} + +strip_xlf() { + INPUT_FILE=$1 + SRC=$2 + TGT=$3 + grep '<source xml:lang=' $INPUT_FILE | sed 's/^<[^<>]*>//g' | sed 's/<[^<>]*>$//g' > $INPUT_FILE.$SRC + grep '<target xml:lang=' $INPUT_FILE | sed 's/^<[^<>]*>//g' | sed 's/<[^<>]*>$//g' > $INPUT_FILE.$TGT +} + +download_and_process_tilde() { + URL=$1 + UNCOMPRESS_CMD=$2 + FILENAME=$3 + LANG=$4 + PROCESS_CMD=$5 + + mkdir -p tilde + cd tilde + wget -nc $URL + $UNCOMPRESS_CMD + echo "executing cmd" + echo $PROCESS_CMD + $PROCESS_CMD + cd .. + ln -sf tilde/$FILENAME.$LANG tilde.$LANG + ln -sf tilde/$FILENAME.en tilde.en +} + +prepare_cs() { + OUTPUT_DIR=$TMP_DIR/cs + mkdir -p $OUTPUT_DIR + cd $OUTPUT_DIR + + #download_and_select europarl "http://www.statmt.org/europarl/v10/training/europarl-v10.cs-en.tsv.gz" "gunzip europarl-v10.cs-en.tsv.gz" cs europarl-v10.cs-en.tsv 1 2 & + #download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release5.1/en-cs.txt.gz" "gunzip en-cs.txt.gz" cs en-cs.txt 2 1 & + #link_commoncrawl cs + #download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.cs-en.tsv.gz" "gunzip news-commentary-v15.cs-en.tsv.gz" cs news-commentary-v15.cs-en.tsv 1 2 & + #download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.cs-en.tsv.gz" "gunzip wikititles-v2.cs-en.tsv.gz" cs wikititles-v2.cs-en.tsv 1 2 & + #download_and_process_tilde "http://data.statmt.org/wmt20/translation-task/rapid/RAPID_2019.cs-en.xlf.gz" "gunzip RAPID_2019.cs-en.xlf.gz" RAPID_2019.cs-en.xlf cs "strip_xlf RAPID_2019.cs-en.xlf cs en" & + #download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.cs-en.langid.tsv.gz" "gunzip WikiMatrix.v1.cs-en.langid.tsv.gz" cs WikiMatrix.v1.cs-en.langid.tsv 2 3 & + + #wait + + # remove previous results + #rm -f all.?? + #find ./ -maxdepth 1 -name "*.cs" | sort -V | xargs cat > all.cs + #find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en + if [ -z $CZENG_CORPUS ] ; + then + echo "Please download CZENG_CORPUS manually and place them at $CZENG_CORPUS. Exitting..." + exit + fi + cat $CZENG_CORPUS | sed '/^$/d' | cut -f5 > all.cs + cat $CZENG_CORPUS | sed '/^$/d' | cut -f6 > all.en + + lid_filter cs all.cs $DEST/train.cs_CZ-en_XX.cs_CZ en all.en $DEST/train.cs_CZ-en_XX.en_XX +} + +prepare_de() { + OUTPUT_DIR=$TMP_DIR/de + mkdir -p $OUTPUT_DIR + cd $OUTPUT_DIR + + download_and_select europarl "http://www.statmt.org/europarl/v10/training/europarl-v10.de-en.tsv.gz" "gunzip europarl-v10.de-en.tsv.gz" de europarl-v10.de-en.tsv 1 2 & + download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release5.1/en-de.txt.gz" "gunzip en-de.txt.gz" de en-de.txt 2 1 & + link_commoncrawl de + download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.de-en.tsv.gz" "gunzip news-commentary-v15.de-en.tsv.gz" de news-commentary-v15.de-en.tsv 1 2 & + download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.de-en.tsv.gz" "gunzip wikititles-v2.de-en.tsv.gz" de wikititles-v2.de-en.tsv 1 2 & + download_and_process_tilde "http://data.statmt.org/wmt20/translation-task/rapid/RAPID_2019.de-en.xlf.gz" "gunzip RAPID_2019.de-en.xlf.gz" RAPID_2019.de-en.xlf de "strip_xlf RAPID_2019.de-en.xlf de en" & + download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.de-en.langid.tsv.gz" "gunzip WikiMatrix.v1.de-en.langid.tsv.gz" de WikiMatrix.v1.de-en.langid.tsv 2 3 & + + wait + + # remove previous results + rm -f all.?? + find ./ -maxdepth 1 -name "*.de" | sort -V | xargs cat > all.de + find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en + lid_filter de all.de $DEST/train.de_DE-en_XX.de_DE en all.en $DEST/train.de_DE-en_XX.en_XX +} + +prepare_tmx() { + TMX_FILE=$1 + git clone https://github.com/amake/TMX2Corpus $UTILS/tmx2corpus + pip install tinysegmenter + + python $UTILS/tmx2corpus/tmx2corpus.py $TMX_FILE +} + +prepare_pl() { + OUTPUT_DIR=$TMP_DIR/pl + mkdir -p $OUTPUT_DIR + cd $OUTPUT_DIR + + # download_and_select europarl "http://www.statmt.org/europarl/v10/training/europarl-v10.pl-en.tsv.gz" "gunzip europarl-v10.pl-en.tsv.gz" pl europarl-v10.pl-en.tsv 1 2 & + # download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release5.1/en-pl.txt.gz" "gunzip en-pl.txt.gz" pl en-pl.txt 2 1 & + # download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.pl-en.tsv.gz" "gunzip wikititles-v2.pl-en.tsv.gz" pl wikititles-v2.pl-en.tsv 1 2 & + download_and_select tilde "https://tilde-model.s3-eu-west-1.amazonaws.com/rapid2019.en-pl.tmx.zip" "gunzip rapid2019.en-pl.tmx.zip" bitext pl "prepare_tmx RAPID_2019.UNIQUE.en-pl.tmx" & + # download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-pl.langid.tsv.gz" "gunzip WikiMatrix.v1.en-pl.langid.tsv.gz" pl WikiMatrix.v1.en-pl.langid.tsv 3 2 & + + wait + + # remove previous results + rm -f all.?? + find ./ -maxdepth 1 -name "*.pl" | sort -V | xargs cat > all.pl + find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en + lid_filter pl all.pl $DEST/train.pl_PL-en_XX.pl_PL en all.en $DEST/train.pl_PL-en_XX.en_XX +} + +prepare_uncorpus() { + $URLS=$1 + $FILES=$2 + + mkdir -p uncorpus + cd uncorpus + + for URL in $URLS; do + wget -nc $URL + done + cat $FILES > uncorpus.tar.gz + tar -zxvf uncorpus.tar.gz + + cd .. + ln -sf uncorpus/en-$LANG/UNv1.0.en-$LANG.$LANG uncorpus.$LANG + ln -sf uncorpus/en-$LANG/UNv1.0.en-$LANG.en uncorpus.en +} + +prepare_yandex() { + mkdir -p yandex + cd yandex + unzip $YANDEX_CORPUS ./ + cd .. + ln -s yandex/corpus.en_ru.1m.en yandex.en + ln -s yandex/corpus.en_ru.1m.ru yandex.ru +} + +prepare_ru() { + OUTPUT_DIR=$TMP_DIR/ru + mkdir -p $OUTPUT_DIR + cd $OUTPUT_DIR + + download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release1/paracrawl-release1.en-ru.zipporah0-dedup-clean.tgz" "tar -zxvf paracrawl-release1.en-ru.zipporah0-dedup-clean.tgz" ru paracrawl-release1.en-ru.zipporah0-dedup-clean & + link_commoncrawl ru + download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.en-ru.tsv.gz" "gunzip news-commentary-v15.en-ru.tsv.gz" ru news-commentary-v15.en-ru.tsv 2 1 & + prepare_yandex & + download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ru-en.tsv.gz" "gunzip wikititles-v2.ru-en.tsv.gz" ru wikititles-v2.ru-en.tsv 1 2 & + prepare_uncorpus "https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.00 https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.01 https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.02" "UNv1.0.en-ru.tar.gz.00 UNv1.0.en-ru.tar.gz.01 UNv1.0.en-ru.tar.gz.02" & + download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-ru.langid.tsv.gz" "gunzip WikiMatrix.v1.en-ru.langid.tsv.gz" ru WikiMatrix.v1.en-ru.langid.tsv 3 2 & + + wait + + # remove previous results + rm -f all.?? + find ./ -maxdepth 1 -name "*.ru" | sort -V | xargs cat > all.ru + find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en + lid_filter ru all.ru $DEST/train.ru_RU-en_XX.ru_RU en all.en $DEST/train.ru_RU-en_XX.en_XX +} + +prepare_ccmt() { + mkdir -p ccmt + cd ccmt + # assume ccmt data is already unzipped under CCMT_DIR folder + cat $CCMT_DIR/datum2017/Book*_cn.txt | sed 's/ //g' > datum2017.detok.zh + cat $CCMT_DIR/datum2017/Book*_en.txt > datum2017.detok.en + cat $CCMT_DIR/casict2011/casict-A_ch.txt $CCMT_DIR/casict2011/casict-B_ch.txt $CCMT_DIR/casict2015/casict2015_ch.txt $CCMT_DIR/datum2015/datum_ch.txt $CCMT_DIR/neu2017/NEU_cn.txt datum2017.detok.zh > ccmt.zh + cat $CCMT_DIR/casict2011/casict-A_en.txt $CCMT_DIR/casict2011/casict-B_en.txt $CCMT_DIR/casict2015/casict2015_en.txt $CCMT_DIR/datum2015/datum_en.txt $CCMT_DIR/neu2017/NEU_en.txt datum2017.detok.en > ccmt.en + cd .. + ln -sf ccmt/ccmt.zh ccmt.zh + ln -sf ccmt/ccmt.en ccmt.en +} + +prepare_zh() { + OUTPUT_DIR=$TMP_DIR/zh + mkdir -p $OUTPUT_DIR + cd $OUTPUT_DIR + + download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.en-zh.tsv.gz" "gunzip news-commentary-v15.en-zh.tsv.gz" zh news-commentary-v15.en-zh.tsv 2 1 & + download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.zh-en.tsv.gz" "gunzip wikititles-v2.zh-en.tsv.gz" zh wikititles-v2.zh-en.tsv 1 2 & + prepare_uncorpus "https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-zh.tar.gz.00 https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-zh.tar.gz.01" "UNv1.0.en-zh.tar.gz.00 UNv1.0.en-zh.tar.gz.01" & + prepare_ccmt & + download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-zh.langid.tsv.gz" "gunzip WikiMatrix.v1.en-zh.langid.tsv.gz" zh WikiMatrix.v1.en-zh.langid.tsv 3 2 & + + wait + + # remove previous results + rm -f all.?? + find ./ -maxdepth 1 -name "*.zh" | sort -V | xargs cat > all.zh + find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en + lid_filter zh all.zh $DEST/train.zh_CN-en_XX.zh_CN en all.en $DEST/train.zh_CN-en_XX.en_XX +} + +prepare_tests() { + OUTPUT_DIR=$TMP_DIR + mkdir -p $OUTPUT_DIR + cd $OUTPUT_DIR + wget -nc http://data.statmt.org/wmt20/translation-task/dev.tgz + tar -zxvf dev.tgz + cd dev + + cat newsdev2020-jaen-src.ja.sgm | $UTILS/strip_sgm.sh > newsdev2020-jaen.ja + cat newsdev2020-jaen-ref.en.sgm | $UTILS/strip_sgm.sh > newsdev2020-jaen.en + split newsdev2020-jaen.ja -a 0 -n r/1/2 > $DEST/valid.ja_XX-en_XX.ja_XX + split newsdev2020-jaen.en -a 0 -n r/1/2 > $DEST/valid.ja_XX-en_XX.en_XX + split newsdev2020-jaen.ja -a 0 -n r/2/2 > $DEST/test.ja_XX-en_XX.ja_XX + split newsdev2020-jaen.en -a 0 -n r/2/2 > $DEST/test.ja_XX-en_XX.en_XX + + cat newsdev2020-iuen-src.iu.sgm | strip_sgm.sh > newsdev2020-iuen.iu + cat newsdev2020-iuen-ref.en.sgm | strip_sgm.sh > newsdev2020-iuen.en + split newsdev2020-iuen.iu -a 0 -n r/1/2 > $DEST/valid.iu_CA-en_XX.iu_CA + split newsdev2020-iuen.en -a 0 -n r/1/2 > $DEST/valid.iu_CA-en_XX.en_XX + split newsdev2020-iuen.iu -a 0 -n r/2/2 > $DEST/test.iu_CA-en_XX.iu_CA + split newsdev2020-iuen.en -a 0 -n r/2/2 > $DEST/test.iu_CA-en_XX.en_XX + + cat newsdev2020-taen-src.ta.sgm | strip_sgm.sh > newsdev2020-taen.ta + cat newsdev2020-taen-ref.en.sgm | strip_sgm.sh > newsdev2020-taen.en + split newsdev2020-taen.ta -a 0 -n r/1/2 > $DEST/valid.ta_IN-en_XX.ta_IN + split newsdev2020-taen.en -a 0 -n r/1/2 > $DEST/valid.ta_IN-en_XX.en_XX + split newsdev2020-taen.ta -a 0 -n r/2/2 > $DEST/test.ta_IN-en_XX.ta_IN + split newsdev2020-taen.en -a 0 -n r/2/2 > $DEST/test.ta_IN-en_XX.en_XX + + cp wikipedia.dev.km-en.km $DEST/valid.km_KH-en_XX.km_KH + cp wikipedia.dev.km-en.en $DEST/valid.km_KH-en_XX.en_XX + cp wikipedia.devtest.km-en.km $DEST/test.km_KH-en_XX.km_KH + cp wikipedia.devtest.km-en.en $DEST/test.km_KH-en_XX.en_XX + + cp wikipedia.dev.ps-en.ps $DEST/valid.ps_AF-en_XX.ps_AF + cp wikipedia.dev.ps-en.en $DEST/valid.ps_AF-en_XX.en_XX + cp wikipedia.devtest.ps-en.ps $DEST/test.ps_AF-en_XX.ps_AF + cp wikipedia.devtest.ps-en.en $DEST/test.ps_AF-en_XX.en_XX + + cat newsdev2020-plen-src.pl.sgm | strip_sgm.sh > newsdev2020-plen.pl + cat newsdev2020-plen-ref.en.sgm | strip_sgm.sh > newsdev2020-plen.en + split newsdev2020-plen.pl -a 0 -n r/1/2 > $DEST/valid.pl_PL-en_XX.pl_PL + split newsdev2020-plen.en -a 0 -n r/1/2 > $DEST/valid.pl_PL-en_XX.en_XX + split newsdev2020-plen.pl -a 0 -n r/2/2 > $DEST/test.pl_PL-en_XX.pl_PL + split newsdev2020-plen.en -a 0 -n r/2/2 > $DEST/test.pl_PL-en_XX.en_XX + + cat newstest2018-encs-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-cs_CZ.en_XX + cat newstest2018-encs-ref.cs.sgm | strip_sgm.sh > $DEST/valid.en_XX-cs_CZ.cs_CZ + cat newstest2019-encs-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-cs_CZ.en_XX + cat newstest2019-encs-ref.cs.sgm | strip_sgm.sh > $DEST/test.en_XX-cs_CZ.cs_CZ + + cat newstest2018-deen-src.de.sgm | strip_sgm.sh > $DEST/valid.de_DE-en_XX.de_DE + cat newstest2018-deen-ref.en.sgm | strip_sgm.sh > $DEST/valid.de_DE-en_XX.en_XX + cat newstest2018-ende-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-de_DE.en_XX + cat newstest2018-ende-ref.de.sgm | strip_sgm.sh > $DEST/valid.en_XX-de_DE.de_DE + cat newstest2019-deen-src.de.sgm | strip_sgm.sh > $DEST/test.de_DE-en_XX.de_DE + cat newstest2019-deen-ref.en.sgm | strip_sgm.sh > $DEST/test.de_DE-en_XX.en_XX + cat newstest2019-ende-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-de_DE.en_XX + cat newstest2019-ende-ref.de.sgm | strip_sgm.sh > $DEST/test.en_XX-de_DE.de_DE + + cat newstest2018-ruen-src.ru.sgm | strip_sgm.sh > $DEST/valid.ru_RU-en_XX.ru_RU + cat newstest2018-ruen-ref.en.sgm | strip_sgm.sh > $DEST/valid.ru_RU-en_XX.en_XX + cat newstest2018-enru-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-ru_RU.en_XX + cat newstest2018-enru-ref.ru.sgm | strip_sgm.sh > $DEST/valid.en_XX-ru_RU.ru_RU + cat newstest2019-ruen-src.ru.sgm | strip_sgm.sh > $DEST/test.ru_RU-en_XX.ru_RU + cat newstest2019-ruen-ref.en.sgm | strip_sgm.sh > $DEST/test.ru_RU-en_XX.en_XX + cat newstest2019-enru-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-ru_RU.en_XX + cat newstest2019-enru-ref.ru.sgm | strip_sgm.sh > $DEST/test.en_XX-ru_RU.ru_RU + + cat newstest2018-zhen-src.zh.sgm | strip_sgm.sh > $DEST/valid.zh_CN-en_XX.zh_CN + cat newstest2018-zhen-ref.en.sgm | strip_sgm.sh > $DEST/valid.zh_CN-en_XX.en_XX + cat newstest2018-enzh-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-zh_CN.en_XX + cat newstest2018-enzh-ref.zh.sgm | strip_sgm.sh > $DEST/valid.en_XX-zh_CN.zh_CN + cat newstest2019-zhen-src.zh.sgm | strip_sgm.sh > $DEST/test.zh_CN-en_XX.zh_CN + cat newstest2019-zhen-ref.en.sgm | strip_sgm.sh > $DEST/test.zh_CN-en_XX.en_XX + cat newstest2019-enzh-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-zh_CN.en_XX + cat newstest2019-enzh-ref.zh.sgm | strip_sgm.sh > $DEST/test.en_XX-zh_CN.zh_CN +} + +mkdir -p $DEST + +prepare_lid +prepare_moses +download_commoncrawl + +prepare_ja & +prepare_ta & +prepare_km & +prepare_ps & +prepare_iu & +prepare_cs & +prepare_de & +prepare_pl & +prepare_ru & +prepare_zh & + +# prepare valid/test set +prepare_tests & + +# wait + +# TODO remove intermediate files +# rm -rf $TMP_DIR diff --git a/data/fairseq/examples/multilingual/data_scripts/preprocess_ML50_v1.sh b/data/fairseq/examples/multilingual/data_scripts/preprocess_ML50_v1.sh new file mode 100644 index 0000000000000000000000000000000000000000..4655936149cab212b3cfa14f306d71153729f9d7 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/preprocess_ML50_v1.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +if [ -z $WORKDIR_ROOT ] ; +then + echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..." + exit +fi + +if [ -z $SPM_PATH ] ; +then + echo "Please install sentence piecence from https://github.com/google/sentencepiece and set SPM_PATH pointing to the installed spm_encode.py. Exitting..." + exit +fi + +ML50=${WORKDIR_ROOT}/ML50 + +mkdir -p $ML50/dedup +mkdir -p $ML50/cleaned_dedup + +python ./dedup_all.py --from-folder $ML50/raw --to-folder $ML50/dedup +python ./remove_valid_test_in_train.py --from-folder $ML50/dedup --to-folder $ML50/clean +python ./binarize.py --raw-folder $ML50/clean \ No newline at end of file diff --git a/data/fairseq/examples/multilingual/data_scripts/remove_valid_test_in_train.py b/data/fairseq/examples/multilingual/data_scripts/remove_valid_test_in_train.py new file mode 100644 index 0000000000000000000000000000000000000000..ef618adef7c7d010f8de38fb5ebeb5a35d2d3cac --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/remove_valid_test_in_train.py @@ -0,0 +1,290 @@ +import os, sys +import glob, itertools +import pandas as pd + +WORKDIR_ROOT = os.environ.get('WORKDIR_ROOT', None) + +if WORKDIR_ROOT is None or not WORKDIR_ROOT.strip(): + print('please specify your working directory root in OS environment variable WORKDIR_ROOT. Exitting..."') + sys.exit(-1) + + +def load_langs(path): + with open(path) as fr: + langs = [l.strip() for l in fr] + return langs + + + +def load_sentences(raw_data, split, direction): + src, tgt = direction.split('-') + src_path = f"{raw_data}/{split}.{direction}.{src}" + tgt_path = f"{raw_data}/{split}.{direction}.{tgt}" + if os.path.exists(src_path) and os.path.exists(tgt_path): + return [(src, open(src_path).read().splitlines()), (tgt, open(tgt_path).read().splitlines())] + else: + return [] + +def swap_direction(d): + src, tgt = d.split('-') + return f'{tgt}-{src}' + +def get_all_test_data(raw_data, directions, split='test'): + test_data = [ + x + for dd in directions + for d in [dd, swap_direction(dd)] + for x in load_sentences(raw_data, split, d) + ] + # all_test_data = {s for _, d in test_data for s in d} + all_test_data = {} + for lang, d in test_data: + for s in d: + s = s.strip() + lgs = all_test_data.get(s, set()) + lgs.add(lang) + all_test_data[s] = lgs + return all_test_data, test_data + +def check_train_sentences(raw_data, direction, all_test_data, mess_up_train={}): + src, tgt = direction.split('-') + tgt_path = f"{raw_data}/train.{direction}.{tgt}" + src_path = f"{raw_data}/train.{direction}.{src}" + print(f'check training data in {raw_data}/train.{direction}') + size = 0 + if not os.path.exists(tgt_path) or not os.path.exists(src_path): + return mess_up_train, size + with open(src_path) as f, open(tgt_path) as g: + for src_line, tgt_line in zip(f, g): + s = src_line.strip() + t = tgt_line.strip() + size += 1 + if s in all_test_data: + langs = mess_up_train.get(s, set()) + langs.add(direction) + mess_up_train[s] = langs + if t in all_test_data: + langs = mess_up_train.get(t, set()) + langs.add(direction) + mess_up_train[t] = langs + return mess_up_train, size + +def check_train_all(raw_data, directions, all_test_data): + mess_up_train = {} + data_sizes = {} + for direction in directions: + _, size = check_train_sentences(raw_data, direction, all_test_data, mess_up_train) + data_sizes[direction] = size + return mess_up_train, data_sizes + +def count_train_in_other_set(mess_up_train): + train_in_others = [(direction, s) for s, directions in mess_up_train.items() for direction in directions] + counts = {} + for direction, s in train_in_others: + counts[direction] = counts.get(direction, 0) + 1 + return counts + +def train_size_if_remove_in_otherset(data_sizes, mess_up_train): + counts_in_other = count_train_in_other_set(mess_up_train) + remain_sizes = [] + for direction, count in counts_in_other.items(): + remain_sizes.append((direction, data_sizes[direction] - count, data_sizes[direction], count, 100 * count / data_sizes[direction] )) + return remain_sizes + + +def remove_messed_up_sentences(raw_data, direction, mess_up_train, mess_up_train_pairs, corrected_langs): + split = 'train' + src_lang, tgt_lang = direction.split('-') + + tgt = f"{raw_data}/{split}.{direction}.{tgt_lang}" + src = f"{raw_data}/{split}.{direction}.{src_lang}" + print(f'working on {direction}: ', src, tgt) + if not os.path.exists(tgt) or not os.path.exists(src) : + return + + corrected_tgt = f"{to_folder}/{split}.{direction}.{tgt_lang}" + corrected_src = f"{to_folder}/{split}.{direction}.{src_lang}" + line_num = 0 + keep_num = 0 + with open(src, encoding='utf8',) as fsrc, \ + open(tgt, encoding='utf8',) as ftgt, \ + open(corrected_src, 'w', encoding='utf8') as fsrc_corrected, \ + open(corrected_tgt, 'w', encoding='utf8') as ftgt_corrected: + for s, t in zip(fsrc, ftgt): + s = s.strip() + t = t.strip() + if t not in mess_up_train \ + and s not in mess_up_train \ + and (s, t) not in mess_up_train_pairs \ + and (t, s) not in mess_up_train_pairs: + corrected_langs.add(direction) + print(s, file=fsrc_corrected) + print(t, file=ftgt_corrected) + keep_num += 1 + line_num += 1 + if line_num % 1000 == 0: + print(f'completed {line_num} lines', end='\r') + return line_num, keep_num + +########## + + +def merge_valid_test_messup(mess_up_train_valid, mess_up_train_test): + merged_mess = [] + for s in set(list(mess_up_train_valid.keys()) + list(mess_up_train_test.keys())): + if not s: + continue + valid = mess_up_train_valid.get(s, set()) + test = mess_up_train_test.get(s, set()) + merged_mess.append((s, valid | test)) + return dict(merged_mess) + + + +######### +def check_train_pairs(raw_data, direction, all_test_data, mess_up_train={}): + src, tgt = direction.split('-') + #a hack; TODO: check the reversed directions + path1 = f"{raw_data}/train.{src}-{tgt}.{src}" + path2 = f"{raw_data}/train.{src}-{tgt}.{tgt}" + if not os.path.exists(path1) or not os.path.exists(path2) : + return + + with open(path1) as f1, open(path2) as f2: + for src_line, tgt_line in zip(f1, f2): + s = src_line.strip() + t = tgt_line.strip() + if (s, t) in all_test_data or (t, s) in all_test_data: + langs = mess_up_train.get( (s, t), set()) + langs.add(src) + langs.add(tgt) + mess_up_train[(s, t)] = langs + + +def load_pairs(raw_data, split, direction): + src, tgt = direction.split('-') + src_f = f"{raw_data}/{split}.{direction}.{src}" + tgt_f = f"{raw_data}/{split}.{direction}.{tgt}" + if tgt != 'en_XX': + src_f, tgt_f = tgt_f, src_f + if os.path.exists(src_f) and os.path.exists(tgt_f): + return list(zip(open(src_f).read().splitlines(), + open(tgt_f).read().splitlines(), + )) + else: + return [] + +# skip_langs = ['cs_CZ', 'en_XX', 'tl_XX', 'tr_TR'] +def get_messed_up_test_pairs(split, directions): + test_pairs = [ + (d, load_pairs(raw_data, split, d)) + for d in directions + ] + # all_test_data = {s for _, d in test_data for s in d} + all_test_pairs = {} + for direction, d in test_pairs: + src, tgt = direction.split('-') + for s in d: + langs = all_test_pairs.get(s, set()) + langs.add(src) + langs.add(tgt) + all_test_pairs[s] = langs + mess_up_train_pairs = {} + for direction in directions: + check_train_pairs(raw_data, direction, all_test_pairs, mess_up_train_pairs) + return all_test_pairs, mess_up_train_pairs + + + +if __name__ == "__main__": + ####### + import argparse + parser = argparse.ArgumentParser() + parser.add_argument( + '--from-folder', + required=True, + type=str) + parser.add_argument( + '--to-folder', + required=True, + type=str) + parser.add_argument( + '--directions', + default=None, + type=str) + + + args = parser.parse_args() + raw_data = args.from_folder + to_folder = args.to_folder + os.makedirs(to_folder, exist_ok=True) + + if args.directions: + directions = args.directions.split(',') + else: + raw_files = itertools.chain( + glob.glob(f'{raw_data}/train*'), + glob.glob(f'{raw_data}/valid*'), + glob.glob(f'{raw_data}/test*'), + ) + directions = [os.path.split(file_path)[-1].split('.')[1] for file_path in raw_files] + print('working on directions: ', directions) + + ########## + + + + all_test_data, test_data = get_all_test_data(raw_data, directions, 'test') + print('==loaded test data==') + all_valid_data, valid_data = get_all_test_data(raw_data, directions, 'valid') + print('==loaded valid data==') + all_valid_test_data = merge_valid_test_messup(all_test_data, all_valid_data) + mess_up_train, data_sizes = check_train_all(raw_data, directions, all_valid_test_data) + print('training messing up with valid, test data:', len(mess_up_train)) + data_situation = train_size_if_remove_in_otherset(data_sizes, mess_up_train) + df = pd.DataFrame(data_situation, columns=['direction', 'train_size_after_remove', 'orig_size', 'num_to_remove', 'remove_percent']) + df.sort_values('remove_percent', ascending=False) + df.to_csv(f'{raw_data}/clean_summary.tsv', sep='\t') + print(f'projected data clean summary in: {raw_data}/clean_summary.tsv') + + # correct the dataset: + all_test_pairs, mess_up_test_train_pairs = get_messed_up_test_pairs('test', directions) + all_valid_pairs, mess_up_valid_train_pairs = get_messed_up_test_pairs('valid', directions) + + all_messed_pairs = set(mess_up_test_train_pairs.keys()).union(set(mess_up_valid_train_pairs.keys())) + corrected_directions = set() + + real_data_situation = [] + for direction in directions: + org_size, new_size = remove_messed_up_sentences(raw_data, direction, mess_up_train, all_messed_pairs, corrected_directions) + if org_size == 0: + print(f"{direction} has size 0") + continue + real_data_situation.append( + (direction, new_size, org_size, org_size - new_size, (org_size - new_size) / org_size * 100) + ) + print('corrected directions: ', corrected_directions) + df = pd.DataFrame(real_data_situation, columns=['direction', 'train_size_after_remove', 'orig_size', 'num_to_remove', 'remove_percent']) + df.sort_values('remove_percent', ascending=False) + df.to_csv(f'{raw_data}/actual_clean_summary.tsv', sep='\t') + print(f'actual data clean summary (which can be different from the projected one because of duplications) in: {raw_data}/actual_clean_summary.tsv') + + import shutil + for direction in directions: + src_lang, tgt_lang = direction.split('-') + for split in ['train', 'valid', 'test']: + # copying valid, test and uncorrected train + if direction in corrected_directions and split == 'train': + continue + tgt = f"{raw_data}/{split}.{direction}.{tgt_lang}" + src = f"{raw_data}/{split}.{direction}.{src_lang}" + if not (os.path.exists(src) and os.path.exists(tgt)): + continue + corrected_tgt = f"{to_folder}/{split}.{direction}.{tgt_lang}" + corrected_src = f"{to_folder}/{split}.{direction}.{src_lang}" + print(f'copying {src} to {corrected_src}') + shutil.copyfile(src, corrected_src) + print(f'copying {tgt} to {corrected_tgt}') + shutil.copyfile(tgt, corrected_tgt) + + print('completed') \ No newline at end of file diff --git a/data/fairseq/examples/multilingual/data_scripts/requirement.txt b/data/fairseq/examples/multilingual/data_scripts/requirement.txt new file mode 100644 index 0000000000000000000000000000000000000000..e85d7d540e08a1407f92dfb2311972a1a5a30123 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/requirement.txt @@ -0,0 +1,2 @@ +wget +pandas \ No newline at end of file diff --git a/data/fairseq/examples/multilingual/data_scripts/utils/dedup.py b/data/fairseq/examples/multilingual/data_scripts/utils/dedup.py new file mode 100644 index 0000000000000000000000000000000000000000..d6fed8c695cf218d3502d6ed8d23015520c0e179 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/utils/dedup.py @@ -0,0 +1,41 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import argparse + +def deup(src_file, tgt_file, src_file_out, tgt_file_out): + seen = set() + dup_count = 0 + with open(src_file, encoding='utf-8') as fsrc, \ + open(tgt_file, encoding='utf-8') as ftgt, \ + open(src_file_out, 'w', encoding='utf-8') as fsrc_out, \ + open(tgt_file_out, 'w', encoding='utf-8') as ftgt_out: + for s, t in zip(fsrc, ftgt): + if (s, t) not in seen: + fsrc_out.write(s) + ftgt_out.write(t) + seen.add((s, t)) + else: + dup_count += 1 + print(f'number of duplication: {dup_count}') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--src-file", type=str, required=True, + help="src file") + parser.add_argument("--tgt-file", type=str, required=True, + help="tgt file") + parser.add_argument("--src-file-out", type=str, required=True, + help="src ouptut file") + parser.add_argument("--tgt-file-out", type=str, required=True, + help="tgt ouput file") + args = parser.parse_args() + deup(args.src_file, args.tgt_file, args.src_file_out, args.tgt_file_out) + + +if __name__ == "__main__": + main() diff --git a/data/fairseq/examples/multilingual/data_scripts/utils/fasttext_multi_filter.py b/data/fairseq/examples/multilingual/data_scripts/utils/fasttext_multi_filter.py new file mode 100644 index 0000000000000000000000000000000000000000..41b38ba5bef20cb043921ac61820db8689189a5a --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/utils/fasttext_multi_filter.py @@ -0,0 +1,63 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +#!/bin/python + +import fasttext +from multiprocessing import Pool +import contextlib +import sys +import argparse +from functools import partial +import io + +model = None +def init(model_path): + global model + model = fasttext.load_model(model_path) + +def pred(lines): + return lines, [model.predict(line.strip())[0][0][9:] for line in lines] + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=str, required=True, + help="model to load") + parser.add_argument("--inputs", nargs="+", default=['-'], + help="input files to filter") + parser.add_argument("--langs", nargs="+", required=True, + help="lang ids of each input file") + parser.add_argument("--outputs", nargs="+", default=['-'], + help="path to save lid filtered outputs") + parser.add_argument("--num-workers", type=int, metavar="N", default=10, + help="number of processes in parallel") + args = parser.parse_args() + + assert len(args.inputs) == len(args.langs) and len(args.inputs) == len(args.outputs) + + with contextlib.ExitStack() as stack: + inputs = [ + stack.enter_context(open(input, "r", encoding="utf-8", newline="\n", errors="replace")) + if input != "-" else io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8', errors="replace") + for input in args.inputs + ] + outputs = [ + stack.enter_context(open(output, "w", encoding="utf-8", newline="\n")) + if output != "-" else sys.stdout + for output in args.outputs + ] + with Pool(args.num_workers, initializer=partial(init, args.model)) as p: + skip_cnt = 0 + for lines, preds in p.imap(pred, list(zip(*inputs)), chunksize=500): + if not all(a == b for a, b in zip(preds, args.langs)): + skip_cnt += 1 + continue + for line, output_h in zip(lines, outputs): + print(line.strip(), file=output_h) + print(f"Skipped {skip_cnt} lines.") + +if __name__ == "__main__": + main() diff --git a/data/fairseq/examples/multilingual/data_scripts/utils/strip_sgm.sh b/data/fairseq/examples/multilingual/data_scripts/utils/strip_sgm.sh new file mode 100644 index 0000000000000000000000000000000000000000..7f4f61d7b1a46f51a1221de6b336cb70b5a0b8b3 --- /dev/null +++ b/data/fairseq/examples/multilingual/data_scripts/utils/strip_sgm.sh @@ -0,0 +1 @@ +grep "seg id" | sed 's/<seg id="[0-9]\+">//g' | sed 's/<\/seg>//g' diff --git a/data/fairseq/examples/multilingual/finetune_multilingual_model.sh b/data/fairseq/examples/multilingual/finetune_multilingual_model.sh new file mode 100644 index 0000000000000000000000000000000000000000..25960c5dc8a02e5580b61837099770a082b4dd83 --- /dev/null +++ b/data/fairseq/examples/multilingual/finetune_multilingual_model.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +path_2_data=$1 # <path to data> which contains binarized data for each directions +lang_list=$2 # <path to a file which contains a list of languages separted by new lines> +lang_pairs=$3 #a list language pairs to train multilingual models, e.g. "en-fr,en-cs,fr-en,cs-en" +# pretrained can be an mBART pretrained model as well +pretrained_model=$4 #<path to a pretrained model> + + +fairseq-train "$path_2_data" \ + --encoder-normalize-before --decoder-normalize-before \ + --arch transformer --layernorm-embedding \ + --task translation_multi_simple_epoch \ + --finetune-from-model "$pretrained_model" \ + --sampling-method "temperature" \ + --sampling-temperature "1.5" \ + --encoder-langtok "src" \ + --decoder-langtok \ + --lang-dict "$lang_list" \ + --lang-pairs "$lang_pairs" \ + --criterion label_smoothed_cross_entropy --label-smoothing 0.2 \ + --optimizer adam --adam-eps 1e-06 --adam-betas '(0.9, 0.98)' \ + --lr-scheduler inverse_sqrt --lr 3e-05 --warmup-updates 2500 --max-update 40000 \ + --dropout 0.3 --attention-dropout 0.1 --weight-decay 0.0 \ + --max-tokens 1024 --update-freq 2 \ + --save-interval 1 --save-interval-updates 5000 --keep-interval-updates 10 --no-epoch-checkpoints \ + --seed 222 --log-format simple --log-interval 2 diff --git a/data/fairseq/examples/multilingual/multilingual_fairseq_gen.sh b/data/fairseq/examples/multilingual/multilingual_fairseq_gen.sh new file mode 100644 index 0000000000000000000000000000000000000000..65aa322d7daaa428015de98abe4664a6a4164bfd --- /dev/null +++ b/data/fairseq/examples/multilingual/multilingual_fairseq_gen.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +lang_pairs="en-fr,en-cs,fr-en,cs-en" +path_2_data=$1 # <path to data> +lang_list=$2 # <path to a file which contains list of languages separted by new lines> +model=$3 # <path to a trained model> +source_lang=cs +target_lang=en + +fairseq-generate "$path_2_data" \ + --path "$model" \ + --task translation_multi_simple_epoch \ + --gen-subset test \ + --source-lang "$source_lang" \ + --target-lang "$target_lang" \ + --sacrebleu --remove-bpe 'sentencepiece'\ + --batch-size 32 \ + --encoder-langtok "src" \ + --decoder-langtok \ + --lang-dict "$lang_list" \ + --lang-pairs "$lang_pairs" diff --git a/data/fairseq/examples/multilingual/train_multilingual_model.sh b/data/fairseq/examples/multilingual/train_multilingual_model.sh new file mode 100644 index 0000000000000000000000000000000000000000..cc050bd3f02de8a2f303737f187442d2eb80e4ef --- /dev/null +++ b/data/fairseq/examples/multilingual/train_multilingual_model.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +path_2_data=$1 # <path to data> which contains binarized data for each directions +lang_list=$2 # <path to a file which contains a list of languages separted by new lines> +lang_pairs=$3 #a list language pairs to train multilingual models, e.g. "en-fr,en-cs,fr-en,cs-en" + +fairseq-train "$path_2_data" \ + --encoder-normalize-before --decoder-normalize-before \ + --arch transformer --layernorm-embedding \ + --task translation_multi_simple_epoch \ + --sampling-method "temperature" \ + --sampling-temperature 1.5 \ + --encoder-langtok "src" \ + --decoder-langtok \ + --lang-dict "$lang_list" \ + --lang-pairs "$lang_pairs" \ + --criterion label_smoothed_cross_entropy --label-smoothing 0.2 \ + --optimizer adam --adam-eps 1e-06 --adam-betas '(0.9, 0.98)' \ + --lr-scheduler inverse_sqrt --lr 3e-05 --warmup-updates 2500 --max-update 40000 \ + --dropout 0.3 --attention-dropout 0.1 --weight-decay 0.0 \ + --max-tokens 1024 --update-freq 2 \ + --save-interval 1 --save-interval-updates 5000 --keep-interval-updates 10 --no-epoch-checkpoints \ + --seed 222 --log-format simple --log-interval 2 diff --git a/data/fairseq/examples/operators/alignment_train_cpu.cpp b/data/fairseq/examples/operators/alignment_train_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..13c015308e320d9ce14909fcdcc135300538e705 --- /dev/null +++ b/data/fairseq/examples/operators/alignment_train_cpu.cpp @@ -0,0 +1,166 @@ +/** + * Copyright 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include <torch/extension.h> // @manual=//caffe2:torch_extension +#include <algorithm> + +namespace { + +template <typename T> +void exclusiveCumprod( + const T* p_choose, + T* cumprod_1mp, + uint32_t bsz, + uint32_t tgt_len, + uint32_t src_len) { + // cumprod_1mp = 1 - p_choose + for (uint32_t b = 0; b < bsz; b++) { + for (uint32_t tgt = 0; tgt < tgt_len; tgt++) { + for (uint32_t src = 0; src < src_len; src++) { + uint32_t idx = b * tgt_len * src_len + tgt * src_len + src; + cumprod_1mp[idx] = 1 - p_choose[idx]; + } + } + } + + // Implementing exclusive cumprod in the innermost dimension + // cumprod_1mp = cumprod(1 - p_choose) + // There is cumprod in pytorch, however there is no exclusive mode. + // cumprod(x) = [x1, x1x2, x2x3x4, ..., prod_{i=1}^n x_i] + // exclusive means + // cumprod(x) = [1, x1, x1x2, x1x2x3, ..., prod_{i=1}^{n-1} x_i] + for (uint32_t b = 0; b < bsz; b++) { + for (uint32_t tgt = 0; tgt < tgt_len; tgt++) { + uint32_t idx_offset = b * tgt_len * src_len + tgt * src_len; + T prev = cumprod_1mp[idx_offset]; + // index [b][tgt][0] + cumprod_1mp[idx_offset] = (T)1.0; + T curr; + for (uint32_t src = 1; src < src_len; src++) { + uint32_t idx = idx_offset + src; + curr = cumprod_1mp[idx]; + cumprod_1mp[idx] = cumprod_1mp[idx - 1] * prev; + prev = curr; + } + } + } +} + +template <typename T> +void clamp( + const T* cumprod_1mp, + T* cumprod_1mp_clamp, + uint32_t bsz, + uint32_t tgt_len, + uint32_t src_len, + T min_val, + T max_val) { + for (uint32_t b = 0; b < bsz; b++) { + for (uint32_t tgt = 0; tgt < tgt_len; tgt++) { + for (uint32_t src = 0; src < src_len; src++) { + uint32_t idx = b * tgt_len * src_len + tgt * src_len + src; + if (cumprod_1mp[idx] < min_val) { + cumprod_1mp_clamp[idx] = min_val; + } else if (cumprod_1mp[idx] > max_val) { + cumprod_1mp_clamp[idx] = max_val; + } else { + cumprod_1mp_clamp[idx] = cumprod_1mp[idx]; + } + } + } + } +} + +template <typename T> +void alignmentTrainCPUImpl( + const T* p_choose, + T* alpha, + uint32_t bsz, + uint32_t tgt_len, + uint32_t src_len, + float eps) { + // p_choose: bsz , tgt_len, src_len + // cumprod_1mp: bsz , tgt_len, src_len + // cumprod_1mp_clamp : bsz, tgt_len, src_len + // alpha: bsz + 1, tgt_len, src_len + + uint32_t elements = bsz * tgt_len * src_len; + T* cumprod_1mp = new T[elements]; + T* cumprod_1mp_clamp = new T[elements]; + + exclusiveCumprod<T>(p_choose, cumprod_1mp, bsz, tgt_len, src_len); + clamp<T>( + cumprod_1mp, cumprod_1mp_clamp, bsz, tgt_len, src_len, (T)eps, (T)1.0); + + // ai = p_i * cumprod(1 − pi) * cumsum(a_i / cumprod(1 − pi)) + + // Initialize alpha [:, 0, 0] + for (uint32_t b = 0; b < bsz; b++) { + alpha[b * tgt_len * src_len] = 1.0; + } + + for (uint32_t tgt = 0; tgt < tgt_len; tgt++) { + for (uint32_t b = 0; b < bsz; b++) { + uint32_t alpha_idx, inout_idx; + T prev_scan = 0, curr_scan, out; + for (uint32_t src = 0; src < src_len; src++) { + // Apply scan/cumsum + if (tgt == 0) { + // alpha index is [b][tgt][src] + alpha_idx = b * tgt_len * src_len + src; + } else { + // alpha index is [b][tgt-1][src] + alpha_idx = b * tgt_len * src_len + (tgt - 1) * src_len + src; + } + // input index is [b][tgt][src] + inout_idx = b * tgt_len * src_len + tgt * src_len + src; + curr_scan = prev_scan + alpha[alpha_idx] / cumprod_1mp_clamp[inout_idx]; + + out = curr_scan * p_choose[inout_idx] * cumprod_1mp[inout_idx]; + alpha[inout_idx] = std::min<T>(std::max<T>(out, 0), 1.0); + prev_scan = curr_scan; + } + } + } + + free(cumprod_1mp); + free(cumprod_1mp_clamp); +} + +void alignmentTrainCPU( + const torch::Tensor& p_choose, + torch::Tensor& alpha, + float eps) { + uint32_t bsz = p_choose.size(0); + uint32_t tgt_len = p_choose.size(1); + uint32_t src_len = p_choose.size(2); + + AT_DISPATCH_FLOATING_TYPES_AND2( + torch::ScalarType::Half, + torch::ScalarType::BFloat16, + p_choose.scalar_type(), + "alignmentCPUImpl", + [&]() { + alignmentTrainCPUImpl<scalar_t>( + p_choose.data_ptr<scalar_t>(), + alpha.data_ptr<scalar_t>(), + bsz, + tgt_len, + src_len, + eps); + }); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def( + "alignment_train_cpu", + &alignmentTrainCPU, + "expected_alignment_from_p_choose (CPU)"); +} + +} // namespace diff --git a/data/fairseq/examples/operators/alignment_train_cuda.cpp b/data/fairseq/examples/operators/alignment_train_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..430e04813923074c5458c11e05dbdab879659bb5 --- /dev/null +++ b/data/fairseq/examples/operators/alignment_train_cuda.cpp @@ -0,0 +1,31 @@ +/** + * Copyright 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "alignment_train_cuda.h" +#include "utils.h" + +namespace { + +void alignmentTrainCUDA( + const torch::Tensor& p_choose, + torch::Tensor& alpha, + float eps) { + CHECK_INPUT(p_choose); + CHECK_INPUT(alpha); + + alignmentTrainCUDAWrapper(p_choose, alpha, eps); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def( + "alignment_train_cuda", + &alignmentTrainCUDA, + "expected_alignment_from_p_choose (CUDA)"); +} + +} // namespace diff --git a/data/fairseq/examples/operators/alignment_train_cuda.h b/data/fairseq/examples/operators/alignment_train_cuda.h new file mode 100644 index 0000000000000000000000000000000000000000..8289d1a69079138de6d73993ff117a59f000df37 --- /dev/null +++ b/data/fairseq/examples/operators/alignment_train_cuda.h @@ -0,0 +1,16 @@ +/** + * Copyright 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include <torch/extension.h> // @manual=//caffe2:torch_extension + +void alignmentTrainCUDAWrapper( + const torch::Tensor& p_choose, + torch::Tensor& alpha, + float eps); diff --git a/data/fairseq/examples/operators/alignment_train_kernel.cu b/data/fairseq/examples/operators/alignment_train_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..efae7cc76f1cce8701dbb5bb7b5ff1e402993953 --- /dev/null +++ b/data/fairseq/examples/operators/alignment_train_kernel.cu @@ -0,0 +1,354 @@ +/** + * Copyright 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include <ATen/ATen.h> +#include <ATen/cuda/CUDAContext.h> // @manual=//caffe2/aten:ATen-cu +#include <cuda_runtime.h> +#include <algorithm> // std::min/max +#include <cub/cub.cuh> + +#include "alignment_train_cuda.h" +#include "utils.h" + +namespace { + +// The thread block length in threads along the X dimension +constexpr int BLOCK_DIM_X = 128; +// The thread block length in threads along the Y dimension +constexpr int BLOCK_DIM_Y = 8; +// The thread block length in threads for scan operation +constexpr int SCAN_BLOCK = 512; + +#define gpuErrchk(ans) \ + { gpuAssert((ans), __FILE__, __LINE__); } + +inline void +gpuAssert(cudaError_t code, const char* file, int line, bool abort = true) { + if (code != cudaSuccess) { + fprintf( + stderr, + "\nGPUassert: %s %s %d\n", + cudaGetErrorString(code), + file, + line); + if (abort) + exit(code); + } +} + +template <typename T> +struct Prod { + /// prod operator, returns <tt>a * b</tt> + __host__ __device__ __forceinline__ T + operator()(const T& a, const T& b) const { + return a * b; + } +}; + +template <typename T> +struct BlockPrefixProdCallbackOp { + // Running prefix + T running_total; + + // Constructor + __device__ BlockPrefixProdCallbackOp(T running_total) + : running_total(running_total) {} + + // Callback operator to be entered by the first warp of threads in the block. + // Thread-0 is responsible for returning a value for seeding the block-wide + // scan. + __device__ T operator()(const T block_aggregate) { + T old_prefix = running_total; + running_total *= block_aggregate; + return old_prefix; + } +}; + +template <typename T> +struct BlockPrefixSumCallbackOp { + // Running prefix + T running_total; + + // Constructor + __device__ BlockPrefixSumCallbackOp(T running_total) + : running_total(running_total) {} + + // Callback operator to be entered by the first warp of threads in the block. + // Thread-0 is responsible for returning a value for seeding the block-wide + // scan. + __device__ T operator()(const T block_aggregate) { + T old_prefix = running_total; + running_total += block_aggregate; + return old_prefix; + } +}; + +template <typename T> +__global__ void oneMinusPKernel( + const T* __restrict__ p_choose, + T* __restrict__ cumprod_1mp, + uint32_t bsz, + uint32_t tgt_len, + uint32_t src_len) { + for (uint32_t b = blockIdx.x; b < bsz; b += gridDim.x) { + for (uint32_t tgt = threadIdx.y; tgt < tgt_len; tgt += blockDim.y) { + for (uint32_t src = threadIdx.x; src < src_len; src += blockDim.x) { + uint32_t idx = b * tgt_len * src_len + tgt * src_len + src; + cumprod_1mp[idx] = 1 - p_choose[idx]; + } + } + } +} + +template <typename T, int TPB> +__global__ void innermostScanKernel( + T* __restrict__ cumprod_1mp, + uint32_t bsz, + uint32_t tgt_len, + uint32_t src_len) { + for (uint32_t b = blockIdx.y; b < bsz; b += gridDim.y) { + for (uint32_t tgt = blockIdx.x; tgt < tgt_len; tgt += gridDim.x) { + // Specialize BlockScan for a 1D block of TPB threads on type T + typedef cub::BlockScan<T, TPB> BlockScan; + // Allocate shared memory for BlockScan + __shared__ typename BlockScan::TempStorage temp_storage; + // Initialize running total + BlockPrefixProdCallbackOp<T> prefix_op(1); + + const uint32_t tid = threadIdx.x; + for (uint32_t block_src = 0; block_src < src_len; + block_src += blockDim.x) { + uint32_t src = block_src + tid; + uint32_t idx = b * tgt_len * src_len + tgt * src_len + src; + T thread_data = (src < src_len) ? cumprod_1mp[idx] : (T)0; + + // Collectively compute the block-wide inclusive prefix sum + BlockScan(temp_storage) + .ExclusiveScan(thread_data, thread_data, Prod<T>(), prefix_op); + __syncthreads(); + + // write the scanned value to output + if (src < src_len) { + cumprod_1mp[idx] = thread_data; + } + } + } + } +} + +template <typename T> +__global__ void clampKernel( + const T* __restrict__ cumprod_1mp, + T* __restrict__ cumprod_1mp_clamp, + uint32_t bsz, + uint32_t tgt_len, + uint32_t src_len, + T min_val, + T max_val) { + for (uint32_t b = blockIdx.x; b < bsz; b += gridDim.x) { + for (uint32_t tgt = threadIdx.y; tgt < tgt_len; tgt += blockDim.y) { + for (uint32_t src = threadIdx.x; src < src_len; src += blockDim.x) { + uint32_t idx = b * tgt_len * src_len + tgt * src_len + src; + if (cumprod_1mp[idx] < min_val) { + cumprod_1mp_clamp[idx] = min_val; + } else if (cumprod_1mp[idx] > max_val) { + cumprod_1mp_clamp[idx] = max_val; + } else { + cumprod_1mp_clamp[idx] = cumprod_1mp[idx]; + } + } + } + } +} + +template <typename T> +__global__ void initAlphaCUDAKernel( + T* alpha, + uint32_t bsz, + uint32_t tgt_len, + uint32_t src_len) { + // alpha[:, 0, 0] = 1.0 + for (uint32_t b = blockIdx.x; b < bsz; b += gridDim.x) { + alpha[b * tgt_len * src_len] = (T)1.0; + } +} + +template <typename T, int TPB> +__global__ void alignmentTrainCUDAKernel( + const T* __restrict__ p_choose, + const T* __restrict__ cumprod_1mp, + const T* __restrict__ cumprod_1mp_clamp, + T* __restrict__ alpha, + uint32_t bsz, + uint32_t tgt_len, + uint32_t src_len, + uint32_t tgt) { + for (uint32_t b = blockIdx.x; b < bsz; b += gridDim.x) { + // Specialize BlockScan for a 1D block of TPB threads on type T + typedef cub::BlockScan<T, TPB> BlockScan; + + // Allocate shared memory for BlockScan + __shared__ typename BlockScan::TempStorage temp_storage; + // Initialize running total + BlockPrefixSumCallbackOp<T> prefix_op(0); + + uint32_t b_offset = b * tgt_len * src_len; + const uint32_t tid = threadIdx.x; + for (uint32_t block_src = 0; block_src < src_len; block_src += blockDim.x) { + uint32_t src = block_src + tid; + // Obtain a segment of consecutive items that are blocked across threads + uint32_t inout_idx, alpha_idx; + if (tgt == 0) { + // both alpha and other input index is [b][0][src] + alpha_idx = b_offset + src; + } else { + // alpha index is [b][tgt-1][src] + alpha_idx = b_offset + (tgt - 1) * src_len + src; + } + inout_idx = b_offset + tgt * src_len + src; + T thread_data = (T)0; + if (src < src_len) { + thread_data = alpha[alpha_idx] / cumprod_1mp_clamp[inout_idx]; + } + + // Collectively compute the block-wide inclusive prefix sum + BlockScan(temp_storage).InclusiveSum(thread_data, thread_data, prefix_op); + __syncthreads(); + + if (src < src_len) { + T out = thread_data * p_choose[inout_idx] * cumprod_1mp[inout_idx]; + // Clamps all elements into the range [ 0, 1.0 ] + alpha[inout_idx] = std::min<T>(std::max<T>(out, 0), (T)1.0); + } + } + } +} + +template <typename T> +void exclusiveCumprod( + const T* p_choose, + T* cumprod_1mp, + uint32_t bsz, + uint32_t tgt_len, + uint32_t src_len, + uint32_t max_grid_x, + uint32_t max_grid_y, + cudaStream_t& stream) { + // cumprod_1mp = 1 - p_choose + dim3 grid(std::min<T>(max_grid_x, bsz), 1, 1); + dim3 block(BLOCK_DIM_X, BLOCK_DIM_Y, 1); + oneMinusPKernel<T><<<grid, block, 0, stream>>>( + p_choose, cumprod_1mp, bsz, tgt_len, src_len); + gpuErrchk(cudaGetLastError()); + + // scan on the innermost dimension of cumprod_1mp + // cumprod_1mp = cumprod(cumprod_1mp) + dim3 grid_scan( + std::min<T>(max_grid_x, tgt_len), std::min<T>(max_grid_y, bsz), 1); + innermostScanKernel<T, SCAN_BLOCK><<<grid_scan, SCAN_BLOCK, 0, stream>>>( + cumprod_1mp, bsz, tgt_len, src_len); + gpuErrchk(cudaGetLastError()); +} + +template <typename T> +void alignmentTrainCUDAImpl( + const T* p_choose, + T* alpha, + uint32_t bsz, + uint32_t tgt_len, + uint32_t src_len, + float eps) { + // p_choose: bsz , tgt_len, src_len + // cumprod_1mp: bsz , tgt_len, src_len + // cumprod_1mp_clamp : bsz, tgt_len, src_len + // alpha: bsz, tgt_len, src_len + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + uint32_t max_grid_x = at::cuda::getCurrentDeviceProperties()->maxGridSize[0]; + uint32_t max_grid_y = at::cuda::getCurrentDeviceProperties()->maxGridSize[1]; + + // Implementing exclusive cumprod. + // cumprod_1mp = cumprod(1 - p_choose) + // There is cumprod in pytorch, however there is no exclusive mode. + // cumprod(x) = [x1, x1x2, x2x3x4, ..., prod_{i=1}^n x_i] + // exclusive means + // cumprod(x) = [1, x1, x1x2, x1x2x3, ..., prod_{i=1}^{n-1} x_i] + uint32_t elements = bsz * tgt_len * src_len; + T* cumprod_1mp; + gpuErrchk(cudaMalloc(&cumprod_1mp, elements * sizeof(T))); + exclusiveCumprod<T>( + p_choose, + cumprod_1mp, + bsz, + tgt_len, + src_len, + max_grid_x, + max_grid_y, + stream); + + // clamp cumprod_1mp to the range [eps, 1.0] + T* cumprod_1mp_clamp; + gpuErrchk(cudaMalloc(&cumprod_1mp_clamp, elements * sizeof(T))); + dim3 grid_clamp(std::min<T>(max_grid_x, bsz), 1, 1); + dim3 block_clamp(BLOCK_DIM_X, BLOCK_DIM_Y, 1); + clampKernel<T><<<grid_clamp, block_clamp, 0, stream>>>( + cumprod_1mp, cumprod_1mp_clamp, bsz, tgt_len, src_len, (T)eps, (T)1.0); + gpuErrchk(cudaGetLastError()); + + // ai = p_i * cumprod(1 − pi) * cumsum(a_i / cumprod(1 − pi)) + dim3 grid_init(std::min<int>(max_grid_x, bsz), 1, 1); + initAlphaCUDAKernel<T> + <<<grid_init, 1, 0, stream>>>(alpha, bsz, tgt_len, src_len); + gpuErrchk(cudaGetLastError()); + + const int grid = std::min(bsz, max_grid_x); + + for (uint32_t i = 0; i < tgt_len; i++) { + alignmentTrainCUDAKernel<T, SCAN_BLOCK><<<grid, SCAN_BLOCK, 0, stream>>>( + p_choose, + cumprod_1mp, + cumprod_1mp_clamp, + alpha, + bsz, + tgt_len, + src_len, + i); + gpuErrchk(cudaGetLastError()); + } + + gpuErrchk(cudaFree(cumprod_1mp)); + gpuErrchk(cudaFree(cumprod_1mp_clamp)); +} + +} // namespace + +void alignmentTrainCUDAWrapper( + const torch::Tensor& p_choose, + torch::Tensor& alpha, + float eps) { + // p_choose dimension: bsz, tgt_len, src_len + uint32_t bsz = p_choose.size(0); + uint32_t tgt_len = p_choose.size(1); + uint32_t src_len = p_choose.size(2); + + cudaSetDevice(p_choose.get_device()); + + AT_DISPATCH_FLOATING_TYPES_AND2( + torch::ScalarType::Half, + torch::ScalarType::BFloat16, + p_choose.scalar_type(), + "alignmentTrainCUDAImpl", + [&]() { + alignmentTrainCUDAImpl<scalar_t>( + p_choose.data_ptr<scalar_t>(), + alpha.data_ptr<scalar_t>(), + bsz, + tgt_len, + src_len, + eps); + }); +} diff --git a/data/fairseq/examples/operators/utils.h b/data/fairseq/examples/operators/utils.h new file mode 100644 index 0000000000000000000000000000000000000000..0ef5b4383f4a8ff05ea7b44d932a3b47e4c7f927 --- /dev/null +++ b/data/fairseq/examples/operators/utils.h @@ -0,0 +1,19 @@ +/** + * Copyright 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include <torch/extension.h> // @manual=//caffe2:torch_extension + +#define CHECK_CUDA(x) \ + TORCH_CHECK(x.type().is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) \ + TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) diff --git a/data/fairseq/examples/roberta/README.custom_classification.md b/data/fairseq/examples/roberta/README.custom_classification.md new file mode 100644 index 0000000000000000000000000000000000000000..7254bb7d178760ef5b847901bbcac3711af33ca2 --- /dev/null +++ b/data/fairseq/examples/roberta/README.custom_classification.md @@ -0,0 +1,168 @@ +# Finetuning RoBERTa on a custom classification task + +This example shows how to finetune RoBERTa on the IMDB dataset, but should illustrate the process for most classification tasks. + +### 1) Get the data + +```bash +wget http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz +tar zxvf aclImdb_v1.tar.gz +``` + + +### 2) Format data + +`IMDB` data has one data-sample in each file, below python code-snippet converts it one file for train and valid each for ease of processing. +```python +import argparse +import os +import random +from glob import glob + +random.seed(0) + +def main(args): + for split in ['train', 'test']: + samples = [] + for class_label in ['pos', 'neg']: + fnames = glob(os.path.join(args.datadir, split, class_label) + '/*.txt') + for fname in fnames: + with open(fname) as fin: + line = fin.readline() + samples.append((line, 1 if class_label == 'pos' else 0)) + random.shuffle(samples) + out_fname = 'train' if split == 'train' else 'dev' + f1 = open(os.path.join(args.datadir, out_fname + '.input0'), 'w') + f2 = open(os.path.join(args.datadir, out_fname + '.label'), 'w') + for sample in samples: + f1.write(sample[0] + '\n') + f2.write(str(sample[1]) + '\n') + f1.close() + f2.close() + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--datadir', default='aclImdb') + args = parser.parse_args() + main(args) +``` + + +### 3) BPE encode + +Run `multiprocessing_bpe_encoder`, you can also do this in previous step for each sample but that might be slower. +```bash +# Download encoder.json and vocab.bpe +wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json' +wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe' + +for SPLIT in train dev; do + python -m examples.roberta.multiprocessing_bpe_encoder \ + --encoder-json encoder.json \ + --vocab-bpe vocab.bpe \ + --inputs "aclImdb/$SPLIT.input0" \ + --outputs "aclImdb/$SPLIT.input0.bpe" \ + --workers 60 \ + --keep-empty +done +``` + + +### 4) Preprocess data + +```bash +# Download fairseq dictionary. +wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt' + +fairseq-preprocess \ + --only-source \ + --trainpref "aclImdb/train.input0.bpe" \ + --validpref "aclImdb/dev.input0.bpe" \ + --destdir "IMDB-bin/input0" \ + --workers 60 \ + --srcdict dict.txt + +fairseq-preprocess \ + --only-source \ + --trainpref "aclImdb/train.label" \ + --validpref "aclImdb/dev.label" \ + --destdir "IMDB-bin/label" \ + --workers 60 + +``` + + +### 5) Run training + +```bash +TOTAL_NUM_UPDATES=7812 # 10 epochs through IMDB for bsz 32 +WARMUP_UPDATES=469 # 6 percent of the number of updates +LR=1e-05 # Peak LR for polynomial LR scheduler. +HEAD_NAME=imdb_head # Custom name for the classification head. +NUM_CLASSES=2 # Number of classes for the classification task. +MAX_SENTENCES=8 # Batch size. +ROBERTA_PATH=/path/to/roberta.large/model.pt + +CUDA_VISIBLE_DEVICES=0 fairseq-train IMDB-bin/ \ + --restore-file $ROBERTA_PATH \ + --max-positions 512 \ + --batch-size $MAX_SENTENCES \ + --max-tokens 4400 \ + --task sentence_prediction \ + --reset-optimizer --reset-dataloader --reset-meters \ + --required-batch-size-multiple 1 \ + --init-token 0 --separator-token 2 \ + --arch roberta_large \ + --criterion sentence_prediction \ + --classification-head-name $HEAD_NAME \ + --num-classes $NUM_CLASSES \ + --dropout 0.1 --attention-dropout 0.1 \ + --weight-decay 0.1 --optimizer adam --adam-betas "(0.9, 0.98)" --adam-eps 1e-06 \ + --clip-norm 0.0 \ + --lr-scheduler polynomial_decay --lr $LR --total-num-update $TOTAL_NUM_UPDATES --warmup-updates $WARMUP_UPDATES \ + --fp16 --fp16-init-scale 4 --threshold-loss-scale 1 --fp16-scale-window 128 \ + --max-epoch 10 \ + --best-checkpoint-metric accuracy --maximize-best-checkpoint-metric \ + --shorten-method "truncate" \ + --find-unused-parameters \ + --update-freq 4 +``` + +The above command will finetune RoBERTa-large with an effective batch-size of 32 +sentences (`--batch-size=8 --update-freq=4`). The expected +`best-validation-accuracy` after 10 epochs is ~96.5%. + +If you run out of GPU memory, try decreasing `--batch-size` and increase +`--update-freq` to compensate. + + +### 6) Load model using hub interface + +Now we can load the trained model checkpoint using the RoBERTa hub interface. + +Assuming your checkpoints are stored in `checkpoints/`: +```python +from fairseq.models.roberta import RobertaModel +roberta = RobertaModel.from_pretrained( + 'checkpoints', + checkpoint_file='checkpoint_best.pt', + data_name_or_path='IMDB-bin' +) +roberta.eval() # disable dropout +``` + +Finally you can make predictions using the `imdb_head` (or whatever you set +`--classification-head-name` to during training): +```python +label_fn = lambda label: roberta.task.label_dictionary.string( + [label + roberta.task.label_dictionary.nspecial] +) + +tokens = roberta.encode('Best movie this year') +pred = label_fn(roberta.predict('imdb_head', tokens).argmax().item()) +assert pred == '1' # positive + +tokens = roberta.encode('Worst movie ever') +pred = label_fn(roberta.predict('imdb_head', tokens).argmax().item()) +assert pred == '0' # negative +``` diff --git a/data/fairseq/examples/roberta/README.glue.md b/data/fairseq/examples/roberta/README.glue.md new file mode 100644 index 0000000000000000000000000000000000000000..4f596d55af99fba3cdf58b1d5ff3d8f8dbf4383d --- /dev/null +++ b/data/fairseq/examples/roberta/README.glue.md @@ -0,0 +1,64 @@ +# Finetuning RoBERTa on GLUE tasks + +### 1) Download the data from GLUE website (https://gluebenchmark.com/tasks) using following commands: +```bash +wget https://gist.githubusercontent.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e/raw/17b8dd0d724281ed7c3b2aeeda662b92809aadd5/download_glue_data.py +python download_glue_data.py --data_dir glue_data --tasks all +``` + +### 2) Preprocess GLUE task data: +```bash +./examples/roberta/preprocess_GLUE_tasks.sh glue_data <glue_task_name> +``` +`glue_task_name` is one of the following: +`{ALL, QQP, MNLI, QNLI, MRPC, RTE, STS-B, SST-2, CoLA}` +Use `ALL` for preprocessing all the glue tasks. + +### 3) Fine-tuning on GLUE task: +Example fine-tuning cmd for `RTE` task +```bash +ROBERTA_PATH=/path/to/roberta/model.pt + +CUDA_VISIBLE_DEVICES=0 fairseq-hydra-train -config-dir examples/roberta/config/finetuning --config-name rte \ +task.data=RTE-bin checkpoint.restore_file=$ROBERTA_PATH +``` + +There are additional config files for each of the GLUE tasks in the examples/roberta/config/finetuning directory. + +**Note:** + +a) Above cmd-args and hyperparams are tested on one Nvidia `V100` GPU with `32gb` of memory for each task. Depending on the GPU memory resources available to you, you can use increase `--update-freq` and reduce `--batch-size`. + +b) All the settings in above table are suggested settings based on our hyperparam search within a fixed search space (for careful comparison across models). You might be able to find better metrics with wider hyperparam search. + +### Inference on GLUE task +After training the model as mentioned in previous step, you can perform inference with checkpoints in `checkpoints/` directory using following python code snippet: + +```python +from fairseq.models.roberta import RobertaModel + +roberta = RobertaModel.from_pretrained( + 'checkpoints/', + checkpoint_file='checkpoint_best.pt', + data_name_or_path='RTE-bin' +) + +label_fn = lambda label: roberta.task.label_dictionary.string( + [label + roberta.task.label_dictionary.nspecial] +) +ncorrect, nsamples = 0, 0 +roberta.cuda() +roberta.eval() +with open('glue_data/RTE/dev.tsv') as fin: + fin.readline() + for index, line in enumerate(fin): + tokens = line.strip().split('\t') + sent1, sent2, target = tokens[1], tokens[2], tokens[3] + tokens = roberta.encode(sent1, sent2) + prediction = roberta.predict('sentence_classification_head', tokens).argmax().item() + prediction_label = label_fn(prediction) + ncorrect += int(prediction_label == target) + nsamples += 1 +print('| Accuracy: ', float(ncorrect)/float(nsamples)) + +``` diff --git a/data/fairseq/examples/roberta/README.md b/data/fairseq/examples/roberta/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ed4d5df52ccea01216276054a1f253d0d16c0409 --- /dev/null +++ b/data/fairseq/examples/roberta/README.md @@ -0,0 +1,296 @@ +# RoBERTa: A Robustly Optimized BERT Pretraining Approach + +https://arxiv.org/abs/1907.11692 + +## Introduction + +RoBERTa iterates on BERT's pretraining procedure, including training the model longer, with bigger batches over more data; removing the next sentence prediction objective; training on longer sequences; and dynamically changing the masking pattern applied to the training data. See the associated paper for more details. + +### What's New: + +- December 2020: German model (GottBERT) is available: [GottBERT](https://github.com/pytorch/fairseq/tree/main/examples/gottbert). +- January 2020: Italian model (UmBERTo) is available from Musixmatch Research: [UmBERTo](https://github.com/musixmatchresearch/umberto). +- November 2019: French model (CamemBERT) is available: [CamemBERT](https://github.com/pytorch/fairseq/tree/main/examples/camembert). +- November 2019: Multilingual encoder (XLM-RoBERTa) is available: [XLM-R](https://github.com/pytorch/fairseq/tree/main/examples/xlmr). +- September 2019: TensorFlow and TPU support via the [transformers library](https://github.com/huggingface/transformers). +- August 2019: RoBERTa is now supported in the [pytorch-transformers library](https://github.com/huggingface/pytorch-transformers). +- August 2019: Added [tutorial for finetuning on WinoGrande](https://github.com/pytorch/fairseq/tree/main/examples/roberta/wsc#roberta-training-on-winogrande-dataset). +- August 2019: Added [tutorial for pretraining RoBERTa using your own data](README.pretraining.md). + +## Pre-trained models + +Model | Description | # params | Download +---|---|---|--- +`roberta.base` | RoBERTa using the BERT-base architecture | 125M | [roberta.base.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/roberta.base.tar.gz) +`roberta.large` | RoBERTa using the BERT-large architecture | 355M | [roberta.large.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/roberta.large.tar.gz) +`roberta.large.mnli` | `roberta.large` finetuned on [MNLI](http://www.nyu.edu/projects/bowman/multinli) | 355M | [roberta.large.mnli.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/roberta.large.mnli.tar.gz) +`roberta.large.wsc` | `roberta.large` finetuned on [WSC](wsc/README.md) | 355M | [roberta.large.wsc.tar.gz](https://dl.fbaipublicfiles.com/fairseq/models/roberta.large.wsc.tar.gz) + +## Results + +**[GLUE (Wang et al., 2019)](https://gluebenchmark.com/)** +_(dev set, single model, single-task finetuning)_ + +Model | MNLI | QNLI | QQP | RTE | SST-2 | MRPC | CoLA | STS-B +---|---|---|---|---|---|---|---|--- +`roberta.base` | 87.6 | 92.8 | 91.9 | 78.7 | 94.8 | 90.2 | 63.6 | 91.2 +`roberta.large` | 90.2 | 94.7 | 92.2 | 86.6 | 96.4 | 90.9 | 68.0 | 92.4 +`roberta.large.mnli` | 90.2 | - | - | - | - | - | - | - + +**[SuperGLUE (Wang et al., 2019)](https://super.gluebenchmark.com/)** +_(dev set, single model, single-task finetuning)_ + +Model | BoolQ | CB | COPA | MultiRC | RTE | WiC | WSC +---|---|---|---|---|---|---|--- +`roberta.large` | 86.9 | 98.2 | 94.0 | 85.7 | 89.5 | 75.6 | - +`roberta.large.wsc` | - | - | - | - | - | - | 91.3 + +**[SQuAD (Rajpurkar et al., 2018)](https://rajpurkar.github.io/SQuAD-explorer/)** +_(dev set, no additional data used)_ + +Model | SQuAD 1.1 EM/F1 | SQuAD 2.0 EM/F1 +---|---|--- +`roberta.large` | 88.9/94.6 | 86.5/89.4 + +**[RACE (Lai et al., 2017)](http://www.qizhexie.com/data/RACE_leaderboard.html)** +_(test set)_ + +Model | Accuracy | Middle | High +---|---|---|--- +`roberta.large` | 83.2 | 86.5 | 81.3 + +**[HellaSwag (Zellers et al., 2019)](https://rowanzellers.com/hellaswag/)** +_(test set)_ + +Model | Overall | In-domain | Zero-shot | ActivityNet | WikiHow +---|---|---|---|---|--- +`roberta.large` | 85.2 | 87.3 | 83.1 | 74.6 | 90.9 + +**[Commonsense QA (Talmor et al., 2019)](https://www.tau-nlp.org/commonsenseqa)** +_(test set)_ + +Model | Accuracy +---|--- +`roberta.large` (single model) | 72.1 +`roberta.large` (ensemble) | 72.5 + +**[Winogrande (Sakaguchi et al., 2019)](https://arxiv.org/abs/1907.10641)** +_(test set)_ + +Model | Accuracy +---|--- +`roberta.large` | 78.1 + +**[XNLI (Conneau et al., 2018)](https://arxiv.org/abs/1809.05053)** +_(TRANSLATE-TEST)_ + +Model | en | fr | es | de | el | bg | ru | tr | ar | vi | th | zh | hi | sw | ur +---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|--- +`roberta.large.mnli` | 91.3 | 82.91 | 84.27 | 81.24 | 81.74 | 83.13 | 78.28 | 76.79 | 76.64 | 74.17 | 74.05 | 77.5 | 70.9 | 66.65 | 66.81 + +## Example usage + +##### Load RoBERTa from torch.hub (PyTorch >= 1.1): +```python +import torch +roberta = torch.hub.load('pytorch/fairseq', 'roberta.large') +roberta.eval() # disable dropout (or leave in train mode to finetune) +``` + +##### Load RoBERTa (for PyTorch 1.0 or custom models): +```python +# Download roberta.large model +wget https://dl.fbaipublicfiles.com/fairseq/models/roberta.large.tar.gz +tar -xzvf roberta.large.tar.gz + +# Load the model in fairseq +from fairseq.models.roberta import RobertaModel +roberta = RobertaModel.from_pretrained('/path/to/roberta.large', checkpoint_file='model.pt') +roberta.eval() # disable dropout (or leave in train mode to finetune) +``` + +##### Apply Byte-Pair Encoding (BPE) to input text: +```python +tokens = roberta.encode('Hello world!') +assert tokens.tolist() == [0, 31414, 232, 328, 2] +roberta.decode(tokens) # 'Hello world!' +``` + +##### Extract features from RoBERTa: +```python +# Extract the last layer's features +last_layer_features = roberta.extract_features(tokens) +assert last_layer_features.size() == torch.Size([1, 5, 1024]) + +# Extract all layer's features (layer 0 is the embedding layer) +all_layers = roberta.extract_features(tokens, return_all_hiddens=True) +assert len(all_layers) == 25 +assert torch.all(all_layers[-1] == last_layer_features) +``` + +##### Use RoBERTa for sentence-pair classification tasks: +```python +# Download RoBERTa already finetuned for MNLI +roberta = torch.hub.load('pytorch/fairseq', 'roberta.large.mnli') +roberta.eval() # disable dropout for evaluation + +# Encode a pair of sentences and make a prediction +tokens = roberta.encode('Roberta is a heavily optimized version of BERT.', 'Roberta is not very optimized.') +roberta.predict('mnli', tokens).argmax() # 0: contradiction + +# Encode another pair of sentences +tokens = roberta.encode('Roberta is a heavily optimized version of BERT.', 'Roberta is based on BERT.') +roberta.predict('mnli', tokens).argmax() # 2: entailment +``` + +##### Register a new (randomly initialized) classification head: +```python +roberta.register_classification_head('new_task', num_classes=3) +logprobs = roberta.predict('new_task', tokens) # tensor([[-1.1050, -1.0672, -1.1245]], grad_fn=<LogSoftmaxBackward>) +``` + +##### Batched prediction: +```python +import torch +from fairseq.data.data_utils import collate_tokens + +roberta = torch.hub.load('pytorch/fairseq', 'roberta.large.mnli') +roberta.eval() + +batch_of_pairs = [ + ['Roberta is a heavily optimized version of BERT.', 'Roberta is not very optimized.'], + ['Roberta is a heavily optimized version of BERT.', 'Roberta is based on BERT.'], + ['potatoes are awesome.', 'I like to run.'], + ['Mars is very far from earth.', 'Mars is very close.'], +] + +batch = collate_tokens( + [roberta.encode(pair[0], pair[1]) for pair in batch_of_pairs], pad_idx=1 +) + +logprobs = roberta.predict('mnli', batch) +print(logprobs.argmax(dim=1)) +# tensor([0, 2, 1, 0]) +``` + +##### Using the GPU: +```python +roberta.cuda() +roberta.predict('new_task', tokens) # tensor([[-1.1050, -1.0672, -1.1245]], device='cuda:0', grad_fn=<LogSoftmaxBackward>) +``` + +## Advanced usage + +#### Filling masks: + +RoBERTa can be used to fill `<mask>` tokens in the input. Some examples from the +[Natural Questions dataset](https://ai.google.com/research/NaturalQuestions/): +```python +roberta.fill_mask('The first Star wars movie came out in <mask>', topk=3) +# [('The first Star wars movie came out in 1977', 0.9504708051681519, ' 1977'), ('The first Star wars movie came out in 1978', 0.009986862540245056, ' 1978'), ('The first Star wars movie came out in 1979', 0.009574787691235542, ' 1979')] + +roberta.fill_mask('Vikram samvat calender is official in <mask>', topk=3) +# [('Vikram samvat calender is official in India', 0.21878819167613983, ' India'), ('Vikram samvat calender is official in Delhi', 0.08547237515449524, ' Delhi'), ('Vikram samvat calender is official in Gujarat', 0.07556215673685074, ' Gujarat')] + +roberta.fill_mask('<mask> is the common currency of the European Union', topk=3) +# [('Euro is the common currency of the European Union', 0.9456493854522705, 'Euro'), ('euro is the common currency of the European Union', 0.025748178362846375, 'euro'), ('€ is the common currency of the European Union', 0.011183084920048714, '€')] +``` + +#### Pronoun disambiguation (Winograd Schema Challenge): + +RoBERTa can be used to disambiguate pronouns. First install spaCy and download the English-language model: +```bash +pip install spacy +python -m spacy download en_core_web_lg +``` + +Next load the `roberta.large.wsc` model and call the `disambiguate_pronoun` +function. The pronoun should be surrounded by square brackets (`[]`) and the +query referent surrounded by underscores (`_`), or left blank to return the +predicted candidate text directly: +```python +roberta = torch.hub.load('pytorch/fairseq', 'roberta.large.wsc', user_dir='examples/roberta/wsc') +roberta.cuda() # use the GPU (optional) + +roberta.disambiguate_pronoun('The _trophy_ would not fit in the brown suitcase because [it] was too big.') +# True +roberta.disambiguate_pronoun('The trophy would not fit in the brown _suitcase_ because [it] was too big.') +# False + +roberta.disambiguate_pronoun('The city councilmen refused the demonstrators a permit because [they] feared violence.') +# 'The city councilmen' +roberta.disambiguate_pronoun('The city councilmen refused the demonstrators a permit because [they] advocated violence.') +# 'demonstrators' +``` + +See the [RoBERTA Winograd Schema Challenge (WSC) README](wsc/README.md) for more details on how to train this model. + +#### Extract features aligned to words: + +By default RoBERTa outputs one feature vector per BPE token. You can instead +realign the features to match [spaCy's word-level tokenization](https://spacy.io/usage/linguistic-features#tokenization) +with the `extract_features_aligned_to_words` method. This will compute a +weighted average of the BPE-level features for each word and expose them in +spaCy's `Token.vector` attribute: +```python +doc = roberta.extract_features_aligned_to_words('I said, "hello RoBERTa."') +assert len(doc) == 10 +for tok in doc: + print('{:10}{} (...)'.format(str(tok), tok.vector[:5])) +# <s> tensor([-0.1316, -0.0386, -0.0832, -0.0477, 0.1943], grad_fn=<SliceBackward>) (...) +# I tensor([ 0.0559, 0.1541, -0.4832, 0.0880, 0.0120], grad_fn=<SliceBackward>) (...) +# said tensor([-0.1565, -0.0069, -0.8915, 0.0501, -0.0647], grad_fn=<SliceBackward>) (...) +# , tensor([-0.1318, -0.0387, -0.0834, -0.0477, 0.1944], grad_fn=<SliceBackward>) (...) +# " tensor([-0.0486, 0.1818, -0.3946, -0.0553, 0.0981], grad_fn=<SliceBackward>) (...) +# hello tensor([ 0.0079, 0.1799, -0.6204, -0.0777, -0.0923], grad_fn=<SliceBackward>) (...) +# RoBERTa tensor([-0.2339, -0.1184, -0.7343, -0.0492, 0.5829], grad_fn=<SliceBackward>) (...) +# . tensor([-0.1341, -0.1203, -0.1012, -0.0621, 0.1892], grad_fn=<SliceBackward>) (...) +# " tensor([-0.1341, -0.1203, -0.1012, -0.0621, 0.1892], grad_fn=<SliceBackward>) (...) +# </s> tensor([-0.0930, -0.0392, -0.0821, 0.0158, 0.0649], grad_fn=<SliceBackward>) (...) +``` + +#### Evaluating the `roberta.large.mnli` model: + +Example python code snippet to evaluate accuracy on the MNLI `dev_matched` set. +```python +label_map = {0: 'contradiction', 1: 'neutral', 2: 'entailment'} +ncorrect, nsamples = 0, 0 +roberta.cuda() +roberta.eval() +with open('glue_data/MNLI/dev_matched.tsv') as fin: + fin.readline() + for index, line in enumerate(fin): + tokens = line.strip().split('\t') + sent1, sent2, target = tokens[8], tokens[9], tokens[-1] + tokens = roberta.encode(sent1, sent2) + prediction = roberta.predict('mnli', tokens).argmax().item() + prediction_label = label_map[prediction] + ncorrect += int(prediction_label == target) + nsamples += 1 +print('| Accuracy: ', float(ncorrect)/float(nsamples)) +# Expected output: 0.9060 +``` + +## Finetuning + +- [Finetuning on GLUE](README.glue.md) +- [Finetuning on custom classification tasks (e.g., IMDB)](README.custom_classification.md) +- [Finetuning on Winograd Schema Challenge (WSC)](wsc/README.md) +- [Finetuning on Commonsense QA (CQA)](commonsense_qa/README.md) + +## Pretraining using your own data + +See the [tutorial for pretraining RoBERTa using your own data](README.pretraining.md). + +## Citation + +```bibtex +@article{liu2019roberta, + title = {RoBERTa: A Robustly Optimized BERT Pretraining Approach}, + author = {Yinhan Liu and Myle Ott and Naman Goyal and Jingfei Du and + Mandar Joshi and Danqi Chen and Omer Levy and Mike Lewis and + Luke Zettlemoyer and Veselin Stoyanov}, + journal={arXiv preprint arXiv:1907.11692}, + year = {2019}, +} +``` diff --git a/data/fairseq/examples/roberta/README.pretraining.md b/data/fairseq/examples/roberta/README.pretraining.md new file mode 100644 index 0000000000000000000000000000000000000000..a4e7453529111fdd198be637d911d1764cb96c0e --- /dev/null +++ b/data/fairseq/examples/roberta/README.pretraining.md @@ -0,0 +1,84 @@ +# Pretraining RoBERTa using your own data + +This tutorial will walk you through pretraining RoBERTa over your own data. + +### 1) Preprocess the data + +Data should be preprocessed following the [language modeling format](/examples/language_model), i.e. each document should be separated by an empty line (only useful with `--sample-break-mode complete_doc`). Lines will be concatenated as a 1D text stream during training. + +We'll use the [WikiText-103 dataset](https://www.salesforce.com/products/einstein/ai-research/the-wikitext-dependency-language-modeling-dataset/) +to demonstrate how to preprocess raw text data with the GPT-2 BPE. Of course +this dataset is quite small, so the resulting pretrained model will perform +poorly, but it gives the general idea. + +First download the dataset: +```bash +wget https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-raw-v1.zip +unzip wikitext-103-raw-v1.zip +``` + +Next encode it with the GPT-2 BPE: +```bash +mkdir -p gpt2_bpe +wget -O gpt2_bpe/encoder.json https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json +wget -O gpt2_bpe/vocab.bpe https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe +for SPLIT in train valid test; do \ + python -m examples.roberta.multiprocessing_bpe_encoder \ + --encoder-json gpt2_bpe/encoder.json \ + --vocab-bpe gpt2_bpe/vocab.bpe \ + --inputs wikitext-103-raw/wiki.${SPLIT}.raw \ + --outputs wikitext-103-raw/wiki.${SPLIT}.bpe \ + --keep-empty \ + --workers 60; \ +done +``` + +Finally preprocess/binarize the data using the GPT-2 fairseq dictionary: +```bash +wget -O gpt2_bpe/dict.txt https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt +fairseq-preprocess \ + --only-source \ + --srcdict gpt2_bpe/dict.txt \ + --trainpref wikitext-103-raw/wiki.train.bpe \ + --validpref wikitext-103-raw/wiki.valid.bpe \ + --testpref wikitext-103-raw/wiki.test.bpe \ + --destdir data-bin/wikitext-103 \ + --workers 60 +``` + +### 2) Train RoBERTa base +```bash +DATA_DIR=data-bin/wikitext-103 + +fairseq-hydra-train -m --config-dir examples/roberta/config/pretraining \ +--config-name base task.data=$DATA_DIR +``` + +**Note:** You can optionally resume training the released RoBERTa base model by +adding `checkpoint.restore_file=/path/to/roberta.base/model.pt`. + +**Note:** The above command assumes training on 8x32GB V100 GPUs. Each GPU uses +a batch size of 16 sequences (`dataset.batch_size`) and accumulates gradients to +further increase the batch size by 16x (`optimization.update_freq`), for a total batch size +of 2048 sequences. If you have fewer GPUs or GPUs with less memory you may need +to reduce `dataset.batch_size` and increase dataset.update_freq to compensate. +Alternatively if you have more GPUs you can decrease `dataset.update_freq` accordingly +to increase training speed. + +**Note:** The learning rate and batch size are tightly connected and need to be +adjusted together. We generally recommend increasing the learning rate as you +increase the batch size according to the following table (although it's also +dataset dependent, so don't rely on the following values too closely): + +batch size | peak learning rate +---|--- +256 | 0.0001 +2048 | 0.0005 +8192 | 0.0007 + +### 3) Load your pretrained model +```python +from fairseq.models.roberta import RobertaModel +roberta = RobertaModel.from_pretrained('checkpoints', 'checkpoint_best.pt', 'path/to/data') +assert isinstance(roberta.model, torch.nn.Module) +``` diff --git a/data/fairseq/examples/roberta/README.race.md b/data/fairseq/examples/roberta/README.race.md new file mode 100644 index 0000000000000000000000000000000000000000..13c917e8eca6621e91dce541c7e41436b38cbdc1 --- /dev/null +++ b/data/fairseq/examples/roberta/README.race.md @@ -0,0 +1,68 @@ +# Finetuning RoBERTa on RACE tasks + +### 1) Download the data from RACE website (http://www.cs.cmu.edu/~glai1/data/race/) + +### 2) Preprocess RACE data: +```bash +python ./examples/roberta/preprocess_RACE.py --input-dir <input-dir> --output-dir <extracted-data-dir> +./examples/roberta/preprocess_RACE.sh <extracted-data-dir> <output-dir> +``` + +### 3) Fine-tuning on RACE: + +```bash +MAX_EPOCH=5 # Number of training epochs. +LR=1e-05 # Peak LR for fixed LR scheduler. +NUM_CLASSES=4 +MAX_SENTENCES=1 # Batch size per GPU. +UPDATE_FREQ=8 # Accumulate gradients to simulate training on 8 GPUs. +DATA_DIR=/path/to/race-output-dir +ROBERTA_PATH=/path/to/roberta/model.pt + +CUDA_VISIBLE_DEVICES=0,1 fairseq-train $DATA_DIR --ddp-backend=legacy_ddp \ + --restore-file $ROBERTA_PATH \ + --reset-optimizer --reset-dataloader --reset-meters \ + --best-checkpoint-metric accuracy --maximize-best-checkpoint-metric \ + --task sentence_ranking \ + --num-classes $NUM_CLASSES \ + --init-token 0 --separator-token 2 \ + --max-option-length 128 \ + --max-positions 512 \ + --shorten-method "truncate" \ + --arch roberta_large \ + --dropout 0.1 --attention-dropout 0.1 --weight-decay 0.01 \ + --criterion sentence_ranking \ + --optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-06 \ + --clip-norm 0.0 \ + --lr-scheduler fixed --lr $LR \ + --fp16 --fp16-init-scale 4 --threshold-loss-scale 1 --fp16-scale-window 128 \ + --batch-size $MAX_SENTENCES \ + --required-batch-size-multiple 1 \ + --update-freq $UPDATE_FREQ \ + --max-epoch $MAX_EPOCH +``` + +**Note:** + +a) As contexts in RACE are relatively long, we are using smaller batch size per GPU while increasing update-freq to achieve larger effective batch size. + +b) Above cmd-args and hyperparams are tested on one Nvidia `V100` GPU with `32gb` of memory for each task. Depending on the GPU memory resources available to you, you can use increase `--update-freq` and reduce `--batch-size`. + +c) The setting in above command is based on our hyperparam search within a fixed search space (for careful comparison across models). You might be able to find better metrics with wider hyperparam search. + +### 4) Evaluation: + +``` +DATA_DIR=/path/to/race-output-dir # data directory used during training +MODEL_PATH=/path/to/checkpoint_best.pt # path to the finetuned model checkpoint +PREDS_OUT=preds.tsv # output file path to save prediction +TEST_SPLIT=test # can be test (Middle) or test1 (High) +fairseq-validate \ + $DATA_DIR \ + --valid-subset $TEST_SPLIT \ + --path $MODEL_PATH \ + --batch-size 1 \ + --task sentence_ranking \ + --criterion sentence_ranking \ + --save-predictions $PREDS_OUT +``` diff --git a/data/fairseq/examples/roberta/commonsense_qa/README.md b/data/fairseq/examples/roberta/commonsense_qa/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7f386decd87d93bf701e2e313c7fea39d982224f --- /dev/null +++ b/data/fairseq/examples/roberta/commonsense_qa/README.md @@ -0,0 +1,99 @@ +# Finetuning RoBERTa on Commonsense QA + +We follow a similar approach to [finetuning RACE](../README.race.md). Specifically +for each question we construct five inputs, one for each of the five candidate +answer choices. Each input is constructed by concatenating the question and +candidate answer. We then encode each input and pass the resulting "[CLS]" +representations through a fully-connected layer to predict the correct answer. +We train with a standard cross-entropy loss. + +We also found it helpful to prepend a prefix of `Q:` to the question and `A:` to +the answer. The complete input format is: +``` +<s> Q: Where would I not want a fox? </s> A: hen house </s> +``` + +Our final submission is based on a hyperparameter search over the learning rate +(1e-5, 2e-5, 3e-5), batch size (8, 16), number of training steps (2000, 3000, +4000) and random seed. We selected the model with the best performance on the +development set after 100 trials. + +### 1) Download data from the Commonsense QA website (https://www.tau-nlp.org/commonsenseqa) +```bash +bash examples/roberta/commonsense_qa/download_cqa_data.sh +``` + +### 2) Finetune + +```bash +MAX_UPDATES=3000 # Number of training steps. +WARMUP_UPDATES=150 # Linearly increase LR over this many steps. +LR=1e-05 # Peak LR for polynomial LR scheduler. +MAX_SENTENCES=16 # Batch size. +SEED=1 # Random seed. +ROBERTA_PATH=/path/to/roberta/model.pt +DATA_DIR=data/CommonsenseQA + +# we use the --user-dir option to load the task from +# the examples/roberta/commonsense_qa directory: +FAIRSEQ_PATH=/path/to/fairseq +FAIRSEQ_USER_DIR=${FAIRSEQ_PATH}/examples/roberta/commonsense_qa + +CUDA_VISIBLE_DEVICES=0 fairseq-train --fp16 --ddp-backend=legacy_ddp \ + $DATA_DIR \ + --user-dir $FAIRSEQ_USER_DIR \ + --restore-file $ROBERTA_PATH \ + --reset-optimizer --reset-dataloader --reset-meters \ + --no-epoch-checkpoints --no-last-checkpoints --no-save-optimizer-state \ + --best-checkpoint-metric accuracy --maximize-best-checkpoint-metric \ + --task commonsense_qa --init-token 0 --bpe gpt2 \ + --arch roberta_large --max-positions 512 \ + --dropout 0.1 --attention-dropout 0.1 --weight-decay 0.01 \ + --criterion sentence_ranking --num-classes 5 \ + --optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-06 --clip-norm 0.0 \ + --lr-scheduler polynomial_decay --lr $LR \ + --warmup-updates $WARMUP_UPDATES --total-num-update $MAX_UPDATES \ + --batch-size $MAX_SENTENCES \ + --max-update $MAX_UPDATES \ + --log-format simple --log-interval 25 \ + --seed $SEED +``` + +The above command assumes training on 1 GPU with 32GB of RAM. For GPUs with +less memory, decrease `--batch-size` and increase `--update-freq` +accordingly to compensate. + +### 3) Evaluate +```python +import json +import torch +from fairseq.models.roberta import RobertaModel +from examples.roberta import commonsense_qa # load the Commonsense QA task +roberta = RobertaModel.from_pretrained('checkpoints', 'checkpoint_best.pt', 'data/CommonsenseQA') +roberta.eval() # disable dropout +roberta.cuda() # use the GPU (optional) +nsamples, ncorrect = 0, 0 +with open('data/CommonsenseQA/valid.jsonl') as h: + for line in h: + example = json.loads(line) + scores = [] + for choice in example['question']['choices']: + input = roberta.encode( + 'Q: ' + example['question']['stem'], + 'A: ' + choice['text'], + no_separator=True + ) + score = roberta.predict('sentence_classification_head', input, return_logits=True) + scores.append(score) + pred = torch.cat(scores).argmax() + answer = ord(example['answerKey']) - ord('A') + nsamples += 1 + if pred == answer: + ncorrect += 1 + +print('Accuracy: ' + str(ncorrect / float(nsamples))) +# Accuracy: 0.7846027846027847 +``` + +The above snippet is not batched, which makes it quite slow. See [instructions +for batched prediction with RoBERTa](https://github.com/pytorch/fairseq/tree/main/examples/roberta#batched-prediction). diff --git a/data/fairseq/examples/roberta/commonsense_qa/__init__.py b/data/fairseq/examples/roberta/commonsense_qa/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..42d21f35eb3dd33a053dcf0edd5eadd2dff11294 --- /dev/null +++ b/data/fairseq/examples/roberta/commonsense_qa/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from . import commonsense_qa_task # noqa diff --git a/data/fairseq/examples/roberta/commonsense_qa/commonsense_qa_task.py b/data/fairseq/examples/roberta/commonsense_qa/commonsense_qa_task.py new file mode 100644 index 0000000000000000000000000000000000000000..7d8f8131b38772d0018dbbb3524325bcc915120d --- /dev/null +++ b/data/fairseq/examples/roberta/commonsense_qa/commonsense_qa_task.py @@ -0,0 +1,190 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import json +import os + +import numpy as np +import torch +from fairseq.data import ( + Dictionary, + IdDataset, + ListDataset, + NestedDictionaryDataset, + NumelDataset, + NumSamplesDataset, + RawLabelDataset, + RightPadDataset, + SortDataset, + data_utils, + encoders, +) +from fairseq.tasks import LegacyFairseqTask, register_task + + +@register_task("commonsense_qa") +class CommonsenseQATask(LegacyFairseqTask): + """Task to finetune RoBERTa for Commonsense QA.""" + + @staticmethod + def add_args(parser): + """Add task-specific arguments to the parser.""" + parser.add_argument( + "data", metavar="DIR", help="path to data directory; we load <split>.jsonl" + ) + parser.add_argument( + "--init-token", + type=int, + default=None, + help="add token at the beginning of each batch item", + ) + parser.add_argument("--num-classes", type=int, default=5) + + def __init__(self, args, vocab): + super().__init__(args) + self.vocab = vocab + self.mask = vocab.add_symbol("<mask>") + + self.bpe = encoders.build_bpe(args) + + @classmethod + def load_dictionary(cls, filename): + """Load the dictionary from the filename + + Args: + filename (str): the filename + """ + dictionary = Dictionary.load(filename) + dictionary.add_symbol("<mask>") + return dictionary + + @classmethod + def setup_task(cls, args, **kwargs): + assert ( + args.criterion == "sentence_ranking" + ), "Must set --criterion=sentence_ranking" + + # load data and label dictionaries + vocab = cls.load_dictionary(os.path.join(args.data, "dict.txt")) + print("| dictionary: {} types".format(len(vocab))) + + return cls(args, vocab) + + def load_dataset( + self, split, epoch=1, combine=False, data_path=None, return_only=False, **kwargs + ): + """Load a given dataset split. + + Args: + split (str): name of the split (e.g., train, valid, test) + """ + + def binarize(s, append_bos=False): + if self.bpe is not None: + s = self.bpe.encode(s) + tokens = self.vocab.encode_line( + s, + append_eos=True, + add_if_not_exist=False, + ).long() + if append_bos and self.args.init_token is not None: + tokens = torch.cat([tokens.new([self.args.init_token]), tokens]) + return tokens + + if data_path is None: + data_path = os.path.join(self.args.data, split + ".jsonl") + if not os.path.exists(data_path): + raise FileNotFoundError("Cannot find data: {}".format(data_path)) + + src_tokens = [[] for i in range(self.args.num_classes)] + src_lengths = [[] for i in range(self.args.num_classes)] + labels = [] + + with open(data_path) as h: + for line in h: + example = json.loads(line.strip()) + if "answerKey" in example: + label = ord(example["answerKey"]) - ord("A") + labels.append(label) + question = example["question"]["stem"] + assert len(example["question"]["choices"]) == self.args.num_classes + # format: `<s> Q: Where would I not want a fox? </s> A: hen house </s>` + question = "Q: " + question + question_toks = binarize(question, append_bos=True) + for i, choice in enumerate(example["question"]["choices"]): + src = "A: " + choice["text"] + src_bin = torch.cat([question_toks, binarize(src)]) + src_tokens[i].append(src_bin) + src_lengths[i].append(len(src_bin)) + assert all( + len(src_tokens[0]) == len(src_tokens[i]) + for i in range(self.args.num_classes) + ) + assert len(src_tokens[0]) == len(src_lengths[0]) + assert len(labels) == 0 or len(labels) == len(src_tokens[0]) + + for i in range(self.args.num_classes): + src_lengths[i] = np.array(src_lengths[i]) + src_tokens[i] = ListDataset(src_tokens[i], src_lengths[i]) + src_lengths[i] = ListDataset(src_lengths[i]) + + dataset = { + "id": IdDataset(), + "nsentences": NumSamplesDataset(), + "ntokens": NumelDataset(src_tokens[0], reduce=True), + } + + for i in range(self.args.num_classes): + dataset.update( + { + "net_input{}".format(i + 1): { + "src_tokens": RightPadDataset( + src_tokens[i], + pad_idx=self.source_dictionary.pad(), + ), + "src_lengths": src_lengths[i], + } + } + ) + + if len(labels) > 0: + dataset.update({"target": RawLabelDataset(labels)}) + + dataset = NestedDictionaryDataset( + dataset, + sizes=[np.maximum.reduce([src_token.sizes for src_token in src_tokens])], + ) + + with data_utils.numpy_seed(self.args.seed): + dataset = SortDataset( + dataset, + # shuffle + sort_order=[np.random.permutation(len(dataset))], + ) + + print("| Loaded {} with {} samples".format(split, len(dataset))) + + self.datasets[split] = dataset + return self.datasets[split] + + def build_model(self, args, from_checkpoint=False): + from fairseq import models + + model = models.build_model(args, self) + + model.register_classification_head( + "sentence_classification_head", + num_classes=1, + ) + + return model + + @property + def source_dictionary(self): + return self.vocab + + @property + def target_dictionary(self): + return self.vocab diff --git a/data/fairseq/examples/roberta/commonsense_qa/download_cqa_data.sh b/data/fairseq/examples/roberta/commonsense_qa/download_cqa_data.sh new file mode 100644 index 0000000000000000000000000000000000000000..5f300093fa0a0feb819d8b6aed307b59e3891d01 --- /dev/null +++ b/data/fairseq/examples/roberta/commonsense_qa/download_cqa_data.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +OUTDIR=data/CommonsenseQA + +mkdir -p $OUTDIR + +wget -O $OUTDIR/train.jsonl https://s3.amazonaws.com/commensenseqa/train_rand_split.jsonl +wget -O $OUTDIR/valid.jsonl https://s3.amazonaws.com/commensenseqa/dev_rand_split.jsonl +wget -O $OUTDIR/test.jsonl https://s3.amazonaws.com/commensenseqa/test_rand_split_no_answers.jsonl +wget -O $OUTDIR/dict.txt https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt diff --git a/data/fairseq/examples/roberta/config/finetuning/cola.yaml b/data/fairseq/examples/roberta/config/finetuning/cola.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac76611201275fcee6311b625599ea0863c92898 --- /dev/null +++ b/data/fairseq/examples/roberta/config/finetuning/cola.yaml @@ -0,0 +1,59 @@ +# @package _group_ + +common: + fp16: true + fp16_init_scale: 4 + threshold_loss_scale: 1 + fp16_scale_window: 128 + log_format: json + log_interval: 200 + +task: + _name: sentence_prediction + data: ??? + init_token: 0 + separator_token: 2 + num_classes: 2 + max_positions: 512 + +checkpoint: + restore_file: ??? + reset_optimizer: true + reset_dataloader: true + reset_meters: true + best_checkpoint_metric: accuracy + maximize_best_checkpoint_metric: true + no_epoch_checkpoints: true + +distributed_training: + find_unused_parameters: true + distributed_world_size: 1 + +criterion: + _name: sentence_prediction + +dataset: + batch_size: 16 + required_batch_size_multiple: 1 + max_tokens: 4400 + +optimizer: + _name: adam + weight_decay: 0.1 + adam_betas: (0.9,0.98) + adam_eps: 1e-06 + +lr_scheduler: + _name: polynomial_decay + warmup_updates: 320 + +optimization: + clip_norm: 0.0 + lr: [1e-05] + max_update: 5336 + max_epoch: 10 + +model: + _name: roberta + dropout: 0.1 + attention_dropout: 0.1 diff --git a/data/fairseq/examples/roberta/config/finetuning/mnli.yaml b/data/fairseq/examples/roberta/config/finetuning/mnli.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5be10c362fdadae49e5a6018ef74095892903914 --- /dev/null +++ b/data/fairseq/examples/roberta/config/finetuning/mnli.yaml @@ -0,0 +1,59 @@ +# @package _group_ + +common: + fp16: true + fp16_init_scale: 4 + threshold_loss_scale: 1 + fp16_scale_window: 128 + log_format: json + log_interval: 200 + +task: + _name: sentence_prediction + data: ??? + init_token: 0 + separator_token: 2 + num_classes: 3 + max_positions: 512 + +checkpoint: + restore_file: ??? + reset_optimizer: true + reset_dataloader: true + reset_meters: true + best_checkpoint_metric: accuracy + maximize_best_checkpoint_metric: true + no_epoch_checkpoints: true + +distributed_training: + find_unused_parameters: true + distributed_world_size: 1 + +criterion: + _name: sentence_prediction + +dataset: + batch_size: 32 + required_batch_size_multiple: 1 + max_tokens: 4400 + +optimizer: + _name: adam + weight_decay: 0.1 + adam_betas: (0.9,0.98) + adam_eps: 1e-06 + +lr_scheduler: + _name: polynomial_decay + warmup_updates: 7432 + +optimization: + clip_norm: 0.0 + lr: [1e-05] + max_update: 123873 + max_epoch: 10 + +model: + _name: roberta + dropout: 0.1 + attention_dropout: 0.1 diff --git a/data/fairseq/examples/roberta/config/finetuning/mrpc.yaml b/data/fairseq/examples/roberta/config/finetuning/mrpc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa8b7db393ed00dd9b403ba009de70bf18a75309 --- /dev/null +++ b/data/fairseq/examples/roberta/config/finetuning/mrpc.yaml @@ -0,0 +1,59 @@ +# @package _group_ + +common: + fp16: true + fp16_init_scale: 4 + threshold_loss_scale: 1 + fp16_scale_window: 128 + log_format: json + log_interval: 200 + +task: + _name: sentence_prediction + data: ??? + init_token: 0 + separator_token: 2 + num_classes: 2 + max_positions: 512 + +checkpoint: + restore_file: ??? + reset_optimizer: true + reset_dataloader: true + reset_meters: true + best_checkpoint_metric: accuracy + maximize_best_checkpoint_metric: true + no_epoch_checkpoints: true + +distributed_training: + find_unused_parameters: true + distributed_world_size: 1 + +criterion: + _name: sentence_prediction + +dataset: + batch_size: 16 + required_batch_size_multiple: 1 + max_tokens: 4400 + +optimizer: + _name: adam + weight_decay: 0.1 + adam_betas: (0.9,0.98) + adam_eps: 1e-06 + +lr_scheduler: + _name: polynomial_decay + warmup_updates: 137 + +optimization: + clip_norm: 0.0 + lr: [1e-05] + max_update: 2296 + max_epoch: 10 + +model: + _name: roberta + dropout: 0.1 + attention_dropout: 0.1 diff --git a/data/fairseq/examples/roberta/config/finetuning/qnli.yaml b/data/fairseq/examples/roberta/config/finetuning/qnli.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4595b090ee23b74bb3924c09704702c4208e395 --- /dev/null +++ b/data/fairseq/examples/roberta/config/finetuning/qnli.yaml @@ -0,0 +1,59 @@ +# @package _group_ + +common: + fp16: true + fp16_init_scale: 4 + threshold_loss_scale: 1 + fp16_scale_window: 128 + log_format: json + log_interval: 200 + +task: + _name: sentence_prediction + data: ??? + init_token: 0 + separator_token: 2 + num_classes: 2 + max_positions: 512 + +checkpoint: + restore_file: ??? + reset_optimizer: true + reset_dataloader: true + reset_meters: true + best_checkpoint_metric: accuracy + maximize_best_checkpoint_metric: true + no_epoch_checkpoints: true + +distributed_training: + find_unused_parameters: true + distributed_world_size: 1 + +criterion: + _name: sentence_prediction + +dataset: + batch_size: 32 + required_batch_size_multiple: 1 + max_tokens: 4400 + +optimizer: + _name: adam + weight_decay: 0.1 + adam_betas: (0.9,0.98) + adam_eps: 1e-06 + +lr_scheduler: + _name: polynomial_decay + warmup_updates: 1986 + +optimization: + clip_norm: 0.0 + lr: [1e-05] + max_update: 33112 + max_epoch: 10 + +model: + _name: roberta + dropout: 0.1 + attention_dropout: 0.1 diff --git a/data/fairseq/examples/roberta/config/finetuning/qqp.yaml b/data/fairseq/examples/roberta/config/finetuning/qqp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a2b2ed743963af1f558927f226d993c66fbd45c --- /dev/null +++ b/data/fairseq/examples/roberta/config/finetuning/qqp.yaml @@ -0,0 +1,59 @@ +# @package _group_ + +common: + fp16: true + fp16_init_scale: 4 + threshold_loss_scale: 1 + fp16_scale_window: 128 + log_format: json + log_interval: 200 + +task: + _name: sentence_prediction + data: ??? + init_token: 0 + separator_token: 2 + num_classes: 2 + max_positions: 512 + +checkpoint: + restore_file: ??? + reset_optimizer: true + reset_dataloader: true + reset_meters: true + best_checkpoint_metric: accuracy + maximize_best_checkpoint_metric: true + no_epoch_checkpoints: true + +distributed_training: + find_unused_parameters: true + distributed_world_size: 1 + +criterion: + _name: sentence_prediction + +dataset: + batch_size: 32 + required_batch_size_multiple: 1 + max_tokens: 4400 + +optimizer: + _name: adam + weight_decay: 0.1 + adam_betas: (0.9,0.98) + adam_eps: 1e-06 + +lr_scheduler: + _name: polynomial_decay + warmup_updates: 28318 + +optimization: + clip_norm: 0.0 + lr: [1e-05] + max_update: 113272 + max_epoch: 10 + +model: + _name: roberta + dropout: 0.1 + attention_dropout: 0.1 diff --git a/data/fairseq/examples/roberta/config/finetuning/rte.yaml b/data/fairseq/examples/roberta/config/finetuning/rte.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73184650117e5f1ce5ec4542a0076eaf3044c2a3 --- /dev/null +++ b/data/fairseq/examples/roberta/config/finetuning/rte.yaml @@ -0,0 +1,59 @@ +# @package _group_ + +common: + fp16: true + fp16_init_scale: 4 + threshold_loss_scale: 1 + fp16_scale_window: 128 + log_format: json + log_interval: 200 + +task: + _name: sentence_prediction + data: ??? + init_token: 0 + separator_token: 2 + num_classes: 2 + max_positions: 512 + +checkpoint: + restore_file: ??? + reset_optimizer: true + reset_dataloader: true + reset_meters: true + best_checkpoint_metric: accuracy + maximize_best_checkpoint_metric: true + no_epoch_checkpoints: true + +distributed_training: + find_unused_parameters: true + distributed_world_size: 1 + +criterion: + _name: sentence_prediction + +dataset: + batch_size: 16 + required_batch_size_multiple: 1 + max_tokens: 4400 + +optimizer: + _name: adam + weight_decay: 0.1 + adam_betas: (0.9,0.98) + adam_eps: 1e-06 + +lr_scheduler: + _name: polynomial_decay + warmup_updates: 122 + +optimization: + clip_norm: 0.0 + lr: [2e-05] + max_update: 2036 + max_epoch: 10 + +model: + _name: roberta + dropout: 0.1 + attention_dropout: 0.1 diff --git a/data/fairseq/examples/roberta/config/finetuning/run_config/local.yaml b/data/fairseq/examples/roberta/config/finetuning/run_config/local.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45595f9eea7c369a1200d802bfa1883c1bdfe573 --- /dev/null +++ b/data/fairseq/examples/roberta/config/finetuning/run_config/local.yaml @@ -0,0 +1,15 @@ +# @package _global_ +hydra: + sweep: + dir: ${env:PWD}/tmp_dbg/${now:%H-%M-%S} + +distributed_training: + distributed_world_size: 1 + nprocs_per_node: 1 + distributed_port: -1 + +common: + log_interval: 1 + +dataset: + num_workers: 0 diff --git a/data/fairseq/examples/roberta/config/finetuning/run_config/slurm_1g.yaml b/data/fairseq/examples/roberta/config/finetuning/run_config/slurm_1g.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bc21854d406ce8eb004c9bcba4c6b88bec0bcf3 --- /dev/null +++ b/data/fairseq/examples/roberta/config/finetuning/run_config/slurm_1g.yaml @@ -0,0 +1,28 @@ + +# @package _global_ + +hydra: + job: + config: + override_dirname: + kv_sep: '_' + item_sep: '/' + exclude_keys: + - run_config + - distributed_training.distributed_port + sweep: + dir: /checkpoint/${env:USER}/roberta_ft/${env:PREFIX}/${hydra.job.config_name}/${env:SUFFIX} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${hydra.sweep.dir}/submitit + timeout_min: 1000 + cpus_per_task: 8 + gpus_per_node: 1 + tasks_per_node: 1 + mem_gb: 60 + nodes: 1 + name: ${env:PREFIX}_${hydra.job.config_name} + partition: devlab,learnlab,learnfair,scavenge + constraint: volta32gb + max_num_timeout: 30 + exclude: learnfair1381,learnfair5192,learnfair2304 diff --git a/data/fairseq/examples/roberta/config/finetuning/run_config/slurm_1g_aws.yaml b/data/fairseq/examples/roberta/config/finetuning/run_config/slurm_1g_aws.yaml new file mode 100644 index 0000000000000000000000000000000000000000..085391cffa4adc2044f2112c212221a95bb50cd9 --- /dev/null +++ b/data/fairseq/examples/roberta/config/finetuning/run_config/slurm_1g_aws.yaml @@ -0,0 +1,25 @@ +# @package _global_ + +hydra: + job: + config: + override_dirname: + kv_sep: '_' + item_sep: '/' + exclude_keys: + - run_config + - distributed_training.distributed_port + sweep: + dir: /fsx-wav2vec/${env:USER}/roberta_ft/${env:PREFIX}/${hydra.job.config_name}/${env:SUFFIX} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${hydra.sweep.dir}/submitit + timeout_min: 1000 + cpus_per_task: 8 + gpus_per_node: 1 + tasks_per_node: 1 + mem_gb: 0 + nodes: 1 + name: ${env:PREFIX}_${hydra.job.config_name} + partition: learnfair,wav2vec + max_num_timeout: 30 diff --git a/data/fairseq/examples/roberta/config/finetuning/sst_2.yaml b/data/fairseq/examples/roberta/config/finetuning/sst_2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a93ad2f22c4c248f043fc18d345d61e9484ed39e --- /dev/null +++ b/data/fairseq/examples/roberta/config/finetuning/sst_2.yaml @@ -0,0 +1,59 @@ +# @package _group_ + +common: + fp16: true + fp16_init_scale: 4 + threshold_loss_scale: 1 + fp16_scale_window: 128 + log_format: json + log_interval: 200 + +task: + _name: sentence_prediction + data: ??? + init_token: 0 + separator_token: 2 + num_classes: 2 + max_positions: 512 + +checkpoint: + restore_file: ??? + reset_optimizer: true + reset_dataloader: true + reset_meters: true + best_checkpoint_metric: accuracy + maximize_best_checkpoint_metric: true + no_epoch_checkpoints: true + +distributed_training: + find_unused_parameters: true + distributed_world_size: 1 + +criterion: + _name: sentence_prediction + +dataset: + batch_size: 32 + required_batch_size_multiple: 1 + max_tokens: 4400 + +optimizer: + _name: adam + weight_decay: 0.1 + adam_betas: (0.9,0.98) + adam_eps: 1e-06 + +lr_scheduler: + _name: polynomial_decay + warmup_updates: 1256 + +optimization: + clip_norm: 0.0 + lr: [1e-05] + max_update: 20935 + max_epoch: 10 + +model: + _name: roberta + dropout: 0.1 + attention_dropout: 0.1 diff --git a/data/fairseq/examples/roberta/config/finetuning/sts_b.yaml b/data/fairseq/examples/roberta/config/finetuning/sts_b.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d495221ad846162c0b3f15ea6e17d723e7ea754 --- /dev/null +++ b/data/fairseq/examples/roberta/config/finetuning/sts_b.yaml @@ -0,0 +1,58 @@ +# @package _group_ + +common: + fp16: true + fp16_init_scale: 4 + threshold_loss_scale: 1 + fp16_scale_window: 128 + log_format: json + log_interval: 200 + +task: + _name: sentence_prediction + data: ??? + init_token: 0 + separator_token: 2 + num_classes: 1 + max_positions: 512 + +checkpoint: + restore_file: ??? + reset_optimizer: true + reset_dataloader: true + reset_meters: true + no_epoch_checkpoints: true + +distributed_training: + find_unused_parameters: true + distributed_world_size: 1 + +criterion: + _name: sentence_prediction + regression_target: true + +dataset: + batch_size: 16 + required_batch_size_multiple: 1 + max_tokens: 4400 + +optimizer: + _name: adam + weight_decay: 0.1 + adam_betas: (0.9,0.98) + adam_eps: 1e-06 + +lr_scheduler: + _name: polynomial_decay + warmup_updates: 214 + +optimization: + clip_norm: 0.0 + lr: [2e-05] + max_update: 3598 + max_epoch: 10 + +model: + _name: roberta + dropout: 0.1 + attention_dropout: 0.1 diff --git a/data/fairseq/examples/roberta/config/pretraining/base.yaml b/data/fairseq/examples/roberta/config/pretraining/base.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97829908f740ba6813c895aa32019cc2760c1eb8 --- /dev/null +++ b/data/fairseq/examples/roberta/config/pretraining/base.yaml @@ -0,0 +1,42 @@ +# @package _group_ +common: + fp16: true + log_format: json + log_interval: 200 + +checkpoint: + no_epoch_checkpoints: true + +task: + _name: masked_lm + data: ??? + sample_break_mode: complete + tokens_per_sample: 512 + +criterion: masked_lm + +dataset: + batch_size: 16 + ignore_unused_valid_subsets: true + +optimizer: + _name: adam + weight_decay: 0.01 + adam_betas: (0.9,0.98) + adam_eps: 1e-06 + +lr_scheduler: + _name: polynomial_decay + warmup_updates: 10000 + +optimization: + clip_norm: 0 + lr: [0.0005] + max_update: 125000 + update_freq: [16] + +model: + _name: roberta + max_positions: 512 + dropout: 0.1 + attention_dropout: 0.1 diff --git a/data/fairseq/examples/roberta/config/pretraining/run_config/local.yaml b/data/fairseq/examples/roberta/config/pretraining/run_config/local.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45595f9eea7c369a1200d802bfa1883c1bdfe573 --- /dev/null +++ b/data/fairseq/examples/roberta/config/pretraining/run_config/local.yaml @@ -0,0 +1,15 @@ +# @package _global_ +hydra: + sweep: + dir: ${env:PWD}/tmp_dbg/${now:%H-%M-%S} + +distributed_training: + distributed_world_size: 1 + nprocs_per_node: 1 + distributed_port: -1 + +common: + log_interval: 1 + +dataset: + num_workers: 0 diff --git a/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_2.yaml b/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..006a0f2116db5a85da558cada65873f1150eb717 --- /dev/null +++ b/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_2.yaml @@ -0,0 +1,37 @@ +# @package _global_ + +hydra: + job: + config: + override_dirname: + kv_sep: ':' + item_sep: '/' + exclude_keys: + - run_config + - distributed_training.distributed_port + - distributed_training.distributed_world_size + - model.pretrained_model_path + - model.target_network_path + - next_script + - task.cache_in_scratch + - task.data + - checkpoint.save_interval_updates + - checkpoint.keep_interval_updates + - checkpoint.save_on_overflow + - common.log_interval + - common.user_dir + sweep: + dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname} + subdir: '' + launcher: + submitit_folder: ${hydra.sweep.dir} + timeout_min: 4320 + cpus_per_task: 80 + gpus_per_node: 8 + tasks_per_node: 1 + mem_gb: 450 + nodes: 2 + name: ${env:PREFIX}_${hydra.job.config_name} + partition: devlab,learnlab,learnfair,scavenge + constraint: volta32gb,ib4 + max_num_timeout: 30 diff --git a/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_2_aws.yaml b/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_2_aws.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5937ea5a825558cb62b618491e013f10a1680a7 --- /dev/null +++ b/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_2_aws.yaml @@ -0,0 +1,39 @@ +# @package _global_ + +hydra: + job: + config: + override_dirname: + kv_sep: ':' + item_sep: '/' + exclude_keys: + - run_config + - distributed_training.distributed_port + - distributed_training.distributed_world_size + - model.pretrained_model_path + - model.target_network_path + - next_script + - task.cache_in_scratch + - task.local_cache_path + - task.data + - task.post_save_script + - checkpoint.save_interval_updates + - checkpoint.keep_interval_updates + - checkpoint.save_on_overflow + - common.log_interval + - common.user_dir + - model.model_path + sweep: + dir: /fsx-wav2vec/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname} + subdir: '' + launcher: + submitit_folder: ${hydra.sweep.dir} + timeout_min: 4320 + cpus_per_task: 10 + gpus_per_node: 8 + tasks_per_node: 8 + mem_gb: 0 + nodes: 2 + name: ${env:PREFIX}_${hydra.job.config_name} + partition: wav2vec + max_num_timeout: 30 diff --git a/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_3.yaml b/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e1555d20f9380b9d82d5852be6799d35fa3d078 --- /dev/null +++ b/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_3.yaml @@ -0,0 +1,36 @@ +# @package _global_ + +hydra: + job: + config: + override_dirname: + kv_sep: ':' + item_sep: '/' + exclude_keys: + - run_config + - distributed_training.distributed_port + - distributed_training.distributed_world_size + - model.pretrained_model_path + - model.target_network_path + - next_script + - task.cache_in_scratch + - task.data + - checkpoint.save_interval_updates + - checkpoint.keep_interval_updates + - checkpoint.save_on_overflow + - common.log_interval + sweep: + dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname} + subdir: '' + launcher: + submitit_folder: ${hydra.sweep.dir} + timeout_min: 4320 + cpus_per_task: 10 + gpus_per_node: 8 + tasks_per_node: 8 + mem_gb: 450 + nodes: 3 + name: ${env:PREFIX}_${hydra.job.config_name} + partition: devlab,learnlab,learnfair,scavenge + constraint: volta32gb,ib4 + max_num_timeout: 30 diff --git a/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_4.yaml b/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c54d735fb2dc0b0af2d467caa7f64405af18ea10 --- /dev/null +++ b/data/fairseq/examples/roberta/config/pretraining/run_config/slurm_4.yaml @@ -0,0 +1,36 @@ +# @package _global_ + +hydra: + job: + config: + override_dirname: + kv_sep: ':' + item_sep: '/' + exclude_keys: + - run_config + - distributed_training.distributed_port + - distributed_training.distributed_world_size + - model.pretrained_model_path + - model.target_network_path + - next_script + - task.cache_in_scratch + - task.data + - checkpoint.save_interval_updates + - checkpoint.keep_interval_updates + - checkpoint.save_on_overflow + - common.log_interval + sweep: + dir: /checkpoint/${env:USER}/${env:PREFIX}/${hydra.job.config_name}_${hydra.launcher.gpus_per_node}/${hydra.job.override_dirname} + subdir: '' + launcher: + submitit_folder: ${hydra.sweep.dir} + timeout_min: 4320 + cpus_per_task: 10 + gpus_per_node: 8 + tasks_per_node: 8 + mem_gb: 450 + nodes: 4 + name: ${env:PREFIX}_${hydra.job.config_name} + partition: devlab,learnlab,learnfair,scavenge + constraint: volta32gb,ib4 + max_num_timeout: 30 diff --git a/data/fairseq/examples/roberta/fb_multilingual/README.multilingual.pretraining.md b/data/fairseq/examples/roberta/fb_multilingual/README.multilingual.pretraining.md new file mode 100644 index 0000000000000000000000000000000000000000..234fd747085d8fa864be2749e4acb11147394bdf --- /dev/null +++ b/data/fairseq/examples/roberta/fb_multilingual/README.multilingual.pretraining.md @@ -0,0 +1,26 @@ +# Multilingual pretraining RoBERTa + +This tutorial will walk you through pretraining multilingual RoBERTa. + +### 1) Preprocess the data + +```bash +DICTIONARY="/private/home/namangoyal/dataset/XLM/wiki/17/175k/vocab" +DATA_LOCATION="/private/home/namangoyal/dataset/XLM/wiki/17/175k" + +for LANG in en es it +do + fairseq-preprocess \ + --only-source \ + --srcdict $DICTIONARY \ + --trainpref "$DATA_LOCATION/train.$LANG" \ + --validpref "$DATA_LOCATION/valid.$LANG" \ + --testpref "$DATA_LOCATION/test.$LANG" \ + --destdir "wiki_17-bin/$LANG" \ + --workers 60; +done +``` + +### 2) Train RoBERTa base + +[COMING UP...] diff --git a/data/fairseq/examples/roberta/multiprocessing_bpe_encoder.py b/data/fairseq/examples/roberta/multiprocessing_bpe_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..43fe0451bf4d5762d734314075b1402c2a8db2bb --- /dev/null +++ b/data/fairseq/examples/roberta/multiprocessing_bpe_encoder.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import argparse +import contextlib +import sys +from collections import Counter +from multiprocessing import Pool + +from fairseq.data.encoders.gpt2_bpe import get_encoder + + +def main(): + """ + Helper script to encode raw text with the GPT-2 BPE using multiple processes. + + The encoder.json and vocab.bpe files can be obtained here: + - https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json + - https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--encoder-json", + help="path to encoder.json", + ) + parser.add_argument( + "--vocab-bpe", + type=str, + help="path to vocab.bpe", + ) + parser.add_argument( + "--inputs", + nargs="+", + default=["-"], + help="input files to filter/encode", + ) + parser.add_argument( + "--outputs", + nargs="+", + default=["-"], + help="path to save encoded outputs", + ) + parser.add_argument( + "--keep-empty", + action="store_true", + help="keep empty lines", + ) + parser.add_argument("--workers", type=int, default=20) + args = parser.parse_args() + + assert len(args.inputs) == len( + args.outputs + ), "number of input and output paths should match" + + with contextlib.ExitStack() as stack: + inputs = [ + stack.enter_context(open(input, "r", encoding="utf-8")) + if input != "-" + else sys.stdin + for input in args.inputs + ] + outputs = [ + stack.enter_context(open(output, "w", encoding="utf-8")) + if output != "-" + else sys.stdout + for output in args.outputs + ] + + encoder = MultiprocessingEncoder(args) + pool = Pool(args.workers, initializer=encoder.initializer) + encoded_lines = pool.imap(encoder.encode_lines, zip(*inputs), 100) + + stats = Counter() + for i, (filt, enc_lines) in enumerate(encoded_lines, start=1): + if filt == "PASS": + for enc_line, output_h in zip(enc_lines, outputs): + print(enc_line, file=output_h) + else: + stats["num_filtered_" + filt] += 1 + if i % 10000 == 0: + print("processed {} lines".format(i), file=sys.stderr) + + for k, v in stats.most_common(): + print("[{}] filtered {} lines".format(k, v), file=sys.stderr) + + +class MultiprocessingEncoder(object): + def __init__(self, args): + self.args = args + + def initializer(self): + global bpe + bpe = get_encoder(self.args.encoder_json, self.args.vocab_bpe) + + def encode(self, line): + global bpe + ids = bpe.encode(line) + return list(map(str, ids)) + + def decode(self, tokens): + global bpe + return bpe.decode(tokens) + + def encode_lines(self, lines): + """ + Encode a set of lines. All lines will be encoded together. + """ + enc_lines = [] + for line in lines: + line = line.strip() + if len(line) == 0 and not self.args.keep_empty: + return ["EMPTY", None] + tokens = self.encode(line) + enc_lines.append(" ".join(tokens)) + return ["PASS", enc_lines] + + def decode_lines(self, lines): + dec_lines = [] + for line in lines: + tokens = map(int, line.strip().split()) + dec_lines.append(self.decode(tokens)) + return ["PASS", dec_lines] + + +if __name__ == "__main__": + main() diff --git a/data/fairseq/examples/roberta/preprocess_GLUE_tasks.sh b/data/fairseq/examples/roberta/preprocess_GLUE_tasks.sh new file mode 100644 index 0000000000000000000000000000000000000000..7f215a3b53e1c4a7b1f0320102915a49d84a5015 --- /dev/null +++ b/data/fairseq/examples/roberta/preprocess_GLUE_tasks.sh @@ -0,0 +1,185 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +# raw glue data as downloaded by glue download script (https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e) +if [[ $# -ne 2 ]]; then + echo "Run as following:" + echo "./examples/roberta/preprocess_GLUE_tasks.sh <glud_data_folder> <task_name>" + exit 1 +fi + +GLUE_DATA_FOLDER=$1 + +# download bpe encoder.json, vocabulary and fairseq dictionary +wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json' +wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe' +wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt' + +TASKS=$2 # QQP + +if [ "$TASKS" = "ALL" ] +then + TASKS="QQP MNLI QNLI MRPC RTE STS-B SST-2 CoLA" +fi + +for TASK in $TASKS +do + echo "Preprocessing $TASK" + + TASK_DATA_FOLDER="$GLUE_DATA_FOLDER/$TASK" + echo "Raw data as downloaded from glue website: $TASK_DATA_FOLDER" + + SPLITS="train dev test" + INPUT_COUNT=2 + if [ "$TASK" = "QQP" ] + then + INPUT_COLUMNS=( 4 5 ) + TEST_INPUT_COLUMNS=( 2 3 ) + LABEL_COLUMN=6 + elif [ "$TASK" = "MNLI" ] + then + SPLITS="train dev_matched dev_mismatched test_matched test_mismatched" + INPUT_COLUMNS=( 9 10 ) + TEST_INPUT_COLUMNS=( 9 10 ) + DEV_LABEL_COLUMN=16 + LABEL_COLUMN=12 + elif [ "$TASK" = "QNLI" ] + then + INPUT_COLUMNS=( 2 3 ) + TEST_INPUT_COLUMNS=( 2 3 ) + LABEL_COLUMN=4 + elif [ "$TASK" = "MRPC" ] + then + INPUT_COLUMNS=( 4 5 ) + TEST_INPUT_COLUMNS=( 4 5 ) + LABEL_COLUMN=1 + elif [ "$TASK" = "RTE" ] + then + INPUT_COLUMNS=( 2 3 ) + TEST_INPUT_COLUMNS=( 2 3 ) + LABEL_COLUMN=4 + elif [ "$TASK" = "STS-B" ] + then + INPUT_COLUMNS=( 8 9 ) + TEST_INPUT_COLUMNS=( 8 9 ) + LABEL_COLUMN=10 + # Following are single sentence tasks. + elif [ "$TASK" = "SST-2" ] + then + INPUT_COLUMNS=( 1 ) + TEST_INPUT_COLUMNS=( 2 ) + LABEL_COLUMN=2 + INPUT_COUNT=1 + elif [ "$TASK" = "CoLA" ] + then + INPUT_COLUMNS=( 4 ) + TEST_INPUT_COLUMNS=( 2 ) + LABEL_COLUMN=2 + INPUT_COUNT=1 + fi + + # Strip out header and filter lines that don't have expected number of fields. + rm -rf "$TASK_DATA_FOLDER/processed" + mkdir -p "$TASK_DATA_FOLDER/processed" + for SPLIT in $SPLITS + do + # CoLA train and dev doesn't have header. + if [[ ( "$TASK" = "CoLA") && ( "$SPLIT" != "test" ) ]] + then + cp "$TASK_DATA_FOLDER/$SPLIT.tsv" "$TASK_DATA_FOLDER/processed/$SPLIT.tsv.temp"; + else + tail -n +2 "$TASK_DATA_FOLDER/$SPLIT.tsv" > "$TASK_DATA_FOLDER/processed/$SPLIT.tsv.temp"; + fi + + # Remove unformatted lines from train and dev files for QQP dataset. + if [[ ( "$TASK" = "QQP") && ( "$SPLIT" != "test" ) ]] + then + awk -F '\t' -v NUM_FIELDS=6 'NF==NUM_FIELDS{print}{}' "$TASK_DATA_FOLDER/processed/$SPLIT.tsv.temp" > "$TASK_DATA_FOLDER/processed/$SPLIT.tsv"; + else + cp "$TASK_DATA_FOLDER/processed/$SPLIT.tsv.temp" "$TASK_DATA_FOLDER/processed/$SPLIT.tsv"; + fi + rm "$TASK_DATA_FOLDER/processed/$SPLIT.tsv.temp"; + done + + # Split into input0, input1 and label + for SPLIT in $SPLITS + do + for INPUT_TYPE in $(seq 0 $((INPUT_COUNT-1))) + do + if [[ "$SPLIT" != test* ]] + then + COLUMN_NUMBER=${INPUT_COLUMNS[$INPUT_TYPE]} + else + COLUMN_NUMBER=${TEST_INPUT_COLUMNS[$INPUT_TYPE]} + fi + cut -f"$COLUMN_NUMBER" "$TASK_DATA_FOLDER/processed/$SPLIT.tsv" > "$TASK_DATA_FOLDER/processed/$SPLIT.raw.input$INPUT_TYPE"; + done + + if [[ "$SPLIT" != test* ]] + then + if [ "$TASK" = "MNLI" ] && [ "$SPLIT" != "train" ] + then + cut -f"$DEV_LABEL_COLUMN" "$TASK_DATA_FOLDER/processed/$SPLIT.tsv" > "$TASK_DATA_FOLDER/processed/$SPLIT.label"; + else + cut -f"$LABEL_COLUMN" "$TASK_DATA_FOLDER/processed/$SPLIT.tsv" > "$TASK_DATA_FOLDER/processed/$SPLIT.label"; + fi + fi + + # BPE encode. + for INPUT_TYPE in $(seq 0 $((INPUT_COUNT-1))) + do + LANG="input$INPUT_TYPE" + echo "BPE encoding $SPLIT/$LANG" + python -m examples.roberta.multiprocessing_bpe_encoder \ + --encoder-json encoder.json \ + --vocab-bpe vocab.bpe \ + --inputs "$TASK_DATA_FOLDER/processed/$SPLIT.raw.$LANG" \ + --outputs "$TASK_DATA_FOLDER/processed/$SPLIT.$LANG" \ + --workers 60 \ + --keep-empty; + done + done + + # Remove output directory. + rm -rf "$TASK-bin" + + DEVPREF="$TASK_DATA_FOLDER/processed/dev.LANG" + TESTPREF="$TASK_DATA_FOLDER/processed/test.LANG" + if [ "$TASK" = "MNLI" ] + then + DEVPREF="$TASK_DATA_FOLDER/processed/dev_matched.LANG,$TASK_DATA_FOLDER/processed/dev_mismatched.LANG" + TESTPREF="$TASK_DATA_FOLDER/processed/test_matched.LANG,$TASK_DATA_FOLDER/processed/test_mismatched.LANG" + fi + + # Run fairseq preprocessing: + for INPUT_TYPE in $(seq 0 $((INPUT_COUNT-1))) + do + LANG="input$INPUT_TYPE" + fairseq-preprocess \ + --only-source \ + --trainpref "$TASK_DATA_FOLDER/processed/train.$LANG" \ + --validpref "${DEVPREF//LANG/$LANG}" \ + --testpref "${TESTPREF//LANG/$LANG}" \ + --destdir "$TASK-bin/$LANG" \ + --workers 60 \ + --srcdict dict.txt; + done + if [[ "$TASK" != "STS-B" ]] + then + fairseq-preprocess \ + --only-source \ + --trainpref "$TASK_DATA_FOLDER/processed/train.label" \ + --validpref "${DEVPREF//LANG/label}" \ + --destdir "$TASK-bin/label" \ + --workers 60; + else + # For STS-B output range is converted to be between: [0.0, 1.0] + mkdir -p "$TASK-bin/label" + awk '{print $1 / 5.0 }' "$TASK_DATA_FOLDER/processed/train.label" > "$TASK-bin/label/train.label" + awk '{print $1 / 5.0 }' "$TASK_DATA_FOLDER/processed/dev.label" > "$TASK-bin/label/valid.label" + fi +done diff --git a/data/fairseq/examples/roberta/preprocess_RACE.py b/data/fairseq/examples/roberta/preprocess_RACE.py new file mode 100644 index 0000000000000000000000000000000000000000..cdd66072718ccb6033304c97926271909a17f9d6 --- /dev/null +++ b/data/fairseq/examples/roberta/preprocess_RACE.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +import argparse +import json +import os +import re + + +class InputExample: + def __init__(self, paragraph, qa_list, label): + self.paragraph = paragraph + self.qa_list = qa_list + self.label = label + + +def get_examples(data_dir, set_type): + """ + Extract paragraph and question-answer list from each json file + """ + examples = [] + + levels = ["middle", "high"] + set_type_c = set_type.split("-") + if len(set_type_c) == 2: + levels = [set_type_c[1]] + set_type = set_type_c[0] + for level in levels: + cur_dir = os.path.join(data_dir, set_type, level) + for filename in os.listdir(cur_dir): + cur_path = os.path.join(cur_dir, filename) + with open(cur_path, "r") as f: + cur_data = json.load(f) + answers = cur_data["answers"] + options = cur_data["options"] + questions = cur_data["questions"] + context = cur_data["article"].replace("\n", " ") + context = re.sub(r"\s+", " ", context) + for i in range(len(answers)): + label = ord(answers[i]) - ord("A") + qa_list = [] + question = questions[i] + for j in range(4): + option = options[i][j] + if "_" in question: + qa_cat = question.replace("_", option) + else: + qa_cat = " ".join([question, option]) + qa_cat = re.sub(r"\s+", " ", qa_cat) + qa_list.append(qa_cat) + examples.append(InputExample(context, qa_list, label)) + + return examples + + +def main(): + """ + Helper script to extract paragraphs questions and answers from RACE datasets. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--input-dir", + help="input directory for downloaded RACE dataset", + ) + parser.add_argument( + "--output-dir", + help="output directory for extracted data", + ) + args = parser.parse_args() + + if not os.path.exists(args.output_dir): + os.makedirs(args.output_dir, exist_ok=True) + + for set_type in ["train", "dev", "test-middle", "test-high"]: + examples = get_examples(args.input_dir, set_type) + qa_file_paths = [ + os.path.join(args.output_dir, set_type + ".input" + str(i + 1)) + for i in range(4) + ] + qa_files = [open(qa_file_path, "w") for qa_file_path in qa_file_paths] + outf_context_path = os.path.join(args.output_dir, set_type + ".input0") + outf_label_path = os.path.join(args.output_dir, set_type + ".label") + outf_context = open(outf_context_path, "w") + outf_label = open(outf_label_path, "w") + for example in examples: + outf_context.write(example.paragraph + "\n") + for i in range(4): + qa_files[i].write(example.qa_list[i] + "\n") + outf_label.write(str(example.label) + "\n") + + for f in qa_files: + f.close() + outf_label.close() + outf_context.close() + + +if __name__ == "__main__": + main() diff --git a/data/fairseq/examples/roberta/preprocess_RACE.sh b/data/fairseq/examples/roberta/preprocess_RACE.sh new file mode 100644 index 0000000000000000000000000000000000000000..932d2ab6e521fecc7d0297f26a8c43857541ef3b --- /dev/null +++ b/data/fairseq/examples/roberta/preprocess_RACE.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +# data should be downloaded and processed with reprocess_RACE.py +if [[ $# -ne 2 ]]; then + echo "Run as following:" + echo "./examples/roberta/preprocess_RACE.sh <race_data_folder> <output_folder>" + exit 1 +fi + +RACE_DATA_FOLDER=$1 +OUT_DATA_FOLDER=$2 + +# download bpe encoder.json, vocabulary and fairseq dictionary +wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json' +wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe' +wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt' + +SPLITS="train dev test-middle test-high" +INPUT_TYPES="input0 input1 input2 input3 input4" +for INPUT_TYPE in $INPUT_TYPES +do + for SPLIT in $SPLITS + do + echo "BPE encoding $SPLIT/$INPUT_TYPE" + python -m examples.roberta.multiprocessing_bpe_encoder \ + --encoder-json encoder.json \ + --vocab-bpe vocab.bpe \ + --inputs "$RACE_DATA_FOLDER/$SPLIT.$INPUT_TYPE" \ + --outputs "$RACE_DATA_FOLDER/$SPLIT.$INPUT_TYPE.bpe" \ + --workers 10 \ + --keep-empty; + + done +done + +for INPUT_TYPE in $INPUT_TYPES + do + LANG="input$INPUT_TYPE" + fairseq-preprocess \ + --only-source \ + --trainpref "$RACE_DATA_FOLDER/train.$INPUT_TYPE.bpe" \ + --validpref "$RACE_DATA_FOLDER/dev.$INPUT_TYPE.bpe" \ + --testpref "$RACE_DATA_FOLDER/test-middle.$INPUT_TYPE.bpe,$RACE_DATA_FOLDER/test-high.$INPUT_TYPE.bpe" \ + --destdir "$OUT_DATA_FOLDER/$INPUT_TYPE" \ + --workers 10 \ + --srcdict dict.txt; +done + +rm -rf "$OUT_DATA_FOLDER/label" +mkdir -p "$OUT_DATA_FOLDER/label" +cp "$RACE_DATA_FOLDER/train.label" "$OUT_DATA_FOLDER/label/" +cp "$RACE_DATA_FOLDER/dev.label" "$OUT_DATA_FOLDER/label/valid.label" +cp "$RACE_DATA_FOLDER/test-middle.label" "$OUT_DATA_FOLDER/label/test.label" +cp "$RACE_DATA_FOLDER/test-high.label" "$OUT_DATA_FOLDER/label/test1.label" diff --git a/data/fairseq/examples/roberta/wsc/README.md b/data/fairseq/examples/roberta/wsc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..21a045d999739836a17574593292e42131315ae9 --- /dev/null +++ b/data/fairseq/examples/roberta/wsc/README.md @@ -0,0 +1,125 @@ +# Finetuning RoBERTa on Winograd Schema Challenge (WSC) data + +The following instructions can be used to finetune RoBERTa on the WSC training +data provided by [SuperGLUE](https://super.gluebenchmark.com/). + +Note that there is high variance in the results. For our GLUE/SuperGLUE +submission we swept over the learning rate (1e-5, 2e-5, 3e-5), batch size (16, +32, 64) and total number of updates (500, 1000, 2000, 3000), as well as the +random seed. Out of ~100 runs we chose the best 7 models and ensembled them. + +**Approach:** The instructions below use a slightly different loss function than +what's described in the original RoBERTa arXiv paper. In particular, +[Kocijan et al. (2019)](https://arxiv.org/abs/1905.06290) introduce a margin +ranking loss between `(query, candidate)` pairs with tunable hyperparameters +alpha and beta. This is supported in our code as well with the `--wsc-alpha` and +`--wsc-beta` arguments. However, we achieved slightly better (and more robust) +results on the development set by instead using a single cross entropy loss term +over the log-probabilities for the query and all mined candidates. **The +candidates are mined using spaCy from each input sentence in isolation, so the +approach remains strictly pointwise.** This reduces the number of +hyperparameters and our best model achieved 92.3% development set accuracy, +compared to ~90% accuracy for the margin loss. Later versions of the RoBERTa +arXiv paper will describe this updated formulation. + +### 1) Download the WSC data from the SuperGLUE website: +```bash +wget https://dl.fbaipublicfiles.com/glue/superglue/data/v2/WSC.zip +unzip WSC.zip + +# we also need to copy the RoBERTa dictionary into the same directory +wget -O WSC/dict.txt https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt +``` + +### 2) Finetune over the provided training data: +```bash +TOTAL_NUM_UPDATES=2000 # Total number of training steps. +WARMUP_UPDATES=250 # Linearly increase LR over this many steps. +LR=2e-05 # Peak LR for polynomial LR scheduler. +MAX_SENTENCES=16 # Batch size per GPU. +SEED=1 # Random seed. +ROBERTA_PATH=/path/to/roberta/model.pt + +# we use the --user-dir option to load the task and criterion +# from the examples/roberta/wsc directory: +FAIRSEQ_PATH=/path/to/fairseq +FAIRSEQ_USER_DIR=${FAIRSEQ_PATH}/examples/roberta/wsc + +CUDA_VISIBLE_DEVICES=0,1,2,3 fairseq-train WSC/ \ + --restore-file $ROBERTA_PATH \ + --reset-optimizer --reset-dataloader --reset-meters \ + --no-epoch-checkpoints --no-last-checkpoints --no-save-optimizer-state \ + --best-checkpoint-metric accuracy --maximize-best-checkpoint-metric \ + --valid-subset val \ + --fp16 --ddp-backend legacy_ddp \ + --user-dir $FAIRSEQ_USER_DIR \ + --task wsc --criterion wsc --wsc-cross-entropy \ + --arch roberta_large --bpe gpt2 --max-positions 512 \ + --dropout 0.1 --attention-dropout 0.1 --weight-decay 0.01 \ + --optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-06 \ + --lr-scheduler polynomial_decay --lr $LR \ + --warmup-updates $WARMUP_UPDATES --total-num-update $TOTAL_NUM_UPDATES \ + --batch-size $MAX_SENTENCES \ + --max-update $TOTAL_NUM_UPDATES \ + --log-format simple --log-interval 100 \ + --seed $SEED +``` + +The above command assumes training on 4 GPUs, but you can achieve the same +results on a single GPU by adding `--update-freq=4`. + +### 3) Evaluate +```python +from fairseq.models.roberta import RobertaModel +from examples.roberta.wsc import wsc_utils # also loads WSC task and criterion +roberta = RobertaModel.from_pretrained('checkpoints', 'checkpoint_best.pt', 'WSC/') +roberta.cuda() +nsamples, ncorrect = 0, 0 +for sentence, label in wsc_utils.jsonl_iterator('WSC/val.jsonl', eval=True): + pred = roberta.disambiguate_pronoun(sentence) + nsamples += 1 + if pred == label: + ncorrect += 1 +print('Accuracy: ' + str(ncorrect / float(nsamples))) +# Accuracy: 0.9230769230769231 +``` + +## RoBERTa training on WinoGrande dataset +We have also provided `winogrande` task and criterion for finetuning on the +[WinoGrande](https://mosaic.allenai.org/projects/winogrande) like datasets +where there are always two candidates and one is correct. +It's more efficient implementation for such subcases. + +```bash +TOTAL_NUM_UPDATES=23750 # Total number of training steps. +WARMUP_UPDATES=2375 # Linearly increase LR over this many steps. +LR=1e-05 # Peak LR for polynomial LR scheduler. +MAX_SENTENCES=32 # Batch size per GPU. +SEED=1 # Random seed. +ROBERTA_PATH=/path/to/roberta/model.pt + +# we use the --user-dir option to load the task and criterion +# from the examples/roberta/wsc directory: +FAIRSEQ_PATH=/path/to/fairseq +FAIRSEQ_USER_DIR=${FAIRSEQ_PATH}/examples/roberta/wsc + +cd fairseq +CUDA_VISIBLE_DEVICES=0 fairseq-train winogrande_1.0/ \ + --restore-file $ROBERTA_PATH \ + --reset-optimizer --reset-dataloader --reset-meters \ + --no-epoch-checkpoints --no-last-checkpoints --no-save-optimizer-state \ + --best-checkpoint-metric accuracy --maximize-best-checkpoint-metric \ + --valid-subset val \ + --fp16 --ddp-backend legacy_ddp \ + --user-dir $FAIRSEQ_USER_DIR \ + --task winogrande --criterion winogrande \ + --wsc-margin-alpha 5.0 --wsc-margin-beta 0.4 \ + --arch roberta_large --bpe gpt2 --max-positions 512 \ + --dropout 0.1 --attention-dropout 0.1 --weight-decay 0.01 \ + --optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-06 \ + --lr-scheduler polynomial_decay --lr $LR \ + --warmup-updates $WARMUP_UPDATES --total-num-update $TOTAL_NUM_UPDATES \ + --batch-size $MAX_SENTENCES \ + --max-update $TOTAL_NUM_UPDATES \ + --log-format simple --log-interval 100 +``` diff --git a/data/fairseq/examples/roberta/wsc/__init__.py b/data/fairseq/examples/roberta/wsc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..78afa4728eeed96142900118f6452730023466c9 --- /dev/null +++ b/data/fairseq/examples/roberta/wsc/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from . import wsc_criterion # noqa +from . import wsc_task # noqa diff --git a/data/fairseq/examples/roberta/wsc/wsc_criterion.py b/data/fairseq/examples/roberta/wsc/wsc_criterion.py new file mode 100644 index 0000000000000000000000000000000000000000..ed0251fdecc3573228ad271f1090aaf914b48cd1 --- /dev/null +++ b/data/fairseq/examples/roberta/wsc/wsc_criterion.py @@ -0,0 +1,167 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import math + +import torch +import torch.nn.functional as F +from fairseq import utils +from fairseq.criterions import LegacyFairseqCriterion, register_criterion +from fairseq.data import encoders + + +@register_criterion("wsc") +class WSCCriterion(LegacyFairseqCriterion): + def __init__(self, args, task): + super().__init__(args, task) + if self.args.save_predictions is not None: + self.prediction_h = open(self.args.save_predictions, "w") + else: + self.prediction_h = None + self.bpe = encoders.build_bpe(args.bpe) + self.tokenizer = encoders.build_tokenizer(args.tokenizer) + + def __del__(self): + if self.prediction_h is not None: + self.prediction_h.close() + + @staticmethod + def add_args(parser): + """Add criterion-specific arguments to the parser.""" + parser.add_argument("--wsc-margin-alpha", type=float, metavar="A", default=1.0) + parser.add_argument("--wsc-margin-beta", type=float, metavar="B", default=0.0) + parser.add_argument( + "--wsc-cross-entropy", + action="store_true", + help="use cross entropy formulation instead of margin loss", + ) + parser.add_argument( + "--save-predictions", metavar="FILE", help="file to save predictions to" + ) + + def get_masked_input(self, tokens, mask): + masked_tokens = tokens.clone() + masked_tokens[mask] = self.task.mask + return masked_tokens + + def get_lprobs(self, model, tokens, mask): + logits, _ = model(src_tokens=self.get_masked_input(tokens, mask)) + lprobs = F.log_softmax(logits, dim=-1, dtype=torch.float) + scores = lprobs.gather(2, tokens.unsqueeze(-1)).squeeze(-1) + mask = mask.type_as(scores) + scores = (scores * mask).sum(dim=-1) / mask.sum(dim=-1) + return scores + + def get_loss(self, query_lprobs, cand_lprobs): + if self.args.wsc_cross_entropy: + return F.cross_entropy( + torch.cat([query_lprobs, cand_lprobs]).unsqueeze(0), + query_lprobs.new([0]).long(), + ) + else: + return ( + -query_lprobs + + self.args.wsc_margin_alpha + * (cand_lprobs - query_lprobs + self.args.wsc_margin_beta).clamp(min=0) + ).sum() + + def forward(self, model, sample, reduce=True): + # compute loss and accuracy + loss, nloss = 0.0, 0 + ncorrect, nqueries = 0, 0 + + for i, label in enumerate(sample["labels"]): + query_lprobs = self.get_lprobs( + model, + sample["query_tokens"][i].unsqueeze(0), + sample["query_masks"][i].unsqueeze(0), + ) + cand_lprobs = self.get_lprobs( + model, + sample["candidate_tokens"][i], + sample["candidate_masks"][i], + ) + + pred = (query_lprobs >= cand_lprobs).all().item() + + if label is not None: + label = 1 if label else 0 + ncorrect += 1 if pred == label else 0 + nqueries += 1 + + if label: + # only compute a loss for positive instances + nloss += 1 + loss += self.get_loss(query_lprobs, cand_lprobs) + + id = sample["id"][i].item() + if self.prediction_h is not None: + print("{}\t{}\t{}".format(id, pred, label), file=self.prediction_h) + + if nloss == 0: + loss = torch.tensor(0.0, requires_grad=True) + + sample_size = nqueries if nqueries > 0 else 1 + logging_output = { + "loss": utils.item(loss.data) if reduce else loss.data, + "ntokens": sample["ntokens"], + "nsentences": sample["nsentences"], + "sample_size": sample_size, + "ncorrect": ncorrect, + "nqueries": nqueries, + } + return loss, sample_size, logging_output + + @staticmethod + def aggregate_logging_outputs(logging_outputs): + """Aggregate logging outputs from data parallel training.""" + loss_sum = sum(log.get("loss", 0) for log in logging_outputs) + ntokens = sum(log.get("ntokens", 0) for log in logging_outputs) + nsentences = sum(log.get("nsentences", 0) for log in logging_outputs) + sample_size = sum(log.get("sample_size", 0) for log in logging_outputs) + + agg_output = { + "loss": loss_sum / sample_size / math.log(2), + "ntokens": ntokens, + "nsentences": nsentences, + "sample_size": sample_size, + } + + ncorrect = sum(log.get("ncorrect", 0) for log in logging_outputs) + nqueries = sum(log.get("nqueries", 0) for log in logging_outputs) + if nqueries > 0: + agg_output["accuracy"] = ncorrect / float(nqueries) + + return agg_output + + +@register_criterion("winogrande") +class WinograndeCriterion(WSCCriterion): + def forward(self, model, sample, reduce=True): + # compute loss and accuracy + query_lprobs = self.get_lprobs( + model, + sample["query_tokens"], + sample["query_masks"], + ) + cand_lprobs = self.get_lprobs( + model, + sample["candidate_tokens"], + sample["candidate_masks"], + ) + pred = query_lprobs >= cand_lprobs + loss = self.get_loss(query_lprobs, cand_lprobs) + + sample_size = sample["query_tokens"].size(0) + ncorrect = pred.sum().item() + logging_output = { + "loss": utils.item(loss.data) if reduce else loss.data, + "ntokens": sample["ntokens"], + "nsentences": sample["nsentences"], + "sample_size": sample_size, + "ncorrect": ncorrect, + "nqueries": sample_size, + } + return loss, sample_size, logging_output diff --git a/data/fairseq/examples/roberta/wsc/wsc_task.py b/data/fairseq/examples/roberta/wsc/wsc_task.py new file mode 100644 index 0000000000000000000000000000000000000000..602ea737ed75a33fddf44dd859e999ecfce2730d --- /dev/null +++ b/data/fairseq/examples/roberta/wsc/wsc_task.py @@ -0,0 +1,401 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import json +import os +import tempfile + +import numpy as np +import torch +import torch.nn.functional as F +from fairseq import utils +from fairseq.data import ( + Dictionary, + IdDataset, + ListDataset, + NestedDictionaryDataset, + NumelDataset, + NumSamplesDataset, + PadDataset, + SortDataset, + data_utils, + encoders, +) +from fairseq.tasks import LegacyFairseqTask, register_task + +from . import wsc_utils + + +@register_task("wsc") +class WSCTask(LegacyFairseqTask): + """Task to finetune RoBERTa for Winograd Schemas.""" + + @staticmethod + def add_args(parser): + """Add task-specific arguments to the parser.""" + parser.add_argument( + "data", metavar="DIR", help="path to data directory; we load <split>.jsonl" + ) + parser.add_argument( + "--init-token", + type=int, + default=None, + help="add token at the beginning of each batch item", + ) + + def __init__(self, args, vocab): + super().__init__(args) + self.vocab = vocab + self.mask = vocab.add_symbol("<mask>") + + self.bpe = encoders.build_bpe(args) + self.tokenizer = encoders.build_tokenizer(args) + + # hack to handle GPT-2 BPE, which includes leading spaces + if args.bpe == "gpt2": + self.leading_space = True + self.trailing_space = False + else: + self.leading_space = False + self.trailing_space = True + + @classmethod + def load_dictionary(cls, filename): + """Load the dictionary from the filename + + Args: + filename (str): the filename + """ + dictionary = Dictionary.load(filename) + dictionary.add_symbol("<mask>") + return dictionary + + @classmethod + def setup_task(cls, args, **kwargs): + assert args.criterion == "wsc", "Must set --criterion=wsc" + + # load data and label dictionaries + vocab = cls.load_dictionary(os.path.join(args.data, "dict.txt")) + print("| dictionary: {} types".format(len(vocab))) + + return cls(args, vocab) + + def binarize(self, s: str, append_eos: bool = False): + if self.tokenizer is not None: + s = self.tokenizer.encode(s) + if self.bpe is not None: + s = self.bpe.encode(s) + tokens = self.vocab.encode_line( + s, + append_eos=append_eos, + add_if_not_exist=False, + ).long() + if self.args.init_token is not None: + tokens = torch.cat([tokens.new([self.args.init_token]), tokens]) + return tokens + + def binarize_with_mask(self, txt, prefix, suffix, leading_space, trailing_space): + toks = self.binarize( + prefix + leading_space + txt + trailing_space + suffix, + append_eos=True, + ) + mask = torch.zeros_like(toks, dtype=torch.bool) + mask_start = len(self.binarize(prefix)) + mask_size = len(self.binarize(leading_space + txt)) + mask[mask_start : mask_start + mask_size] = 1 + return toks, mask + + def load_dataset( + self, split, epoch=1, combine=False, data_path=None, return_only=False, **kwargs + ): + """Load a given dataset split. + + Args: + split (str): name of the split (e.g., train, valid, test) + """ + if data_path is None: + data_path = os.path.join(self.args.data, split + ".jsonl") + if not os.path.exists(data_path): + raise FileNotFoundError("Cannot find data: {}".format(data_path)) + + query_tokens = [] + query_masks = [] + query_lengths = [] + candidate_tokens = [] + candidate_masks = [] + candidate_lengths = [] + labels = [] + + for sentence, pronoun_span, query, label in wsc_utils.jsonl_iterator(data_path): + prefix = sentence[: pronoun_span.start].text + suffix = sentence[pronoun_span.end :].text_with_ws + + # spaCy spans include trailing spaces, but we need to know about + # leading spaces for the GPT-2 BPE + leading_space = ( + " " if sentence[: pronoun_span.start].text_with_ws.endswith(" ") else "" + ) + trailing_space = " " if pronoun_span.text_with_ws.endswith(" ") else "" + + # get noun phrases, excluding pronouns and anything overlapping with the query + cand_spans = wsc_utils.filter_noun_chunks( + wsc_utils.extended_noun_chunks(sentence), + exclude_pronouns=True, + exclude_query=query, + exact_match=False, + ) + + if query is not None: + query_toks, query_mask = self.binarize_with_mask( + query, prefix, suffix, leading_space, trailing_space + ) + query_len = len(query_toks) + else: + query_toks, query_mask, query_len = None, None, 0 + + query_tokens.append(query_toks) + query_masks.append(query_mask) + query_lengths.append(query_len) + + cand_toks, cand_masks = [], [] + for cand_span in cand_spans: + toks, mask = self.binarize_with_mask( + cand_span.text, + prefix, + suffix, + leading_space, + trailing_space, + ) + cand_toks.append(toks) + cand_masks.append(mask) + + # collate candidates + cand_toks = data_utils.collate_tokens(cand_toks, pad_idx=self.vocab.pad()) + cand_masks = data_utils.collate_tokens(cand_masks, pad_idx=0) + assert cand_toks.size() == cand_masks.size() + + candidate_tokens.append(cand_toks) + candidate_masks.append(cand_masks) + candidate_lengths.append(cand_toks.size(1)) + + labels.append(label) + + query_lengths = np.array(query_lengths) + query_tokens = ListDataset(query_tokens, query_lengths) + query_masks = ListDataset(query_masks, query_lengths) + + candidate_lengths = np.array(candidate_lengths) + candidate_tokens = ListDataset(candidate_tokens, candidate_lengths) + candidate_masks = ListDataset(candidate_masks, candidate_lengths) + + labels = ListDataset(labels, [1] * len(labels)) + + dataset = { + "id": IdDataset(), + "query_tokens": query_tokens, + "query_masks": query_masks, + "candidate_tokens": candidate_tokens, + "candidate_masks": candidate_masks, + "labels": labels, + "nsentences": NumSamplesDataset(), + "ntokens": NumelDataset(query_tokens, reduce=True), + } + + nested_dataset = NestedDictionaryDataset( + dataset, + sizes=[query_lengths], + ) + + with data_utils.numpy_seed(self.args.seed): + shuffle = np.random.permutation(len(query_tokens)) + dataset = SortDataset( + nested_dataset, + # shuffle + sort_order=[shuffle], + ) + + if return_only: + return dataset + + self.datasets[split] = dataset + return self.datasets[split] + + def build_dataset_for_inference(self, sample_json): + with tempfile.NamedTemporaryFile(buffering=0) as h: + h.write((json.dumps(sample_json) + "\n").encode("utf-8")) + dataset = self.load_dataset( + "disambiguate_pronoun", + data_path=h.name, + return_only=True, + ) + return dataset + + def disambiguate_pronoun(self, model, sentence, use_cuda=False): + sample_json = wsc_utils.convert_sentence_to_json(sentence) + dataset = self.build_dataset_for_inference(sample_json) + sample = dataset.collater([dataset[0]]) + if use_cuda: + sample = utils.move_to_cuda(sample) + + def get_masked_input(tokens, mask): + masked_tokens = tokens.clone() + masked_tokens[mask.bool()] = self.mask + return masked_tokens + + def get_lprobs(tokens, mask): + logits, _ = model(src_tokens=get_masked_input(tokens, mask)) + lprobs = F.log_softmax(logits, dim=-1, dtype=torch.float) + scores = lprobs.gather(2, tokens.unsqueeze(-1)).squeeze(-1) + mask = mask.type_as(scores) + scores = (scores * mask).sum(dim=-1) / mask.sum(dim=-1) + return scores + + cand_lprobs = get_lprobs( + sample["candidate_tokens"][0], + sample["candidate_masks"][0], + ) + if sample["query_tokens"][0] is not None: + query_lprobs = get_lprobs( + sample["query_tokens"][0].unsqueeze(0), + sample["query_masks"][0].unsqueeze(0), + ) + return (query_lprobs >= cand_lprobs).all().item() == 1 + else: + best_idx = cand_lprobs.argmax().item() + full_cand = sample["candidate_tokens"][0][best_idx] + mask = sample["candidate_masks"][0][best_idx] + toks = full_cand[mask.bool()] + return self.bpe.decode(self.source_dictionary.string(toks)).strip() + + @property + def source_dictionary(self): + return self.vocab + + @property + def target_dictionary(self): + return self.vocab + + +@register_task("winogrande") +class WinograndeTask(WSCTask): + """ + Task for WinoGrande dataset. Efficient implementation for Winograd schema + tasks with exactly two candidates, one of which is correct. + """ + + @classmethod + def setup_task(cls, args, **kwargs): + assert args.criterion == "winogrande", "Must set --criterion=winogrande" + + # load data and label dictionaries + vocab = cls.load_dictionary(os.path.join(args.data, "dict.txt")) + print("| dictionary: {} types".format(len(vocab))) + + return cls(args, vocab) + + def load_dataset( + self, split, epoch=1, combine=False, data_path=None, return_only=False, **kwargs + ): + """Load a given dataset split. + + Args: + split (str): name of the split (e.g., train, valid, test) + """ + if data_path is None: + data_path = os.path.join(self.args.data, split + ".jsonl") + if not os.path.exists(data_path): + raise FileNotFoundError("Cannot find data: {}".format(data_path)) + + query_tokens = [] + query_masks = [] + query_lengths = [] + candidate_tokens = [] + candidate_masks = [] + candidate_lengths = [] + + itr = wsc_utils.winogrande_jsonl_iterator(data_path, eval=(split == "test")) + + for sample in itr: + sentence, pronoun_span, query, cand_text = sample + prefix = sentence[: pronoun_span[0]].rstrip() + suffix = sentence[pronoun_span[1] :] + + leading_space = " " if sentence[: pronoun_span[0]].endswith(" ") else "" + trailing_space = "" + + if query is not None: + query_toks, query_mask = self.binarize_with_mask( + query, + prefix, + suffix, + leading_space, + trailing_space, + ) + query_len = len(query_toks) + else: + query_toks, query_mask, query_len = None, None, 0 + + query_tokens.append(query_toks) + query_masks.append(query_mask) + query_lengths.append(query_len) + + cand_toks, cand_mask = self.binarize_with_mask( + cand_text, + prefix, + suffix, + leading_space, + trailing_space, + ) + + candidate_tokens.append(cand_toks) + candidate_masks.append(cand_mask) + candidate_lengths.append(cand_toks.size(0)) + + query_lengths = np.array(query_lengths) + + def get_pad_dataset_fn(tokens, length, pad_idx): + return PadDataset( + ListDataset(tokens, length), + pad_idx=pad_idx, + left_pad=False, + ) + + query_tokens = get_pad_dataset_fn(query_tokens, query_lengths, self.vocab.pad()) + query_masks = get_pad_dataset_fn(query_masks, query_lengths, 0) + + candidate_lengths = np.array(candidate_lengths) + candidate_tokens = get_pad_dataset_fn( + candidate_tokens, candidate_lengths, self.vocab.pad() + ) + candidate_masks = get_pad_dataset_fn(candidate_masks, candidate_lengths, 0) + + dataset = { + "id": IdDataset(), + "query_tokens": query_tokens, + "query_masks": query_masks, + "candidate_tokens": candidate_tokens, + "candidate_masks": candidate_masks, + "nsentences": NumSamplesDataset(), + "ntokens": NumelDataset(query_tokens, reduce=True), + } + + nested_dataset = NestedDictionaryDataset( + dataset, + sizes=[query_lengths], + ) + + with data_utils.numpy_seed(self.args.seed): + shuffle = np.random.permutation(len(query_tokens)) + dataset = SortDataset( + nested_dataset, + # shuffle + sort_order=[shuffle], + ) + + if return_only: + return dataset + + self.datasets[split] = dataset + return self.datasets[split] diff --git a/data/fairseq/examples/roberta/wsc/wsc_utils.py b/data/fairseq/examples/roberta/wsc/wsc_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..da6ba74383a2490e1108609f315f44ad4b3bf002 --- /dev/null +++ b/data/fairseq/examples/roberta/wsc/wsc_utils.py @@ -0,0 +1,241 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import json +from functools import lru_cache + + +def convert_sentence_to_json(sentence): + if "_" in sentence: + prefix, rest = sentence.split("_", 1) + query, rest = rest.split("_", 1) + query_index = len(prefix.rstrip().split(" ")) + else: + query, query_index = None, None + + prefix, rest = sentence.split("[", 1) + pronoun, rest = rest.split("]", 1) + pronoun_index = len(prefix.rstrip().split(" ")) + + sentence = sentence.replace("_", "").replace("[", "").replace("]", "") + + return { + "idx": 0, + "text": sentence, + "target": { + "span1_index": query_index, + "span1_text": query, + "span2_index": pronoun_index, + "span2_text": pronoun, + }, + } + + +def extended_noun_chunks(sentence): + noun_chunks = {(np.start, np.end) for np in sentence.noun_chunks} + np_start, cur_np = 0, "NONE" + for i, token in enumerate(sentence): + np_type = token.pos_ if token.pos_ in {"NOUN", "PROPN"} else "NONE" + if np_type != cur_np: + if cur_np != "NONE": + noun_chunks.add((np_start, i)) + if np_type != "NONE": + np_start = i + cur_np = np_type + if cur_np != "NONE": + noun_chunks.add((np_start, len(sentence))) + return [sentence[s:e] for (s, e) in sorted(noun_chunks)] + + +def find_token(sentence, start_pos): + found_tok = None + for tok in sentence: + if tok.idx == start_pos: + found_tok = tok + break + return found_tok + + +def find_span(sentence, search_text, start=0): + search_text = search_text.lower() + for tok in sentence[start:]: + remainder = sentence[tok.i :].text.lower() + if remainder.startswith(search_text): + len_to_consume = len(search_text) + start_idx = tok.idx + for next_tok in sentence[tok.i :]: + end_idx = next_tok.idx + len(next_tok.text) + if end_idx - start_idx == len_to_consume: + span = sentence[tok.i : next_tok.i + 1] + return span + return None + + +@lru_cache(maxsize=1) +def get_detokenizer(): + from sacremoses import MosesDetokenizer + + detok = MosesDetokenizer(lang="en") + return detok + + +@lru_cache(maxsize=1) +def get_spacy_nlp(): + import en_core_web_lg + + nlp = en_core_web_lg.load() + return nlp + + +def jsonl_iterator(input_fname, positive_only=False, ngram_order=3, eval=False): + detok = get_detokenizer() + nlp = get_spacy_nlp() + + with open(input_fname) as fin: + for line in fin: + sample = json.loads(line.strip()) + + if positive_only and "label" in sample and not sample["label"]: + # only consider examples where the query is correct + continue + + target = sample["target"] + + # clean up the query + query = target["span1_text"] + if query is not None: + if "\n" in query: + continue + if query.endswith(".") or query.endswith(","): + query = query[:-1] + + # split tokens + tokens = sample["text"].split(" ") + + def strip_pronoun(x): + return x.rstrip('.,"') + + # find the pronoun + pronoun_idx = target["span2_index"] + pronoun = strip_pronoun(target["span2_text"]) + if strip_pronoun(tokens[pronoun_idx]) != pronoun: + # hack: sometimes the index is misaligned + if strip_pronoun(tokens[pronoun_idx + 1]) == pronoun: + pronoun_idx += 1 + else: + raise Exception("Misaligned pronoun!") + assert strip_pronoun(tokens[pronoun_idx]) == pronoun + + # split tokens before and after the pronoun + before = tokens[:pronoun_idx] + after = tokens[pronoun_idx + 1 :] + + # the GPT BPE attaches leading spaces to tokens, so we keep track + # of whether we need spaces before or after the pronoun + leading_space = " " if pronoun_idx > 0 else "" + trailing_space = " " if len(after) > 0 else "" + + # detokenize + before = detok.detokenize(before, return_str=True) + pronoun = detok.detokenize([pronoun], return_str=True) + after = detok.detokenize(after, return_str=True) + + # hack: when the pronoun ends in a period (or comma), move the + # punctuation to the "after" part + if pronoun.endswith(".") or pronoun.endswith(","): + after = pronoun[-1] + trailing_space + after + pronoun = pronoun[:-1] + + # hack: when the "after" part begins with a comma or period, remove + # the trailing space + if after.startswith(".") or after.startswith(","): + trailing_space = "" + + # parse sentence with spacy + sentence = nlp(before + leading_space + pronoun + trailing_space + after) + + # find pronoun span + start = len(before + leading_space) + first_pronoun_tok = find_token(sentence, start_pos=start) + pronoun_span = find_span(sentence, pronoun, start=first_pronoun_tok.i) + assert pronoun_span.text == pronoun + + if eval: + # convert to format where pronoun is surrounded by "[]" and + # query is surrounded by "_" + query_span = find_span(sentence, query) + query_with_ws = "_{}_{}".format( + query_span.text, + (" " if query_span.text_with_ws.endswith(" ") else ""), + ) + pronoun_with_ws = "[{}]{}".format( + pronoun_span.text, + (" " if pronoun_span.text_with_ws.endswith(" ") else ""), + ) + if query_span.start < pronoun_span.start: + first = (query_span, query_with_ws) + second = (pronoun_span, pronoun_with_ws) + else: + first = (pronoun_span, pronoun_with_ws) + second = (query_span, query_with_ws) + sentence = ( + sentence[: first[0].start].text_with_ws + + first[1] + + sentence[first[0].end : second[0].start].text_with_ws + + second[1] + + sentence[second[0].end :].text + ) + yield sentence, sample.get("label", None) + else: + yield sentence, pronoun_span, query, sample.get("label", None) + + +def winogrande_jsonl_iterator(input_fname, eval=False): + with open(input_fname) as fin: + for line in fin: + sample = json.loads(line.strip()) + sentence, option1, option2 = ( + sample["sentence"], + sample["option1"], + sample["option2"], + ) + + pronoun_span = (sentence.index("_"), sentence.index("_") + 1) + + if eval: + query, cand = option1, option2 + else: + query = option1 if sample["answer"] == "1" else option2 + cand = option2 if sample["answer"] == "1" else option1 + yield sentence, pronoun_span, query, cand + + +def filter_noun_chunks( + chunks, exclude_pronouns=False, exclude_query=None, exact_match=False +): + if exclude_pronouns: + chunks = [ + np + for np in chunks + if (np.lemma_ != "-PRON-" and not all(tok.pos_ == "PRON" for tok in np)) + ] + + if exclude_query is not None: + excl_txt = [exclude_query.lower()] + filtered_chunks = [] + for chunk in chunks: + lower_chunk = chunk.text.lower() + found = False + for excl in excl_txt: + if ( + not exact_match and (lower_chunk in excl or excl in lower_chunk) + ) or lower_chunk == excl: + found = True + break + if not found: + filtered_chunks.append(chunk) + chunks = filtered_chunks + + return chunks diff --git a/data/fairseq/examples/stories/README.md b/data/fairseq/examples/stories/README.md new file mode 100644 index 0000000000000000000000000000000000000000..588941eddc5f0280f5254affd40ef49de874c885 --- /dev/null +++ b/data/fairseq/examples/stories/README.md @@ -0,0 +1,66 @@ +# Hierarchical Neural Story Generation (Fan et al., 2018) + +The following commands provide an example of pre-processing data, training a model, and generating text for story generation with the WritingPrompts dataset. + +## Pre-trained models + +Description | Dataset | Model | Test set(s) +---|---|---|--- +Stories with Convolutional Model <br> ([Fan et al., 2018](https://arxiv.org/abs/1805.04833)) | [WritingPrompts](https://dl.fbaipublicfiles.com/fairseq/data/writingPrompts.tar.gz) | [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/stories_checkpoint.tar.bz2) | [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/stories_test.tar.bz2) + +We provide sample stories generated by the [convolutional seq2seq model](https://dl.fbaipublicfiles.com/fairseq/data/seq2seq_stories.txt) and [fusion model](https://dl.fbaipublicfiles.com/fairseq/data/fusion_stories.txt) from [Fan et al., 2018](https://arxiv.org/abs/1805.04833). The corresponding prompts for the fusion model can be found [here](https://dl.fbaipublicfiles.com/fairseq/data/fusion_prompts.txt). Note that there are unk in the file, as we modeled a small full vocabulary (no BPE or pre-training). We did not use these unk prompts for human evaluation. + +## Dataset + +The dataset can be downloaded like this: + +```bash +cd examples/stories +curl https://dl.fbaipublicfiles.com/fairseq/data/writingPrompts.tar.gz | tar xvzf - +``` + +and contains a train, test, and valid split. The dataset is described here: https://arxiv.org/abs/1805.04833. We model only the first 1000 words of each story, including one newLine token. + +## Example usage + +First we will preprocess the dataset. Note that the dataset release is the full data, but the paper models the first 1000 words of each story. Here is example code that trims the dataset to the first 1000 words of each story: +```python +data = ["train", "test", "valid"] +for name in data: + with open(name + ".wp_target") as f: + stories = f.readlines() + stories = [" ".join(i.split()[0:1000]) for i in stories] + with open(name + ".wp_target", "w") as o: + for line in stories: + o.write(line.strip() + "\n") +``` + +Once we've trimmed the data we can binarize it and train our model: +```bash +# Binarize the dataset: +export TEXT=examples/stories/writingPrompts +fairseq-preprocess --source-lang wp_source --target-lang wp_target \ + --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \ + --destdir data-bin/writingPrompts --padding-factor 1 --thresholdtgt 10 --thresholdsrc 10 + +# Train the model: +fairseq-train data-bin/writingPrompts -a fconv_self_att_wp --lr 0.25 --optimizer nag --clip-norm 0.1 --max-tokens 1500 --lr-scheduler reduce_lr_on_plateau --decoder-attention True --encoder-attention False --criterion label_smoothed_cross_entropy --weight-decay .0000001 --label-smoothing 0 --source-lang wp_source --target-lang wp_target --gated-attention True --self-attention True --project-input True --pretrained False + +# Train a fusion model: +# add the arguments: --pretrained True --pretrained-checkpoint path/to/checkpoint + +# Generate: +# Note: to load the pretrained model at generation time, you need to pass in a model-override argument to communicate to the fusion model at generation time where you have placed the pretrained checkpoint. By default, it will load the exact path of the fusion model's pretrained model from training time. You should use model-override if you have moved the pretrained model (or are using our provided models). If you are generating from a non-fusion model, the model-override argument is not necessary. + +fairseq-generate data-bin/writingPrompts --path /path/to/trained/model/checkpoint_best.pt --batch-size 32 --beam 1 --sampling --sampling-topk 10 --temperature 0.8 --nbest 1 --model-overrides "{'pretrained_checkpoint':'/path/to/pretrained/model/checkpoint'}" +``` + +## Citation +```bibtex +@inproceedings{fan2018hierarchical, + title = {Hierarchical Neural Story Generation}, + author = {Fan, Angela and Lewis, Mike and Dauphin, Yann}, + booktitle = {Conference of the Association for Computational Linguistics (ACL)}, + year = 2018, +} +``` diff --git a/data/fairseq/examples/translation/README.md b/data/fairseq/examples/translation/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2941f5eb8482dab61dca5eca27a71abd7ee5bf5c --- /dev/null +++ b/data/fairseq/examples/translation/README.md @@ -0,0 +1,301 @@ +# Neural Machine Translation + +This README contains instructions for [using pretrained translation models](#example-usage-torchhub) +as well as [training new models](#training-a-new-model). + +## Pre-trained models + +Model | Description | Dataset | Download +---|---|---|--- +`conv.wmt14.en-fr` | Convolutional <br> ([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) | model: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt14.v2.en-fr.fconv-py.tar.bz2) <br> newstest2014: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.v2.en-fr.newstest2014.tar.bz2) <br> newstest2012/2013: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.v2.en-fr.ntst1213.tar.bz2) +`conv.wmt14.en-de` | Convolutional <br> ([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT14 English-German](http://statmt.org/wmt14/translation-task.html#Download) | model: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt14.en-de.fconv-py.tar.bz2) <br> newstest2014: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.en-de.newstest2014.tar.bz2) +`conv.wmt17.en-de` | Convolutional <br> ([Gehring et al., 2017](https://arxiv.org/abs/1705.03122)) | [WMT17 English-German](http://statmt.org/wmt17/translation-task.html#Download) | model: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt17.v2.en-de.fconv-py.tar.bz2) <br> newstest2014: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt17.v2.en-de.newstest2014.tar.bz2) +`transformer.wmt14.en-fr` | Transformer <br> ([Ott et al., 2018](https://arxiv.org/abs/1806.00187)) | [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) | model: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt14.en-fr.joined-dict.transformer.tar.bz2) <br> newstest2014: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt14.en-fr.joined-dict.newstest2014.tar.bz2) +`transformer.wmt16.en-de` | Transformer <br> ([Ott et al., 2018](https://arxiv.org/abs/1806.00187)) | [WMT16 English-German](https://drive.google.com/uc?export=download&id=0B_bZck-ksdkpM25jRUN2X2UxMm8) | model: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/models/wmt16.en-de.joined-dict.transformer.tar.bz2) <br> newstest2014: <br> [download (.tar.bz2)](https://dl.fbaipublicfiles.com/fairseq/data/wmt16.en-de.joined-dict.newstest2014.tar.bz2) +`transformer.wmt18.en-de` | Transformer <br> ([Edunov et al., 2018](https://arxiv.org/abs/1808.09381)) <br> WMT'18 winner | [WMT'18 English-German](http://www.statmt.org/wmt18/translation-task.html) | model: <br> [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt18.en-de.ensemble.tar.gz) <br> See NOTE in the archive +`transformer.wmt19.en-de` | Transformer <br> ([Ng et al., 2019](https://arxiv.org/abs/1907.06616)) <br> WMT'19 winner | [WMT'19 English-German](http://www.statmt.org/wmt19/translation-task.html) | model: <br> [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-de.joined-dict.ensemble.tar.gz) +`transformer.wmt19.de-en` | Transformer <br> ([Ng et al., 2019](https://arxiv.org/abs/1907.06616)) <br> WMT'19 winner | [WMT'19 German-English](http://www.statmt.org/wmt19/translation-task.html) | model: <br> [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.de-en.joined-dict.ensemble.tar.gz) +`transformer.wmt19.en-ru` | Transformer <br> ([Ng et al., 2019](https://arxiv.org/abs/1907.06616)) <br> WMT'19 winner | [WMT'19 English-Russian](http://www.statmt.org/wmt19/translation-task.html) | model: <br> [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.en-ru.ensemble.tar.gz) +`transformer.wmt19.ru-en` | Transformer <br> ([Ng et al., 2019](https://arxiv.org/abs/1907.06616)) <br> WMT'19 winner | [WMT'19 Russian-English](http://www.statmt.org/wmt19/translation-task.html) | model: <br> [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt19.ru-en.ensemble.tar.gz) + +## Example usage (torch.hub) + +We require a few additional Python dependencies for preprocessing: +```bash +pip install fastBPE sacremoses subword_nmt +``` + +Interactive translation via PyTorch Hub: +```python +import torch + +# List available models +torch.hub.list('pytorch/fairseq') # [..., 'transformer.wmt16.en-de', ... ] + +# Load a transformer trained on WMT'16 En-De +# Note: WMT'19 models use fastBPE instead of subword_nmt, see instructions below +en2de = torch.hub.load('pytorch/fairseq', 'transformer.wmt16.en-de', + tokenizer='moses', bpe='subword_nmt') +en2de.eval() # disable dropout + +# The underlying model is available under the *models* attribute +assert isinstance(en2de.models[0], fairseq.models.transformer.TransformerModel) + +# Move model to GPU for faster translation +en2de.cuda() + +# Translate a sentence +en2de.translate('Hello world!') +# 'Hallo Welt!' + +# Batched translation +en2de.translate(['Hello world!', 'The cat sat on the mat.']) +# ['Hallo Welt!', 'Die Katze saß auf der Matte.'] +``` + +Loading custom models: +```python +from fairseq.models.transformer import TransformerModel +zh2en = TransformerModel.from_pretrained( + '/path/to/checkpoints', + checkpoint_file='checkpoint_best.pt', + data_name_or_path='data-bin/wmt17_zh_en_full', + bpe='subword_nmt', + bpe_codes='data-bin/wmt17_zh_en_full/zh.code' +) +zh2en.translate('你好 世界') +# 'Hello World' +``` + +If you are using a `transformer.wmt19` models, you will need to set the `bpe` +argument to `'fastbpe'` and (optionally) load the 4-model ensemble: +```python +en2de = torch.hub.load('pytorch/fairseq', 'transformer.wmt19.en-de', + checkpoint_file='model1.pt:model2.pt:model3.pt:model4.pt', + tokenizer='moses', bpe='fastbpe') +en2de.eval() # disable dropout +``` + +## Example usage (CLI tools) + +Generation with the binarized test sets can be run in batch mode as follows, e.g. for WMT 2014 English-French on a GTX-1080ti: +```bash +mkdir -p data-bin +curl https://dl.fbaipublicfiles.com/fairseq/models/wmt14.v2.en-fr.fconv-py.tar.bz2 | tar xvjf - -C data-bin +curl https://dl.fbaipublicfiles.com/fairseq/data/wmt14.v2.en-fr.newstest2014.tar.bz2 | tar xvjf - -C data-bin +fairseq-generate data-bin/wmt14.en-fr.newstest2014 \ + --path data-bin/wmt14.en-fr.fconv-py/model.pt \ + --beam 5 --batch-size 128 --remove-bpe | tee /tmp/gen.out +# ... +# | Translated 3003 sentences (96311 tokens) in 166.0s (580.04 tokens/s) +# | Generate test with beam=5: BLEU4 = 40.83, 67.5/46.9/34.4/25.5 (BP=1.000, ratio=1.006, syslen=83262, reflen=82787) + +# Compute BLEU score +grep ^H /tmp/gen.out | cut -f3- > /tmp/gen.out.sys +grep ^T /tmp/gen.out | cut -f2- > /tmp/gen.out.ref +fairseq-score --sys /tmp/gen.out.sys --ref /tmp/gen.out.ref +# BLEU4 = 40.83, 67.5/46.9/34.4/25.5 (BP=1.000, ratio=1.006, syslen=83262, reflen=82787) +``` + +## Training a new model + +### IWSLT'14 German to English (Transformer) + +The following instructions can be used to train a Transformer model on the [IWSLT'14 German to English dataset](http://workshop2014.iwslt.org/downloads/proceeding.pdf). + +First download and preprocess the data: +```bash +# Download and prepare the data +cd examples/translation/ +bash prepare-iwslt14.sh +cd ../.. + +# Preprocess/binarize the data +TEXT=examples/translation/iwslt14.tokenized.de-en +fairseq-preprocess --source-lang de --target-lang en \ + --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \ + --destdir data-bin/iwslt14.tokenized.de-en \ + --workers 20 +``` + +Next we'll train a Transformer translation model over this data: +```bash +CUDA_VISIBLE_DEVICES=0 fairseq-train \ + data-bin/iwslt14.tokenized.de-en \ + --arch transformer_iwslt_de_en --share-decoder-input-output-embed \ + --optimizer adam --adam-betas '(0.9, 0.98)' --clip-norm 0.0 \ + --lr 5e-4 --lr-scheduler inverse_sqrt --warmup-updates 4000 \ + --dropout 0.3 --weight-decay 0.0001 \ + --criterion label_smoothed_cross_entropy --label-smoothing 0.1 \ + --max-tokens 4096 \ + --eval-bleu \ + --eval-bleu-args '{"beam": 5, "max_len_a": 1.2, "max_len_b": 10}' \ + --eval-bleu-detok moses \ + --eval-bleu-remove-bpe \ + --eval-bleu-print-samples \ + --best-checkpoint-metric bleu --maximize-best-checkpoint-metric +``` + +Finally we can evaluate our trained model: +```bash +fairseq-generate data-bin/iwslt14.tokenized.de-en \ + --path checkpoints/checkpoint_best.pt \ + --batch-size 128 --beam 5 --remove-bpe +``` + +### WMT'14 English to German (Convolutional) + +The following instructions can be used to train a Convolutional translation model on the WMT English to German dataset. +See the [Scaling NMT README](../scaling_nmt/README.md) for instructions to train a Transformer translation model on this data. + +The WMT English to German dataset can be preprocessed using the `prepare-wmt14en2de.sh` script. +By default it will produce a dataset that was modeled after [Attention Is All You Need (Vaswani et al., 2017)](https://arxiv.org/abs/1706.03762), but with additional news-commentary-v12 data from WMT'17. + +To use only data available in WMT'14 or to replicate results obtained in the original [Convolutional Sequence to Sequence Learning (Gehring et al., 2017)](https://arxiv.org/abs/1705.03122) paper, please use the `--icml17` option. + +```bash +# Download and prepare the data +cd examples/translation/ +# WMT'17 data: +bash prepare-wmt14en2de.sh +# or to use WMT'14 data: +# bash prepare-wmt14en2de.sh --icml17 +cd ../.. + +# Binarize the dataset +TEXT=examples/translation/wmt17_en_de +fairseq-preprocess \ + --source-lang en --target-lang de \ + --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \ + --destdir data-bin/wmt17_en_de --thresholdtgt 0 --thresholdsrc 0 \ + --workers 20 + +# Train the model +mkdir -p checkpoints/fconv_wmt_en_de +fairseq-train \ + data-bin/wmt17_en_de \ + --arch fconv_wmt_en_de \ + --dropout 0.2 \ + --criterion label_smoothed_cross_entropy --label-smoothing 0.1 \ + --optimizer nag --clip-norm 0.1 \ + --lr 0.5 --lr-scheduler fixed --force-anneal 50 \ + --max-tokens 4000 \ + --save-dir checkpoints/fconv_wmt_en_de + +# Evaluate +fairseq-generate data-bin/wmt17_en_de \ + --path checkpoints/fconv_wmt_en_de/checkpoint_best.pt \ + --beam 5 --remove-bpe +``` + +### WMT'14 English to French +```bash +# Download and prepare the data +cd examples/translation/ +bash prepare-wmt14en2fr.sh +cd ../.. + +# Binarize the dataset +TEXT=examples/translation/wmt14_en_fr +fairseq-preprocess \ + --source-lang en --target-lang fr \ + --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \ + --destdir data-bin/wmt14_en_fr --thresholdtgt 0 --thresholdsrc 0 \ + --workers 60 + +# Train the model +mkdir -p checkpoints/fconv_wmt_en_fr +fairseq-train \ + data-bin/wmt14_en_fr \ + --arch fconv_wmt_en_fr \ + --dropout 0.1 \ + --criterion label_smoothed_cross_entropy --label-smoothing 0.1 \ + --optimizer nag --clip-norm 0.1 \ + --lr 0.5 --lr-scheduler fixed --force-anneal 50 \ + --max-tokens 3000 \ + --save-dir checkpoints/fconv_wmt_en_fr + +# Evaluate +fairseq-generate \ + data-bin/fconv_wmt_en_fr \ + --path checkpoints/fconv_wmt_en_fr/checkpoint_best.pt \ + --beam 5 --remove-bpe +``` + +## Multilingual Translation + +We also support training multilingual translation models. In this example we'll +train a multilingual `{de,fr}-en` translation model using the IWSLT'17 datasets. + +Note that we use slightly different preprocessing here than for the IWSLT'14 +En-De data above. In particular we learn a joint BPE code for all three +languages and use fairseq-interactive and sacrebleu for scoring the test set. + +```bash +# First install sacrebleu and sentencepiece +pip install sacrebleu sentencepiece + +# Then download and preprocess the data +cd examples/translation/ +bash prepare-iwslt17-multilingual.sh +cd ../.. + +# Binarize the de-en dataset +TEXT=examples/translation/iwslt17.de_fr.en.bpe16k +fairseq-preprocess --source-lang de --target-lang en \ + --trainpref $TEXT/train.bpe.de-en \ + --validpref $TEXT/valid0.bpe.de-en,$TEXT/valid1.bpe.de-en,$TEXT/valid2.bpe.de-en,$TEXT/valid3.bpe.de-en,$TEXT/valid4.bpe.de-en,$TEXT/valid5.bpe.de-en \ + --destdir data-bin/iwslt17.de_fr.en.bpe16k \ + --workers 10 + +# Binarize the fr-en dataset +# NOTE: it's important to reuse the en dictionary from the previous step +fairseq-preprocess --source-lang fr --target-lang en \ + --trainpref $TEXT/train.bpe.fr-en \ + --validpref $TEXT/valid0.bpe.fr-en,$TEXT/valid1.bpe.fr-en,$TEXT/valid2.bpe.fr-en,$TEXT/valid3.bpe.fr-en,$TEXT/valid4.bpe.fr-en,$TEXT/valid5.bpe.fr-en \ + --tgtdict data-bin/iwslt17.de_fr.en.bpe16k/dict.en.txt \ + --destdir data-bin/iwslt17.de_fr.en.bpe16k \ + --workers 10 + +# Train a multilingual transformer model +# NOTE: the command below assumes 1 GPU, but accumulates gradients from +# 8 fwd/bwd passes to simulate training on 8 GPUs +mkdir -p checkpoints/multilingual_transformer +CUDA_VISIBLE_DEVICES=0 fairseq-train data-bin/iwslt17.de_fr.en.bpe16k/ \ + --max-epoch 50 \ + --ddp-backend=legacy_ddp \ + --task multilingual_translation --lang-pairs de-en,fr-en \ + --arch multilingual_transformer_iwslt_de_en \ + --share-decoders --share-decoder-input-output-embed \ + --optimizer adam --adam-betas '(0.9, 0.98)' \ + --lr 0.0005 --lr-scheduler inverse_sqrt \ + --warmup-updates 4000 --warmup-init-lr '1e-07' \ + --label-smoothing 0.1 --criterion label_smoothed_cross_entropy \ + --dropout 0.3 --weight-decay 0.0001 \ + --save-dir checkpoints/multilingual_transformer \ + --max-tokens 4000 \ + --update-freq 8 + +# Generate and score the test set with sacrebleu +SRC=de +sacrebleu --test-set iwslt17 --language-pair ${SRC}-en --echo src \ + | python scripts/spm_encode.py --model examples/translation/iwslt17.de_fr.en.bpe16k/sentencepiece.bpe.model \ + > iwslt17.test.${SRC}-en.${SRC}.bpe +cat iwslt17.test.${SRC}-en.${SRC}.bpe \ + | fairseq-interactive data-bin/iwslt17.de_fr.en.bpe16k/ \ + --task multilingual_translation --lang-pairs de-en,fr-en \ + --source-lang ${SRC} --target-lang en \ + --path checkpoints/multilingual_transformer/checkpoint_best.pt \ + --buffer-size 2000 --batch-size 128 \ + --beam 5 --remove-bpe=sentencepiece \ + > iwslt17.test.${SRC}-en.en.sys +grep ^H iwslt17.test.${SRC}-en.en.sys | cut -f3 \ + | sacrebleu --test-set iwslt17 --language-pair ${SRC}-en +``` + +##### Argument format during inference + +During inference it is required to specify a single `--source-lang` and +`--target-lang`, which indicates the inference langauge direction. +`--lang-pairs`, `--encoder-langtok`, `--decoder-langtok` have to be set to +the same value as training. diff --git a/data/fairseq/examples/translation/prepare-iwslt14.sh b/data/fairseq/examples/translation/prepare-iwslt14.sh new file mode 100644 index 0000000000000000000000000000000000000000..2fb6643fbccb58701dcbb77d91430e68a821ba38 --- /dev/null +++ b/data/fairseq/examples/translation/prepare-iwslt14.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# +# Adapted from https://github.com/facebookresearch/MIXER/blob/master/prepareData.sh + +echo 'Cloning Moses github repository (for tokenization scripts)...' +git clone https://github.com/moses-smt/mosesdecoder.git + +echo 'Cloning Subword NMT repository (for BPE pre-processing)...' +git clone https://github.com/rsennrich/subword-nmt.git + +SCRIPTS=mosesdecoder/scripts +TOKENIZER=$SCRIPTS/tokenizer/tokenizer.perl +LC=$SCRIPTS/tokenizer/lowercase.perl +CLEAN=$SCRIPTS/training/clean-corpus-n.perl +BPEROOT=subword-nmt/subword_nmt +BPE_TOKENS=10000 + +URL="http://dl.fbaipublicfiles.com/fairseq/data/iwslt14/de-en.tgz" +GZ=de-en.tgz + +if [ ! -d "$SCRIPTS" ]; then + echo "Please set SCRIPTS variable correctly to point to Moses scripts." + exit +fi + +src=de +tgt=en +lang=de-en +prep=iwslt14.tokenized.de-en +tmp=$prep/tmp +orig=orig + +mkdir -p $orig $tmp $prep + +echo "Downloading data from ${URL}..." +cd $orig +wget "$URL" + +if [ -f $GZ ]; then + echo "Data successfully downloaded." +else + echo "Data not successfully downloaded." + exit +fi + +tar zxvf $GZ +cd .. + +echo "pre-processing train data..." +for l in $src $tgt; do + f=train.tags.$lang.$l + tok=train.tags.$lang.tok.$l + + cat $orig/$lang/$f | \ + grep -v '<url>' | \ + grep -v '<talkid>' | \ + grep -v '<keywords>' | \ + sed -e 's/<title>//g' | \ + sed -e 's/<\/title>//g' | \ + sed -e 's/<description>//g' | \ + sed -e 's/<\/description>//g' | \ + perl $TOKENIZER -threads 8 -l $l > $tmp/$tok + echo "" +done +perl $CLEAN -ratio 1.5 $tmp/train.tags.$lang.tok $src $tgt $tmp/train.tags.$lang.clean 1 175 +for l in $src $tgt; do + perl $LC < $tmp/train.tags.$lang.clean.$l > $tmp/train.tags.$lang.$l +done + +echo "pre-processing valid/test data..." +for l in $src $tgt; do + for o in `ls $orig/$lang/IWSLT14.TED*.$l.xml`; do + fname=${o##*/} + f=$tmp/${fname%.*} + echo $o $f + grep '<seg id' $o | \ + sed -e 's/<seg id="[0-9]*">\s*//g' | \ + sed -e 's/\s*<\/seg>\s*//g' | \ + sed -e "s/\’/\'/g" | \ + perl $TOKENIZER -threads 8 -l $l | \ + perl $LC > $f + echo "" + done +done + + +echo "creating train, valid, test..." +for l in $src $tgt; do + awk '{if (NR%23 == 0) print $0; }' $tmp/train.tags.de-en.$l > $tmp/valid.$l + awk '{if (NR%23 != 0) print $0; }' $tmp/train.tags.de-en.$l > $tmp/train.$l + + cat $tmp/IWSLT14.TED.dev2010.de-en.$l \ + $tmp/IWSLT14.TEDX.dev2012.de-en.$l \ + $tmp/IWSLT14.TED.tst2010.de-en.$l \ + $tmp/IWSLT14.TED.tst2011.de-en.$l \ + $tmp/IWSLT14.TED.tst2012.de-en.$l \ + > $tmp/test.$l +done + +TRAIN=$tmp/train.en-de +BPE_CODE=$prep/code +rm -f $TRAIN +for l in $src $tgt; do + cat $tmp/train.$l >> $TRAIN +done + +echo "learn_bpe.py on ${TRAIN}..." +python $BPEROOT/learn_bpe.py -s $BPE_TOKENS < $TRAIN > $BPE_CODE + +for L in $src $tgt; do + for f in train.$L valid.$L test.$L; do + echo "apply_bpe.py to ${f}..." + python $BPEROOT/apply_bpe.py -c $BPE_CODE < $tmp/$f > $prep/$f + done +done diff --git a/data/fairseq/examples/translation/prepare-iwslt17-multilingual.sh b/data/fairseq/examples/translation/prepare-iwslt17-multilingual.sh new file mode 100644 index 0000000000000000000000000000000000000000..23be87555322bc03b13e9d95951d88b1a442f97a --- /dev/null +++ b/data/fairseq/examples/translation/prepare-iwslt17-multilingual.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +SRCS=( + "de" + "fr" +) +TGT=en + +ROOT=$(dirname "$0") +SCRIPTS=$ROOT/../../scripts +SPM_TRAIN=$SCRIPTS/spm_train.py +SPM_ENCODE=$SCRIPTS/spm_encode.py + +BPESIZE=16384 +ORIG=$ROOT/iwslt17_orig +DATA=$ROOT/iwslt17.de_fr.en.bpe16k +mkdir -p "$ORIG" "$DATA" + +TRAIN_MINLEN=1 # remove sentences with <1 BPE token +TRAIN_MAXLEN=250 # remove sentences with >250 BPE tokens + +URLS=( + "https://wit3.fbk.eu/archive/2017-01-trnted/texts/de/en/de-en.tgz" + "https://wit3.fbk.eu/archive/2017-01-trnted/texts/fr/en/fr-en.tgz" +) +ARCHIVES=( + "de-en.tgz" + "fr-en.tgz" +) +VALID_SETS=( + "IWSLT17.TED.dev2010.de-en IWSLT17.TED.tst2010.de-en IWSLT17.TED.tst2011.de-en IWSLT17.TED.tst2012.de-en IWSLT17.TED.tst2013.de-en IWSLT17.TED.tst2014.de-en IWSLT17.TED.tst2015.de-en" + "IWSLT17.TED.dev2010.fr-en IWSLT17.TED.tst2010.fr-en IWSLT17.TED.tst2011.fr-en IWSLT17.TED.tst2012.fr-en IWSLT17.TED.tst2013.fr-en IWSLT17.TED.tst2014.fr-en IWSLT17.TED.tst2015.fr-en" +) + +# download and extract data +for ((i=0;i<${#URLS[@]};++i)); do + ARCHIVE=$ORIG/${ARCHIVES[i]} + if [ -f "$ARCHIVE" ]; then + echo "$ARCHIVE already exists, skipping download" + else + URL=${URLS[i]} + wget -P "$ORIG" "$URL" + if [ -f "$ARCHIVE" ]; then + echo "$URL successfully downloaded." + else + echo "$URL not successfully downloaded." + exit 1 + fi + fi + FILE=${ARCHIVE: -4} + if [ -e "$FILE" ]; then + echo "$FILE already exists, skipping extraction" + else + tar -C "$ORIG" -xzvf "$ARCHIVE" + fi +done + +echo "pre-processing train data..." +for SRC in "${SRCS[@]}"; do + for LANG in "${SRC}" "${TGT}"; do + cat "$ORIG/${SRC}-${TGT}/train.tags.${SRC}-${TGT}.${LANG}" \ + | grep -v '<url>' \ + | grep -v '<talkid>' \ + | grep -v '<keywords>' \ + | grep -v '<speaker>' \ + | grep -v '<reviewer' \ + | grep -v '<translator' \ + | grep -v '<doc' \ + | grep -v '</doc>' \ + | sed -e 's/<title>//g' \ + | sed -e 's/<\/title>//g' \ + | sed -e 's/<description>//g' \ + | sed -e 's/<\/description>//g' \ + | sed 's/^\s*//g' \ + | sed 's/\s*$//g' \ + > "$DATA/train.${SRC}-${TGT}.${LANG}" + done +done + +echo "pre-processing valid data..." +for ((i=0;i<${#SRCS[@]};++i)); do + SRC=${SRCS[i]} + VALID_SET=(${VALID_SETS[i]}) + for ((j=0;j<${#VALID_SET[@]};++j)); do + FILE=${VALID_SET[j]} + for LANG in "$SRC" "$TGT"; do + grep '<seg id' "$ORIG/${SRC}-${TGT}/${FILE}.${LANG}.xml" \ + | sed -e 's/<seg id="[0-9]*">\s*//g' \ + | sed -e 's/\s*<\/seg>\s*//g' \ + | sed -e "s/\’/\'/g" \ + > "$DATA/valid${j}.${SRC}-${TGT}.${LANG}" + done + done +done + +# learn BPE with sentencepiece +TRAIN_FILES=$(for SRC in "${SRCS[@]}"; do echo $DATA/train.${SRC}-${TGT}.${SRC}; echo $DATA/train.${SRC}-${TGT}.${TGT}; done | tr "\n" ",") +echo "learning joint BPE over ${TRAIN_FILES}..." +python "$SPM_TRAIN" \ + --input=$TRAIN_FILES \ + --model_prefix=$DATA/sentencepiece.bpe \ + --vocab_size=$BPESIZE \ + --character_coverage=1.0 \ + --model_type=bpe + +# encode train/valid +echo "encoding train with learned BPE..." +for SRC in "${SRCS[@]}"; do + python "$SPM_ENCODE" \ + --model "$DATA/sentencepiece.bpe.model" \ + --output_format=piece \ + --inputs $DATA/train.${SRC}-${TGT}.${SRC} $DATA/train.${SRC}-${TGT}.${TGT} \ + --outputs $DATA/train.bpe.${SRC}-${TGT}.${SRC} $DATA/train.bpe.${SRC}-${TGT}.${TGT} \ + --min-len $TRAIN_MINLEN --max-len $TRAIN_MAXLEN +done + +echo "encoding valid with learned BPE..." +for ((i=0;i<${#SRCS[@]};++i)); do + SRC=${SRCS[i]} + VALID_SET=(${VALID_SETS[i]}) + for ((j=0;j<${#VALID_SET[@]};++j)); do + python "$SPM_ENCODE" \ + --model "$DATA/sentencepiece.bpe.model" \ + --output_format=piece \ + --inputs $DATA/valid${j}.${SRC}-${TGT}.${SRC} $DATA/valid${j}.${SRC}-${TGT}.${TGT} \ + --outputs $DATA/valid${j}.bpe.${SRC}-${TGT}.${SRC} $DATA/valid${j}.bpe.${SRC}-${TGT}.${TGT} + done +done diff --git a/data/fairseq/examples/translation/prepare-wmt14en2de.sh b/data/fairseq/examples/translation/prepare-wmt14en2de.sh new file mode 100644 index 0000000000000000000000000000000000000000..6702c88b568c9e680b525593ff0c9fb0a474825d --- /dev/null +++ b/data/fairseq/examples/translation/prepare-wmt14en2de.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# Adapted from https://github.com/facebookresearch/MIXER/blob/master/prepareData.sh + +echo 'Cloning Moses github repository (for tokenization scripts)...' +git clone https://github.com/moses-smt/mosesdecoder.git + +echo 'Cloning Subword NMT repository (for BPE pre-processing)...' +git clone https://github.com/rsennrich/subword-nmt.git + +SCRIPTS=mosesdecoder/scripts +TOKENIZER=$SCRIPTS/tokenizer/tokenizer.perl +CLEAN=$SCRIPTS/training/clean-corpus-n.perl +NORM_PUNC=$SCRIPTS/tokenizer/normalize-punctuation.perl +REM_NON_PRINT_CHAR=$SCRIPTS/tokenizer/remove-non-printing-char.perl +BPEROOT=subword-nmt/subword_nmt +BPE_TOKENS=40000 + +URLS=( + "http://statmt.org/wmt13/training-parallel-europarl-v7.tgz" + "http://statmt.org/wmt13/training-parallel-commoncrawl.tgz" + "http://data.statmt.org/wmt17/translation-task/training-parallel-nc-v12.tgz" + "http://data.statmt.org/wmt17/translation-task/dev.tgz" + "http://statmt.org/wmt14/test-full.tgz" +) +FILES=( + "training-parallel-europarl-v7.tgz" + "training-parallel-commoncrawl.tgz" + "training-parallel-nc-v12.tgz" + "dev.tgz" + "test-full.tgz" +) +CORPORA=( + "training/europarl-v7.de-en" + "commoncrawl.de-en" + "training/news-commentary-v12.de-en" +) + +# This will make the dataset compatible to the one used in "Convolutional Sequence to Sequence Learning" +# https://arxiv.org/abs/1705.03122 +if [ "$1" == "--icml17" ]; then + URLS[2]="http://statmt.org/wmt14/training-parallel-nc-v9.tgz" + FILES[2]="training-parallel-nc-v9.tgz" + CORPORA[2]="training/news-commentary-v9.de-en" + OUTDIR=wmt14_en_de +else + OUTDIR=wmt17_en_de +fi + +if [ ! -d "$SCRIPTS" ]; then + echo "Please set SCRIPTS variable correctly to point to Moses scripts." + exit +fi + +src=en +tgt=de +lang=en-de +prep=$OUTDIR +tmp=$prep/tmp +orig=orig +dev=dev/newstest2013 + +mkdir -p $orig $tmp $prep + +cd $orig + +for ((i=0;i<${#URLS[@]};++i)); do + file=${FILES[i]} + if [ -f $file ]; then + echo "$file already exists, skipping download" + else + url=${URLS[i]} + wget "$url" + if [ -f $file ]; then + echo "$url successfully downloaded." + else + echo "$url not successfully downloaded." + exit -1 + fi + if [ ${file: -4} == ".tgz" ]; then + tar zxvf $file + elif [ ${file: -4} == ".tar" ]; then + tar xvf $file + fi + fi +done +cd .. + +echo "pre-processing train data..." +for l in $src $tgt; do + rm $tmp/train.tags.$lang.tok.$l + for f in "${CORPORA[@]}"; do + cat $orig/$f.$l | \ + perl $NORM_PUNC $l | \ + perl $REM_NON_PRINT_CHAR | \ + perl $TOKENIZER -threads 8 -a -l $l >> $tmp/train.tags.$lang.tok.$l + done +done + +echo "pre-processing test data..." +for l in $src $tgt; do + if [ "$l" == "$src" ]; then + t="src" + else + t="ref" + fi + grep '<seg id' $orig/test-full/newstest2014-deen-$t.$l.sgm | \ + sed -e 's/<seg id="[0-9]*">\s*//g' | \ + sed -e 's/\s*<\/seg>\s*//g' | \ + sed -e "s/\’/\'/g" | \ + perl $TOKENIZER -threads 8 -a -l $l > $tmp/test.$l + echo "" +done + +echo "splitting train and valid..." +for l in $src $tgt; do + awk '{if (NR%100 == 0) print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/valid.$l + awk '{if (NR%100 != 0) print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/train.$l +done + +TRAIN=$tmp/train.de-en +BPE_CODE=$prep/code +rm -f $TRAIN +for l in $src $tgt; do + cat $tmp/train.$l >> $TRAIN +done + +echo "learn_bpe.py on ${TRAIN}..." +python $BPEROOT/learn_bpe.py -s $BPE_TOKENS < $TRAIN > $BPE_CODE + +for L in $src $tgt; do + for f in train.$L valid.$L test.$L; do + echo "apply_bpe.py to ${f}..." + python $BPEROOT/apply_bpe.py -c $BPE_CODE < $tmp/$f > $tmp/bpe.$f + done +done + +perl $CLEAN -ratio 1.5 $tmp/bpe.train $src $tgt $prep/train 1 250 +perl $CLEAN -ratio 1.5 $tmp/bpe.valid $src $tgt $prep/valid 1 250 + +for L in $src $tgt; do + cp $tmp/bpe.test.$L $prep/test.$L +done diff --git a/data/fairseq/examples/translation/prepare-wmt14en2fr.sh b/data/fairseq/examples/translation/prepare-wmt14en2fr.sh new file mode 100644 index 0000000000000000000000000000000000000000..2ac97a5b76fab255449493488ed8bd67350a7bac --- /dev/null +++ b/data/fairseq/examples/translation/prepare-wmt14en2fr.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# Adapted from https://github.com/facebookresearch/MIXER/blob/master/prepareData.sh + +echo 'Cloning Moses github repository (for tokenization scripts)...' +git clone https://github.com/moses-smt/mosesdecoder.git + +echo 'Cloning Subword NMT repository (for BPE pre-processing)...' +git clone https://github.com/rsennrich/subword-nmt.git + +SCRIPTS=mosesdecoder/scripts +TOKENIZER=$SCRIPTS/tokenizer/tokenizer.perl +CLEAN=$SCRIPTS/training/clean-corpus-n.perl +NORM_PUNC=$SCRIPTS/tokenizer/normalize-punctuation.perl +REM_NON_PRINT_CHAR=$SCRIPTS/tokenizer/remove-non-printing-char.perl +BPEROOT=subword-nmt/subword_nmt +BPE_TOKENS=40000 + +URLS=( + "http://statmt.org/wmt13/training-parallel-europarl-v7.tgz" + "http://statmt.org/wmt13/training-parallel-commoncrawl.tgz" + "http://statmt.org/wmt13/training-parallel-un.tgz" + "http://statmt.org/wmt14/training-parallel-nc-v9.tgz" + "http://statmt.org/wmt10/training-giga-fren.tar" + "http://statmt.org/wmt14/test-full.tgz" +) +FILES=( + "training-parallel-europarl-v7.tgz" + "training-parallel-commoncrawl.tgz" + "training-parallel-un.tgz" + "training-parallel-nc-v9.tgz" + "training-giga-fren.tar" + "test-full.tgz" +) +CORPORA=( + "training/europarl-v7.fr-en" + "commoncrawl.fr-en" + "un/undoc.2000.fr-en" + "training/news-commentary-v9.fr-en" + "giga-fren.release2.fixed" +) + +if [ ! -d "$SCRIPTS" ]; then + echo "Please set SCRIPTS variable correctly to point to Moses scripts." + exit +fi + +src=en +tgt=fr +lang=en-fr +prep=wmt14_en_fr +tmp=$prep/tmp +orig=orig + +mkdir -p $orig $tmp $prep + +cd $orig + +for ((i=0;i<${#URLS[@]};++i)); do + file=${FILES[i]} + if [ -f $file ]; then + echo "$file already exists, skipping download" + else + url=${URLS[i]} + wget "$url" + if [ -f $file ]; then + echo "$url successfully downloaded." + else + echo "$url not successfully downloaded." + exit -1 + fi + if [ ${file: -4} == ".tgz" ]; then + tar zxvf $file + elif [ ${file: -4} == ".tar" ]; then + tar xvf $file + fi + fi +done + +gunzip giga-fren.release2.fixed.*.gz +cd .. + +echo "pre-processing train data..." +for l in $src $tgt; do + rm $tmp/train.tags.$lang.tok.$l + for f in "${CORPORA[@]}"; do + cat $orig/$f.$l | \ + perl $NORM_PUNC $l | \ + perl $REM_NON_PRINT_CHAR | \ + perl $TOKENIZER -threads 8 -a -l $l >> $tmp/train.tags.$lang.tok.$l + done +done + +echo "pre-processing test data..." +for l in $src $tgt; do + if [ "$l" == "$src" ]; then + t="src" + else + t="ref" + fi + grep '<seg id' $orig/test-full/newstest2014-fren-$t.$l.sgm | \ + sed -e 's/<seg id="[0-9]*">\s*//g' | \ + sed -e 's/\s*<\/seg>\s*//g' | \ + sed -e "s/\’/\'/g" | \ + perl $TOKENIZER -threads 8 -a -l $l > $tmp/test.$l + echo "" +done + +echo "splitting train and valid..." +for l in $src $tgt; do + awk '{if (NR%1333 == 0) print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/valid.$l + awk '{if (NR%1333 != 0) print $0; }' $tmp/train.tags.$lang.tok.$l > $tmp/train.$l +done + +TRAIN=$tmp/train.fr-en +BPE_CODE=$prep/code +rm -f $TRAIN +for l in $src $tgt; do + cat $tmp/train.$l >> $TRAIN +done + +echo "learn_bpe.py on ${TRAIN}..." +python $BPEROOT/learn_bpe.py -s $BPE_TOKENS < $TRAIN > $BPE_CODE + +for L in $src $tgt; do + for f in train.$L valid.$L test.$L; do + echo "apply_bpe.py to ${f}..." + python $BPEROOT/apply_bpe.py -c $BPE_CODE < $tmp/$f > $tmp/bpe.$f + done +done + +perl $CLEAN -ratio 1.5 $tmp/bpe.train $src $tgt $prep/train 1 250 +perl $CLEAN -ratio 1.5 $tmp/bpe.valid $src $tgt $prep/valid 1 250 + +for L in $src $tgt; do + cp $tmp/bpe.test.$L $prep/test.$L +done diff --git a/data/fairseq/examples/wmt20/README.md b/data/fairseq/examples/wmt20/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b4f2874652f8be19998a65faa1d9276d8017ec59 --- /dev/null +++ b/data/fairseq/examples/wmt20/README.md @@ -0,0 +1,72 @@ +# WMT 20 + +This page provides pointers to the models of Facebook-FAIR's WMT'20 news translation task submission [(Chen et al., 2020)](https://arxiv.org/abs/2011.08298). + +## Single best MT models (after finetuning on part of WMT20 news dev set) + +Model | Description | Download +---|---|--- +`transformer.wmt20.ta-en` | Ta->En | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.ta-en.single.tar.gz) +`transformer.wmt20.en-ta` | En->Ta | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.en-ta.single.tar.gz) +`transformer.wmt20.iu-en.news` | Iu->En (News domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.iu-en.news.single.tar.gz) +`transformer.wmt20.en-iu.news` | En->Iu (News domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.en-iu.news.single.tar.gz) +`transformer.wmt20.iu-en.nh` | Iu->En (Nunavut Hansard domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.iu-en.nh.single.tar.gz) +`transformer.wmt20.en-iu.nh` | En->Iu (Nunavut Hansard domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.en-iu.nh.single.tar.gz) + +## Language models +Model | Description | Download +---|---|--- +`transformer_lm.wmt20.en` | En Language Model | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.en.tar.gz) +`transformer_lm.wmt20.ta` | Ta Language Model | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.ta.tar.gz) +`transformer_lm.wmt20.iu.news` | Iu Language Model (News domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.iu.news.tar.gz) +`transformer_lm.wmt20.iu.nh` | Iu Language Model (Nunavut Hansard domain) | [download (.tar.gz)](https://dl.fbaipublicfiles.com/fairseq/models/wmt20.iu.nh.tar.gz) + +## Example usage (torch.hub) + +#### Translation + +```python +import torch + +# English to Tamil translation +en2ta = torch.hub.load('pytorch/fairseq', 'transformer.wmt20.en-ta') +en2ta.translate("Machine learning is great!") # 'இயந்திரக் கற்றல் அருமை!' + +# Tamil to English translation +ta2en = torch.hub.load('pytorch/fairseq', 'transformer.wmt20.ta-en') +ta2en.translate("இயந்திரக் கற்றல் அருமை!") # 'Machine learning is great!' + +# English to Inuktitut translation +en2iu = torch.hub.load('pytorch/fairseq', 'transformer.wmt20.en-iu.news') +en2iu.translate("machine learning is great!") # 'ᖃᒧᑕᐅᔭᓄᑦ ᐃᓕᓐᓂᐊᕐᓂᖅ ᐱᐅᔪᒻᒪᕆᒃ!' + +# Inuktitut to English translation +iu2en = torch.hub.load('pytorch/fairseq', 'transformer.wmt20.iu-en.news') +iu2en.translate("ᖃᒧᑕᐅᔭᓄᑦ ᐃᓕᓐᓂᐊᕐᓂᖅ ᐱᐅᔪᒻᒪᕆᒃ!") # 'Machine learning excellence!' +``` + +#### Language Modeling + +```python +# Sample from the English LM +en_lm = torch.hub.load('pytorch/fairseq', 'transformer_lm.wmt20.en') +en_lm.sample("Machine learning is") # 'Machine learning is a type of artificial intelligence that uses machine learning to learn from data and make predictions.' + +# Sample from the Tamil LM +ta_lm = torch.hub.load('pytorch/fairseq', 'transformer_lm.wmt20.ta') +ta_lm.sample("இயந்திரக் கற்றல் என்பது செயற்கை நுண்ணறிவின்") # 'இயந்திரக் கற்றல் என்பது செயற்கை நுண்ணறிவின் ஒரு பகுதியாகும்.' + +# Sample from the Inuktitut LM +iu_lm = torch.hub.load('pytorch/fairseq', 'transformer_lm.wmt20.iu.news') +iu_lm.sample("ᖃᒧᑕᐅᔭᓄᑦ ᐃᓕᓐᓂᐊᕐᓂᖅ") # 'ᖃᒧᑕᐅᔭᓄᑦ ᐃᓕᓐᓂᐊᕐᓂᖅ, ᐊᒻᒪᓗ ᓯᓚᐅᑉ ᐊᓯᙳᖅᐸᓪᓕᐊᓂᖓᓄᑦ ᖃᓄᐃᓕᐅᕈᑎᒃᓴᑦ, ᐃᓚᖃᖅᖢᑎᒃ ᐅᑯᓂᖓ:' +``` + +## Citation +```bibtex +@inproceedings{chen2020facebook + title={Facebook AI's WMT20 News Translation Task Submission}, + author={Peng-Jen Chen and Ann Lee and Changhan Wang and Naman Goyal and Angela Fan and Mary Williamson and Jiatao Gu}, + booktitle={Proc. of WMT}, + year={2020}, +} +```