edit

README.md

@@ -0,0 +1,4 @@
+This is a tiny random  pegasus-cnn_dailymail model used for testing
+
+See `make-pegasus-cnn_dailymail-tiny-random.py` for how it was created.
+

config.json

@@ -0,0 +1,56 @@
+{
+  "activation_dropout": 0.1,
+  "activation_function": "relu",
+  "add_bias_logits": false,
+  "add_final_layer_norm": true,
+  "architectures": [
+    "PegasusForConditionalGeneration"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 0,
+  "classif_dropout": 0.0,
+  "classifier_dropout": 0.0,
+  "d_model": 64,
+  "decoder_attention_heads": 2,
+  "decoder_ffn_dim": 64,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 2,
+  "decoder_start_token_id": 0,
+  "dropout": 0.1,
+  "encoder_attention_heads": 16,
+  "encoder_ffn_dim": 64,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 2,
+  "eos_token_id": 1,
+  "extra_pos_embeddings": 1,
+  "forced_eos_token_id": 1,
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "length_penalty": 0.8,
+  "max_length": 128,
+  "max_position_embeddings": 1024,
+  "min_length": 32,
+  "model_type": "pegasus",
+  "normalize_before": true,
+  "normalize_embedding": false,
+  "num_beams": 8,
+  "num_hidden_layers": 2,
+  "pad_token_id": 0,
+  "scale_embedding": true,
+  "static_position_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.9.0.dev0",
+  "use_cache": true,
+  "vocab_size": 5103
+}

make-pegasus-cnn_dailymail-tiny-random.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# coding: utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script creates a smallish random model, with a few layers to test things like MP/PP, where
+# tiny and tiner models are too too small
+#
+# It will be used then as "stas/mt5-tiny-random"
+
+# To build:
+# 1. clone sentencepiece into this dir
+# git clone https://github.com/google/sentencepiece
+#
+# 2. run this script
+
+from pathlib import Path
+import json
+import tempfile
+
+from transformers import PegasusTokenizer, PegasusTokenizerFast, PegasusConfig, PegasusForConditionalGeneration
+#from transformers.models.t5.tokenization_t5 import VOCAB_FILES_NAMES
+
+mname_from = "google/pegasus-cnn_dailymail"
+mname_very_small = "pegasus-cnn_dailymail-tiny-random"
+
+tokenizer = PegasusTokenizer.from_pretrained(mname_from)
+config = PegasusConfig.from_pretrained(mname_from)
+#tokenizer_fast = PegasusTokenizerFast.from_pretrained(mname_from)
+
+# Shrink the vocab of orig
+import sys
+# HACK: need the sentencepiece source to get sentencepiece_model_pb2, as it doesn't get installed
+# git clone https://github.com/google/sentencepiece
+sys.path.append("./sentencepiece/python/src/sentencepiece")
+import sentencepiece_model_pb2 as model
+
+tmp_dir = "/tmp/pegasus-tiny"
+tokenizer.save_pretrained(tmp_dir)
+file = tmp_dir + "/spiece.model"
+with open(file, 'rb') as f: data = f.read()
+
+# adapted from https://blog.ceshine.net/post/trim-down-sentencepiece-vocabulary/
+m = model.ModelProto()
+m.ParseFromString(data)
+
+keep_items = 5000
+
+print("Shrinking vocab")
+print(f"original dict {len(m.pieces)}")
+for i in range(len(m.pieces)-keep_items): _ = m.pieces.pop()
+print(f"new dict {len(m.pieces)}")
+
+with open(tmp_dir + "/spiece-short.model", 'wb') as f:
+    f.write(m.SerializeToString())
+
+tokenizer = PegasusTokenizer(vocab_file=tmp_dir + "/spiece-short.model")
+
+config.update(dict(
+    vocab_size=keep_items+12,
+    d_model=64,
+    decoder_attention_heads=2,
+    decoder_ffn_dim=64,
+    decoder_layers=2,
+    encoder_attention_heads=16,
+    encoder_ffn_dim=64,
+    encoder_layers=2,
+    num_hidden_layers=2,
+))
+print("new config", config)
+
+very_small_model = PegasusForConditionalGeneration(config)
+print(f"num of params {very_small_model.num_parameters()}")
+very_small_model.resize_token_embeddings(len(tokenizer))
+
+# Test
+src_texts = ["A long paragraph for summarization.", "Another paragraph for summarization."]
+tgt_texts = ["Summary of the text.", "Another summary."]
+
+batch = tokenizer.prepare_seq2seq_batch(src_texts, tgt_texts, return_tensors="pt")
+outputs = very_small_model(**batch)
+
+print("test output:", len(outputs.logits[0]))
+
+# Save
+very_small_model.half() # makes it smaller
+very_small_model.save_pretrained(mname_very_small)
+config.save_pretrained(mname_very_small)
+tokenizer.save_pretrained(mname_very_small)
+#tokenizer_fast.save_pretrained(mname_very_small)
+
+print(f"Generated {mname_very_small}")
+
+# Upload
+# transformers-cli repo create pegasus-cnn_dailymail-tiny-random
+# clone and add files

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:170d2a8b3c7499cdc5abe22abda269ccd3020bf1e5571e1f53364b11d4bd601d
+size 1227089

special_tokens_map.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": "<mask_2>", "additional_special_tokens": ["<mask_1>", "<unk_2>", "<unk_3>", "<unk_4>", "<unk_5>", "<unk_6>", "<unk_7>", "<unk_8>", "<unk_9>", "<unk_10>", "<unk_11>", "<unk_12>", "<unk_13>", "<unk_14>", "<unk_15>", "<unk_16>", "<unk_17>", "<unk_18>", "<unk_19>", "<unk_20>", "<unk_21>", "<unk_22>", "<unk_23>", "<unk_24>", "<unk_25>", "<unk_26>", "<unk_27>", "<unk_28>", "<unk_29>", "<unk_30>", "<unk_31>", "<unk_32>", "<unk_33>", "<unk_34>", "<unk_35>", "<unk_36>", "<unk_37>", "<unk_38>", "<unk_39>", "<unk_40>", "<unk_41>", "<unk_42>", "<unk_43>", "<unk_44>", "<unk_45>", "<unk_46>", "<unk_47>", "<unk_48>", "<unk_49>", "<unk_50>", "<unk_51>", "<unk_52>", "<unk_53>", "<unk_54>", "<unk_55>", "<unk_56>", "<unk_57>", "<unk_58>", "<unk_59>", "<unk_60>", "<unk_61>", "<unk_62>", "<unk_63>", "<unk_64>", "<unk_65>", "<unk_66>", "<unk_67>", "<unk_68>", "<unk_69>", "<unk_70>", "<unk_71>", "<unk_72>", "<unk_73>", "<unk_74>", "<unk_75>", "<unk_76>", "<unk_77>", "<unk_78>", "<unk_79>", "<unk_80>", "<unk_81>", "<unk_82>", "<unk_83>", "<unk_84>", "<unk_85>", "<unk_86>", "<unk_87>", "<unk_88>", "<unk_89>", "<unk_90>", "<unk_91>", "<unk_92>", "<unk_93>", "<unk_94>", "<unk_95>", "<unk_96>", "<unk_97>", "<unk_98>", "<unk_99>", "<unk_100>", "<unk_101>", "<unk_102>"]}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:987dae5fe18ba4b1884e60e9c2e6eb0ee6b6aea7de7515983aee5aae69dde75a
+size 326384

tokenizer_config.json

@@ -0,0 +1 @@
+{"eos_token": "</s>", "unk_token": "<unk>", "mask_token": "<mask_2>", "pad_token": "<pad>", "mask_token_sent": "<mask_1>", "offset": 103, "additional_special_tokens": ["<mask_1>", "<unk_2>", "<unk_3>", "<unk_4>", "<unk_5>", "<unk_6>", "<unk_7>", "<unk_8>", "<unk_9>", "<unk_10>", "<unk_11>", "<unk_12>", "<unk_13>", "<unk_14>", "<unk_15>", "<unk_16>", "<unk_17>", "<unk_18>", "<unk_19>", "<unk_20>", "<unk_21>", "<unk_22>", "<unk_23>", "<unk_24>", "<unk_25>", "<unk_26>", "<unk_27>", "<unk_28>", "<unk_29>", "<unk_30>", "<unk_31>", "<unk_32>", "<unk_33>", "<unk_34>", "<unk_35>", "<unk_36>", "<unk_37>", "<unk_38>", "<unk_39>", "<unk_40>", "<unk_41>", "<unk_42>", "<unk_43>", "<unk_44>", "<unk_45>", "<unk_46>", "<unk_47>", "<unk_48>", "<unk_49>", "<unk_50>", "<unk_51>", "<unk_52>", "<unk_53>", "<unk_54>", "<unk_55>", "<unk_56>", "<unk_57>", "<unk_58>", "<unk_59>", "<unk_60>", "<unk_61>", "<unk_62>", "<unk_63>", "<unk_64>", "<unk_65>", "<unk_66>", "<unk_67>", "<unk_68>", "<unk_69>", "<unk_70>", "<unk_71>", "<unk_72>", "<unk_73>", "<unk_74>", "<unk_75>", "<unk_76>", "<unk_77>", "<unk_78>", "<unk_79>", "<unk_80>", "<unk_81>", "<unk_82>", "<unk_83>", "<unk_84>", "<unk_85>", "<unk_86>", "<unk_87>", "<unk_88>", "<unk_89>", "<unk_90>", "<unk_91>", "<unk_92>", "<unk_93>", "<unk_94>", "<unk_95>", "<unk_96>", "<unk_97>", "<unk_98>", "<unk_99>", "<unk_100>", "<unk_101>", "<unk_102>"], "sp_model_kwargs": {}, "tokenizer_class": "PegasusTokenizer"}