updates

README.md

@@ -0,0 +1 @@
+This is a tiny-electra random model to be used for basic testing.

config.json

@@ -0,0 +1,27 @@
+{
+  "architectures": [
+    "ElectraForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "embedding_size": 64,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 64,
+  "initializer_range": 0.02,
+  "intermediate_size": 64,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 64,
+  "model_type": "electra",
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "summary_activation": "gelu",
+  "summary_last_dropout": 0.1,
+  "summary_type": "first",
+  "summary_use_proj": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.9.0.dev0",
+  "type_vocab_size": 2,
+  "vocab_size": 30522
+}

make-tiny-electra.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# coding: utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script creates a tiny random model
+#
+# It will be used then as "hf-internal-testing/tiny-electra"
+
+# ***To build from scratch***
+#
+# 1. clone sentencepiece into a parent dir
+# git clone https://github.com/google/sentencepiece
+#
+# 2. create a new repo at https://huggingface.co/new
+# make sure to choose 'hf-internal-testing' as the Owner
+#
+# 3. clone
+# git clone https://huggingface.co/hf-internal-testing/tiny-electra
+# cd tiny-electra
+
+# 4. start with some pre-existing script from one of the https://huggingface.co/hf-internal-testing/ tiny model repos, e.g.
+# wget https://huggingface.co/hf-internal-testing/tiny-electra/raw/main/make-xlm-roberta.py
+# chmod a+x ./make-tiny-electra.py
+# mv ./make-tiny-xlm-roberta.py ./make-tiny-electra.py
+#
+# 5. automatically rename things from the old names to new ones
+# perl -pi -e 's|XLMRoberta|Electra|g' make-tiny-electra.py
+# perl -pi -e 's|xlm-roberta|electra|g' make-tiny-electra.py
+#
+# 6. edit and re-run this script while fixing it up
+# ./make-tiny-electra.py
+#
+# 7. add/commit/push
+# git add *
+# git commit -m "new tiny model"
+# git push
+
+# ***To update***
+#
+# 1. clone the existing repo
+# git clone https://huggingface.co/hf-internal-testing/tiny-electra
+# cd tiny-electra
+#
+# 2. edit and re-run this script after doing whatever changes are needed
+# ./make-tiny-electra.py
+#
+# 3. commit/push
+# git commit -m "new tiny model"
+# git push
+
+import sys
+import os
+
+from transformers import ElectraTokenizer, ElectraTokenizerFast, ElectraConfig, ElectraForMaskedLM
+
+mname_orig = "google/electra-small-generator"
+mname_tiny = "tiny-electra"
+
+### Tokenizer
+
+tokenizer_fast_tiny = ElectraTokenizerFast.from_pretrained(mname_orig)
+tokenizer_tiny = ElectraTokenizer.from_pretrained(mname_orig)
+
+### Config
+
+config_tiny = ElectraConfig.from_pretrained(mname_orig)
+print(config_tiny)
+# remember to update this to the actual config as each model is different and then shrink the numbers
+config_tiny.update(dict(
+    embedding_size=64,
+    hidden_size=64,
+    intermediate_size=64,
+    max_position_embeddings=64,
+    num_attention_heads=2,
+    num_hidden_layers=2,
+))
+print("New config", config_tiny)
+
+### Model
+
+model_tiny = ElectraForMaskedLM(config_tiny)
+print(f"{mname_tiny}: num of params {model_tiny.num_parameters()}")
+model_tiny.resize_token_embeddings(len(tokenizer_tiny))
+
+
+inputs = tokenizer_tiny("The capital of France is [MASK].", return_tensors="pt")
+labels = tokenizer_tiny("The capital of France is Paris.", return_tensors="pt")["input_ids"]
+outputs = model_tiny(**inputs, labels=labels)
+print("Test with normal tokenizer:", len(outputs.logits[0]))
+
+inputs = tokenizer_fast_tiny("The capital of France is [MASK].", return_tensors="pt")
+labels = tokenizer_fast_tiny("The capital of France is Paris.", return_tensors="pt")["input_ids"]
+outputs = model_tiny(**inputs, labels=labels)
+print("Test with normal tokenizer:", len(outputs.logits[0]))
+
+# Save
+model_tiny.half() # makes it smaller
+model_tiny.save_pretrained(".")
+tokenizer_tiny.save_pretrained(".")
+tokenizer_fast_tiny.save_pretrained(".")
+
+readme = "README.md"
+if not os.path.exists(readme):
+    with open(readme, "w") as f:
+        f.write(f"This is a {mname_tiny} random model to be used for basic testing.\n")
+
+print(f"Generated {mname_tiny}")

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d055d70e8a7ca5b19773a812c87e5206436b1381bfcb3d0ee6ca2b653cc929a2
+size 4102372

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "google/electra-small-generator", "tokenizer_class": "ElectraTokenizer"}