Transformer CNN Emotion Recognition (code, models)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+models/ailia-models/code/03-01-01-01-01-01-01.wav filter=lfs diff=lfs merge=lfs -text

code/transformer-cnn-emotion-recognition.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fcb6f514212fa50eeb1dff06a8a8b8e47c8017d913de0e5f1ecb9f0857fdca5
+size 20363810

models/ailia-models/code/03-01-01-01-01-01-01.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c467aad2ca80b089dbd4c2ba0dd16be508e4ce08230f46d36a208b859dc7fbac
+size 375720

models/ailia-models/code/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Ilia Zenkov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

models/ailia-models/code/README.md

@@ -0,0 +1,56 @@
+# transformer-cnn-emotion-recognition
+
+## Input
+
+audio file
+
+```
+03-01-01-01-01-01-01.wav
+RAVDESS Dataset
+https://smartlaboratory.org/ravdess/
+```
+
+## Output
+
+emotion label
+
+```
+Emotion: neutral
+Confidence: 0.99993193
+```
+
+## Labels
+
+```
+"surprised", "neutral", "calm", "happy",
+"sad", "angry", "fearful", "disgust"
+```
+
+## Requirements
+This model requires additional module.
+
+```
+pip3 install librosa
+```
+
+## Usage
+
+```bash
+$ python3 transformer-cnn-emotion-recognition.py -i input.wav
+```
+
+## Reference
+
+[Combining Spatial and Temporal Feature Representions of Speech Emotion by Parallelizing CNNs and Transformer-Encoders](https://github.com/IliaZenkov/transformer-cnn-emotion-recognition)
+
+## Framework
+
+PyTorch 1.6.0
+
+## Model Format
+
+ONNX opset = 11
+
+## Netron
+
+[parallel_is_all_you_want_ep428.onnx.prototxt](https://netron.app/?url=https://storage.googleapis.com/ailia-models/parallel_is_all_you_want/parallel_is_all_you_want_ep428.onnx.prototxt)

models/ailia-models/code/std_mean.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d216d69500fa7e18652821e466e7c6e6a2dfcfc49a37495966e1f63eb191ceaf
+size 90368

models/ailia-models/code/std_var.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f61153aa67beca6da05315e5d8eda4ede63fb3095a469cfe85dbe70cb47f042
+size 90368

models/ailia-models/code/transformer-cnn-emotion-recognition.py

@@ -0,0 +1,167 @@
+import time
+import sys
+
+import numpy as np
+import librosa
+
+import ailia
+
+# import original modules
+sys.path.append('../../util')
+from arg_utils import get_base_parser, update_parser  # noqa: E402
+from model_utils import check_and_download_models  # noqa: E402
+
+# logger
+from logging import getLogger  # noqa: E402
+
+logger = getLogger(__name__)
+
+# ======================
+# PARAMETERS
+# ======================
+# https://smartlaboratory.org/ravdess/
+WAVE_PATH = "03-01-01-01-01-01-01.wav"
+
+WEIGHT_PATH = "parallel_is_all_you_want_ep428.onnx"
+MODEL_PATH = "parallel_is_all_you_want_ep428.onnx.prototxt"
+REMOTE_PATH = "https://storage.googleapis.com/ailia-models/parallel_is_all_you_want/"
+
+LABELS = [
+    "surprised", "neutral", "calm", "happy",
+    "sad", "angry", "fearful", "disgust",
+]
+
+# ======================
+# Arguemnt Parser Config
+# ======================
+parser = get_base_parser(
+    'Parallel_is_All_You_Want.', WAVE_PATH, None, input_ftype='audio')
+args = update_parser(parser)
+
+
+# ======================
+# Utils
+# ======================
+
+def get_waveforms(file, sample_rate=48000):
+    # load an individual sample audio file
+    # read the full 3 seconds of the file, cut off the first 0.5s of silence; native sample rate = 48k
+    # don't need to store the sample rate that librosa.load returns
+    waveform, _ = librosa.load(file, duration=3, offset=0.5, sr=sample_rate)
+
+    # make sure waveform vectors are homogenous by defining explicitly
+    waveform_homo = np.zeros((int(sample_rate * 3, )))
+    waveform_homo[:len(waveform)] = waveform
+
+    # return a single file's waveform
+    return waveform_homo
+
+
+def feature_mfcc(
+        waveform,
+        sample_rate,
+        n_mfcc=40,
+        fft=1024,
+        winlen=512,
+        window='hamming',
+        # hop=256, # increases # of time steps; was not helpful
+        mels=128):
+    # Compute the MFCCs for all STFT frames
+    # 40 mel filterbanks (n_mfcc) = 40 coefficients
+    mfc_coefficients = librosa.feature.mfcc(
+        y=waveform,
+        sr=sample_rate,
+        n_mfcc=n_mfcc,
+        n_fft=fft,
+        win_length=winlen,
+        window=window,
+        # hop_length=hop,
+        n_mels=mels,
+        fmax=sample_rate / 2
+    )
+
+    return mfc_coefficients
+
+
+def preprocess(x):
+    c, h, w = x.shape
+    x = x.reshape(-1)
+
+    mean = np.load('std_mean.npy')
+    var = np.load('std_var.npy')
+    x = (x - mean) / np.sqrt(var)
+
+    x = x.reshape((c, h, w))
+
+    return x
+
+
+# ======================
+# Main function
+# ======================
+
+def predict(net, waveform):
+    sample_rate = 48000
+    features = feature_mfcc(waveform, sample_rate)
+    features = np.expand_dims(features, axis=0)
+
+    x = preprocess(features)
+
+    # feedforward
+    x = np.expand_dims(x, axis=0)
+    output = net.predict([x])
+
+    output_logits, output_softmax = output
+    output_softmax = output_softmax[0]
+
+    label = np.argmax(output_softmax)
+    conf = output_softmax[label]
+
+    return label, conf
+
+
+def recognize_from_audio(net):
+    for input_path in args.input:
+        logger.info(f'input: {input_path}')
+
+        # load audio
+        waveform = get_waveforms(input_path)
+
+        # inference
+        logger.info('Start inference...')
+        if args.benchmark:
+            logger.info('BENCHMARK mode')
+            total_time_estimation = 0
+            for i in range(args.benchmark_count):
+                start = int(round(time.time() * 1000))
+                label, conf = predict(net, waveform)
+                end = int(round(time.time() * 1000))
+                estimation_time = (end - start)
+
+                # Loggin
+                logger.info(f'\tailia processing estimation time {estimation_time} ms')
+                if i != 0:
+                    total_time_estimation = total_time_estimation + estimation_time
+
+        else:
+            label, conf = predict(net, waveform)
+
+        label = LABELS[label]
+        logger.info("Emotion: %s" % label)
+        logger.info("Confidence: %s" % conf)
+
+    logger.info('Script finished successfully.')
+
+
+def main():
+    # model files check and download
+    check_and_download_models(WEIGHT_PATH, MODEL_PATH, REMOTE_PATH)
+
+    # initialize
+    net = ailia.Net(MODEL_PATH, WEIGHT_PATH, env_id=args.env_id)
+
+    recognize_from_audio(net)
+
+
+if __name__ == "__main__":
+    main()

models/ailia-models/parallel_is_all_you_want_ep428.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea26e8632d8db39edd80118981ea26e4ab7bad459b2e774d47d90116c2eac8e3
+size 1018228

models/ailia-models/parallel_is_all_you_want_ep428.onnx.prototxt

@@ -0,0 +1,4341 @@
+ir_version: 6
+producer_name: "pytorch"
+producer_version: "1.6"
+model_version: 0
+graph {
+  name: "torch-jit-export"
+  node {
+    input: "features"
+    input: "conv2Dblock1.0.weight"
+    input: "conv2Dblock1.0.bias"
+    output: "93"
+    name: "Conv_0"
+    op_type: "Conv"
+    attribute {
+      name: "dilations"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "group"
+      i: 1
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 3
+      ints: 3
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 1
+      ints: 1
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+  }
+  node {
+    input: "93"
+    input: "conv2Dblock1.1.weight"
+    input: "conv2Dblock1.1.bias"
+    input: "conv2Dblock1.1.running_mean"
+    input: "conv2Dblock1.1.running_var"
+    output: "94"
+    name: "BatchNormalization_1"
+    op_type: "BatchNormalization"
+    attribute {
+      name: "epsilon"
+      f: 9.999999747378752e-06
+      type: FLOAT
+    }
+    attribute {
+      name: "momentum"
+      f: 0.8999999761581421
+      type: FLOAT
+    }
+  }
+  node {
+    input: "94"
+    output: "95"
+    name: "Relu_2"
+    op_type: "Relu"
+  }
+  node {
+    input: "95"
+    output: "96"
+    name: "MaxPool_3"
+    op_type: "MaxPool"
+    attribute {
+      name: "ceil_mode"
+      i: 0
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 2
+      ints: 2
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 0
+      ints: 0
+      ints: 0
+      ints: 0
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 2
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    input: "96"
+    input: "conv2Dblock1.5.weight"
+    input: "conv2Dblock1.5.bias"
+    output: "97"
+    name: "Conv_4"
+    op_type: "Conv"
+    attribute {
+      name: "dilations"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "group"
+      i: 1
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 3
+      ints: 3
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 1
+      ints: 1
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+  }
+  node {
+    input: "97"
+    input: "conv2Dblock1.6.weight"
+    input: "conv2Dblock1.6.bias"
+    input: "conv2Dblock1.6.running_mean"
+    input: "conv2Dblock1.6.running_var"
+    output: "98"
+    name: "BatchNormalization_5"
+    op_type: "BatchNormalization"
+    attribute {
+      name: "epsilon"
+      f: 9.999999747378752e-06
+      type: FLOAT
+    }
+    attribute {
+      name: "momentum"
+      f: 0.8999999761581421
+      type: FLOAT
+    }
+  }
+  node {
+    input: "98"
+    output: "99"
+    name: "Relu_6"
+    op_type: "Relu"
+  }
+  node {
+    input: "99"
+    output: "100"
+    name: "MaxPool_7"
+    op_type: "MaxPool"
+    attribute {
+      name: "ceil_mode"
+      i: 0
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 4
+      ints: 4
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 0
+      ints: 0
+      ints: 0
+      ints: 0
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 4
+      ints: 4
+      type: INTS
+    }
+  }
+  node {
+    input: "100"
+    input: "conv2Dblock1.10.weight"
+    input: "conv2Dblock1.10.bias"
+    output: "101"
+    name: "Conv_8"
+    op_type: "Conv"
+    attribute {
+      name: "dilations"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "group"
+      i: 1
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 3
+      ints: 3
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 1
+      ints: 1
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+  }
+  node {
+    input: "101"
+    input: "conv2Dblock1.11.weight"
+    input: "conv2Dblock1.11.bias"
+    input: "conv2Dblock1.11.running_mean"
+    input: "conv2Dblock1.11.running_var"
+    output: "102"
+    name: "BatchNormalization_9"
+    op_type: "BatchNormalization"
+    attribute {
+      name: "epsilon"
+      f: 9.999999747378752e-06
+      type: FLOAT
+    }
+    attribute {
+      name: "momentum"
+      f: 0.8999999761581421
+      type: FLOAT
+    }
+  }
+  node {
+    input: "102"
+    output: "103"
+    name: "Relu_10"
+    op_type: "Relu"
+  }
+  node {
+    input: "103"
+    output: "104"
+    name: "MaxPool_11"
+    op_type: "MaxPool"
+    attribute {
+      name: "ceil_mode"
+      i: 0
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 4
+      ints: 4
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 0
+      ints: 0
+      ints: 0
+      ints: 0
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 4
+      ints: 4
+      type: INTS
+    }
+  }
+  node {
+    input: "104"
+    output: "105"
+    name: "Flatten_12"
+    op_type: "Flatten"
+    attribute {
+      name: "axis"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "features"
+    input: "conv2Dblock2.0.weight"
+    input: "conv2Dblock2.0.bias"
+    output: "106"
+    name: "Conv_13"
+    op_type: "Conv"
+    attribute {
+      name: "dilations"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "group"
+      i: 1
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 3
+      ints: 3
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 1
+      ints: 1
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+  }
+  node {
+    input: "106"
+    input: "conv2Dblock2.1.weight"
+    input: "conv2Dblock2.1.bias"
+    input: "conv2Dblock2.1.running_mean"
+    input: "conv2Dblock2.1.running_var"
+    output: "107"
+    name: "BatchNormalization_14"
+    op_type: "BatchNormalization"
+    attribute {
+      name: "epsilon"
+      f: 9.999999747378752e-06
+      type: FLOAT
+    }
+    attribute {
+      name: "momentum"
+      f: 0.8999999761581421
+      type: FLOAT
+    }
+  }
+  node {
+    input: "107"
+    output: "108"
+    name: "Relu_15"
+    op_type: "Relu"
+  }
+  node {
+    input: "108"
+    output: "109"
+    name: "MaxPool_16"
+    op_type: "MaxPool"
+    attribute {
+      name: "ceil_mode"
+      i: 0
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 2
+      ints: 2
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 0
+      ints: 0
+      ints: 0
+      ints: 0
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 2
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    input: "109"
+    input: "conv2Dblock2.5.weight"
+    input: "conv2Dblock2.5.bias"
+    output: "110"
+    name: "Conv_17"
+    op_type: "Conv"
+    attribute {
+      name: "dilations"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "group"
+      i: 1
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 3
+      ints: 3
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 1
+      ints: 1
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+  }
+  node {
+    input: "110"
+    input: "conv2Dblock2.6.weight"
+    input: "conv2Dblock2.6.bias"
+    input: "conv2Dblock2.6.running_mean"
+    input: "conv2Dblock2.6.running_var"
+    output: "111"
+    name: "BatchNormalization_18"
+    op_type: "BatchNormalization"
+    attribute {
+      name: "epsilon"
+      f: 9.999999747378752e-06
+      type: FLOAT
+    }
+    attribute {
+      name: "momentum"
+      f: 0.8999999761581421
+      type: FLOAT
+    }
+  }
+  node {
+    input: "111"
+    output: "112"
+    name: "Relu_19"
+    op_type: "Relu"
+  }
+  node {
+    input: "112"
+    output: "113"
+    name: "MaxPool_20"
+    op_type: "MaxPool"
+    attribute {
+      name: "ceil_mode"
+      i: 0
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 4
+      ints: 4
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 0
+      ints: 0
+      ints: 0
+      ints: 0
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 4
+      ints: 4
+      type: INTS
+    }
+  }
+  node {
+    input: "113"
+    input: "conv2Dblock2.10.weight"
+    input: "conv2Dblock2.10.bias"
+    output: "114"
+    name: "Conv_21"
+    op_type: "Conv"
+    attribute {
+      name: "dilations"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "group"
+      i: 1
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 3
+      ints: 3
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 1
+      ints: 1
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 1
+      ints: 1
+      type: INTS
+    }
+  }
+  node {
+    input: "114"
+    input: "conv2Dblock2.11.weight"
+    input: "conv2Dblock2.11.bias"
+    input: "conv2Dblock2.11.running_mean"
+    input: "conv2Dblock2.11.running_var"
+    output: "115"
+    name: "BatchNormalization_22"
+    op_type: "BatchNormalization"
+    attribute {
+      name: "epsilon"
+      f: 9.999999747378752e-06
+      type: FLOAT
+    }
+    attribute {
+      name: "momentum"
+      f: 0.8999999761581421
+      type: FLOAT
+    }
+  }
+  node {
+    input: "115"
+    output: "116"
+    name: "Relu_23"
+    op_type: "Relu"
+  }
+  node {
+    input: "116"
+    output: "117"
+    name: "MaxPool_24"
+    op_type: "MaxPool"
+    attribute {
+      name: "ceil_mode"
+      i: 0
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 4
+      ints: 4
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 0
+      ints: 0
+      ints: 0
+      ints: 0
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 4
+      ints: 4
+      type: INTS
+    }
+  }
+  node {
+    input: "117"
+    output: "118"
+    name: "Flatten_25"
+    op_type: "Flatten"
+    attribute {
+      name: "axis"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "features"
+    output: "119"
+    name: "MaxPool_26"
+    op_type: "MaxPool"
+    attribute {
+      name: "ceil_mode"
+      i: 0
+      type: INT
+    }
+    attribute {
+      name: "kernel_shape"
+      ints: 1
+      ints: 4
+      type: INTS
+    }
+    attribute {
+      name: "pads"
+      ints: 0
+      ints: 0
+      ints: 0
+      ints: 0
+      type: INTS
+    }
+    attribute {
+      name: "strides"
+      ints: 1
+      ints: 4
+      type: INTS
+    }
+  }
+  node {
+    input: "119"
+    output: "120"
+    name: "Squeeze_27"
+    op_type: "Squeeze"
+    attribute {
+      name: "axes"
+      ints: 1
+      type: INTS
+    }
+  }
+  node {
+    input: "120"
+    output: "121"
+    name: "Transpose_28"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 2
+      ints: 0
+      ints: 1
+      type: INTS
+    }
+  }
+  node {
+    input: "121"
+    output: "122"
+    name: "Shape_29"
+    op_type: "Shape"
+  }
+  node {
+    output: "123"
+    name: "Constant_30"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "122"
+    input: "123"
+    output: "124"
+    name: "Gather_31"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "121"
+    output: "125"
+    name: "Shape_32"
+    op_type: "Shape"
+  }
+  node {
+    output: "126"
+    name: "Constant_33"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "125"
+    input: "126"
+    output: "127"
+    name: "Gather_34"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "121"
+    output: "128"
+    name: "Shape_35"
+    op_type: "Shape"
+  }
+  node {
+    output: "129"
+    name: "Constant_36"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "128"
+    input: "129"
+    output: "130"
+    name: "Gather_37"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    output: "131"
+    name: "Constant_38"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "130"
+    input: "131"
+    output: "132"
+    name: "Div_39"
+    op_type: "Div"
+  }
+  node {
+    input: "132"
+    output: "133"
+    name: "Cast_40"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 7
+      type: INT
+    }
+  }
+  node {
+    input: "133"
+    output: "134"
+    name: "Cast_41"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 7
+      type: INT
+    }
+  }
+  node {
+    input: "121"
+    input: "502"
+    output: "136"
+    name: "MatMul_42"
+    op_type: "MatMul"
+  }
+  node {
+    input: "136"
+    input: "transformer_encoder.layers.0.self_attn.in_proj_bias"
+    output: "137"
+    name: "Add_43"
+    op_type: "Add"
+  }
+  node {
+    input: "137"
+    output: "138"
+    output: "139"
+    output: "140"
+    name: "Split_44"
+    op_type: "Split"
+    attribute {
+      name: "axis"
+      i: -1
+      type: INT
+    }
+    attribute {
+      name: "split"
+      ints: 40
+      ints: 40
+      ints: 40
+      type: INTS
+    }
+  }
+  node {
+    output: "141"
+    name: "Constant_45"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "138"
+    input: "141"
+    output: "142"
+    name: "Mul_46"
+    op_type: "Mul"
+  }
+  node {
+    output: "143"
+    name: "Constant_47"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "127"
+    input: "143"
+    output: "144"
+    name: "Mul_48"
+    op_type: "Mul"
+  }
+  node {
+    input: "124"
+    output: "145"
+    name: "Unsqueeze_49"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "144"
+    output: "146"
+    name: "Unsqueeze_50"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "134"
+    output: "147"
+    name: "Unsqueeze_51"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "145"
+    input: "146"
+    input: "147"
+    output: "148"
+    name: "Concat_52"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "142"
+    input: "148"
+    output: "149"
+    name: "Reshape_53"
+    op_type: "Reshape"
+  }
+  node {
+    input: "149"
+    output: "150"
+    name: "Transpose_54"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    output: "151"
+    name: "Constant_55"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "127"
+    input: "151"
+    output: "152"
+    name: "Mul_56"
+    op_type: "Mul"
+  }
+  node {
+    input: "152"
+    output: "155"
+    name: "Unsqueeze_57"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "134"
+    output: "156"
+    name: "Unsqueeze_58"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "503"
+    input: "155"
+    input: "156"
+    output: "157"
+    name: "Concat_59"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "139"
+    input: "157"
+    output: "158"
+    name: "Reshape_60"
+    op_type: "Reshape"
+  }
+  node {
+    output: "159"
+    name: "Constant_61"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "127"
+    input: "159"
+    output: "160"
+    name: "Mul_62"
+    op_type: "Mul"
+  }
+  node {
+    input: "160"
+    output: "163"
+    name: "Unsqueeze_63"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "134"
+    output: "164"
+    name: "Unsqueeze_64"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "504"
+    input: "163"
+    input: "164"
+    output: "165"
+    name: "Concat_65"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "140"
+    input: "165"
+    output: "166"
+    name: "Reshape_66"
+    op_type: "Reshape"
+  }
+  node {
+    input: "166"
+    output: "167"
+    name: "Transpose_67"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    input: "158"
+    output: "168"
+    name: "Transpose_68"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 2
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "150"
+    input: "168"
+    output: "169"
+    name: "MatMul_69"
+    op_type: "MatMul"
+  }
+  node {
+    input: "169"
+    output: "170"
+    name: "Softmax_70"
+    op_type: "Softmax"
+    attribute {
+      name: "axis"
+      i: 2
+      type: INT
+    }
+  }
+  node {
+    input: "170"
+    input: "167"
+    output: "171"
+    name: "MatMul_71"
+    op_type: "MatMul"
+  }
+  node {
+    input: "171"
+    output: "172"
+    name: "Transpose_72"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    input: "124"
+    output: "173"
+    name: "Unsqueeze_73"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "127"
+    output: "174"
+    name: "Unsqueeze_74"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "130"
+    output: "175"
+    name: "Unsqueeze_75"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "173"
+    input: "174"
+    input: "175"
+    output: "176"
+    name: "Concat_76"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "172"
+    input: "176"
+    output: "177"
+    name: "Reshape_77"
+    op_type: "Reshape"
+  }
+  node {
+    input: "177"
+    input: "505"
+    output: "179"
+    name: "MatMul_78"
+    op_type: "MatMul"
+  }
+  node {
+    input: "179"
+    input: "transformer_encoder.layers.0.self_attn.out_proj.bias"
+    output: "180"
+    name: "Add_79"
+    op_type: "Add"
+  }
+  node {
+    input: "121"
+    input: "180"
+    output: "181"
+    name: "Add_80"
+    op_type: "Add"
+  }
+  node {
+    input: "181"
+    output: "183"
+    name: "ReduceMean_81"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    input: "181"
+    input: "183"
+    output: "184"
+    name: "Sub_82"
+    op_type: "Sub"
+  }
+  node {
+    input: "184"
+    output: "185"
+    name: "Cast_83"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "185"
+    input: "506"
+    output: "187"
+    name: "Pow_84"
+    op_type: "Pow"
+  }
+  node {
+    input: "187"
+    output: "188"
+    name: "ReduceMean_85"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    output: "189"
+    name: "Constant_86"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "188"
+    input: "189"
+    output: "190"
+    name: "Add_87"
+    op_type: "Add"
+  }
+  node {
+    input: "190"
+    output: "191"
+    name: "Sqrt_88"
+    op_type: "Sqrt"
+  }
+  node {
+    input: "184"
+    input: "191"
+    output: "192"
+    name: "Div_89"
+    op_type: "Div"
+  }
+  node {
+    input: "192"
+    input: "transformer_encoder.layers.0.norm1.weight"
+    output: "193"
+    name: "Mul_90"
+    op_type: "Mul"
+  }
+  node {
+    input: "193"
+    input: "transformer_encoder.layers.0.norm1.bias"
+    output: "194"
+    name: "Add_91"
+    op_type: "Add"
+  }
+  node {
+    input: "194"
+    input: "507"
+    output: "196"
+    name: "MatMul_92"
+    op_type: "MatMul"
+  }
+  node {
+    input: "196"
+    input: "transformer_encoder.layers.0.linear1.bias"
+    output: "197"
+    name: "Add_93"
+    op_type: "Add"
+  }
+  node {
+    input: "197"
+    output: "198"
+    name: "Relu_94"
+    op_type: "Relu"
+  }
+  node {
+    input: "198"
+    input: "508"
+    output: "200"
+    name: "MatMul_95"
+    op_type: "MatMul"
+  }
+  node {
+    input: "200"
+    input: "transformer_encoder.layers.0.linear2.bias"
+    output: "201"
+    name: "Add_96"
+    op_type: "Add"
+  }
+  node {
+    input: "194"
+    input: "201"
+    output: "202"
+    name: "Add_97"
+    op_type: "Add"
+  }
+  node {
+    input: "202"
+    output: "204"
+    name: "ReduceMean_98"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    input: "202"
+    input: "204"
+    output: "205"
+    name: "Sub_99"
+    op_type: "Sub"
+  }
+  node {
+    input: "205"
+    output: "206"
+    name: "Cast_100"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "206"
+    input: "509"
+    output: "208"
+    name: "Pow_101"
+    op_type: "Pow"
+  }
+  node {
+    input: "208"
+    output: "209"
+    name: "ReduceMean_102"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    output: "210"
+    name: "Constant_103"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "209"
+    input: "210"
+    output: "211"
+    name: "Add_104"
+    op_type: "Add"
+  }
+  node {
+    input: "211"
+    output: "212"
+    name: "Sqrt_105"
+    op_type: "Sqrt"
+  }
+  node {
+    input: "205"
+    input: "212"
+    output: "213"
+    name: "Div_106"
+    op_type: "Div"
+  }
+  node {
+    input: "213"
+    input: "transformer_encoder.layers.0.norm2.weight"
+    output: "214"
+    name: "Mul_107"
+    op_type: "Mul"
+  }
+  node {
+    input: "214"
+    input: "transformer_encoder.layers.0.norm2.bias"
+    output: "215"
+    name: "Add_108"
+    op_type: "Add"
+  }
+  node {
+    input: "215"
+    output: "216"
+    name: "Shape_109"
+    op_type: "Shape"
+  }
+  node {
+    output: "217"
+    name: "Constant_110"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "216"
+    input: "217"
+    output: "218"
+    name: "Gather_111"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "215"
+    output: "219"
+    name: "Shape_112"
+    op_type: "Shape"
+  }
+  node {
+    output: "220"
+    name: "Constant_113"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "219"
+    input: "220"
+    output: "221"
+    name: "Gather_114"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "215"
+    output: "222"
+    name: "Shape_115"
+    op_type: "Shape"
+  }
+  node {
+    output: "223"
+    name: "Constant_116"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "222"
+    input: "223"
+    output: "224"
+    name: "Gather_117"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    output: "225"
+    name: "Constant_118"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "224"
+    input: "225"
+    output: "226"
+    name: "Div_119"
+    op_type: "Div"
+  }
+  node {
+    input: "226"
+    output: "227"
+    name: "Cast_120"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 7
+      type: INT
+    }
+  }
+  node {
+    input: "227"
+    output: "228"
+    name: "Cast_121"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 7
+      type: INT
+    }
+  }
+  node {
+    input: "215"
+    input: "510"
+    output: "230"
+    name: "MatMul_122"
+    op_type: "MatMul"
+  }
+  node {
+    input: "230"
+    input: "transformer_encoder.layers.1.self_attn.in_proj_bias"
+    output: "231"
+    name: "Add_123"
+    op_type: "Add"
+  }
+  node {
+    input: "231"
+    output: "232"
+    output: "233"
+    output: "234"
+    name: "Split_124"
+    op_type: "Split"
+    attribute {
+      name: "axis"
+      i: -1
+      type: INT
+    }
+    attribute {
+      name: "split"
+      ints: 40
+      ints: 40
+      ints: 40
+      type: INTS
+    }
+  }
+  node {
+    output: "235"
+    name: "Constant_125"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "232"
+    input: "235"
+    output: "236"
+    name: "Mul_126"
+    op_type: "Mul"
+  }
+  node {
+    output: "237"
+    name: "Constant_127"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "221"
+    input: "237"
+    output: "238"
+    name: "Mul_128"
+    op_type: "Mul"
+  }
+  node {
+    input: "218"
+    output: "239"
+    name: "Unsqueeze_129"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "238"
+    output: "240"
+    name: "Unsqueeze_130"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "228"
+    output: "241"
+    name: "Unsqueeze_131"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "239"
+    input: "240"
+    input: "241"
+    output: "242"
+    name: "Concat_132"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "236"
+    input: "242"
+    output: "243"
+    name: "Reshape_133"
+    op_type: "Reshape"
+  }
+  node {
+    input: "243"
+    output: "244"
+    name: "Transpose_134"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    output: "245"
+    name: "Constant_135"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "221"
+    input: "245"
+    output: "246"
+    name: "Mul_136"
+    op_type: "Mul"
+  }
+  node {
+    input: "246"
+    output: "249"
+    name: "Unsqueeze_137"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "228"
+    output: "250"
+    name: "Unsqueeze_138"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "511"
+    input: "249"
+    input: "250"
+    output: "251"
+    name: "Concat_139"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "233"
+    input: "251"
+    output: "252"
+    name: "Reshape_140"
+    op_type: "Reshape"
+  }
+  node {
+    output: "253"
+    name: "Constant_141"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "221"
+    input: "253"
+    output: "254"
+    name: "Mul_142"
+    op_type: "Mul"
+  }
+  node {
+    input: "254"
+    output: "257"
+    name: "Unsqueeze_143"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "228"
+    output: "258"
+    name: "Unsqueeze_144"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "512"
+    input: "257"
+    input: "258"
+    output: "259"
+    name: "Concat_145"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "234"
+    input: "259"
+    output: "260"
+    name: "Reshape_146"
+    op_type: "Reshape"
+  }
+  node {
+    input: "260"
+    output: "261"
+    name: "Transpose_147"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    input: "252"
+    output: "262"
+    name: "Transpose_148"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 2
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "244"
+    input: "262"
+    output: "263"
+    name: "MatMul_149"
+    op_type: "MatMul"
+  }
+  node {
+    input: "263"
+    output: "264"
+    name: "Softmax_150"
+    op_type: "Softmax"
+    attribute {
+      name: "axis"
+      i: 2
+      type: INT
+    }
+  }
+  node {
+    input: "264"
+    input: "261"
+    output: "265"
+    name: "MatMul_151"
+    op_type: "MatMul"
+  }
+  node {
+    input: "265"
+    output: "266"
+    name: "Transpose_152"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    input: "218"
+    output: "267"
+    name: "Unsqueeze_153"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "221"
+    output: "268"
+    name: "Unsqueeze_154"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "224"
+    output: "269"
+    name: "Unsqueeze_155"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "267"
+    input: "268"
+    input: "269"
+    output: "270"
+    name: "Concat_156"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "266"
+    input: "270"
+    output: "271"
+    name: "Reshape_157"
+    op_type: "Reshape"
+  }
+  node {
+    input: "271"
+    input: "513"
+    output: "273"
+    name: "MatMul_158"
+    op_type: "MatMul"
+  }
+  node {
+    input: "273"
+    input: "transformer_encoder.layers.1.self_attn.out_proj.bias"
+    output: "274"
+    name: "Add_159"
+    op_type: "Add"
+  }
+  node {
+    input: "215"
+    input: "274"
+    output: "275"
+    name: "Add_160"
+    op_type: "Add"
+  }
+  node {
+    input: "275"
+    output: "277"
+    name: "ReduceMean_161"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    input: "275"
+    input: "277"
+    output: "278"
+    name: "Sub_162"
+    op_type: "Sub"
+  }
+  node {
+    input: "278"
+    output: "279"
+    name: "Cast_163"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "279"
+    input: "514"
+    output: "281"
+    name: "Pow_164"
+    op_type: "Pow"
+  }
+  node {
+    input: "281"
+    output: "282"
+    name: "ReduceMean_165"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    output: "283"
+    name: "Constant_166"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "282"
+    input: "283"
+    output: "284"
+    name: "Add_167"
+    op_type: "Add"
+  }
+  node {
+    input: "284"
+    output: "285"
+    name: "Sqrt_168"
+    op_type: "Sqrt"
+  }
+  node {
+    input: "278"
+    input: "285"
+    output: "286"
+    name: "Div_169"
+    op_type: "Div"
+  }
+  node {
+    input: "286"
+    input: "transformer_encoder.layers.1.norm1.weight"
+    output: "287"
+    name: "Mul_170"
+    op_type: "Mul"
+  }
+  node {
+    input: "287"
+    input: "transformer_encoder.layers.1.norm1.bias"
+    output: "288"
+    name: "Add_171"
+    op_type: "Add"
+  }
+  node {
+    input: "288"
+    input: "515"
+    output: "290"
+    name: "MatMul_172"
+    op_type: "MatMul"
+  }
+  node {
+    input: "290"
+    input: "transformer_encoder.layers.1.linear1.bias"
+    output: "291"
+    name: "Add_173"
+    op_type: "Add"
+  }
+  node {
+    input: "291"
+    output: "292"
+    name: "Relu_174"
+    op_type: "Relu"
+  }
+  node {
+    input: "292"
+    input: "516"
+    output: "294"
+    name: "MatMul_175"
+    op_type: "MatMul"
+  }
+  node {
+    input: "294"
+    input: "transformer_encoder.layers.1.linear2.bias"
+    output: "295"
+    name: "Add_176"
+    op_type: "Add"
+  }
+  node {
+    input: "288"
+    input: "295"
+    output: "296"
+    name: "Add_177"
+    op_type: "Add"
+  }
+  node {
+    input: "296"
+    output: "298"
+    name: "ReduceMean_178"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    input: "296"
+    input: "298"
+    output: "299"
+    name: "Sub_179"
+    op_type: "Sub"
+  }
+  node {
+    input: "299"
+    output: "300"
+    name: "Cast_180"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "300"
+    input: "517"
+    output: "302"
+    name: "Pow_181"
+    op_type: "Pow"
+  }
+  node {
+    input: "302"
+    output: "303"
+    name: "ReduceMean_182"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    output: "304"
+    name: "Constant_183"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "303"
+    input: "304"
+    output: "305"
+    name: "Add_184"
+    op_type: "Add"
+  }
+  node {
+    input: "305"
+    output: "306"
+    name: "Sqrt_185"
+    op_type: "Sqrt"
+  }
+  node {
+    input: "299"
+    input: "306"
+    output: "307"
+    name: "Div_186"
+    op_type: "Div"
+  }
+  node {
+    input: "307"
+    input: "transformer_encoder.layers.1.norm2.weight"
+    output: "308"
+    name: "Mul_187"
+    op_type: "Mul"
+  }
+  node {
+    input: "308"
+    input: "transformer_encoder.layers.1.norm2.bias"
+    output: "309"
+    name: "Add_188"
+    op_type: "Add"
+  }
+  node {
+    input: "309"
+    output: "310"
+    name: "Shape_189"
+    op_type: "Shape"
+  }
+  node {
+    output: "311"
+    name: "Constant_190"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "310"
+    input: "311"
+    output: "312"
+    name: "Gather_191"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "309"
+    output: "313"
+    name: "Shape_192"
+    op_type: "Shape"
+  }
+  node {
+    output: "314"
+    name: "Constant_193"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "313"
+    input: "314"
+    output: "315"
+    name: "Gather_194"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "309"
+    output: "316"
+    name: "Shape_195"
+    op_type: "Shape"
+  }
+  node {
+    output: "317"
+    name: "Constant_196"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "316"
+    input: "317"
+    output: "318"
+    name: "Gather_197"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    output: "319"
+    name: "Constant_198"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "318"
+    input: "319"
+    output: "320"
+    name: "Div_199"
+    op_type: "Div"
+  }
+  node {
+    input: "320"
+    output: "321"
+    name: "Cast_200"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 7
+      type: INT
+    }
+  }
+  node {
+    input: "321"
+    output: "322"
+    name: "Cast_201"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 7
+      type: INT
+    }
+  }
+  node {
+    input: "309"
+    input: "518"
+    output: "324"
+    name: "MatMul_202"
+    op_type: "MatMul"
+  }
+  node {
+    input: "324"
+    input: "transformer_encoder.layers.2.self_attn.in_proj_bias"
+    output: "325"
+    name: "Add_203"
+    op_type: "Add"
+  }
+  node {
+    input: "325"
+    output: "326"
+    output: "327"
+    output: "328"
+    name: "Split_204"
+    op_type: "Split"
+    attribute {
+      name: "axis"
+      i: -1
+      type: INT
+    }
+    attribute {
+      name: "split"
+      ints: 40
+      ints: 40
+      ints: 40
+      type: INTS
+    }
+  }
+  node {
+    output: "329"
+    name: "Constant_205"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "326"
+    input: "329"
+    output: "330"
+    name: "Mul_206"
+    op_type: "Mul"
+  }
+  node {
+    output: "331"
+    name: "Constant_207"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "315"
+    input: "331"
+    output: "332"
+    name: "Mul_208"
+    op_type: "Mul"
+  }
+  node {
+    input: "312"
+    output: "333"
+    name: "Unsqueeze_209"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "332"
+    output: "334"
+    name: "Unsqueeze_210"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "322"
+    output: "335"
+    name: "Unsqueeze_211"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "333"
+    input: "334"
+    input: "335"
+    output: "336"
+    name: "Concat_212"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "330"
+    input: "336"
+    output: "337"
+    name: "Reshape_213"
+    op_type: "Reshape"
+  }
+  node {
+    input: "337"
+    output: "338"
+    name: "Transpose_214"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    output: "339"
+    name: "Constant_215"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "315"
+    input: "339"
+    output: "340"
+    name: "Mul_216"
+    op_type: "Mul"
+  }
+  node {
+    input: "340"
+    output: "343"
+    name: "Unsqueeze_217"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "322"
+    output: "344"
+    name: "Unsqueeze_218"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "519"
+    input: "343"
+    input: "344"
+    output: "345"
+    name: "Concat_219"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "327"
+    input: "345"
+    output: "346"
+    name: "Reshape_220"
+    op_type: "Reshape"
+  }
+  node {
+    output: "347"
+    name: "Constant_221"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "315"
+    input: "347"
+    output: "348"
+    name: "Mul_222"
+    op_type: "Mul"
+  }
+  node {
+    input: "348"
+    output: "351"
+    name: "Unsqueeze_223"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "322"
+    output: "352"
+    name: "Unsqueeze_224"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "520"
+    input: "351"
+    input: "352"
+    output: "353"
+    name: "Concat_225"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "328"
+    input: "353"
+    output: "354"
+    name: "Reshape_226"
+    op_type: "Reshape"
+  }
+  node {
+    input: "354"
+    output: "355"
+    name: "Transpose_227"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    input: "346"
+    output: "356"
+    name: "Transpose_228"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 2
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "338"
+    input: "356"
+    output: "357"
+    name: "MatMul_229"
+    op_type: "MatMul"
+  }
+  node {
+    input: "357"
+    output: "358"
+    name: "Softmax_230"
+    op_type: "Softmax"
+    attribute {
+      name: "axis"
+      i: 2
+      type: INT
+    }
+  }
+  node {
+    input: "358"
+    input: "355"
+    output: "359"
+    name: "MatMul_231"
+    op_type: "MatMul"
+  }
+  node {
+    input: "359"
+    output: "360"
+    name: "Transpose_232"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    input: "312"
+    output: "361"
+    name: "Unsqueeze_233"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "315"
+    output: "362"
+    name: "Unsqueeze_234"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "318"
+    output: "363"
+    name: "Unsqueeze_235"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "361"
+    input: "362"
+    input: "363"
+    output: "364"
+    name: "Concat_236"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "360"
+    input: "364"
+    output: "365"
+    name: "Reshape_237"
+    op_type: "Reshape"
+  }
+  node {
+    input: "365"
+    input: "521"
+    output: "367"
+    name: "MatMul_238"
+    op_type: "MatMul"
+  }
+  node {
+    input: "367"
+    input: "transformer_encoder.layers.2.self_attn.out_proj.bias"
+    output: "368"
+    name: "Add_239"
+    op_type: "Add"
+  }
+  node {
+    input: "309"
+    input: "368"
+    output: "369"
+    name: "Add_240"
+    op_type: "Add"
+  }
+  node {
+    input: "369"
+    output: "371"
+    name: "ReduceMean_241"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    input: "369"
+    input: "371"
+    output: "372"
+    name: "Sub_242"
+    op_type: "Sub"
+  }
+  node {
+    input: "372"
+    output: "373"
+    name: "Cast_243"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "373"
+    input: "522"
+    output: "375"
+    name: "Pow_244"
+    op_type: "Pow"
+  }
+  node {
+    input: "375"
+    output: "376"
+    name: "ReduceMean_245"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    output: "377"
+    name: "Constant_246"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "376"
+    input: "377"
+    output: "378"
+    name: "Add_247"
+    op_type: "Add"
+  }
+  node {
+    input: "378"
+    output: "379"
+    name: "Sqrt_248"
+    op_type: "Sqrt"
+  }
+  node {
+    input: "372"
+    input: "379"
+    output: "380"
+    name: "Div_249"
+    op_type: "Div"
+  }
+  node {
+    input: "380"
+    input: "transformer_encoder.layers.2.norm1.weight"
+    output: "381"
+    name: "Mul_250"
+    op_type: "Mul"
+  }
+  node {
+    input: "381"
+    input: "transformer_encoder.layers.2.norm1.bias"
+    output: "382"
+    name: "Add_251"
+    op_type: "Add"
+  }
+  node {
+    input: "382"
+    input: "523"
+    output: "384"
+    name: "MatMul_252"
+    op_type: "MatMul"
+  }
+  node {
+    input: "384"
+    input: "transformer_encoder.layers.2.linear1.bias"
+    output: "385"
+    name: "Add_253"
+    op_type: "Add"
+  }
+  node {
+    input: "385"
+    output: "386"
+    name: "Relu_254"
+    op_type: "Relu"
+  }
+  node {
+    input: "386"
+    input: "524"
+    output: "388"
+    name: "MatMul_255"
+    op_type: "MatMul"
+  }
+  node {
+    input: "388"
+    input: "transformer_encoder.layers.2.linear2.bias"
+    output: "389"
+    name: "Add_256"
+    op_type: "Add"
+  }
+  node {
+    input: "382"
+    input: "389"
+    output: "390"
+    name: "Add_257"
+    op_type: "Add"
+  }
+  node {
+    input: "390"
+    output: "392"
+    name: "ReduceMean_258"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    input: "390"
+    input: "392"
+    output: "393"
+    name: "Sub_259"
+    op_type: "Sub"
+  }
+  node {
+    input: "393"
+    output: "394"
+    name: "Cast_260"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "394"
+    input: "525"
+    output: "396"
+    name: "Pow_261"
+    op_type: "Pow"
+  }
+  node {
+    input: "396"
+    output: "397"
+    name: "ReduceMean_262"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    output: "398"
+    name: "Constant_263"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "397"
+    input: "398"
+    output: "399"
+    name: "Add_264"
+    op_type: "Add"
+  }
+  node {
+    input: "399"
+    output: "400"
+    name: "Sqrt_265"
+    op_type: "Sqrt"
+  }
+  node {
+    input: "393"
+    input: "400"
+    output: "401"
+    name: "Div_266"
+    op_type: "Div"
+  }
+  node {
+    input: "401"
+    input: "transformer_encoder.layers.2.norm2.weight"
+    output: "402"
+    name: "Mul_267"
+    op_type: "Mul"
+  }
+  node {
+    input: "402"
+    input: "transformer_encoder.layers.2.norm2.bias"
+    output: "403"
+    name: "Add_268"
+    op_type: "Add"
+  }
+  node {
+    input: "403"
+    output: "404"
+    name: "Shape_269"
+    op_type: "Shape"
+  }
+  node {
+    output: "405"
+    name: "Constant_270"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "404"
+    input: "405"
+    output: "406"
+    name: "Gather_271"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "403"
+    output: "407"
+    name: "Shape_272"
+    op_type: "Shape"
+  }
+  node {
+    output: "408"
+    name: "Constant_273"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "407"
+    input: "408"
+    output: "409"
+    name: "Gather_274"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "403"
+    output: "410"
+    name: "Shape_275"
+    op_type: "Shape"
+  }
+  node {
+    output: "411"
+    name: "Constant_276"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "410"
+    input: "411"
+    output: "412"
+    name: "Gather_277"
+    op_type: "Gather"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    output: "413"
+    name: "Constant_278"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "412"
+    input: "413"
+    output: "414"
+    name: "Div_279"
+    op_type: "Div"
+  }
+  node {
+    input: "414"
+    output: "415"
+    name: "Cast_280"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 7
+      type: INT
+    }
+  }
+  node {
+    input: "415"
+    output: "416"
+    name: "Cast_281"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 7
+      type: INT
+    }
+  }
+  node {
+    input: "403"
+    input: "526"
+    output: "418"
+    name: "MatMul_282"
+    op_type: "MatMul"
+  }
+  node {
+    input: "418"
+    input: "transformer_encoder.layers.3.self_attn.in_proj_bias"
+    output: "419"
+    name: "Add_283"
+    op_type: "Add"
+  }
+  node {
+    input: "419"
+    output: "420"
+    output: "421"
+    output: "422"
+    name: "Split_284"
+    op_type: "Split"
+    attribute {
+      name: "axis"
+      i: -1
+      type: INT
+    }
+    attribute {
+      name: "split"
+      ints: 40
+      ints: 40
+      ints: 40
+      type: INTS
+    }
+  }
+  node {
+    output: "423"
+    name: "Constant_285"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "420"
+    input: "423"
+    output: "424"
+    name: "Mul_286"
+    op_type: "Mul"
+  }
+  node {
+    output: "425"
+    name: "Constant_287"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "409"
+    input: "425"
+    output: "426"
+    name: "Mul_288"
+    op_type: "Mul"
+  }
+  node {
+    input: "406"
+    output: "427"
+    name: "Unsqueeze_289"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "426"
+    output: "428"
+    name: "Unsqueeze_290"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "416"
+    output: "429"
+    name: "Unsqueeze_291"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "427"
+    input: "428"
+    input: "429"
+    output: "430"
+    name: "Concat_292"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "424"
+    input: "430"
+    output: "431"
+    name: "Reshape_293"
+    op_type: "Reshape"
+  }
+  node {
+    input: "431"
+    output: "432"
+    name: "Transpose_294"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    output: "433"
+    name: "Constant_295"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "409"
+    input: "433"
+    output: "434"
+    name: "Mul_296"
+    op_type: "Mul"
+  }
+  node {
+    input: "434"
+    output: "437"
+    name: "Unsqueeze_297"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "416"
+    output: "438"
+    name: "Unsqueeze_298"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "527"
+    input: "437"
+    input: "438"
+    output: "439"
+    name: "Concat_299"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "421"
+    input: "439"
+    output: "440"
+    name: "Reshape_300"
+    op_type: "Reshape"
+  }
+  node {
+    output: "441"
+    name: "Constant_301"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 7
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "409"
+    input: "441"
+    output: "442"
+    name: "Mul_302"
+    op_type: "Mul"
+  }
+  node {
+    input: "442"
+    output: "445"
+    name: "Unsqueeze_303"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "416"
+    output: "446"
+    name: "Unsqueeze_304"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "528"
+    input: "445"
+    input: "446"
+    output: "447"
+    name: "Concat_305"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "422"
+    input: "447"
+    output: "448"
+    name: "Reshape_306"
+    op_type: "Reshape"
+  }
+  node {
+    input: "448"
+    output: "449"
+    name: "Transpose_307"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    input: "440"
+    output: "450"
+    name: "Transpose_308"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 2
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "432"
+    input: "450"
+    output: "451"
+    name: "MatMul_309"
+    op_type: "MatMul"
+  }
+  node {
+    input: "451"
+    output: "452"
+    name: "Softmax_310"
+    op_type: "Softmax"
+    attribute {
+      name: "axis"
+      i: 2
+      type: INT
+    }
+  }
+  node {
+    input: "452"
+    input: "449"
+    output: "453"
+    name: "MatMul_311"
+    op_type: "MatMul"
+  }
+  node {
+    input: "453"
+    output: "454"
+    name: "Transpose_312"
+    op_type: "Transpose"
+    attribute {
+      name: "perm"
+      ints: 1
+      ints: 0
+      ints: 2
+      type: INTS
+    }
+  }
+  node {
+    input: "406"
+    output: "455"
+    name: "Unsqueeze_313"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "409"
+    output: "456"
+    name: "Unsqueeze_314"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "412"
+    output: "457"
+    name: "Unsqueeze_315"
+    op_type: "Unsqueeze"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+  }
+  node {
+    input: "455"
+    input: "456"
+    input: "457"
+    output: "458"
+    name: "Concat_316"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "454"
+    input: "458"
+    output: "459"
+    name: "Reshape_317"
+    op_type: "Reshape"
+  }
+  node {
+    input: "459"
+    input: "529"
+    output: "461"
+    name: "MatMul_318"
+    op_type: "MatMul"
+  }
+  node {
+    input: "461"
+    input: "transformer_encoder.layers.3.self_attn.out_proj.bias"
+    output: "462"
+    name: "Add_319"
+    op_type: "Add"
+  }
+  node {
+    input: "403"
+    input: "462"
+    output: "463"
+    name: "Add_320"
+    op_type: "Add"
+  }
+  node {
+    input: "463"
+    output: "465"
+    name: "ReduceMean_321"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    input: "463"
+    input: "465"
+    output: "466"
+    name: "Sub_322"
+    op_type: "Sub"
+  }
+  node {
+    input: "466"
+    output: "467"
+    name: "Cast_323"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "467"
+    input: "530"
+    output: "469"
+    name: "Pow_324"
+    op_type: "Pow"
+  }
+  node {
+    input: "469"
+    output: "470"
+    name: "ReduceMean_325"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    output: "471"
+    name: "Constant_326"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "470"
+    input: "471"
+    output: "472"
+    name: "Add_327"
+    op_type: "Add"
+  }
+  node {
+    input: "472"
+    output: "473"
+    name: "Sqrt_328"
+    op_type: "Sqrt"
+  }
+  node {
+    input: "466"
+    input: "473"
+    output: "474"
+    name: "Div_329"
+    op_type: "Div"
+  }
+  node {
+    input: "474"
+    input: "transformer_encoder.layers.3.norm1.weight"
+    output: "475"
+    name: "Mul_330"
+    op_type: "Mul"
+  }
+  node {
+    input: "475"
+    input: "transformer_encoder.layers.3.norm1.bias"
+    output: "476"
+    name: "Add_331"
+    op_type: "Add"
+  }
+  node {
+    input: "476"
+    input: "531"
+    output: "478"
+    name: "MatMul_332"
+    op_type: "MatMul"
+  }
+  node {
+    input: "478"
+    input: "transformer_encoder.layers.3.linear1.bias"
+    output: "479"
+    name: "Add_333"
+    op_type: "Add"
+  }
+  node {
+    input: "479"
+    output: "480"
+    name: "Relu_334"
+    op_type: "Relu"
+  }
+  node {
+    input: "480"
+    input: "532"
+    output: "482"
+    name: "MatMul_335"
+    op_type: "MatMul"
+  }
+  node {
+    input: "482"
+    input: "transformer_encoder.layers.3.linear2.bias"
+    output: "483"
+    name: "Add_336"
+    op_type: "Add"
+  }
+  node {
+    input: "476"
+    input: "483"
+    output: "484"
+    name: "Add_337"
+    op_type: "Add"
+  }
+  node {
+    input: "484"
+    output: "486"
+    name: "ReduceMean_338"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    input: "484"
+    input: "486"
+    output: "487"
+    name: "Sub_339"
+    op_type: "Sub"
+  }
+  node {
+    input: "487"
+    output: "488"
+    name: "Cast_340"
+    op_type: "Cast"
+    attribute {
+      name: "to"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "488"
+    input: "533"
+    output: "490"
+    name: "Pow_341"
+    op_type: "Pow"
+  }
+  node {
+    input: "490"
+    output: "491"
+    name: "ReduceMean_342"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: -1
+      type: INTS
+    }
+  }
+  node {
+    output: "492"
+    name: "Constant_343"
+    op_type: "Constant"
+    attribute {
+      name: "value"
+      t {
+        data_type: 1
+      }
+      type: TENSOR
+    }
+  }
+  node {
+    input: "491"
+    input: "492"
+    output: "493"
+    name: "Add_344"
+    op_type: "Add"
+  }
+  node {
+    input: "493"
+    output: "494"
+    name: "Sqrt_345"
+    op_type: "Sqrt"
+  }
+  node {
+    input: "487"
+    input: "494"
+    output: "495"
+    name: "Div_346"
+    op_type: "Div"
+  }
+  node {
+    input: "495"
+    input: "transformer_encoder.layers.3.norm2.weight"
+    output: "496"
+    name: "Mul_347"
+    op_type: "Mul"
+  }
+  node {
+    input: "496"
+    input: "transformer_encoder.layers.3.norm2.bias"
+    output: "497"
+    name: "Add_348"
+    op_type: "Add"
+  }
+  node {
+    input: "497"
+    output: "498"
+    name: "ReduceMean_349"
+    op_type: "ReduceMean"
+    attribute {
+      name: "axes"
+      ints: 0
+      type: INTS
+    }
+    attribute {
+      name: "keepdims"
+      i: 0
+      type: INT
+    }
+  }
+  node {
+    input: "105"
+    input: "118"
+    input: "498"
+    output: "499"
+    name: "Concat_350"
+    op_type: "Concat"
+    attribute {
+      name: "axis"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "499"
+    input: "fc1_linear.weight"
+    input: "fc1_linear.bias"
+    output: "output_logits"
+    name: "Gemm_351"
+    op_type: "Gemm"
+    attribute {
+      name: "alpha"
+      f: 1.0
+      type: FLOAT
+    }
+    attribute {
+      name: "beta"
+      f: 1.0
+      type: FLOAT
+    }
+    attribute {
+      name: "transB"
+      i: 1
+      type: INT
+    }
+  }
+  node {
+    input: "output_logits"
+    output: "output_softmax"
+    name: "Softmax_352"
+    op_type: "Softmax"
+    attribute {
+      name: "axis"
+      i: 1
+      type: INT
+    }
+  }
+  initializer {
+      dims: 40
+      dims: 120
+      data_type: 1
+      name: "502"
+  }
+  initializer {
+      dims: 1
+      data_type: 7
+      name: "503"
+  }
+  initializer {
+      dims: 1
+      data_type: 7
+      name: "504"
+  }
+  initializer {
+      dims: 40
+      dims: 40
+      data_type: 1
+      name: "505"
+  }
+  initializer {
+      data_type: 1
+      name: "506"
+  }
+  initializer {
+      dims: 40
+      dims: 512
+      data_type: 1
+      name: "507"
+  }
+  initializer {
+      dims: 512
+      dims: 40
+      data_type: 1
+      name: "508"
+  }
+  initializer {
+      data_type: 1
+      name: "509"
+  }
+  initializer {
+      dims: 40
+      dims: 120
+      data_type: 1
+      name: "510"
+  }
+  initializer {
+      dims: 1
+      data_type: 7
+      name: "511"
+  }
+  initializer {
+      dims: 1
+      data_type: 7
+      name: "512"
+  }
+  initializer {
+      dims: 40
+      dims: 40
+      data_type: 1
+      name: "513"
+  }
+  initializer {
+      data_type: 1
+      name: "514"
+  }
+  initializer {
+      dims: 40
+      dims: 512
+      data_type: 1
+      name: "515"
+  }
+  initializer {
+      dims: 512
+      dims: 40
+      data_type: 1
+      name: "516"
+  }
+  initializer {
+      data_type: 1
+      name: "517"
+  }
+  initializer {
+      dims: 40
+      dims: 120
+      data_type: 1
+      name: "518"
+  }
+  initializer {
+      dims: 1
+      data_type: 7
+      name: "519"
+  }
+  initializer {
+      dims: 1
+      data_type: 7
+      name: "520"
+  }
+  initializer {
+      dims: 40
+      dims: 40
+      data_type: 1
+      name: "521"
+  }
+  initializer {
+      data_type: 1
+      name: "522"
+  }
+  initializer {
+      dims: 40
+      dims: 512
+      data_type: 1
+      name: "523"
+  }
+  initializer {
+      dims: 512
+      dims: 40
+      data_type: 1
+      name: "524"
+  }
+  initializer {
+      data_type: 1
+      name: "525"
+  }
+  initializer {
+      dims: 40
+      dims: 120
+      data_type: 1
+      name: "526"
+  }
+  initializer {
+      dims: 1
+      data_type: 7
+      name: "527"
+  }
+  initializer {
+      dims: 1
+      data_type: 7
+      name: "528"
+  }
+  initializer {
+      dims: 40
+      dims: 40
+      data_type: 1
+      name: "529"
+  }
+  initializer {
+      data_type: 1
+      name: "530"
+  }
+  initializer {
+      dims: 40
+      dims: 512
+      data_type: 1
+      name: "531"
+  }
+  initializer {
+      dims: 512
+      dims: 40
+      data_type: 1
+      name: "532"
+  }
+  initializer {
+      data_type: 1
+      name: "533"
+  }
+  initializer {
+      dims: 16
+      data_type: 1
+      name: "conv2Dblock1.0.bias"
+  }
+  initializer {
+      dims: 16
+      dims: 1
+      dims: 3
+      dims: 3
+      data_type: 1
+      name: "conv2Dblock1.0.weight"
+  }
+  initializer {
+      dims: 16
+      data_type: 1
+      name: "conv2Dblock1.1.bias"
+  }
+  initializer {
+      dims: 16
+      data_type: 1
+      name: "conv2Dblock1.1.running_mean"
+  }
+  initializer {
+      dims: 16
+      data_type: 1
+      name: "conv2Dblock1.1.running_var"
+  }
+  initializer {
+      dims: 16
+      data_type: 1
+      name: "conv2Dblock1.1.weight"
+  }
+  initializer {
+      dims: 64
+      data_type: 1
+      name: "conv2Dblock1.10.bias"
+  }
+  initializer {
+      dims: 64
+      dims: 32
+      dims: 3
+      dims: 3
+      data_type: 1
+      name: "conv2Dblock1.10.weight"
+  }
+  initializer {
+      dims: 64
+      data_type: 1
+      name: "conv2Dblock1.11.bias"
+  }
+  initializer {
+      dims: 64
+      data_type: 1
+      name: "conv2Dblock1.11.running_mean"
+  }
+  initializer {
+      dims: 64
+      data_type: 1
+      name: "conv2Dblock1.11.running_var"
+  }
+  initializer {
+      dims: 64
+      data_type: 1
+      name: "conv2Dblock1.11.weight"
+  }
+  initializer {
+      dims: 32
+      data_type: 1
+      name: "conv2Dblock1.5.bias"
+  }
+  initializer {
+      dims: 32
+      dims: 16
+      dims: 3
+      dims: 3
+      data_type: 1
+      name: "conv2Dblock1.5.weight"
+  }
+  initializer {
+      dims: 32
+      data_type: 1
+      name: "conv2Dblock1.6.bias"
+  }
+  initializer {
+      dims: 32
+      data_type: 1
+      name: "conv2Dblock1.6.running_mean"
+  }
+  initializer {
+      dims: 32
+      data_type: 1
+      name: "conv2Dblock1.6.running_var"
+  }
+  initializer {
+      dims: 32
+      data_type: 1
+      name: "conv2Dblock1.6.weight"
+  }
+  initializer {
+      dims: 16
+      data_type: 1
+      name: "conv2Dblock2.0.bias"
+  }
+  initializer {
+      dims: 16
+      dims: 1
+      dims: 3
+      dims: 3
+      data_type: 1
+      name: "conv2Dblock2.0.weight"
+  }
+  initializer {
+      dims: 16
+      data_type: 1
+      name: "conv2Dblock2.1.bias"
+  }
+  initializer {
+      dims: 16
+      data_type: 1
+      name: "conv2Dblock2.1.running_mean"
+  }
+  initializer {
+      dims: 16
+      data_type: 1
+      name: "conv2Dblock2.1.running_var"
+  }
+  initializer {
+      dims: 16
+      data_type: 1
+      name: "conv2Dblock2.1.weight"
+  }
+  initializer {
+      dims: 64
+      data_type: 1
+      name: "conv2Dblock2.10.bias"
+  }
+  initializer {
+      dims: 64
+      dims: 32
+      dims: 3
+      dims: 3
+      data_type: 1
+      name: "conv2Dblock2.10.weight"
+  }
+  initializer {
+      dims: 64
+      data_type: 1
+      name: "conv2Dblock2.11.bias"
+  }
+  initializer {
+      dims: 64
+      data_type: 1
+      name: "conv2Dblock2.11.running_mean"
+  }
+  initializer {
+      dims: 64
+      data_type: 1
+      name: "conv2Dblock2.11.running_var"
+  }
+  initializer {
+      dims: 64
+      data_type: 1
+      name: "conv2Dblock2.11.weight"
+  }
+  initializer {
+      dims: 32
+      data_type: 1
+      name: "conv2Dblock2.5.bias"
+  }
+  initializer {
+      dims: 32
+      dims: 16
+      dims: 3
+      dims: 3
+      data_type: 1
+      name: "conv2Dblock2.5.weight"
+  }
+  initializer {
+      dims: 32
+      data_type: 1
+      name: "conv2Dblock2.6.bias"
+  }
+  initializer {
+      dims: 32
+      data_type: 1
+      name: "conv2Dblock2.6.running_mean"
+  }
+  initializer {
+      dims: 32
+      data_type: 1
+      name: "conv2Dblock2.6.running_var"
+  }
+  initializer {
+      dims: 32
+      data_type: 1
+      name: "conv2Dblock2.6.weight"
+  }
+  initializer {
+      dims: 8
+      data_type: 1
+      name: "fc1_linear.bias"
+  }
+  initializer {
+      dims: 8
+      dims: 1064
+      data_type: 1
+      name: "fc1_linear.weight"
+  }
+  initializer {
+      dims: 512
+      data_type: 1
+      name: "transformer_encoder.layers.0.linear1.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.0.linear2.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.0.norm1.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.0.norm1.weight"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.0.norm2.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.0.norm2.weight"
+  }
+  initializer {
+      dims: 120
+      data_type: 1
+      name: "transformer_encoder.layers.0.self_attn.in_proj_bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.0.self_attn.out_proj.bias"
+  }
+  initializer {
+      dims: 512
+      data_type: 1
+      name: "transformer_encoder.layers.1.linear1.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.1.linear2.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.1.norm1.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.1.norm1.weight"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.1.norm2.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.1.norm2.weight"
+  }
+  initializer {
+      dims: 120
+      data_type: 1
+      name: "transformer_encoder.layers.1.self_attn.in_proj_bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.1.self_attn.out_proj.bias"
+  }
+  initializer {
+      dims: 512
+      data_type: 1
+      name: "transformer_encoder.layers.2.linear1.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.2.linear2.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.2.norm1.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.2.norm1.weight"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.2.norm2.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.2.norm2.weight"
+  }
+  initializer {
+      dims: 120
+      data_type: 1
+      name: "transformer_encoder.layers.2.self_attn.in_proj_bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.2.self_attn.out_proj.bias"
+  }
+  initializer {
+      dims: 512
+      data_type: 1
+      name: "transformer_encoder.layers.3.linear1.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.3.linear2.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.3.norm1.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.3.norm1.weight"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.3.norm2.bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.3.norm2.weight"
+  }
+  initializer {
+      dims: 120
+      data_type: 1
+      name: "transformer_encoder.layers.3.self_attn.in_proj_bias"
+  }
+  initializer {
+      dims: 40
+      data_type: 1
+      name: "transformer_encoder.layers.3.self_attn.out_proj.bias"
+  }
+  input {
+    name: "features"
+    type {
+      tensor_type {
+        elem_type: 1
+        shape {
+          dim {
+            dim_param: "N"
+          }
+          dim {
+            dim_value: 1
+          }
+          dim {
+            dim_value: 40
+          }
+          dim {
+            dim_value: 282
+          }
+        }
+      }
+    }
+  }
+  output {
+    name: "output_logits"
+    type {
+      tensor_type {
+        elem_type: 1
+        shape {
+          dim {
+            dim_param: "N"
+          }
+          dim {
+            dim_value: 8
+          }
+        }
+      }
+    }
+  }
+  output {
+    name: "output_softmax"
+    type {
+      tensor_type {
+        elem_type: 1
+        shape {
+          dim {
+            dim_param: "N"
+          }
+          dim {
+            dim_value: 8
+          }
+        }
+      }
+    }
+  }
+}
+opset_import {
+  version: 11
+}

models/ailia-models/source.txt

@@ -0,0 +1,4 @@
+https://github.com/axinc-ai/ailia-models/tree/master/audio_processing/transformer-cnn-emotion-recognition
+
+https://storage.googleapis.com/ailia-models/parallel_is_all_you_want/parallel_is_all_you_want_ep428.onnx
+https://storage.googleapis.com/ailia-models/parallel_is_all_you_want/parallel_is_all_you_want_ep428.onnx.prototxt