Use only ReLU

chatbot_constructor.py

@@ -71,11 +71,11 @@ def train(message: str = "", regularization: float = 0.0001, dropout: float = 0.
         dense1_layer = Dense(512, activation="linear", kernel_regularizer=L1(regularization))(dropout2_layer)
         prelu1_layer = PReLU()(dense1_layer)
         dropout3_layer = Dropout(dropout)(prelu1_layer)
-        dense2_layer = Dense(256, activation="tanh", kernel_regularizer=L1(regularization))(dropout3_layer)
+        dense2_layer = Dense(256, activation="relu", kernel_regularizer=L1(regularization))(dropout3_layer)
         dropout4_layer = Dropout(dropout)(dense2_layer)
         dense3_layer = Dense(256, activation="relu", kernel_regularizer=L1(regularization))(dropout4_layer)
         dropout5_layer = Dropout(dropout)(dense3_layer)
-        dense4_layer = Dense(100, activation="tanh", kernel_regularizer=L1(regularization))(dropout5_layer)
+        dense4_layer = Dense(100, activation="relu", kernel_regularizer=L1(regularization))(dropout5_layer)
         concat2_layer = Concatenate()([dense4_layer, prelu1_layer, attn_flatten_layer, conv1_flatten_layer])
         dense4_layer = Dense(resps_len, activation=end_activation, kernel_regularizer=L1(regularization))(concat2_layer)
         model = Model(inputs=input_layer, outputs=dense4_layer)