Add all file

Maybe keras will mess up

.gitignore

@@ -0,0 +1 @@
+**/__pycache__/

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-slim
+FROM python:3.12-slim
 
 WORKDIR /app
 
@@ -18,4 +18,4 @@ EXPOSE 8501
 
 HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health
 
-ENTRYPOINT ["streamlit", "run", "src/streamlit_app.py", "--server.port=8501", "--server.address=0.0.0.0"]
+ENTRYPOINT ["streamlit", "run", "src/streamlit_app.py", "--server.port=8501", "--server.address=0.0.0.0"]

requirements.txt

@@ -1,3 +1,7 @@
-altair
-pandas
-streamlit
+streamlit
+streamlit-webrtc
+numpy
+tensorflow-gpu
+keras
+Pillow
+opencv-python

src/classification.py

@@ -0,0 +1,120 @@
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+from nets import get_model_from_name
+from utils.utils import (cvtColor, get_classes, letterbox_image,
+                         preprocess_input)
+                         
+from huggingface_hub import hf_hub_download
+
+#--------------------------------------------#
+#   使用自己训练好的模型预测需要修改4个参数
+#   model_path和classes_path、backbone
+#   和alpha都需要修改！
+#--------------------------------------------#
+class Classification(object):
+    _defaults = {
+        #--------------------------------------------------------------------------#
+        #   使用自己训练好的模型进行预测一定要修改model_path和classes_path！
+        #   model_path指向logs文件夹下的权值文件，classes_path指向model_data下的txt
+        #   如果出现shape不匹配，同时要注意训练时的model_path和classes_path参数的修改
+        #--------------------------------------------------------------------------#
+        # "model_path"    : 'model_data/mobilenet_2_5_224_tf_no_top.h5',
+        "model_path"    : hf_hub_download(repo_id="username/model-name", filename="model.h5"),
+        "classes_path"  : 'model_data/cls_classes.txt',
+        #--------------------------------------------------------------------#
+        #   输入的图片大小
+        #--------------------------------------------------------------------#
+        "input_shape"   : [224, 224],
+        #--------------------------------------------------------------------#
+        #   所用模型种类：
+        #   mobilenet、resnet50、vgg16是常用的分类网络
+        #--------------------------------------------------------------------#
+        "backbone"      : 'vgg16',
+        #--------------------------------------------------------------------#
+        #   当使用mobilenet的alpha值
+        #   仅在backbone='mobilenet'的时候有效
+        #--------------------------------------------------------------------#
+        "alpha"         : 0.25
+    }
+
+    @classmethod
+    def get_defaults(cls, n):
+        if n in cls._defaults:
+            return cls._defaults[n]
+        else:
+            return "Unrecognized attribute name '" + n + "'"
+
+    #---------------------------------------------------#
+    #   初始化classification
+    #---------------------------------------------------#
+    def __init__(self, **kwargs):
+        self.__dict__.update(self._defaults)
+        for name, value in kwargs.items():
+            setattr(self, name, value)
+
+        #---------------------------------------------------#
+        #   获得种类
+        #---------------------------------------------------#
+        self.class_names, self.num_classes = get_classes(self.classes_path)
+        self.generate()
+
+    #---------------------------------------------------#
+    #   载入模型
+    #---------------------------------------------------#
+    def generate(self):
+        model_path = os.path.expanduser(self.model_path)
+        assert model_path.endswith('.h5'), 'Keras model or weights must be a .h5 file.'
+        
+        #---------------------------------------------------#
+        #   载入模型与权值
+        #---------------------------------------------------#
+        if self.backbone == "mobilenet":
+            self.model = get_model_from_name[self.backbone](input_shape = [self.input_shape[0], self.input_shape[1], 3], classes = self.num_classes, alpha = self.alpha)
+        else:
+            self.model = get_model_from_name[self.backbone](input_shape = [self.input_shape[0], self.input_shape[1], 3], classes = self.num_classes)
+        self.model.load_weights(self.model_path)
+        print('{} model, and classes {} loaded.'.format(model_path, self.class_names))
+
+    #---------------------------------------------------#
+    #   检测图片
+    #---------------------------------------------------#
+    def detect_image(self, image):
+        #---------------------------------------------------------#
+        #   在这里将图像转换成RGB图像，防止灰度图在预测时报错。
+        #   代码仅仅支持RGB图像的预测，所有其它类型的图像都会转化成RGB
+        #---------------------------------------------------------#
+        image       = cvtColor(image)
+        # 查看数据类型
+        # print(type(image))
+        #---------------------------------------------------#
+        #   对图片进行不失真的resize
+        #---------------------------------------------------#
+        image_data  = letterbox_image(image, [self.input_shape[1], self.input_shape[0]])
+        #---------------------------------------------------------#
+        #   归一化+添加上batch_size维度
+        #---------------------------------------------------------#
+        image_data  = np.expand_dims(preprocess_input(np.array(image_data, np.float32)), 0)
+        
+        #---------------------------------------------------#
+        #   图片传入网络进行预测
+        #---------------------------------------------------#
+        preds       = self.model.predict(image_data)[0]
+        #---------------------------------------------------#
+        #   获得所属种类
+        #---------------------------------------------------#
+        class_name  = self.class_names[np.argmax(preds)]
+        probability = np.max(preds)
+
+        #---------------------------------------------------#
+        #   绘图并写字
+        #---------------------------------------------------#
+
+        # plt.subplot(1, 1, 1)
+        # plt.imshow(np.array(image))
+        # plt.title('Class:%s Probability:%.3f' %(class_name, probability))
+        # plt.show()
+
+        return class_name, probability

src/model_data/cls_classes.txt

@@ -0,0 +1,7 @@
+disgust
+fear
+happiness
+others
+repression
+sadness
+surprise

src/model_data/mobilenet_2_5_224_tf_no_top.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbdb03ee2a22fd895301636cd328b234bb3a9952358f436d82f46b81e0d5b0bf
+size 2108140

src/model_data/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfe5187d0a272bed55ba430631598124cff8e880b98d38c9e56c8d66032abdc1
+size 58889256

src/nets/Loss.py

@@ -0,0 +1,113 @@
+import os
+from keras import backend as K
+import tensorflow as tf
+import numpy as np
+
+def multi_category_focal_loss2(gamma=2., alpha=1):
+    """
+    focal loss for multi category of multi label problem
+    适用于多分类或多标签问题的focal loss
+    alpha控制真值y_true为1/0时的权重
+        1的权重为alpha, 0的权重为1-alpha
+    当你的模型欠拟合，学习存在困难时，可以尝试适用本函数作为loss
+    当模型过于激进(无论何时总是倾向于预测出1),尝试将alpha调小
+    当模型过于惰性(无论何时总是倾向于预测出0,或是某一个固定的常数,说明没有学到有效特征)
+        尝试将alpha调大,鼓励模型进行预测出1。
+    Usage:
+     model.compile(loss=[multi_category_focal_loss2(alpha=0.25, gamma=2)], metrics=["accuracy"], optimizer=adam)
+    """
+    epsilon = 1.e-7
+    gamma = float(gamma)
+    alpha = tf.constant(alpha, dtype=tf.float32)
+
+    def multi_category_focal_loss2_fixed(y_true, y_pred):
+        y_true = tf.cast(y_true, tf.float32)
+        y_pred = tf.clip_by_value(y_pred, epsilon, 1. - epsilon)
+
+        alpha_t = y_true * alpha + (tf.ones_like(y_true) - y_true) * (1 - alpha)
+        y_t = tf.multiply(y_true, y_pred) + tf.multiply(1 - y_true, 1 - y_pred)
+        ce = -tf.log(y_t)
+        weight = tf.pow(tf.subtract(1., y_t), gamma)
+        fl = tf.multiply(tf.multiply(weight, ce), alpha_t)
+        loss = tf.reduce_mean(fl)
+        return loss
+
+    return multi_category_focal_loss2_fixed
+
+def multi_category_focal_loss1(alpha, gamma=2.0):
+    """
+    focal loss for multi category of multi label problem
+    适用于多分类或多标签问题的focal loss
+    alpha用于指定不同类别/标签的权重，数组大小需要与类别个数一致
+    当你的数据集不同类别/标签之间存在偏斜，可以尝试适用本函数作为loss
+    Usage:
+     model.compile(loss=[multi_category_focal_loss1(alpha=[1,2,3,2], gamma=2)], metrics=["accuracy"], optimizer=adam)
+    """
+    epsilon = 1.e-7
+    alpha = tf.constant(alpha, dtype=tf.float32)
+    #alpha = tf.constant([[1],[1],[1],[1],[1]], dtype=tf.float32)
+    #alpha = tf.constant_initializer(alpha)
+    gamma = float(gamma)
+    def multi_category_focal_loss1_fixed(y_true, y_pred):
+        y_true = tf.cast(y_true, tf.float32)
+        y_pred = tf.clip_by_value(y_pred, epsilon, 1. - epsilon)
+        y_t = tf.multiply(y_true, y_pred) + tf.multiply(1-y_true, 1-y_pred)
+        ce = -tf.log(y_t)
+        weight = tf.pow(tf.subtract(1., y_t), gamma)
+        fl = tf.matmul(tf.multiply(weight, ce), alpha)
+        loss = tf.reduce_mean(fl)
+        return loss
+    return multi_category_focal_loss1_fixed
+
+
+def Cross_entropy_loss(y_true, y_pred):
+    '''
+    :param y_true: ont-hot encoding ,shape is [batch_size,nums_classes]
+    :param y_pred: shape is [batch_size,nums_classes],each example defined as probability for per class
+    :return:shape is [batch_size,], a list include cross_entropy for per example
+    '''
+    y_pred = K.clip(y_pred, K.epsilon(), 1.0 - K.epsilon())
+    crossEntropyLoss = -y_true * tf.log(y_pred)
+
+    return tf.reduce_sum(crossEntropyLoss, -1)
+
+# focal loss with multi label
+def focal_loss(classes_num, gamma=2., alpha=.25, e=0.1):
+    # classes_num contains sample number of each classes
+    def focal_loss_fixed(target_tensor, prediction_tensor):
+        '''
+        prediction_tensor is the output tensor with shape [None, 100], where 100 is the number of classes
+        target_tensor is the label tensor, same shape as predcition_tensor
+        '''
+        import tensorflow as tf
+        from tensorflow.python.ops import array_ops
+        from keras import backend as K
+
+        #1# get focal loss with no balanced weight which presented in paper function (4)
+        zeros = array_ops.zeros_like(prediction_tensor, dtype=prediction_tensor.dtype)
+        one_minus_p = array_ops.where(tf.greater(target_tensor,zeros), target_tensor - prediction_tensor, zeros)
+        FT = -1 * (one_minus_p ** gamma) * tf.log(tf.clip_by_value(prediction_tensor, 1e-8, 1.0))
+
+        #2# get balanced weight alpha
+        classes_weight = array_ops.zeros_like(prediction_tensor, dtype=prediction_tensor.dtype)
+
+        total_num = float(sum(classes_num))
+        classes_w_t1 = [ total_num / ff for ff in classes_num ]
+        sum_ = sum(classes_w_t1)
+        classes_w_t2 = [ ff/sum_ for ff in classes_w_t1 ]   #scale
+        classes_w_tensor = tf.convert_to_tensor(classes_w_t2, dtype=prediction_tensor.dtype)
+        classes_weight += classes_w_tensor
+
+        alpha = array_ops.where(tf.greater(target_tensor, zeros), classes_weight, zeros)
+
+        #3# get balanced focal loss
+        balanced_fl = alpha * FT
+        balanced_fl = tf.reduce_mean(balanced_fl)
+
+        #4# add other op to prevent overfit
+        # reference : https://spaces.ac.cn/archives/4493
+        nb_classes = len(classes_num)
+        fianal_loss = (1-e) * balanced_fl + e * K.categorical_crossentropy(K.ones_like(prediction_tensor)/nb_classes, prediction_tensor)
+
+        return fianal_loss
+    return focal_loss_fixed

src/nets/__init__.py

@@ -0,0 +1,16 @@
+from .mobilenet import MobileNet
+from .resnet50 import ResNet50
+from .vgg16 import VGG16
+
+get_model_from_name = {
+    "mobilenet"     : MobileNet,
+    "resnet50"      : ResNet50,
+    "vgg16"         : VGG16,
+}
+
+freeze_layers = {
+    "mobilenet"     : 81,
+    "resnet50"      : 173,
+    "vgg16"         : 19,
+    "cspdarknet53"  : 60,
+}

src/nets/mobilenet.py

@@ -0,0 +1,105 @@
+from keras import backend as K
+from keras.layers import (Activation, BatchNormalization, Conv2D,
+                          DepthwiseConv2D, Dropout, GlobalAveragePooling2D,
+                          Input, Reshape)
+from keras.models import Model
+
+
+def _conv_block(inputs, filters, alpha, kernel=(3, 3), strides=(1, 1)):
+    filters = int(filters * alpha)
+    x = Conv2D(filters, kernel,
+                      padding='same',
+                      use_bias=False,
+                      strides=strides,
+                      name='conv1')(inputs)
+    x = BatchNormalization(name='conv1_bn')(x)
+    return Activation(relu6, name='conv1_relu')(x)
+
+
+def _depthwise_conv_block(inputs, pointwise_conv_filters, alpha,
+                          depth_multiplier=1, strides=(1, 1), block_id=1):
+
+    pointwise_conv_filters = int(pointwise_conv_filters * alpha)
+
+    x = DepthwiseConv2D((3, 3),
+                        padding='same',
+                        depth_multiplier=depth_multiplier,
+                        strides=strides,
+                        use_bias=False,
+                        name='conv_dw_%d' % block_id)(inputs)
+
+    x = BatchNormalization(name='conv_dw_%d_bn' % block_id)(x)
+    x = Activation(relu6, name='conv_dw_%d_relu' % block_id)(x)
+
+    x = Conv2D(pointwise_conv_filters, (1, 1),
+               padding='same',
+               use_bias=False,
+               strides=(1, 1),
+               name='conv_pw_%d' % block_id)(x)
+    x = BatchNormalization(name='conv_pw_%d_bn' % block_id)(x)
+    return Activation(relu6, name='conv_pw_%d_relu' % block_id)(x)
+
+def MobileNet(input_shape=None,
+              alpha=1.0,
+              depth_multiplier=1,
+              dropout=1e-3,
+              classes=1000):
+
+    img_input = Input(shape=input_shape)
+
+    # 224,224,3 -> 112,112,32  
+    x = _conv_block(img_input, 32, alpha, strides=(2, 2))
+    
+    # 112,112,32 -> 112,112,64
+    x = _depthwise_conv_block(x, 64, alpha, depth_multiplier, block_id=1)
+
+
+    # 112,112,64 -> 56,56,128
+    x = _depthwise_conv_block(x, 128, alpha, depth_multiplier,
+                              strides=(2, 2), block_id=2)
+    x = _depthwise_conv_block(x, 128, alpha, depth_multiplier, block_id=3)
+
+
+    # 56,56,128 -> 28,28,256
+    x = _depthwise_conv_block(x, 256, alpha, depth_multiplier,
+                              strides=(2, 2), block_id=4)
+    x = _depthwise_conv_block(x, 256, alpha, depth_multiplier, block_id=5)
+    
+
+    # 28,28,256 -> 14,14,512
+    x = _depthwise_conv_block(x, 512, alpha, depth_multiplier,
+                              strides=(2, 2), block_id=6)
+    x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=7)
+    x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=8)
+    x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=9)
+    x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=10)
+    x = _depthwise_conv_block(x, 512, alpha, depth_multiplier, block_id=11)
+
+    # 14,14,512 -> 7,7,1024
+    x = _depthwise_conv_block(x, 1024, alpha, depth_multiplier,
+                              strides=(2, 2), block_id=12)
+    x = _depthwise_conv_block(x, 1024, alpha, depth_multiplier, block_id=13)
+
+    # 7,7,1024 -> 1,1,1024
+    x = GlobalAveragePooling2D()(x)
+
+    shape = (1, 1, int(1024 * alpha))
+
+    x = Reshape(shape, name='reshape_1')(x)
+    x = Dropout(dropout, name='dropout')(x)
+
+    x = Conv2D(classes, (1, 1),padding='same', name='conv_preds')(x)
+    x = Activation('softmax', name='act_softmax')(x)
+    x = Reshape((classes,), name='reshape_2')(x)
+
+    inputs = img_input
+
+    model = Model(inputs, x, name='mobilenet_%0.2f' % (alpha))
+    return model
+
+def relu6(x):
+    return K.relu(x, max_value=6)
+
+if __name__ == '__main__':
+    model = MobileNet(input_shape=(224, 224, 3))
+    model.summary()

src/nets/resnet50.py

@@ -0,0 +1,118 @@
+from keras import layers
+from keras.layers import (Activation, AveragePooling2D, BatchNormalization,
+                          Conv2D, Dense, Flatten, Input, MaxPooling2D,
+                          ZeroPadding2D)
+from keras.models import Model
+
+
+def identity_block(input_tensor, kernel_size, filters, stage, block):
+
+    filters1, filters2, filters3 = filters
+
+    conv_name_base = 'res' + str(stage) + block + '_branch'
+    bn_name_base = 'bn' + str(stage) + block + '_branch'
+
+    # 减少通道数
+    x = Conv2D(filters1, (1, 1), name=conv_name_base + '2a')(input_tensor)
+    x = BatchNormalization(name=bn_name_base + '2a')(x)
+    x = Activation('relu')(x)
+
+    # 3x3卷积
+    x = Conv2D(filters2, kernel_size,padding='same', name=conv_name_base + '2b')(x)
+    x = BatchNormalization(name=bn_name_base + '2b')(x)
+    x = Activation('relu')(x)
+    
+    # 上升通道数
+    x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)
+    x = BatchNormalization(name=bn_name_base + '2c')(x)
+
+    x = layers.add([x, input_tensor])
+    x = Activation('relu')(x)
+    return x
+
+
+def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)):
+    filters1, filters2, filters3 = filters
+
+    conv_name_base = 'res' + str(stage) + block + '_branch'
+    bn_name_base = 'bn' + str(stage) + block + '_branch'
+
+    # 减少通道数
+    x = Conv2D(filters1, (1, 1), strides=strides, name=conv_name_base + '2a')(input_tensor)
+    x = BatchNormalization(name=bn_name_base + '2a')(x)
+    x = Activation('relu')(x)
+
+    # 3x3卷积
+    x = Conv2D(filters2, kernel_size, padding='same', name=conv_name_base + '2b')(x)
+    x = BatchNormalization(name=bn_name_base + '2b')(x)
+    x = Activation('relu')(x)
+
+    # 上升通道数
+    x = Conv2D(filters3, (1, 1), name=conv_name_base + '2c')(x)
+    x = BatchNormalization(name=bn_name_base + '2c')(x)
+
+    # 残差边
+    shortcut = Conv2D(filters3, (1, 1), strides=strides,
+                      name=conv_name_base + '1')(input_tensor)
+    shortcut = BatchNormalization(name=bn_name_base + '1')(shortcut)
+
+    x = layers.add([x, shortcut])
+    x = Activation('relu')(x)
+    return x
+
+
+def ResNet50(input_shape=[224,224,3], classes=1000):
+    img_input = Input(shape=input_shape)
+
+    x = ZeroPadding2D((3, 3))(img_input)
+    # 224,224,3 -> 112,112,64
+    x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)
+    x = BatchNormalization(name='bn_conv1')(x)
+    x = Activation('relu')(x)
+
+    x = ZeroPadding2D((1, 1))(x)
+    # 112,112,64 -> 56,56,64
+    x = MaxPooling2D((3, 3), strides=(2, 2))(x)
+
+    # 56,56,64 -> 56,56,256
+    x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
+    x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
+    x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
+
+    # 56,56,256 -> 28,28,512
+    x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
+    x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
+    x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
+    x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
+
+    # 28,28,512 -> 14,14,1024
+    x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
+    x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
+
+    # 14,14,1024 -> 7,7,2048
+    x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
+    x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
+    x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
+
+    # 1,1,2048
+    x = AveragePooling2D((7, 7), name='avg_pool')(x)
+
+    # 进行预测
+    # 2048
+    x = Flatten()(x)
+
+    # num_classes
+    x = Dense(classes, activation='softmax', name='fc1000')(x)
+
+    model = Model(img_input, x, name='resnet50')
+
+    return model
+
+
+if __name__ == '__main__':
+    model = ResNet50()
+    model.summary()

src/nets/vgg16.py

@@ -0,0 +1,98 @@
+from keras.layers import Conv2D, Dense, Flatten, Input, MaxPooling2D
+from keras.models import Model        #导入包Conv2D是卷积核 Flatten是展开 Input输入  MaxPooling2D最大卷积核
+
+
+def VGG16(input_shape=None, classes=1000): #def 就是开始定义VGG16的网络
+    img_input = Input(shape=input_shape)  # 224, 224, 3
+
+    # Block 1
+    # 224, 224, 3 -> 224, 224, 64
+    x = Conv2D(64, (3, 3),  #开始第一个卷积核进行特征提取，64为卷积核的个数；(3, 3)是卷积核的大小
+                      activation='relu', #relu激活函数
+                      padding='same',  #padding='same' 尺寸不变
+                      name='block1_conv1')(img_input)  #dui juanjihe jinxing mingming  # x= 224*224*64
+    x = Conv2D(64, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block1_conv2')(x)    #224*224*64
+
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)  #112*122*64
+
+    # Block 2
+
+    x = Conv2D(128, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block2_conv1')(x) #112*112*128
+    x = Conv2D(128, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block2_conv2')(x)
+
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
+
+    # Block 3
+
+    x = Conv2D(256, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block3_conv1')(x)
+    x = Conv2D(256, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block3_conv2')(x)
+    x = Conv2D(256, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block3_conv3')(x)
+
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
+
+    # Block 4
+
+    x = Conv2D(512, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block4_conv1')(x)
+    x = Conv2D(512, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block4_conv2')(x)
+    x = Conv2D(512, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block4_conv3')(x)
+                      
+
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
+
+    # Block 5
+
+    x = Conv2D(512, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block5_conv1')(x)
+    x = Conv2D(512, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block5_conv2')(x)
+    x = Conv2D(512, (3, 3),
+                      activation='relu',
+                      padding='same',
+                      name='block5_conv3')(x)
+    # 14, 14, 512 -> 7, 7, 512
+    x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
+
+    x = Flatten(name='flatten')(x)
+    x = Dense(4096, activation='relu', name='fc1')(x)
+    x = Dense(4096, activation='relu', name='fc2')(x)
+    x = Dense(classes, activation='softmax', name='predictions')(x) #激活函数
+
+    inputs = img_input
+
+    model = Model(inputs, x, name='vgg16')
+    return model
+
+if __name__ == '__main__':
+    model = VGG16(input_shape=(224, 224, 3))
+    model.summary()

src/streamlit_app.py

@@ -1,40 +1,173 @@
-import altair as alt
+import os
+import subprocess
+
+import cv2
 import numpy as np
-import pandas as pd
+from PIL import Image
+
 import streamlit as st
+from streamlit_webrtc import VideoProcessorBase, webrtc_streamer
+
+from classification import Classification
+
+classificator = Classification()
+face_cascade = cv2.CascadeClassifier(
+    os.path.join('model_data', 'haarcascade_frontalface_alt.xml')
+)
+
+# Streamlit Title
+st.title("Real-Time Micro-Emotion Recognition")
+
+# Only Live Emotion Detection Mode
+st.write("Turn on your camera and detect emotions in real-time.")
+
+# Camera selection UI
+st.sidebar.header("Camera Settings")
+def get_connected_cameras():
+    try:
+        result = subprocess.run(
+            ['v4l2-ctl', '--list-devices'], 
+            capture_output=True, 
+            text=True, 
+            check=True)
+        devices = result.stdout.split('\n\n')
+        camera_indices = []
+        for device in devices:
+            if "Camera" in device or "camera" in device:
+                lines = device.split('\n')
+                if len(lines) > 1:
+                    index_line = lines[1]
+                    index_str = index_line.strip().split(':')[0].strip()
+                    try:
+                        index = int(index_str[4:])
+                        camera_indices.append(index)
+                    except (ValueError, IndexError):
+                        pass
+        return camera_indices
+    except FileNotFoundError:
+        return [0]  # Fallback to default camera if v4l2-ctl is not available
+    except subprocess.CalledProcessError:
+        return [0]
+
+available_cameras = get_connected_cameras()
+
+if len(available_cameras) > 1:
+    camera_index = st.sidebar.selectbox(
+        "Select Camera Index",
+        options=available_cameras,
+        index=0,
+        format_func=lambda x: f"Camera {x}"
+    )
+else:
+    camera_index = 0
+    st.sidebar.write("Only one camera detected. Using default camera.")
+
+# --- Face detection and augmentation functions ---
+def face_detect(img):
+    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    faces = face_cascade.detectMultiScale(
+        img_gray,
+        scaleFactor=1.1,
+        minNeighbors=1,
+        minSize=(30, 30)
+    )
+    return img, img_gray, faces
+
+# --- Emotion class mapping ---
+def map_emotion_to_class(emotion):
+    positive = ['happiness', 'happy']
+    negative = ['disgust', 'sadness', 'fear', 'sad', 'angry', 'disgusted']
+    surprise = ['surprise']
+    others = ['repression', 'tense', 'neutral', 'others']
+    e = emotion.lower()
+    if any(p in e for p in positive):
+        return 'Positive'
+    elif any(n in e for n in negative):
+        return 'Negative'
+    elif any(s in e for s in surprise):
+        return 'Surprise'
+    else:
+        return 'Others'
+
+# --- Streamlit session state for emotion tracking ---
+if 'emotion_history' not in st.session_state:
+    st.session_state['emotion_history'] = []
+
+# Video Processing Class
+class EmotionRecognitionProcessor(VideoProcessorBase):
+    def __init__(self):
+        self.last_class = None
+        self.rapid_change_count = 0
+
+    def recv(self, frame):
+        border_color = (255, 0, 0)  # Rectangle color (blue in BGR)
+        font_color = (0, 0, 255)    # Text color (red in BGR)
+        img = frame.to_ndarray(format="bgr24")
+        img_disp, img_gray, faces = face_detect(img)
+        current_class = None
+
+        if len(faces) == 0:
+            cv2.putText(
+                img_disp, 'No Face Detect.', (2, 20),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 0, 255), 1
+            )
+
+        for (x, y, w, h) in faces:
+            x1, y1 = max(x - 10, 0), max(y - 10, 0)
+            x2 = min(x + w + 10, img_disp.shape[1]) 
+            y2 = min(y + h + 10, img_disp.shape[0])
+            
+            face_img_gray = img_gray[y1:y2, x1:x2]
+            if face_img_gray.size == 0:
+                continue
+            face_img_pil = Image.fromarray(face_img_gray)
+            emotion, probability = classificator.detect_image(face_img_pil)
+            emotion_class = map_emotion_to_class(emotion)
+
+            cv2.rectangle(
+                img_disp, 
+                (x1, y1), 
+                (x2, y2), 
+                border_color, 
+                thickness=2
+            )
+            cv2.putText(
+                img_disp, emotion, (x + 30, y - 30),
+                cv2.FONT_HERSHEY_SIMPLEX, 1, font_color, 1
+            )
+            # Show probability
+            cv2.putText(
+                img_disp, str(round(probability, 3)), (x + 30, y - 50),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.3, font_color, 1
+            )
+            current_class = emotion_class
+
+        # Track emotion class changes
+        if current_class:
+            history = st.session_state['emotion_history']
+            history.append(current_class)
+            if len(history) > 10:
+                history.pop(0)
+            # Detect rapid changes
+            if len(history) >= 3 and len(set(history[-3:])) > 1:
+                self.rapid_change_count += 1
+            else:
+                self.rapid_change_count = 0
+
+        return frame.from_ndarray(img_disp, format="bgr24")
+
+webrtc_streamer(
+    key="emotion-detection",
+    video_processor_factory=EmotionRecognitionProcessor,
+)
 
-"""
-# Welcome to Streamlit!
-
-Edit `/streamlit_app.py` to customize this app to your heart's desire :heart:.
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
-
-In the meantime, below is an example of what you can do with just a few lines of code:
-"""
-
-num_points = st.slider("Number of points in spiral", 1, 10000, 1100)
-num_turns = st.slider("Number of turns in spiral", 1, 300, 31)
-
-indices = np.linspace(0, 1, num_points)
-theta = 2 * np.pi * num_turns * indices
-radius = indices
-
-x = radius * np.cos(theta)
-y = radius * np.sin(theta)
-
-df = pd.DataFrame({
-    "x": x,
-    "y": y,
-    "idx": indices,
-    "rand": np.random.randn(num_points),
-})
-
-st.altair_chart(alt.Chart(df, height=700, width=700)
-    .mark_point(filled=True)
-    .encode(
-        x=alt.X("x", axis=None),
-        y=alt.Y("y", axis=None),
-        color=alt.Color("idx", legend=None, scale=alt.Scale()),
-        size=alt.Size("rand", legend=None, scale=alt.Scale(range=[1, 150])),
-    ))
+# --- Streamlit alert for rapid emotion changes ---
+history = st.session_state['emotion_history']
+if len(history) >= 3 and len(set(history[-3:])) > 1:
+    st.warning(
+        "⚠️ Rapid changes in your detected emotional state were observed. "
+        "Micro-expressions may not always reflect your true feelings. "
+        "If you feel emotionally unstable or distressed, " \
+        "consider reaching out to a mental health professional, "
+        "talking it over with a close person or taking a break."
+    )

src/utils/__init__.py

@@ -0,0 +1 @@
+#

src/utils/backend/__init__.py

@@ -0,0 +1 @@
+from .tensorflow_backend import *  # noqa: F401,F403

src/utils/backend/tensorflow_backend.py

@@ -0,0 +1,100 @@
+import tensorflow
+
+
+def disable_tensorflow_v2_behavior():
+    """ See https://www.tensorflow.org/api_docs/python/tf/compat/v1/disable_tensorflow_v2_behavior .
+    """
+    tensorflow.compat.v1.disable_v2_behavior()
+
+
+def ones(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/ones .
+    """
+    return tensorflow.ones(*args, **kwargs)
+
+
+def transpose(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/transpose .
+    """
+    return tensorflow.transpose(*args, **kwargs)
+
+
+def map_fn(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/map_fn .
+    """
+    return tensorflow.map_fn(*args, **kwargs)
+
+
+def pad(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/pad .
+    """
+    return tensorflow.pad(*args, **kwargs)
+
+
+def top_k(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/nn/top_k .
+    """
+    return tensorflow.nn.top_k(*args, **kwargs)
+
+
+def clip_by_value(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/clip_by_value .
+    """
+    return tensorflow.clip_by_value(*args, **kwargs)
+
+
+def resize_images(images, size, method='bilinear', align_corners=False):
+    """ See https://www.tensorflow.org/versions/r1.14/api_docs/python/tf/image/resize_images .
+
+    Args
+        method: The method used for interpolation. One of ('bilinear', 'nearest', 'bicubic', 'area').
+    """
+    methods = {
+        'bilinear': tensorflow.image.ResizeMethod.BILINEAR,
+        'nearest' : tensorflow.image.ResizeMethod.NEAREST_NEIGHBOR,
+        'bicubic' : tensorflow.image.ResizeMethod.BICUBIC,
+        'area'    : tensorflow.image.ResizeMethod.AREA,
+    }
+    return tensorflow.compat.v1.image.resize_images(images, size, tensorflow.image.ResizeMethod.NEAREST_NEIGHBOR, False)
+
+
+def non_max_suppression(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/image/non_max_suppression .
+    """
+    return tensorflow.image.non_max_suppression(*args, **kwargs)
+
+
+def range(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/range .
+    """
+    return tensorflow.range(*args, **kwargs)
+
+
+def scatter_nd(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/scatter_nd .
+    """
+    return tensorflow.scatter_nd(*args, **kwargs)
+
+
+def gather_nd(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/gather_nd .
+    """
+    return tensorflow.gather_nd(*args, **kwargs)
+
+
+def meshgrid(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/meshgrid .
+    """
+    return tensorflow.meshgrid(*args, **kwargs)
+
+
+def where(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/where .
+    """
+    return tensorflow.where(*args, **kwargs)
+
+
+def unstack(*args, **kwargs):
+    """ See https://www.tensorflow.org/api_docs/python/tf/unstack .
+    """
+    return tensorflow.unstack(*args, **kwargs)

src/utils/callbacks.py

@@ -0,0 +1,86 @@
+import os
+
+import keras
+import matplotlib
+matplotlib.use('Agg')
+from matplotlib import pyplot as plt
+import scipy.signal
+import tensorflow as tf
+
+
+class LossHistory(keras.callbacks.Callback):
+    def __init__(self, log_dir):
+        import datetime
+        curr_time = datetime.datetime.now()
+        time_str = datetime.datetime.strftime(curr_time,'%Y_%m_%d_%H_%M_%S')
+        self.log_dir    = log_dir
+        self.time_str   = time_str
+        self.save_path  = os.path.join(self.log_dir, "loss_" + str(self.time_str))  
+        self.losses     = []
+        self.val_loss   = []
+        
+        os.makedirs(self.save_path)
+
+    def on_epoch_end(self, batch, logs={}):
+        self.losses.append(logs.get('loss'))
+        self.val_loss.append(logs.get('val_loss'))
+        with open(os.path.join(self.save_path, "epoch_loss_" + str(self.time_str) + ".txt"), 'a') as f:
+            f.write(str(logs.get('loss')))
+            f.write("\n")
+        with open(os.path.join(self.save_path, "epoch_val_loss_" + str(self.time_str) + ".txt"), 'a') as f:
+            f.write(str(logs.get('val_loss')))
+            f.write("\n")
+        self.loss_plot()
+
+    def loss_plot(self):
+        iters = range(len(self.losses))
+
+        plt.figure()
+        plt.plot(iters, self.losses, 'red', linewidth = 2, label='train loss')
+        plt.plot(iters, self.val_loss, 'coral', linewidth = 2, label='val loss')
+        try:
+            if len(self.losses) < 25:
+                num = 5
+            else:
+                num = 15
+            
+            plt.plot(iters, scipy.signal.savgol_filter(self.losses, num, 3), 'green', linestyle = '--', linewidth = 2, label='smooth train loss')
+            plt.plot(iters, scipy.signal.savgol_filter(self.val_loss, num, 3), '#8B4513', linestyle = '--', linewidth = 2, label='smooth val loss')
+        except:
+            pass
+
+        plt.grid(True)
+        plt.xlabel('Epoch')
+        plt.ylabel('Loss')
+        plt.title('A Loss Curve')
+        plt.legend(loc="upper right")
+
+        plt.savefig(os.path.join(self.save_path, "epoch_loss_" + str(self.time_str) + ".png"))
+
+        plt.cla()
+        plt.close("all")
+
+class ExponentDecayScheduler(keras.callbacks.Callback):
+    def __init__(self,
+                 decay_rate,
+                 verbose=0):
+        super(ExponentDecayScheduler, self).__init__()
+        self.decay_rate         = decay_rate
+        self.verbose            = verbose
+        self.learning_rates     = []
+
+    def on_epoch_end(self, batch, logs=None):
+        lr = self.model.optimizer.learning_rate
+        try:
+            current_lr = tf.keras.backend.get_value(lr)
+        except Exception:
+            current_lr = lr
+            
+        new_lr = current_lr * self.decay_rate
+        try:
+            tf.keras.backend.set_value(lr, new_lr)
+        except Exception:
+            print("Warning: Could not set learning rate dynamically.")
+        
+        if self.verbose > 0:
+            print('Setting learning rate to %s.' % (new_lr))

src/utils/dataloader.py

@@ -0,0 +1,128 @@
+import math
+from random import shuffle
+
+import cv2
+import keras
+import numpy as np
+from keras.utils import to_categorical
+from PIL import Image
+
+from .utils import cvtColor, preprocess_input
+
+
+class ClsDatasets(keras.utils.Sequence):
+    def __init__(self, annotation_lines, input_shape, batch_size, num_classes, train, **kwargs):
+        super().__init__()
+        self.annotation_lines   = annotation_lines
+        self.length             = len(self.annotation_lines)
+
+        self.input_shape        = input_shape
+        self.batch_size         = batch_size
+        self.num_classes        = num_classes
+        self.train              = train
+
+    def __len__(self):
+        return int(math.ceil(self.length / float(self.batch_size)))
+
+    def __getitem__(self, index):
+        X_train = []
+        Y_train = []
+        start = index * self.batch_size
+        end = min((index + 1) * self.batch_size, self.length)
+        for i in range(start, end):
+            annotation_path = self.annotation_lines[i].split(';')[1].split()[0]
+            image = Image.open(annotation_path)
+            image = self.get_random_data(image, self.input_shape, random=self.train)
+            image = preprocess_input(np.array(image).astype(np.float32))
+
+            X_train.append(image)
+            Y_train.append(int(self.annotation_lines[i].split(';')[0]))
+
+        X_train = np.array(X_train)
+        Y_train = to_categorical(np.array(Y_train), num_classes = self.num_classes)   
+        return X_train, Y_train
+
+    def on_epoch_end(self):
+        if self.train:
+            np.random.shuffle(self.annotation_lines)
+
+    def rand(self, a=0, b=1):
+        return np.random.rand()*(b-a) + a
+
+    def get_random_data(self, image, input_shape, jitter=.3, hue=.1, sat=1.5, val=1.5, random=True):
+        #------------------------------#
+        #   读取图像并转换成RGB图像
+        #------------------------------#
+        image   = cvtColor(image)
+        #------------------------------#
+        #   获得图像的高宽与目标高宽
+        #------------------------------#
+        iw, ih  = image.size
+        h, w    = input_shape
+
+        if not random:
+            scale = min(w/iw, h/ih)
+            nw = int(iw*scale)
+            nh = int(ih*scale)
+            dx = (w-nw)//2
+            dy = (h-nh)//2
+
+            #---------------------------------#
+            #   将图像多余的部分加上灰条
+            #---------------------------------#
+            image       = image.resize((nw,nh), Image.BICUBIC)
+            new_image   = Image.new('RGB', (w,h), (128,128,128))
+            new_image.paste(image, (dx, dy))
+            image_data  = np.array(new_image, np.float32)
+
+            return image_data
+
+        #------------------------------------------#
+        #   对图像进行缩放并且进行长和宽的扭曲
+        #------------------------------------------#
+        new_ar = w/h * self.rand(1-jitter,1+jitter)/self.rand(1-jitter,1+jitter)
+        scale = self.rand(.75, 1.25)
+        if new_ar < 1:
+            nh = int(scale*h)
+            nw = int(nh*new_ar)
+        else:
+            nw = int(scale*w)
+            nh = int(nw/new_ar)
+        image = image.resize((nw,nh), Image.BICUBIC)
+
+        #------------------------------------------#
+        #   将图像多余的部分加上灰条
+        #------------------------------------------#
+        dx = int(self.rand(0, w-nw))
+        dy = int(self.rand(0, h-nh))
+        new_image = Image.new('RGB', (w,h), (128,128,128))
+        new_image.paste(image, (dx, dy))
+        image = new_image
+
+        #------------------------------------------#
+        #   翻转图像
+        #------------------------------------------#
+        flip = self.rand()<.5
+        if flip: image = image.transpose(Image.FLIP_LEFT_RIGHT)
+        
+        rotate = self.rand()<.5
+        if rotate: 
+            angle = np.random.randint(-15,15)
+            a,b = w/2,h/2
+            M = cv2.getRotationMatrix2D((a,b),angle,1)
+            image = cv2.warpAffine(np.array(image), M, (w,h), borderValue=[128,128,128]) 
+
+        #------------------------------------------#
+        #   色域扭曲
+        #------------------------------------------#
+        hue = self.rand(-hue, hue)
+        sat = self.rand(1, sat) if self.rand()<.5 else 1/self.rand(1, sat)
+        val = self.rand(1, val) if self.rand()<.5 else 1/self.rand(1, val)
+        x = cv2.cvtColor(np.array(image,np.float32)/255, cv2.COLOR_RGB2HSV)
+        x[..., 1] *= sat
+        x[..., 2] *= val
+        x[x[:,:, 0]>360, 0] = 360
+        x[:, :, 1:][x[:, :, 1:]>1] = 1
+        x[x<0] = 0
+        image_data = cv2.cvtColor(x, cv2.COLOR_HSV2RGB)*255
+        return image_data

src/utils/utils.py

@@ -0,0 +1,47 @@
+import numpy as np
+from PIL import Image
+
+#---------------------------------------------------#
+#   不失真的resize
+#---------------------------------------------------#
+def letterbox_image(image, size):
+    iw, ih  = image.size
+    w, h    = size
+
+    scale   = min(w/iw, h/ih)
+    nw      = int(iw*scale)
+    nh      = int(ih*scale)
+
+    image   = image.resize((nw,nh), Image.BICUBIC)
+    new_image = Image.new('RGB', size, (128,128,128))
+    new_image.paste(image, ((w-nw)//2, (h-nh)//2))
+
+    return new_image
+
+#---------------------------------------------------#
+#   获得类
+#---------------------------------------------------#
+def get_classes(classes_path):
+    with open(classes_path, encoding='utf-8') as f:
+        class_names = f.readlines()
+    class_names = [c.strip() for c in class_names]
+    return class_names, len(class_names)
+
+#---------------------------------------------------------#
+#   将图像转换成RGB图像，防止灰度图在预测时报错。
+#   代码仅仅支持RGB图像的预测，所有其它类型的图像都会转化成RGB
+#---------------------------------------------------------#
+def cvtColor(image):
+    if len(np.shape(image)) == 3 and np.shape(image)[2] == 3:
+        return image 
+    else:
+        image = image.convert('RGB')
+        return image 
+
+#----------------------------------------#
+#   预处理训练图片
+#----------------------------------------#
+def preprocess_input(x):
+    x /= 127.5
+    x -= 1.
+    return x