Spaces:
Sleeping
Sleeping
Update inference.py
Browse files- inference.py +25 -15
inference.py
CHANGED
|
@@ -1,7 +1,13 @@
|
|
| 1 |
-
|
|
|
|
| 2 |
import tensorflow as tf
|
|
|
|
| 3 |
from preprocessing import VideoPreprocessor
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
class LipReadingModel:
|
| 6 |
def __init__(self, model_path='best_model_1_WER.keras'):
|
| 7 |
# Initialize character mappings before loading the model
|
|
@@ -16,9 +22,10 @@ class LipReadingModel:
|
|
| 16 |
model_path,
|
| 17 |
custom_objects={'CTCLoss': self.CTCLoss} # Include custom loss if needed
|
| 18 |
)
|
| 19 |
-
|
| 20 |
except Exception as e:
|
| 21 |
-
|
|
|
|
| 22 |
|
| 23 |
@staticmethod
|
| 24 |
def CTCLoss(y_true, y_pred):
|
|
@@ -69,24 +76,28 @@ class LipReadingModel:
|
|
| 69 |
# Final dense layer
|
| 70 |
model.add(tf.keras.layers.Dense(self.char_to_num.vocabulary_size() + 1, kernel_initializer='he_normal', activation='softmax'))
|
| 71 |
|
|
|
|
| 72 |
return model
|
| 73 |
|
| 74 |
def predict(self, normalized_frames):
|
| 75 |
if normalized_frames is None or int(tf.size(normalized_frames)) == 0:
|
| 76 |
return "No frames extracted from the video. Please ensure the video contains a clear view of the face and lips."
|
|
|
|
|
|
|
|
|
|
| 77 |
|
| 78 |
-
|
| 79 |
yhat = self.model.predict(frames, verbose=0)
|
| 80 |
|
| 81 |
input_length = [yhat.shape[1]] # batch size of 1
|
| 82 |
# Perform CTC decoding
|
| 83 |
decoded_tf = tf.keras.backend.ctc_decode(yhat, input_length=input_length, greedy=True)[0][0]
|
| 84 |
decoded = decoded_tf.numpy().flatten() # Convert to numpy array and flatten
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
# Convert numerical predictions to characters
|
| 91 |
prediction = ''.join(
|
| 92 |
[
|
|
@@ -95,26 +106,25 @@ class LipReadingModel:
|
|
| 95 |
if int(num) != -1
|
| 96 |
]
|
| 97 |
)
|
| 98 |
-
|
| 99 |
-
return prediction
|
| 100 |
-
|
| 101 |
-
|
| 102 |
|
|
|
|
| 103 |
|
| 104 |
-
def predict_from_video(video_path=None, frames=None, model=None):
|
| 105 |
"""
|
| 106 |
Predicts the text from a video file or webcam frames using the provided model.
|
| 107 |
Args:
|
| 108 |
video_path (str, optional): Path to the video file. Defaults to None.
|
| 109 |
frames (List[np.ndarray], optional): List of frames from webcam. Defaults to None.
|
| 110 |
model (LipReadingModel, optional): An instance of the LipReadingModel. Defaults to None.
|
|
|
|
| 111 |
Returns:
|
| 112 |
str: Predicted text.
|
| 113 |
"""
|
| 114 |
if model is None:
|
| 115 |
model = LipReadingModel()
|
| 116 |
|
| 117 |
-
preprocessor
|
|
|
|
| 118 |
|
| 119 |
if video_path:
|
| 120 |
# Preprocess video from file
|
|
|
|
| 1 |
+
# inference.py
|
| 2 |
+
|
| 3 |
import tensorflow as tf
|
| 4 |
+
import logging
|
| 5 |
from preprocessing import VideoPreprocessor
|
| 6 |
|
| 7 |
+
# Configure Logging
|
| 8 |
+
logging.basicConfig(level=logging.INFO)
|
| 9 |
+
logger = logging.getLogger(__name__)
|
| 10 |
+
|
| 11 |
class LipReadingModel:
|
| 12 |
def __init__(self, model_path='best_model_1_WER.keras'):
|
| 13 |
# Initialize character mappings before loading the model
|
|
|
|
| 22 |
model_path,
|
| 23 |
custom_objects={'CTCLoss': self.CTCLoss} # Include custom loss if needed
|
| 24 |
)
|
| 25 |
+
logger.info("Model loaded successfully.")
|
| 26 |
except Exception as e:
|
| 27 |
+
logger.error(f"Error loading model: {e}")
|
| 28 |
+
self.model = self.load_model() # Fallback to building the model if loading fails
|
| 29 |
|
| 30 |
@staticmethod
|
| 31 |
def CTCLoss(y_true, y_pred):
|
|
|
|
| 76 |
# Final dense layer
|
| 77 |
model.add(tf.keras.layers.Dense(self.char_to_num.vocabulary_size() + 1, kernel_initializer='he_normal', activation='softmax'))
|
| 78 |
|
| 79 |
+
logger.info("Built the model architecture successfully.")
|
| 80 |
return model
|
| 81 |
|
| 82 |
def predict(self, normalized_frames):
|
| 83 |
if normalized_frames is None or int(tf.size(normalized_frames)) == 0:
|
| 84 |
return "No frames extracted from the video. Please ensure the video contains a clear view of the face and lips."
|
| 85 |
+
|
| 86 |
+
# Add batch dimension
|
| 87 |
+
frames = tf.expand_dims(normalized_frames, axis=0) # Shape: (1, num_frames, 85, 85, 1)
|
| 88 |
|
| 89 |
+
# Perform prediction
|
| 90 |
yhat = self.model.predict(frames, verbose=0)
|
| 91 |
|
| 92 |
input_length = [yhat.shape[1]] # batch size of 1
|
| 93 |
# Perform CTC decoding
|
| 94 |
decoded_tf = tf.keras.backend.ctc_decode(yhat, input_length=input_length, greedy=True)[0][0]
|
| 95 |
decoded = decoded_tf.numpy().flatten() # Convert to numpy array and flatten
|
| 96 |
+
|
| 97 |
+
logger.debug(f"decoded type: {type(decoded)}")
|
| 98 |
+
logger.debug(f"decoded shape: {decoded.shape}")
|
| 99 |
+
logger.debug(f"decoded contents: {decoded}")
|
| 100 |
+
|
| 101 |
# Convert numerical predictions to characters
|
| 102 |
prediction = ''.join(
|
| 103 |
[
|
|
|
|
| 106 |
if int(num) != -1
|
| 107 |
]
|
| 108 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
|
| 110 |
+
return prediction
|
| 111 |
|
| 112 |
+
def predict_from_video(video_path=None, frames=None, model=None, preprocessor=None):
|
| 113 |
"""
|
| 114 |
Predicts the text from a video file or webcam frames using the provided model.
|
| 115 |
Args:
|
| 116 |
video_path (str, optional): Path to the video file. Defaults to None.
|
| 117 |
frames (List[np.ndarray], optional): List of frames from webcam. Defaults to None.
|
| 118 |
model (LipReadingModel, optional): An instance of the LipReadingModel. Defaults to None.
|
| 119 |
+
preprocessor (VideoPreprocessor, optional): An instance of the VideoPreprocessor. Defaults to None.
|
| 120 |
Returns:
|
| 121 |
str: Predicted text.
|
| 122 |
"""
|
| 123 |
if model is None:
|
| 124 |
model = LipReadingModel()
|
| 125 |
|
| 126 |
+
if preprocessor is None:
|
| 127 |
+
preprocessor = VideoPreprocessor()
|
| 128 |
|
| 129 |
if video_path:
|
| 130 |
# Preprocess video from file
|