Spaces:
Sleeping
Sleeping
Update inference.py
Browse files- inference.py +39 -49
inference.py
CHANGED
|
@@ -1,13 +1,7 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
import tensorflow as tf
|
| 4 |
-
import logging
|
| 5 |
from preprocessing import VideoPreprocessor
|
| 6 |
|
| 7 |
-
# Configure Logging
|
| 8 |
-
logging.basicConfig(level=logging.INFO)
|
| 9 |
-
logger = logging.getLogger(__name__)
|
| 10 |
-
|
| 11 |
class LipReadingModel:
|
| 12 |
def __init__(self, model_path='best_model_1_WER.keras'):
|
| 13 |
# Initialize character mappings before loading the model
|
|
@@ -22,10 +16,9 @@ class LipReadingModel:
|
|
| 22 |
model_path,
|
| 23 |
custom_objects={'CTCLoss': self.CTCLoss} # Include custom loss if needed
|
| 24 |
)
|
| 25 |
-
|
| 26 |
except Exception as e:
|
| 27 |
-
|
| 28 |
-
self.model = self.load_model() # Fallback to building the model if loading fails
|
| 29 |
|
| 30 |
@staticmethod
|
| 31 |
def CTCLoss(y_true, y_pred):
|
|
@@ -76,28 +69,24 @@ class LipReadingModel:
|
|
| 76 |
# Final dense layer
|
| 77 |
model.add(tf.keras.layers.Dense(self.char_to_num.vocabulary_size() + 1, kernel_initializer='he_normal', activation='softmax'))
|
| 78 |
|
| 79 |
-
logger.info("Built the model architecture successfully.")
|
| 80 |
return model
|
| 81 |
|
| 82 |
def predict(self, normalized_frames):
|
| 83 |
if normalized_frames is None or int(tf.size(normalized_frames)) == 0:
|
| 84 |
return "No frames extracted from the video. Please ensure the video contains a clear view of the face and lips."
|
| 85 |
-
|
| 86 |
-
# Add batch dimension
|
| 87 |
-
frames = tf.expand_dims(normalized_frames, axis=0) # Shape: (1, num_frames, 85, 85, 1)
|
| 88 |
|
| 89 |
-
#
|
| 90 |
yhat = self.model.predict(frames, verbose=0)
|
| 91 |
|
| 92 |
input_length = [yhat.shape[1]] # batch size of 1
|
| 93 |
# Perform CTC decoding
|
| 94 |
decoded_tf = tf.keras.backend.ctc_decode(yhat, input_length=input_length, greedy=True)[0][0]
|
| 95 |
decoded = decoded_tf.numpy().flatten() # Convert to numpy array and flatten
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
# Convert numerical predictions to characters
|
| 102 |
prediction = ''.join(
|
| 103 |
[
|
|
@@ -106,34 +95,35 @@ class LipReadingModel:
|
|
| 106 |
if int(num) != -1
|
| 107 |
]
|
| 108 |
)
|
| 109 |
-
|
| 110 |
return prediction
|
| 111 |
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
|
|
|
|
|
| 1 |
+
import os
|
|
|
|
| 2 |
import tensorflow as tf
|
|
|
|
| 3 |
from preprocessing import VideoPreprocessor
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
class LipReadingModel:
|
| 6 |
def __init__(self, model_path='best_model_1_WER.keras'):
|
| 7 |
# Initialize character mappings before loading the model
|
|
|
|
| 16 |
model_path,
|
| 17 |
custom_objects={'CTCLoss': self.CTCLoss} # Include custom loss if needed
|
| 18 |
)
|
| 19 |
+
print("Model loaded successfully.")
|
| 20 |
except Exception as e:
|
| 21 |
+
print(f"Error loading model: {e}")
|
|
|
|
| 22 |
|
| 23 |
@staticmethod
|
| 24 |
def CTCLoss(y_true, y_pred):
|
|
|
|
| 69 |
# Final dense layer
|
| 70 |
model.add(tf.keras.layers.Dense(self.char_to_num.vocabulary_size() + 1, kernel_initializer='he_normal', activation='softmax'))
|
| 71 |
|
|
|
|
| 72 |
return model
|
| 73 |
|
| 74 |
def predict(self, normalized_frames):
|
| 75 |
if normalized_frames is None or int(tf.size(normalized_frames)) == 0:
|
| 76 |
return "No frames extracted from the video. Please ensure the video contains a clear view of the face and lips."
|
|
|
|
|
|
|
|
|
|
| 77 |
|
| 78 |
+
frames = tf.expand_dims(normalized_frames, axis=0) # Add batch dimension
|
| 79 |
yhat = self.model.predict(frames, verbose=0)
|
| 80 |
|
| 81 |
input_length = [yhat.shape[1]] # batch size of 1
|
| 82 |
# Perform CTC decoding
|
| 83 |
decoded_tf = tf.keras.backend.ctc_decode(yhat, input_length=input_length, greedy=True)[0][0]
|
| 84 |
decoded = decoded_tf.numpy().flatten() # Convert to numpy array and flatten
|
| 85 |
+
|
| 86 |
+
print("decoded type:", type(decoded))
|
| 87 |
+
print("decoded shape:", decoded.shape)
|
| 88 |
+
print("decoded contents:", decoded)
|
| 89 |
+
|
| 90 |
# Convert numerical predictions to characters
|
| 91 |
prediction = ''.join(
|
| 92 |
[
|
|
|
|
| 95 |
if int(num) != -1
|
| 96 |
]
|
| 97 |
)
|
| 98 |
+
|
| 99 |
return prediction
|
| 100 |
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def predict_from_video(video_path=None, frames=None, model=None):
|
| 105 |
+
"""
|
| 106 |
+
Predicts the text from a video file or webcam frames using the provided model.
|
| 107 |
+
Args:
|
| 108 |
+
video_path (str, optional): Path to the video file. Defaults to None.
|
| 109 |
+
frames (List[np.ndarray], optional): List of frames from webcam. Defaults to None.
|
| 110 |
+
model (LipReadingModel, optional): An instance of the LipReadingModel. Defaults to None.
|
| 111 |
+
Returns:
|
| 112 |
+
str: Predicted text.
|
| 113 |
+
"""
|
| 114 |
+
if model is None:
|
| 115 |
+
model = LipReadingModel()
|
| 116 |
+
|
| 117 |
+
preprocessor = VideoPreprocessor()
|
| 118 |
+
|
| 119 |
+
if video_path:
|
| 120 |
+
# Preprocess video from file
|
| 121 |
+
normalized_frames = preprocessor.preprocess_video(video_path)
|
| 122 |
+
elif frames is not None:
|
| 123 |
+
# Preprocess frames from webcam
|
| 124 |
+
normalized_frames = preprocessor.preprocess_frames(frames)
|
| 125 |
+
else:
|
| 126 |
+
return "No video or frames provided for prediction."
|
| 127 |
+
|
| 128 |
+
prediction = model.predict(normalized_frames)
|
| 129 |
+
return prediction
|