Upload 14 files

Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.10
+
+WORKDIR /code
+COPY . .
+
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

app/calibration_task.py

@@ -0,0 +1,66 @@
+import cv2
+import time
+import mediapipe as mp
+import numpy as np
+import json
+from microexpression_tracker import track_microexpressions
+
+mp_face_mesh = mp.solutions.face_mesh
+face_mesh = mp_face_mesh.FaceMesh(static_image_mode=False, max_num_faces=2)
+CALIBRATION_DURATION = 10 # seconds
+
+print("\n🟩 Welcome to SafeSpace!")
+print("Press SPACE to begin calibration. Position your face within the webcam frame and keep it still.")
+cap = cv2.VideoCapture(0)
+
+# Wait for SPACE key to start
+while True:
+    ret, frame = cap.read()
+    if not ret:
+        continue
+    cv2.putText(frame, "Press SPACE to start calibration", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,255,0), 2)
+    cv2.imshow("Calibration", frame)
+    if cv2.waitKey(1) & 0xFF == 32:
+        break
+
+print("\n🟡 Calibration started. Please follow the instructions for 5 seconds.")
+
+start_time = time.time()
+eye_centers = []
+
+while time.time() - start_time < CALIBRATION_DURATION:
+    ret, frame = cap.read()
+    if not ret:
+        continue
+
+    micro, face_bbox, multiple_faces = track_microexpressions(frame, face_mesh, {})
+
+    h, w, _ = frame.shape
+
+    if face_bbox:
+        cv2.rectangle(frame, (face_bbox[0], face_bbox[1]), (face_bbox[2], face_bbox[3]), (0,255,0), 2)
+    if multiple_faces:
+        cv2.putText(frame, "Multiple faces detected!", (20, h-40), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 3)
+
+    if face_bbox:
+        eye_centers.append((face_bbox[0] + face_bbox[2]) / 2 / w)
+
+    elapsed = int(time.time() - start_time)
+    cv2.putText(frame, f"Calibration: {CALIBRATION_DURATION-elapsed}s left", (20, 450), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,0,0), 2)
+    cv2.imshow("Calibration", frame)
+    if cv2.waitKey(1) & 0xFF == ord('q'):
+        break
+
+print("\n✅ Calibration finished. Thank you!")
+cap.release()
+cv2.destroyAllWindows()
+
+user_calib = {
+    'eye_left': np.percentile(eye_centers, 5) if eye_centers else 0.35,
+    'eye_right': np.percentile(eye_centers, 95) if eye_centers else 0.65
+}
+
+with open('user_calibration.json', 'w') as f:
+    json.dump(user_calib, f)
+
+print("\n🟢 Ready for session. Please start the main stress detection.")

app/final_facial_model.py

@@ -0,0 +1,133 @@
+import cv2
+import joblib
+import time
+import json
+from microexpression_tracker import track_microexpressions, get_lip_engagement
+from collections import Counter
+import mediapipe as mp
+import numpy as np
+
+# --- Load microexpression calibration ---
+with open('user_calibration.json', 'r') as f:
+    calibration_ref = json.load(f)
+
+# --- Load your ELM model and scaler ---
+# (Commented out for now)
+# elm_model = joblib.load("src/model_fer.pkl")
+# scaler = joblib.load("src/scaler_fer.pkl")
+
+IMG_SIZE = 96  # use same as in training
+SESSION_DURATION = 15  # seconds
+EYE_AWAY_THRESHOLD = 20
+HEAD_TURN_THRESHOLD = 20
+
+def preprocess_for_model(frame):
+    img = cv2.resize(frame, (IMG_SIZE, IMG_SIZE))
+    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+    img = img.astype(np.float32) / 255.0
+    img = img.flatten().reshape(1, -1)
+    return img
+
+cap = cv2.VideoCapture(0)
+
+print("\n📸 Welcome! Press SPACE to start stress & engagement analysis.")
+while True:
+    ret, frame = cap.read()
+    if not ret:
+        continue
+    cv2.putText(frame, "Press SPACE to start", (60, 60), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,255,0), 2)
+    cv2.imshow("Webcam", frame)
+    if cv2.waitKey(1) & 0xFF == 32:
+        break
+
+print("\n🟡 Session started. Look at the screen for 15 seconds.")
+session_start = time.time()
+
+mp_face_mesh = mp.solutions.face_mesh
+face_mesh = mp_face_mesh.FaceMesh(static_image_mode=False, max_num_faces=1)
+
+eye_away_count = 0
+head_turn_count = 0
+engagement_predictions = []  # Not used if ELM is commented out, but kept for completeness
+lip_engagement_predictions = []
+
+while time.time() - session_start < SESSION_DURATION:
+    ret, frame = cap.read()
+    if not ret:
+        continue
+
+    # --- Microexpression detection ---
+    micro, face_bbox, multiple_faces = track_microexpressions(frame, face_mesh, calibration_ref)
+
+    # --- Lip engagement detection ---
+    h, w, _ = frame.shape
+    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+    results = face_mesh.process(frame_rgb)
+    lip_engagement = "No Face"
+    if results.multi_face_landmarks:
+        landmarks = [(lm.x, lm.y) for lm in results.multi_face_landmarks[0].landmark]
+        lip_engagement = get_lip_engagement(landmarks)
+        lip_engagement_predictions.append(lip_engagement)
+        if face_bbox:
+            cv2.rectangle(frame, (face_bbox[0], face_bbox[1]), (face_bbox[2], face_bbox[3]), (0,255,0), 2)
+    else:
+        lip_engagement_predictions.append("No Face")
+        landmarks = None
+
+    if micro["eye_away"]:
+        eye_away_count += 1
+    if micro["head_turn"]:
+        head_turn_count += 1
+
+    if multiple_faces:
+        cv2.putText(frame, "Multiple faces detected!", (20, 450), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 3)
+
+    elapsed = int(time.time() - session_start)
+    cv2.putText(frame, f"Time left: {SESSION_DURATION-elapsed}s", (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2)
+    cv2.putText(frame, f"Engagement: {lip_engagement}", (20, 130), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,255,0), 2)
+    cv2.imshow('SafeSpace Session', frame)
+
+    print(f"Frame: {len(lip_engagement_predictions)} | Engagement: {lip_engagement}")
+
+    if cv2.waitKey(1) & 0xFF == ord('q'):
+        break
+
+cap.release()
+cv2.destroyAllWindows()
+
+# --- Summarize Lip Engagement Results (Safe & Robust) ---
+lip_counts = Counter(lip_engagement_predictions)
+# Remove "No Face" for reporting majority, if present
+lip_counts_no_face = Counter({k: v for k, v in lip_counts.items() if k != "No Face"})
+lip_total = sum(lip_counts_no_face.values())
+
+if lip_total > 0:
+    for label in ["Engaged", "Partially Engaged", "Not Engaged"]:
+        print(f"Engagement {label}: {lip_counts_no_face.get(label,0)} frames ({(lip_counts_no_face.get(label,0)/lip_total)*100:.1f}%)")
+
+    lip_majority_label = lip_counts_no_face.most_common(1)[0][0]
+else:
+    print("No valid engagement predictions to summarize.")
+    lip_majority_label = "No Face"
+
+# --- Microexpression-based feedback ---
+if 10 < eye_away_count < 20 and 10 < head_turn_count < 20:
+    print(f"\nEye distraction detected some times in session.")
+    print(f"Head turn detected some times in session.")
+
+if eye_away_count > EYE_AWAY_THRESHOLD:
+    print(f"\nEye distraction detected many times in session.")
+if head_turn_count > HEAD_TURN_THRESHOLD:
+    print(f"Head turn detected many times in session.")
+
+# --- Final Hybrid Result (using only lips & microexpression, as ELM is off) ---
+hybrid_result = lip_majority_label
+
+# Apply microexpression override
+if eye_away_count > EYE_AWAY_THRESHOLD or head_turn_count > HEAD_TURN_THRESHOLD:
+    if hybrid_result == "Engaged":
+        hybrid_result = "Partially Engaged"
+    elif hybrid_result == "Partially Engaged":
+        hybrid_result = "Not Engaged"
+
+print(f"\n✅ Final Conclusion (with Microexpressions): {hybrid_result}")

app/main.py

@@ -0,0 +1,42 @@
+# app/main.py
+
+from fastapi import FastAPI, UploadFile
+import numpy as np
+import cv2
+from app.calibration_task import generate_calibration_data
+from app.final_facial_model import predict_stress_level
+
+app = FastAPI()
+
+CALIBRATION_SECONDS = 10
+ANALYSIS_SECONDS = 30
+FPS = 10
+
+calib_frames = []
+analysis_frames = []
+
+@app.post("/calibrate")
+async def calibrate(image: UploadFile):
+    contents = await image.read()
+    np_arr = np.frombuffer(contents, np.uint8)
+    frame = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
+    calib_frames.append(frame)
+
+    if len(calib_frames) >= CALIBRATION_SECONDS * FPS:
+        generate_calibration_data(calib_frames)
+        calib_frames.clear()
+        return {"status": "Calibration Complete"}
+    return {"status": f"Calibrating... {len(calib_frames)//FPS}s"}
+
+@app.post("/analyze")
+async def analyze(image: UploadFile):
+    contents = await image.read()
+    np_arr = np.frombuffer(contents, np.uint8)
+    frame = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
+    analysis_frames.append(frame)
+
+    if len(analysis_frames) >= ANALYSIS_SECONDS * FPS:
+        result = predict_stress_level(analysis_frames)
+        analysis_frames.clear()
+        return {"result": result}
+    return {"status": f"Analyzing... {len(analysis_frames)//FPS}s"}

app/microexpression_tracker.py

@@ -0,0 +1,74 @@
+import numpy as np
+import cv2
+import mediapipe as mp
+
+
+LEFT_EYE = [33, 133]
+RIGHT_EYE = [362, 263]
+NOSE = 1
+
+mp_face_mesh = mp.solutions.face_mesh
+def get_lip_engagement(landmarks):
+    TOP_LIP = 13
+    BOTTOM_LIP = 14
+    LIP_LEFT = 78
+    LIP_RIGHT = 308
+    top_lip = landmarks[TOP_LIP]
+    bottom_lip = landmarks[BOTTOM_LIP]
+    left_corner = landmarks[LIP_LEFT]
+    right_corner = landmarks[LIP_RIGHT]
+    lip_opening = abs(top_lip[1] - bottom_lip[1])
+    lip_width = abs(right_corner[0] - left_corner[0])
+
+    # print(f"[DEBUG] lip_opening: {lip_opening:.3f}, lip_width: {lip_width:.3f}")
+
+    # Example, adjust as per your actual values!
+    # This logic: high opening OR high width = Engaged (smile/mouth open)
+    # very small both = Not Engaged, everything else = Partially Engaged
+    if lip_opening > 0.01 or lip_width > 0.18:
+        return "Engaged"
+    elif lip_opening < 0.002 or lip_width < 0.04:
+        return "Not Engaged"
+    else:
+        return "Partially Engaged"
+
+
+
+
+
+def track_microexpressions(frame, face_mesh, calibration_ref=None):
+    if calibration_ref is None:
+        calibration_ref = {}
+    h, w, _ = frame.shape
+    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+    results = face_mesh.process(frame_rgb)
+    micro = {
+             "eye_away": False,
+             "head_turn": False,
+             }
+    face_bbox = None
+    multiple_faces = False
+
+    if results.multi_face_landmarks:
+        if len(results.multi_face_landmarks) > 1:
+            multiple_faces = True
+
+        lm = results.multi_face_landmarks[0].landmark
+        xs = [p.x for p in lm]
+        ys = [p.y for p in lm]
+        xmin, xmax = min(xs)*w, max(xs)*w
+        ymin, ymax = min(ys)*h, max(ys)*h
+        face_bbox = [int(xmin), int(ymin), int(xmax), int(ymax)]
+
+        eye_x = (lm[LEFT_EYE[0]].x + lm[RIGHT_EYE[0]].x) / 2
+        nose_x = lm[NOSE].x
+
+        margin = 0.07
+        eye_left_th = calibration_ref.get('eye_left', 0.30)
+        eye_right_th = calibration_ref.get('eye_right', 0.70)
+        if eye_x < (eye_left_th - margin) or eye_x > (eye_right_th + margin):
+            micro["eye_away"] = True
+        if nose_x < (eye_left_th - margin) or nose_x > (eye_right_th + margin):
+            micro["head_turn"] = True
+
+    return micro, face_bbox, multiple_faces

app/model_files/label_encoder_fer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec6798a50709c0ea32b0d6ac9817d4b45f2645d02b4f3cea8da303c3ae9438da
+size 531

app/model_files/model_fer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f70ac0fc7d4c7783efbdfb5e7ed7c2b46dfd0e2ab0db3a7d4f75498005226bdf
+size 6780643

app/model_files/scaler_fer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cedfac8ecc967834e963ddb617ff07f815c96bd3655629542870decf1363211
+size 68199

app/model_files/user_calibration.json

@@ -0,0 +1 @@
+{"eye_left": 0.54296875, "eye_right": 0.559375}

app/simple_elm.py

@@ -0,0 +1,46 @@
+
+import numpy as np
+
+class SimpleELMClassifier:
+    def __init__(self, n_hidden=300, activation='relu', random_state=42):
+        self.n_hidden = n_hidden
+        self.activation = activation
+        self.random_state = random_state
+        self.is_fitted = False
+
+    def _activation(self, X):
+        if self.activation == 'sigmoid':
+            return 1 / (1 + np.exp(-X))
+        elif self.activation == 'tanh':
+            return np.tanh(X)
+        elif self.activation == 'relu':
+            return np.maximum(0, X)
+        else:
+            raise ValueError('Unsupported activation')
+
+    def fit(self, X, y):
+        np.random.seed(self.random_state)
+        n_samples, n_features = X.shape
+        n_classes = len(np.unique(y))
+        self.classes_ = np.unique(y)
+        Y = np.zeros((n_samples, n_classes))
+        for i, label in enumerate(self.classes_):
+            Y[y == label, i] = 1
+
+        self.W = np.random.randn(n_features, self.n_hidden)
+        self.b = np.random.randn(self.n_hidden)
+        H = self._activation(np.dot(X, self.W) + self.b)
+        self.beta = np.dot(np.linalg.pinv(H), Y)
+        self.is_fitted = True
+
+    def predict_proba(self, X):
+        if not self.is_fitted:
+            raise Exception("Model not fitted yet.")
+        H = self._activation(np.dot(X, self.W) + self.b)
+        logits = np.dot(H, self.beta)
+        e_logits = np.exp(logits - np.max(logits, axis=1, keepdims=True))
+        return e_logits / np.sum(e_logits, axis=1, keepdims=True)
+
+    def predict(self, X):
+        proba = self.predict_proba(X)
+        return np.argmax(proba, axis=1)

app/utils.py

@@ -0,0 +1,61 @@
+import numpy as np
+from tensorflow.keras.applications.resnet50 import preprocess_input
+from tensorflow.keras.preprocessing.image import img_to_array
+from tensorflow.keras.applications import ResNet50
+import joblib
+from simple_elm import SimpleELMClassifier
+import cv2
+from microexpression_tracker import track_microexpressions
+import time
+import json
+
+class_names = ['anger', 'contempt', 'disgust', 'fear', 'happiness', 'neutral', 'sadness', 'surprise']
+
+resnet = ResNet50(weights='imagenet', include_top=False, input_shape=(96, 96, 3), pooling='avg')
+
+model_fer = joblib.load("src/model_fer.pkl")
+scaler_fer = joblib.load("src/scaler_fer.pkl")
+# model_ck = joblib.load("src/model_ck.pkl")
+# scaler_ck = joblib.load("src/scaler_ck.pkl")
+
+def map_emotion_to_engagement(emotion_label):
+    # fer returns: 'angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'
+    if emotion_label in ['happy', 'surprise']:
+        return "Engaged"
+    elif emotion_label in ['neutral']:
+        return "Partially Engaged"
+    else:
+        return "Not Engaged"
+
+
+def preprocess_image_for_resnet(image, target_size=(96, 96)):
+    import cv2
+    image = cv2.resize(image, target_size)
+    image = img_to_array(image)
+    image = np.expand_dims(image, axis=0)
+    return preprocess_input(image)
+
+def predict_emotion(frame):
+    try:
+        preprocessed = preprocess_image_for_resnet(frame)
+        features = resnet.predict(preprocessed, verbose=0).flatten().reshape(1, -1)
+        feat_fer = scaler_fer.transform(features)
+        probs_fer = model_fer.predict_proba(feat_fer)[0]
+        # Get the index of the class with highest probability
+        idx = np.argmax(probs_fer)
+        final_label = class_names[idx]
+        print(f"[DEBUG] Emotion probabilities: {dict(zip(class_names, np.round(probs_fer, 3)))}")
+        print(f"[DEBUG] Detected emotion: {final_label}")
+        return final_label
+    except Exception as e:
+        print(f"[ERROR] Prediction failed: {e}")
+        return "error"
+
+
+def predict_engagement_class(frame):
+    emotion_label = predict_emotion(frame)
+    print(f"[DEBUG] Detected emotion: {emotion_label}")
+    engagement_label = map_emotion_to_engagement(emotion_label)
+    return engagement_label
+
+

app/webcam_inference.py

@@ -0,0 +1,92 @@
+# webcam_inference.py
+
+import cv2
+import numpy as np
+import joblib
+from tensorflow.keras.applications import ResNet50
+from tensorflow.keras.applications.resnet50 import preprocess_input
+from tensorflow.keras.preprocessing.image import img_to_array
+from utils import map_emotion_to_engagement
+import os
+from hpelm import ELM
+import joblib
+from simple_elm import SimpleELMClassifier
+import joblib
+model_fer = joblib.load("src/model_fer.pkl")
+scaler_fer = joblib.load("src/scaler_fer.pkl")
+
+# Load models and scalers
+# elm=ELM(2048,7)
+# model_fer = joblib.load("src/model_fer.pkl")
+# scaler_fer = joblib.load("src/scaler_fer.pkl")
+# model_ck = joblib.load("src/model_ck.pkl")
+# scaler_ck = joblib.load("src/scaler_ck.pkl")
+
+# Class index to emotion label
+class_names = ['anger', 'contempt', 'disgust', 'fear', 'happiness', 'neutral', 'sadness', 'surprise']
+
+# Load ResNet50 model
+resnet = ResNet50(weights='imagenet', include_top=False, input_shape=(96, 96, 3), pooling='avg')
+
+# Preprocess frame
+def preprocess_frame(frame):
+    frame = cv2.resize(frame, (96, 96))
+    frame = img_to_array(frame)
+    frame = np.expand_dims(frame, axis=0)
+    return preprocess_input(frame)
+
+# Predict function with neutral class bypass logic
+def predict_emotion(frame):
+    try:
+        # Extract features
+        preprocessed = preprocess_frame(frame)
+        features = resnet.predict(preprocessed, verbose=0).flatten().reshape(1, -1)
+
+        # Scale features
+        feat_fer = scaler_fer.transform(features)
+        feat_ck = scaler_ck.transform(features)
+
+        # Get probabilities
+        probs_fer = model_fer.predict_proba(feat_fer)[0]
+        # probs_ck = model_ck.predict_proba(feat_ck)[0]
+
+        label_fer = class_names[np.argmax(probs_fer)]
+
+        # If FER predicts neutral, use only FER
+        final_probs = probs_fer
+
+        final_label = class_names[np.argmax(final_probs)]
+        return final_label
+
+    except Exception as e:
+        print(f"[ERROR] Prediction failed: {e}")
+        return "error"
+
+# Main webcam loop
+def run_webcam():
+    cap = cv2.VideoCapture(0)
+    print("[INFO] Webcam started. Press 'q' to quit.")
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        flipped = cv2.flip(frame, 1)  # Flip for mirror view
+        emotion = predict_emotion(flipped)
+        engagement = map_emotion_to_engagement(emotion)
+
+        # Overlay
+        cv2.putText(flipped, f"Emotion: {emotion}", (10, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 0), 2)
+        cv2.putText(flipped, f"Engagement: {engagement}", (10, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 255), 2)
+
+        cv2.imshow("FINAL FACIAL MODEL FOR SAFESPACE", flipped)
+
+        if cv2.waitKey(1) & 0xFF == ord('q'):
+            break
+
+    cap.release()
+    cv2.destroyAllWindows()
+
+if __name__ == "__main__":
+    run_webcam()

huggingface.yaml

@@ -0,0 +1 @@
+sdk: docker

requirements.txt

@@ -0,0 +1,19 @@
+
+mediapipe
+tensorflow
+transformers
+matplotlib
+tqdm
+imutils
+dlib
+Pillow
+
+
+fastapi
+uvicorn
+numpy
+opencv-python
+scikit-learn
+joblib
+dlib
+imutils