Upload 6 files

LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Aditya Singh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,13 +1,107 @@
----
-title: Face Recognition App
-emoji: 🐠
-colorFrom: gray
-colorTo: red
-sdk: streamlit
-sdk_version: 1.41.1
-app_file: app.py
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# Face Recognition Using MTCNN, VGG-Face2 and Pinecone DB
+
+An implementation of siamease neural networks on one shot learning tasks for face recognition tasks utilising MTCNN, FaceNet and Pinecone DB for building an interactive and easy to use application to store and detect faces from images as well as camera inputs accurately.
+
+
+## Deployment
+
+To use this project application run
+
+```bash
+  https://face-recognition-tf-1.onrender.com
+```
+Note : Loading may take some time due to limited  server processing capability.
+
+
+
+## Outputs
+<img width="956" alt="356581473-3f98e226-d9a5-4036-9c35-06481f292c8a" src="https://github.com/user-attachments/assets/60c7ea7f-1604-421b-8265-5b171da0edaa">
+<img width="956" alt="image" src="https://github.com/user-attachments/assets/1bc6a9c0-2698-483c-b478-80bf8d29f442">
+
+
+## Key Features
+
+__1)__ __Face detection using MTCNN__ :     
+Used MTCNN for accurate face detection on images.          
+
+__2)__ __Embedding extraction using FacNet__ :  
+created embeddings vectors of shape (512,1) using a Tensorflow pretrained model on face datasets.  
+
+__3)__  __Pinecone Database for efficent storage and retreival__:   
+Created a pinecone DB index for storing embdeeings by creating relevent metadata and upserting it to the index with cosine similarity as serach parameter.    
+
+__4)__ __Siamease Network Architehture__:   
+Implemented a Siamease Network like architehture to acheive one shot learning for face recognition using a combination of face detection and face recognition.  
+
+__5)__ __Streamlit application__:   
+Created and deployed an interactive streamlit application to interact with the project. Deployment was done on render.  
+
+__6)__ __Multiface Detection capabilities__:    
+The model is able to detect and recognise multiple faces in an image however for creating a new entry in the database for a person, an induidual image is required to ensure integrity of the data in the database. 
+
+__7)__ __Modularity of code__:  
+The project is cretated so that the induvidual blocks can be changed to suit the detection needs, eg. MTCNN can be replace with YOLO detection for faster results for applications such as ANPR after replacing FaceNet with a suitably trained model to generate embeddings.   
+
+
+## Pinecone API Reference
+
+##### Refer to Pinecone Documentation at : [pc docs](https://docs.pinecone.io/reference/api/introduction)
+
+#### Pinecone env variables used :
+
+| Parameter | Type     | Description                |
+| :-------- | :------- | :------------------------- |
+| `Pinecone key` | `string` | <Your API key>|
+| `Pinecone index`      | `string` | Name of Index |
+
+
+#### Installing pinecone client
+```bash
+pip install pinecone --upgrade pinecone-client
+```
+#### Connecting to index and upserting
+
+```bash
+from pinecone import Pinecone
+pc = Pinecone(os.environ["PINECONE_API_KEY"])
+index = pc.Index(os.environ["PINECONE_IDX"])
+```
+```bash
+vectors = 'Create Vectors'
+index.upsert(vectors)
+```
+
+#### Query top k vectors
+```bash
+out = index.query(
+      vector = vectors.tolist(),
+      top_k = k,
+      include_metadata = True
+  )
+```
+
+
+## Dependencies
+- OpenCV
+- Tensorflow
+- MTCNN
+- FaceNet
+- Pinecone
+- streamlit
+- os
+- dotenv
+- numpy
+#### Installation :
+After pulling this repo, run:
+```bash
+pip install requirements
+```
+#### env file setup :
+setup an env file to store api keys and index information as the template :
+```bash
+pinecone_key = "your_api_key"
+pinecone_index = "your_index_name"
+```
+
+

app.py

@@ -0,0 +1,54 @@
+import streamlit as st
+import cv2
+import base64
+from detection import *
+
+background_image = """
+<style>
+[data-testid="stAppViewContainer"] > .main {
+    background-image: url("https://images.pexels.com/photos/1054218/pexels-photo-1054218.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2");
+    background-size: cover;
+    background-position: center;  
+    background-repeat: no-repeat;
+}
+</style>
+"""
+
+st.markdown(background_image, unsafe_allow_html=True)
+
+st.html("<h1><center>Face Detection Web App</h1></center>")
+vis = True
+db_mode = st.checkbox("Add to data base")
+if db_mode:
+    vis = False
+if not vis:
+    st.html("<h3><center>Storing Image to Database</h3></center>")
+input_method = st.checkbox("Camera Mode")
+cam_inp , file_inp = None , None
+if input_method:
+    cam_inp = st.camera_input(label="Say Cheese!")
+else:
+    file_inp = st.file_uploader("Upload a Photo!")
+
+if cam_inp != None:
+    file = cam_inp
+else:
+    file = file_inp
+
+if file and vis:
+    min_conf = st.number_input("Enter the desired confidence value:" , min_value=0.0 , max_value=1.0, step=0.10, disabled= not vis , value=0.65)
+    if min_conf:
+        img , labels = detect_and_fetch(file , min_confidence= min_conf)
+        st.image(img , caption='Detections' , use_column_width=True)
+        st.header('Detected:')
+        labels = set(labels)
+        for i in labels:
+            if i is not "Unknown":
+                st.text(i)
+else:
+    name = st.text_input("Write the Name of Person" , disabled= vis)
+    if name:
+        process = write_and_upsert(file , name)
+        if process:
+            st.text("Uploaded Sucessfully!")
+

detection.py

@@ -0,0 +1,132 @@
+import cv2
+from pinecone import Pinecone
+import os
+from dotenv import load_dotenv , dotenv_values
+import numpy as np
+from mtcnn import MTCNN
+import matplotlib.pyplot as plt
+import scipy # for MTCNN Dependencies
+from keras_facenet import FaceNet
+
+detector = MTCNN()
+embedder = FaceNet()
+load_dotenv()
+pc = Pinecone(os.getenv("pinecone_key"))
+index = pc.Index(os.getenv("pinecone_index"))
+
+def get_embedding(file):
+  img = plt.imread(file)
+  img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+  try:
+    faces = detector.detect_faces(img)
+    if len(faces) > 1:
+      return print("Multiple Faces")
+    else:
+      faces = faces[0]
+      x1, y1, width, height = faces['box']
+      x1, y1 = abs(x1), abs(y1)
+      img = img[y1:y1+height, x1:x1+width]
+      img = cv2.resize(img, (224, 224))
+      img = np.expand_dims(img, axis=0)
+      print(f"Detected Face in {file}")
+      return embedder.embeddings(img)[0]
+  except:
+    print(f"Failed to detect faces in {file}")
+    return None
+  
+def make_embeddings(file , name):
+  embeddings = {}
+  emb = get_embedding(file)
+  if emb is not None:
+    embeddings[name] = emb
+  return embeddings
+
+def make_meta(file , i , name):
+  meta = {'image_id':f"Image_{i}"}
+  meta['label'] = name
+  return meta
+
+def upsert_embeddings(face_embeddings , metadata):
+  assert len(face_embeddings) == len(metadata) , f"Dimesnion mismatch, got embeddings len {len(face_embeddings)} and metadata length {len(metadata)}"
+  assert face_embeddings is not None and metadata is not None , 'Input vectors cannot be None'
+  assert face_embeddings[0].shape[0] == index.describe_index_stats()['dimension'] , 'Dimension mismatch'
+  a = index.describe_index_stats()['total_vector_count'] + 1
+  upsert_data = [(str(i + a), face_embeddings[i].tolist(), metadata[i]) for i in range(len(face_embeddings))] # index starts from last vector in database
+  index.upsert(vectors=upsert_data)
+  return True
+
+def fetch_embeddings(embeddings):
+  assert embeddings.shape[0] == index.describe_index_stats()['dimension'] , f"Expected{embeddings.shape[0]} got {index.describe_index_stats()['dimension']} instead"
+  assert embeddings is not None , 'Input vectors cannot be None'
+
+  out = index.query(
+      vector=embeddings.tolist(),
+      top_k=1,
+      include_metadata=True
+  )
+  name , confidence = out['matches'][0]['metadata']['label'] , out['matches'][0]['score']
+  return name , confidence
+
+def multiple_faces(file):
+  imga = plt.imread(file)
+  #imga = cv2.cvtColor(imga, cv2.COLOR_BGR2RGB)
+  try:
+    faces = detector.detect_faces(imga)
+    embs = []
+    boxes = []
+    for face in faces:
+      x1, y1, width, height = face['box']
+      x1, y1 = abs(x1), abs(y1)
+      img = imga[y1:y1+height, x1:x1+width]
+      img = cv2.resize(img, (224, 224))
+      img = np.expand_dims(img, axis=0)
+      embs.append(embedder.embeddings(img)[0])
+      boxes.append([x1, y1, width, height])
+    return embs, boxes
+  except Exception as e:
+      print(f"Failed to detect faces in {file} due to {e}")
+  return None
+
+def draw_boxes(file, boxes ,labels, probs):
+  img = plt.imread(file)
+  #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+  img = np.copy(img)
+  for box , label , prob in zip(boxes,labels, probs):
+    x1, y1, width, height = box
+    img = cv2.rectangle(img, (x1, y1), (x1+width, y1+height), (0, 255, 0), 1)
+    if label == 'Unknown':
+      img = cv2.putText(img, f"{label}", (x1, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 0, 255), 1)
+    else:
+      img = cv2.putText(img, f"{label} {np.round(prob , decimals = 2)}", (x1, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 255, 0), 1)
+  #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+  return img
+
+def write_and_upsert(dir ,  Name, upsert = True):
+  embeddings = make_embeddings(dir , Name)
+  metadata = [make_meta(dir , 1 , Name)]
+  if upsert:
+    upsert_embeddings(list(embeddings.values()), metadata)
+  return True
+
+def detect_and_fetch(dir , directory = False , min_confidence = 0):
+  files = []
+  files.append(dir)
+  for file in files:
+    try:
+      embs , boxes = multiple_faces(file)
+      labels = []
+      probs = []
+      for emb in embs:
+        name , confidence = fetch_embeddings(emb)
+        if confidence > min_confidence:
+          labels.append(name)
+          probs.append(confidence)
+        else:
+          labels.append('Unknown')
+          probs.append('unk')
+      img = draw_boxes(file, boxes, labels, probs)
+    except Exception as e:
+      print(f"Failed to detect faces in {file} due to {e}")
+  return img , labels 
+
+

packages.txt

@@ -0,0 +1,2 @@
+libgl1-mesa-glx
+libglib2.0-0

requirements.txt

@@ -0,0 +1,10 @@
+tensorflow
+streamlit
+matplotlib
+opencv-python
+pinecone
+scipy
+mtcnn
+keras-facenet
+python-dotenv
+scikit-learn