Spaces:

BitBasher
/

EduConnect

Runtime error

App Files Files Community

dtyago commited on Feb 20, 2024

Commit

97db660

1 Parent(s): 4f116c5

Implement Admin db clean up

Browse files

Files changed (6) hide show

app/admin/admin_functions.py +35 -0
app/admin/templates/data_management.html +66 -0
app/admin/templates/user_registration.html +4 -0
app/main.py +25 -3
app/utils/chat_rag.py +1 -1
requirements.txt +1 -1

app/admin/admin_functions.py CHANGED Viewed

@@ -3,6 +3,14 @@ from typing import Optional
 import bcrypt
 import os
 import shutil
 from ..utils import get_user_cropped_image_from_photo
@@ -68,6 +76,7 @@ def verify_admin_password(submitted_user: str, submitted_password: str) -> bool:
     return False
 def get_disk_usage(path="/home/user/data"):
     total, used, free = shutil.disk_usage(path)
     # Convert bytes to MB by dividing by 2^20
@@ -83,3 +92,29 @@ def get_disk_usage(path="/home/user/data"):
 # - Moderating chat messages or viewing chat history.
 # - Managing system settings or configurations.

 import bcrypt
 import os
 import shutil
+# Import vector store for database operations
+from langchain.vectorstores import Chroma
+# Import embeddings module from langchain for vector representations of text
+from langchain.embeddings import HuggingFaceEmbeddings
+from app.main import CHROMADB_LOC
+from app.utils.chat_rag import sanitize_collection_name
 from ..utils import get_user_cropped_image_from_photo
     return False
+# Get disk usage
 def get_disk_usage(path="/home/user/data"):
     total, used, free = shutil.disk_usage(path)
     # Convert bytes to MB by dividing by 2^20
 # - Moderating chat messages or viewing chat history.
 # - Managing system settings or configurations.
+# Display all faces in collection
+def faces_count(client, db):
+    return {
+        "face_count"        : db.count(),
+        "all_faces"         : db.get(),
+        "all_collections"   : client.list_collections() # List all collections at this location
+    }
+# Delete all faces in collection
+def remove_all_faces(client, user_faces_collection="user_faces_db"):
+    # Fetch all user IDs from the user_faces_db collection
+    all_user_ids = client.get_all_ids(collection_name=user_faces_collection)
+    # Loop through all user IDs and delete associated collections
+    for user_id in all_user_ids:
+        sanitized_collection_name = sanitize_collection_name(user_id)
+        vectordb = Chroma(
+            collection_name=sanitized_collection_name,
+            embedding_function=HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2'),
+            persist_directory=f"{CHROMADB_LOC}/{sanitized_collection_name}", # Optional: Separate directory for each user's data
+            )
+        all_ids = vectordb._collection.get()
+        vectordb._collection.delete(ids=all_ids)
+    # Finally, delete the user_faces_db collection itself
+    client.delete_collection(user_faces_collection)
+    print(f"All user collections and {user_faces_collection} have been removed.")

app/admin/templates/data_management.html ADDED Viewed

	@@ -0,0 +1,66 @@

+<!DOCTYPE html>
+<html>
+<head>
+    <title>Econnect Data Management</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link href="static/css/mvp.css" rel="stylesheet" media="screen">
+    <!-- Bootstrap CSS -->
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css">
+</head>
+<body>
+    <h1>Face DB</h1>
+    <div>
+        <p>Total Faces: {{ faces.face_count }}</p>
+        <h2>Registered faces:</h2>
+        <p>{{ faces.all_faces }}</p>
+        <hr>
+        <h2>All collections:</h2>
+        <p>{{ faces.all_collections }}</p>
+    </div>
+    <hr>
+    <div class="container">
+        <h2>User Registration</h2>
+        <form id="deleteForm" action="/admin/delete_faces" method="post" enctype="multipart/form-data">
+            <h2>Delete and clean vectordb:</h2>
+            <!-- Updated button to only trigger the modal -->
+            <button type="button" class="btn btn-default" data-toggle="modal" data-target="#confirmationModal">Delete ALL</button>
+        </form>
+        {% if error %}
+            <p class="error"><strong>Error:</strong> {{ error }}</p>
+        {% endif %}
+    </div>
+    <!-- Modal -->
+    <div class="modal fade" id="confirmationModal" tabindex="-1" role="dialog" aria-labelledby="confirmationModalLabel" aria-hidden="true">
+        <div class="modal-dialog" role="document">
+            <div class="modal-content">
+                <div class="modal-header">
+                    <h5 class="modal-title" id="confirmationModalLabel">Confirm Deletion</h5>
+                    <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                        <span aria-hidden="true">&times;</span>
+                    </button>
+                </div>
+                <div class="modal-body">
+                    Are you sure you want to delete ALL faces?
+                </div>
+                <div class="modal-footer">
+                    <button type="button" class="btn btn-secondary" data-dismiss="modal">Cancel</button>
+                    <button type="button" class="btn btn-danger" id="confirmDelete">Delete All</button>
+                </div>
+            </div>
+        </div>
+    </div>
+    <!-- Bootstrap and jQuery scripts -->
+    <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"></script>
+    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"></script>
+    <script>
+        document.addEventListener('DOMContentLoaded', (event) => {
+            document.getElementById('confirmDelete').addEventListener('click', function() {
+                document.getElementById('deleteForm').submit();
+            });
+        });
+    </script>
+</body>
+</html>

app/admin/templates/user_registration.html CHANGED Viewed

@@ -30,5 +30,9 @@
         <p class="error"><strong>Error:</strong> {{ error }}
     {% endif %}
     </div>
 </body>
 </html>

         <p class="error"><strong>Error:</strong> {{ error }}
     {% endif %}
     </div>
+    <hr>
+    <div>
+       <a href="/admin/data_management">Data Management</a>
+    </div>
 </body>
 </html>

app/main.py CHANGED Viewed

@@ -1,16 +1,16 @@
 import os
 import chromadb
-from fastapi import FastAPI, Request, Form, File, UploadFile, Depends
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
-from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse
 from fastapi.templating import Jinja2Templates
 from .admin import admin_functions as admin
 from .utils.db import UserFaceEmbeddingFunction,ChromaDBFaceHelper
 from .api import userlogin, userlogout, userchat, userupload
-from .utils.db import tinydb_helper, ChromaDBFaceHelper
 CHROMADB_LOC = "/home/user/data/chromadb"
@@ -28,6 +28,7 @@ app.add_middleware(
 # Persitent storage for chromadb setup in /data volume
 ec_client = chromadb.PersistentClient(CHROMADB_LOC)
 # The following collection reference is needed for admin function to register face
 user_faces_db = ec_client.get_or_create_collection(name="user_faces_db", embedding_function=UserFaceEmbeddingFunction())
@@ -85,6 +86,27 @@ async def handle_user_registration(request: Request, email: str = Form(...), nam
         # Reload registration page with error message
         return templates.TemplateResponse("user_registration.html", {"request": request, "error": "Registration failed"})
 app.include_router(userlogin.router)
 app.include_router(userlogout.router)
 app.include_router(userchat.router)

 import os
 import chromadb
+from fastapi import FastAPI, Request, Form, File, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
+from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.templating import Jinja2Templates
 from .admin import admin_functions as admin
 from .utils.db import UserFaceEmbeddingFunction,ChromaDBFaceHelper
 from .api import userlogin, userlogout, userchat, userupload
+from .utils.db import ChromaDBFaceHelper
 CHROMADB_LOC = "/home/user/data/chromadb"
 # Persitent storage for chromadb setup in /data volume
 ec_client = chromadb.PersistentClient(CHROMADB_LOC)
 # The following collection reference is needed for admin function to register face
 user_faces_db = ec_client.get_or_create_collection(name="user_faces_db", embedding_function=UserFaceEmbeddingFunction())
         # Reload registration page with error message
         return templates.TemplateResponse("user_registration.html", {"request": request, "error": "Registration failed"})
+# To display admin utilities
+@app.get("/admin/data_management", response_class=HTMLResponse)
+async def get_db_details(request: Request):
+    # Render the Chroma DB details
+    faces = admin.faces_count(user_faces_db)
+    return templates.TemplateResponse("data_management.html", {
+        "request": request,
+        "faces" : faces
+    })
+@app.post("/admin/delete_faces")
+async def delete_faces(request: Request):
+    try:
+        # Call your function to remove all faces
+        admin.remove_all_faces(ec_client)
+        request.session['flash'] = "All user data successfully deleted."
+    except Exception as e:
+        request.session['flash'] = f"Failed to delete user data: {str(e)}"
+    return RedirectResponse(url="/admin/data_management", status_code=303)
 app.include_router(userlogin.router)
 app.include_router(userlogout.router)
 app.include_router(userchat.router)

app/utils/chat_rag.py CHANGED Viewed

@@ -51,7 +51,7 @@ def get_vectordb_for_user(user_collection_name):
     vectordb = Chroma(
         collection_name=user_collection_name,
         embedding_function=HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2'),
-        persist_directory=f"{CHROMADB_LOC}/{user_collection_name}", # Optional: Separate directory for each user's data
     )
     return vectordb

     vectordb = Chroma(
         collection_name=user_collection_name,
         embedding_function=HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2'),
+        persist_directory=f"{CHROMADB_LOC}" # Optional: Separate directory for each user's data
     )
     return vectordb

requirements.txt CHANGED Viewed

@@ -18,4 +18,4 @@ tinydb                         # The in memory database for storing JWT tokens
 langchain                      # Langgchain for RAG
 llama-cpp-python               # To load the model
 sentence-transformers          # For text embeddings
-pypdf                          # Handling PDF files

 langchain                      # Langgchain for RAG
 llama-cpp-python               # To load the model
 sentence-transformers          # For text embeddings
+pypdf                          # Handling PDF files