make sync dataset function doesnot delete pkl files

deepface/DeepFace.py

@@ -565,8 +565,21 @@ def sync_datasets():
     os.makedirs(missing_dir, exist_ok=True)
     os.makedirs(founded_dir, exist_ok=True)
 
-    cloudservice.delete_pkl_files(missing_dir)
     missing_people = cloudservice.sync_folder('missing_people', missing_dir)
-    cloudservice.delete_pkl_files(founded_dir)
+
     founded_people = cloudservice.sync_folder('founded_people', founded_dir)
 
+def delete_pkls():
+    # Set the local directories
+    base_dir = os_path.get_main_directory()
+
+    missing_dir = os.path.join(base_dir, 'mafqoud', 'images', 'missing_people')
+    founded_dir = os.path.join(base_dir, 'mafqoud', 'images', 'founded_people')
+
+    # Ensure the directories exist
+    os.makedirs(missing_dir, exist_ok=True)
+    os.makedirs(founded_dir, exist_ok=True)
+
+    cloudservice.delete_pkl_files(missing_dir)
+    cloudservice.delete_pkl_files(founded_dir)
+

deepface/api/src/modules/core/routes.py

@@ -142,7 +142,7 @@ def find():
         response.status_code = 400
         return response
     
-    
+    print(img_path)
     if not os.path.exists(img_path) or not os.path.isfile(img_path):
         # If the image does not exist, return a JSON response with status code 404
         response = jsonify({'error': 'Image not found'})
@@ -198,4 +198,10 @@ def find():
 @blueprint.route("/dataset/sync", methods=["GET"])
 def sync_datasets():
     result = service.sync_datasets()
+    return jsonify(result)
+
+
+@blueprint.route("/delete/pkls", methods=["GET"])
+def delete_pkls():
+    result = service.delete_pkls()
     return jsonify(result)

deepface/api/src/modules/core/service.py

@@ -73,5 +73,12 @@ def sync_datasets():
     try:
         DeepFace.sync_datasets()
         return {'data': 'synced successfully'}, 200
+    except Exception as e:
+        return {'error': str(e)}, 400
+
+def delete_pkls():
+    try:
+        DeepFace.delete_pkls()
+        return {'data': 'pkl files deleted successfully'}, 200
     except Exception as e:
         return {'error': str(e)}, 400

deepface/modules/cloudservice.py

@@ -1,3 +1,70 @@
+# from flask import Flask, jsonify
+# import cloudinary
+# import cloudinary.api
+# import cloudinary.uploader
+# import os
+# import glob
+# import requests
+# from dotenv import load_dotenv
+# from deepface.commons.logger import Logger
+
+# logger = Logger(module="modules/cloudservice.py")
+
+# load_dotenv()
+
+# # Configure Cloudinary
+# cloudinary.config(
+#     cloud_name=os.getenv('CLOUDINARY_CLOUD_NAME'),
+#     api_key=os.getenv('CLOUDINARY_API_KEY'),
+#     api_secret=os.getenv('CLOUDINARY_API_SECRET')
+# )
+
+
+# def fetch_cloudinary_images(folder_name):
+
+#     resources = []
+
+#     res = cloudinary.api.resources(type='upload',  resource_type='image', prefix=f'mafqoud/images/{folder_name}')
+#     resources.extend(res.get('resources', []))
+#     return resources
+
+# def download_image(url, local_path):
+#     response = requests.get(url, stream=True)
+#     if response.status_code == 200:
+#         with open(local_path, 'wb') as file:
+#             for chunk in response:
+#                 file.write(chunk)
+
+# def sync_folder(folder_name, local_dir):
+#     cloudinary_images = fetch_cloudinary_images(folder_name)
+#     cloudinary_urls = {img['secure_url']: img['public_id'] for img in cloudinary_images}
+
+#     # Download new images and track downloaded image paths
+#     downloaded_paths = []
+#     for img in cloudinary_images:
+#         url = img['secure_url']
+#         public_id = img['public_id']
+#         file_name = url.split('/')[-1]  # Get the actual file name
+#         local_path = os.path.join(local_dir, file_name)
+#         print(local_path)
+        
+#         if not os.path.exists(local_path):
+#             download_image(url, local_path)
+#         downloaded_paths.append(local_path)
+    
+#     # Remove old images
+#     local_images = [os.path.join(local_dir, f) for f in os.listdir(local_dir) if os.path.isfile(os.path.join(local_dir, f))]
+#     for local_path in local_images:
+#         if local_path not in downloaded_paths and (local_path.endswith('.jpg') or local_path.endswith('.jpeg') or local_path.endswith('.png')):
+#             os.remove(local_path)
+
+# def delete_pkl_files(directory):
+#     """Delete all .pkl files in the specified directory."""
+#     pkl_files = glob.glob(os.path.join(directory, '*.pkl'))
+#     for pkl_file in pkl_files:
+#         os.remove(pkl_file)
+
+
 from flask import Flask, jsonify
 import cloudinary
 import cloudinary.api
@@ -7,6 +74,7 @@ import glob
 import requests
 from dotenv import load_dotenv
 from deepface.commons.logger import Logger
+from deepface import DeepFace  # Assuming this is the correct import for training
 
 logger = Logger(module="modules/cloudservice.py")
 
@@ -21,11 +89,32 @@ cloudinary.config(
 
 
 def fetch_cloudinary_images(folder_name):
-
     resources = []
+    next_cursor = None
+
+    while True:
+        if next_cursor:
+            res = cloudinary.api.resources(
+                type='upload', 
+                resource_type='image', 
+                prefix=f'mafqoud/images/{folder_name}', 
+                max_results=500, 
+                next_cursor=next_cursor
+            )
+        else:
+            res = cloudinary.api.resources(
+                type='upload', 
+                resource_type='image', 
+                prefix=f'mafqoud/images/{folder_name}', 
+                max_results=500
+            )
+
+        resources.extend(res.get('resources', []))
+        next_cursor = res.get('next_cursor')
+
+        if not next_cursor:
+            break
 
-    res = cloudinary.api.resources(type='upload',  resource_type='image', prefix=f'mafqoud/images/{folder_name}')
-    resources.extend(res.get('resources', []))
     return resources
 
 def download_image(url, local_path):
@@ -46,7 +135,7 @@ def sync_folder(folder_name, local_dir):
         public_id = img['public_id']
         file_name = url.split('/')[-1]  # Get the actual file name
         local_path = os.path.join(local_dir, file_name)
-        
+        print(local_path)
         
         if not os.path.exists(local_path):
             download_image(url, local_path)
@@ -55,7 +144,7 @@ def sync_folder(folder_name, local_dir):
     # Remove old images
     local_images = [os.path.join(local_dir, f) for f in os.listdir(local_dir) if os.path.isfile(os.path.join(local_dir, f))]
     for local_path in local_images:
-        if local_path not in downloaded_paths:
+        if local_path not in downloaded_paths and (local_path.endswith('.jpg') or local_path.endswith('.jpeg') or local_path.endswith('.png')):
             os.remove(local_path)
 
 def delete_pkl_files(directory):
@@ -63,6 +152,3 @@ def delete_pkl_files(directory):
     pkl_files = glob.glob(os.path.join(directory, '*.pkl'))
     for pkl_file in pkl_files:
         os.remove(pkl_file)
-
-
-