AdarshDRC commited on
Commit
d12d95f
·
1 Parent(s): 68c0f26

fix : fixing the RESET button

Browse files
Files changed (1) hide show
  1. main.py +56 -12
main.py CHANGED
@@ -40,11 +40,12 @@ except ImportError:
40
  ai = None
41
  p = inflect.engine()
42
 
43
- MAX_CONCURRENT_INFERENCES = int(os.getenv("MAX_CONCURRENT_INFERENCES", "6"))
44
  _inference_sem: asyncio.Semaphore
45
  _pinecone_pool = OrderedDict()
46
  _POOL_MAX = 64
47
  IDX_FACES = "enterprise-faces"
 
48
  IDX_OBJECTS = "enterprise-objects"
49
 
50
  # ════════════════════════════════════════════════════════════════
@@ -297,6 +298,9 @@ async def upload_new_images(
297
  log("ERROR", "upload.missing_keys", user_id=user_id or "anonymous", ip=ip, mode=mode)
298
  raise HTTPException(400, "API Keys are missing. If you are a guest, the server is missing its DEFAULT_ secrets in Hugging Face.")
299
 
 
 
 
300
  folder = standardize_category_name(folder_name)
301
  creds = get_cloudinary_creds(actual_cld_url)
302
  if not creds.get("cloud_name"):
@@ -670,6 +674,34 @@ def _cld_remove_folder(folder: str, creds: dict):
670
  api_key=creds["api_key"], api_secret=creds["api_secret"], cloud_name=creds["cloud_name"])
671
  except Exception: pass
672
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
673
  @app.post("/api/delete-folder")
674
  async def delete_folder(
675
  request: Request,
@@ -736,20 +768,26 @@ async def reset_database(
736
  creds = get_cloudinary_creds(user_cloudinary_url)
737
  if not creds.get("cloud_name"): raise HTTPException(400, "Invalid Cloudinary URL.")
738
 
 
739
  try:
740
- await asyncio.to_thread(lambda: cloudinary.api.delete_all_resources(
741
- api_key=creds["api_key"], api_secret=creds["api_secret"], cloud_name=creds["cloud_name"]))
742
  except Exception as e:
743
  _log_fn("WARNING", f"Cloudinary wipe: {e}")
744
 
745
- # Delete Cloudinary folders too
746
  try:
747
  folders_res = await asyncio.to_thread(_cld_root_folders, creds)
748
- for folder in folders_res.get("folders", []):
749
- await asyncio.to_thread(_cld_remove_folder, folder["name"], creds)
 
 
 
 
 
750
  except Exception as e:
751
  _log_fn("WARNING", f"Cloudinary folder cleanup: {e}")
752
 
 
753
  try:
754
  pc = _get_pinecone(user_pinecone_key)
755
  existing = {idx.name for idx in await asyncio.to_thread(pc.list_indexes)}
@@ -757,7 +795,7 @@ async def reset_database(
757
  if IDX_OBJECTS in existing: tasks.append(asyncio.to_thread(pc.delete_index, IDX_OBJECTS))
758
  if IDX_FACES in existing: tasks.append(asyncio.to_thread(pc.delete_index, IDX_FACES))
759
  if tasks: await asyncio.gather(*tasks)
760
- await asyncio.sleep(2)
761
  await asyncio.gather(
762
  asyncio.to_thread(pc.create_index, name=IDX_OBJECTS, dimension=1536, metric="cosine",
763
  spec=ServerlessSpec(cloud="aws", region="us-east-1")),
@@ -797,20 +835,26 @@ async def delete_account(
797
  raise HTTPException(403, "Account deletion is not allowed on the shared demo database.")
798
 
799
  creds = get_cloudinary_creds(user_cloudinary_url)
 
 
800
  try:
801
- await asyncio.to_thread(lambda: cloudinary.api.delete_all_resources(
802
- api_key=creds["api_key"], api_secret=creds["api_secret"], cloud_name=creds["cloud_name"]))
803
  except Exception as e:
804
  _log_fn("WARNING", f"Account delete Cloudinary: {e}")
805
 
806
- # Delete Cloudinary folders
807
  try:
808
  folders_res = await asyncio.to_thread(_cld_root_folders, creds)
809
- for folder in folders_res.get("folders", []):
810
- await asyncio.to_thread(_cld_remove_folder, folder["name"], creds)
 
 
 
 
811
  except Exception as e:
812
  _log_fn("WARNING", f"Account delete Cloudinary folders: {e}")
813
 
 
814
  try:
815
  pc = _get_pinecone(user_pinecone_key)
816
  existing = {idx.name for idx in await asyncio.to_thread(pc.list_indexes)}
 
40
  ai = None
41
  p = inflect.engine()
42
 
43
+ MAX_CONCURRENT_INFERENCES = int(os.getenv("MAX_CONCURRENT_INFERENCES", "1")) # InsightFace ONNX is NOT thread-safe
44
  _inference_sem: asyncio.Semaphore
45
  _pinecone_pool = OrderedDict()
46
  _POOL_MAX = 64
47
  IDX_FACES = "enterprise-faces"
48
+ MAX_FILES_PER_UPLOAD = 20 # cap to prevent memory corruption on large batches
49
  IDX_OBJECTS = "enterprise-objects"
50
 
51
  # ════════════════════════════════════════════════════════════════
 
298
  log("ERROR", "upload.missing_keys", user_id=user_id or "anonymous", ip=ip, mode=mode)
299
  raise HTTPException(400, "API Keys are missing. If you are a guest, the server is missing its DEFAULT_ secrets in Hugging Face.")
300
 
301
+ if len(files) > MAX_FILES_PER_UPLOAD:
302
+ raise HTTPException(400, f"Maximum {MAX_FILES_PER_UPLOAD} files per upload. Please split into smaller batches.")
303
+
304
  folder = standardize_category_name(folder_name)
305
  creds = get_cloudinary_creds(actual_cld_url)
306
  if not creds.get("cloud_name"):
 
674
  api_key=creds["api_key"], api_secret=creds["api_secret"], cloud_name=creds["cloud_name"])
675
  except Exception: pass
676
 
677
+ def _cld_delete_all_paginated(creds: dict):
678
+ """Delete ALL Cloudinary resources in batches of 100 until none left."""
679
+ deleted = 0
680
+ while True:
681
+ try:
682
+ res = cloudinary.api.resources(
683
+ type="upload", max_results=100,
684
+ api_key=creds["api_key"], api_secret=creds["api_secret"],
685
+ cloud_name=creds["cloud_name"],
686
+ )
687
+ resources = res.get("resources", [])
688
+ if not resources:
689
+ break
690
+ public_ids = [r["public_id"] for r in resources]
691
+ cloudinary.api.delete_resources(
692
+ public_ids,
693
+ api_key=creds["api_key"], api_secret=creds["api_secret"],
694
+ cloud_name=creds["cloud_name"],
695
+ )
696
+ deleted += len(public_ids)
697
+ print(f"🗑️ Deleted {deleted} resources so far...")
698
+ if not res.get("next_cursor"):
699
+ break
700
+ except Exception as e:
701
+ print(f"Cloudinary batch delete error: {e}")
702
+ break
703
+ return deleted
704
+
705
  @app.post("/api/delete-folder")
706
  async def delete_folder(
707
  request: Request,
 
768
  creds = get_cloudinary_creds(user_cloudinary_url)
769
  if not creds.get("cloud_name"): raise HTTPException(400, "Invalid Cloudinary URL.")
770
 
771
+ # ── Cloudinary: paginated delete ALL resources then folders ────
772
  try:
773
+ deleted = await asyncio.to_thread(_cld_delete_all_paginated, creds)
774
+ _log_fn("INFO", f"Cloudinary: deleted {deleted} resources")
775
  except Exception as e:
776
  _log_fn("WARNING", f"Cloudinary wipe: {e}")
777
 
 
778
  try:
779
  folders_res = await asyncio.to_thread(_cld_root_folders, creds)
780
+ # Delete all folders in parallel
781
+ folder_tasks = [
782
+ asyncio.to_thread(_cld_remove_folder, f["name"], creds)
783
+ for f in folders_res.get("folders", [])
784
+ ]
785
+ if folder_tasks:
786
+ await asyncio.gather(*folder_tasks, return_exceptions=True)
787
  except Exception as e:
788
  _log_fn("WARNING", f"Cloudinary folder cleanup: {e}")
789
 
790
+ # ── Pinecone: delete both indexes + recreate ─────────────────
791
  try:
792
  pc = _get_pinecone(user_pinecone_key)
793
  existing = {idx.name for idx in await asyncio.to_thread(pc.list_indexes)}
 
795
  if IDX_OBJECTS in existing: tasks.append(asyncio.to_thread(pc.delete_index, IDX_OBJECTS))
796
  if IDX_FACES in existing: tasks.append(asyncio.to_thread(pc.delete_index, IDX_FACES))
797
  if tasks: await asyncio.gather(*tasks)
798
+ await asyncio.sleep(3) # wait for Pinecone to fully delete
799
  await asyncio.gather(
800
  asyncio.to_thread(pc.create_index, name=IDX_OBJECTS, dimension=1536, metric="cosine",
801
  spec=ServerlessSpec(cloud="aws", region="us-east-1")),
 
835
  raise HTTPException(403, "Account deletion is not allowed on the shared demo database.")
836
 
837
  creds = get_cloudinary_creds(user_cloudinary_url)
838
+
839
+ # ── Cloudinary: paginated delete ALL resources then folders ────
840
  try:
841
+ deleted = await asyncio.to_thread(_cld_delete_all_paginated, creds)
842
+ _log_fn("INFO", f"Account delete Cloudinary: {deleted} resources removed")
843
  except Exception as e:
844
  _log_fn("WARNING", f"Account delete Cloudinary: {e}")
845
 
 
846
  try:
847
  folders_res = await asyncio.to_thread(_cld_root_folders, creds)
848
+ folder_tasks = [
849
+ asyncio.to_thread(_cld_remove_folder, f["name"], creds)
850
+ for f in folders_res.get("folders", [])
851
+ ]
852
+ if folder_tasks:
853
+ await asyncio.gather(*folder_tasks, return_exceptions=True)
854
  except Exception as e:
855
  _log_fn("WARNING", f"Account delete Cloudinary folders: {e}")
856
 
857
+ # ── Pinecone: delete both indexes ────────────────────────────
858
  try:
859
  pc = _get_pinecone(user_pinecone_key)
860
  existing = {idx.name for idx in await asyncio.to_thread(pc.list_indexes)}