Upload supabase_storage.py

supabase_storage.py

@@ -11,8 +11,10 @@ import hashlib
 import numpy as np
 from typing import Optional, List, Dict, Any
 
-# Supabase credentials
-SUPABASE_URL = os.environ.get('SUPABASE_URL', 'https://cqnyibiopjcwuxmyqbgy.supabase.co')
+# Supabase credentials - URL needs trailing slash for storage API
+SUPABASE_URL = os.environ.get('SUPABASE_URL', 'https://cqnyibiopjcwuxmyqbgy.supabase.co/')
+if not SUPABASE_URL.endswith('/'):
+    SUPABASE_URL = SUPABASE_URL + '/'
 SUPABASE_KEY = os.environ.get('SUPABASE_KEY', '')
 BUCKET_NAME = 'datasets'
 
@@ -49,9 +51,9 @@ def is_supabase_available() -> bool:
 
 def _get_dataset_registry(client) -> List[str]:
     """
-    DEPRECATED: Registry file is no longer used.
-    Datasets are now discovered by listing folders directly via list_datasets_from_supabase().
-    Keeping this function for backwards compatibility only.
+    Get the list of dataset names from the registry file.
+    Returns None if there's an error reading (to prevent accidental overwrite).
+    Returns [] only if file doesn't exist yet.
     """
     try:
         storage = client.storage.from_(BUCKET_NAME)
@@ -71,11 +73,7 @@ def _get_dataset_registry(client) -> List[str]:
 
 
 def _update_dataset_registry(client, dataset_name: str, action: str = 'add'):
-    """
-    DEPRECATED: Registry file is no longer used.
-    Datasets are now discovered by listing folders directly.
-    Keeping this function for backwards compatibility only.
-    """
+    """Update the registry file with dataset names."""
     try:
         storage = client.storage.from_(BUCKET_NAME)
 
@@ -170,7 +168,8 @@ def save_dataset_to_supabase(
         )
         print(f"[Supabase] Uploaded {metadata_path}")
 
-        # No registry update needed - datasets are discovered by listing folders directly
+        # 4. Update the registry file (list of all dataset names)
+        _update_dataset_registry(client, dataset_name, action='add')
 
         print(f"[Supabase] Dataset '{dataset_name}' saved successfully")
         return True
@@ -226,103 +225,65 @@ def load_dataset_from_supabase(dataset_name: str) -> Optional[Dict[str, Any]]:
 
 def list_datasets_from_supabase() -> List[Dict[str, Any]]:
     """
-    List all datasets stored in Supabase by listing folders directly.
-    No registry file needed - just lists all folders in the bucket.
+    List all datasets by scanning folders directly in Supabase Storage.
+    No registry file needed - each folder = a dataset.
 
     Returns:
         List of dataset metadata dictionaries
     """
     client = get_supabase_client()
     if not client:
-        print("[Supabase] Client not available")
+        print("[Supabase] Client not available", flush=True)
         return []
 
     try:
         storage = client.storage.from_(BUCKET_NAME)
-        dataset_names = []
 
-        # Method 1: Try storage.list() with empty path
-        try:
-            items = storage.list(path="")
-            print(f"[Supabase] Method 1 (list path=''): {len(items)} items")
-            for item in items:
-                name = item.get('name', '')
-                if name and not name.startswith('_') and not name.startswith('.'):
-                    item_metadata = item.get('metadata')
-                    is_file = item_metadata and item_metadata.get('mimetype')
-                    if not is_file:
-                        dataset_names.append(name)
-                        print(f"[Supabase] Found via list(): {name}")
-        except Exception as e:
-            print(f"[Supabase] Method 1 failed: {e}")
+        # List root of bucket - each folder is a dataset
+        items = storage.list(path="")
+        print(f"[Supabase] storage.list() returned {len(items)} items", flush=True)
 
-        # Method 2: If list() returned nothing, try listing without path argument
-        if not dataset_names:
-            try:
-                items = storage.list()
-                print(f"[Supabase] Method 2 (list no args): {len(items)} items")
-                for item in items:
-                    name = item.get('name', '')
-                    if name and not name.startswith('_') and not name.startswith('.'):
-                        item_metadata = item.get('metadata')
-                        is_file = item_metadata and item_metadata.get('mimetype')
-                        if not is_file:
-                            dataset_names.append(name)
-                            print(f"[Supabase] Found via list(): {name}")
-            except Exception as e:
-                print(f"[Supabase] Method 2 failed: {e}")
+        datasets = []
 
-        # Method 3: Fallback - read from registry file if exists
-        if not dataset_names:
-            print("[Supabase] List methods returned empty, trying registry fallback...")
-            try:
-                response = storage.download("_registry.json")
-                registry = json.loads(response.decode('utf-8'))
-                dataset_names = registry.get('datasets', [])
-                print(f"[Supabase] Found via registry: {dataset_names}")
-            except Exception as e:
-                print(f"[Supabase] Registry fallback failed: {e}")
-
-        # Method 4: Ultimate fallback - probe for known dataset names
-        if not dataset_names:
-            print("[Supabase] Trying direct probe for datasets...")
-            potential_names = ['testing', 'only_ariya___siglip', 'onlyariya_clip']
-            for name in potential_names:
-                try:
-                    storage.download(f"{name}/metadata.json")
-                    dataset_names.append(name)
-                    print(f"[Supabase] Found via probe: {name}")
-                except:
-                    pass
-
-        print(f"[Supabase] Total datasets found: {len(dataset_names)} - {dataset_names}")
+        for item in items:
+            name = item.get("name", "")
+            if not name:
+                continue
+
+            # Skip hidden files and registry file
+            if name.startswith("_") or name.startswith("."):
+                print(f"[Supabase] Skipping: {name}", flush=True)
+                continue
+
+            folder_name = name
+            print(f"[Supabase] Found folder: {folder_name}", flush=True)
 
-        datasets = []
-        for folder_name in dataset_names:
             try:
+                # Load metadata.json to verify it's a valid dataset
                 metadata_path = f"{folder_name}/metadata.json"
                 metadata_response = storage.download(metadata_path)
-                metadata = json.loads(metadata_response.decode('utf-8'))
-                metadata['folder_name'] = folder_name
-                metadata['source'] = 'supabase'
+                metadata = json.loads(metadata_response.decode("utf-8"))
+                metadata["folder_name"] = folder_name
+                metadata["source"] = "supabase"
                 datasets.append(metadata)
-                print(f"[Supabase] Loaded metadata for {folder_name}")
+                print(f"[Supabase] Loaded dataset: {folder_name}", flush=True)
+
             except Exception as e:
-                print(f"[Supabase] Could not load metadata for {folder_name}: {e}")
-                # Add basic info without full metadata
+                # Still add as dataset even without metadata
+                print(f"[Supabase] No metadata for {folder_name}: {e}", flush=True)
                 datasets.append({
-                    'name': folder_name,
-                    'folder_name': folder_name,
-                    'source': 'supabase',
-                    'total_photos': 0,
-                    'created_at': None
+                    "name": folder_name,
+                    "folder_name": folder_name,
+                    "source": "supabase",
+                    "total_photos": 0,
+                    "created_at": None
                 })
 
-        print(f"[Supabase] Found {len(datasets)} datasets")
+        print(f"[Supabase] Total datasets found: {len(datasets)}", flush=True)
         return datasets
 
     except Exception as e:
-        print(f"[Supabase] Error listing datasets: {e}")
+        print(f"[Supabase] Error listing datasets: {e}", flush=True)
         import traceback
         traceback.print_exc()
         return []
@@ -354,7 +315,8 @@ def delete_dataset_from_supabase(dataset_name: str) -> bool:
             client.storage.from_(BUCKET_NAME).remove(file_paths)
             print(f"[Supabase] Deleted {len(file_paths)} files from '{dataset_name}'")
 
-        # No registry update needed - datasets are discovered by listing folders directly
+        # Remove from registry
+        _update_dataset_registry(client, dataset_name, action='remove')
 
         print(f"[Supabase] Dataset '{dataset_name}' deleted successfully")
         return True