fix: use HuggingFace Hub API for file uploads

app/services/hf_storage_adapter.py

@@ -15,9 +15,11 @@ class HFStorageAdapter(StoragePort):
     
     def __init__(self):
         self.bucket_path = settings.HF_BUCKET_PATH
+        # Extract repo_id from bucket path: hf://buckets/Peterase/Ragora-doc-store -> Peterase/Ragora-doc-store
+        self.repo_id = self.bucket_path.replace("hf://buckets/", "")
         self.local_cache = Path("/tmp/ragora_cache")
         self.local_cache.mkdir(exist_ok=True)
-        logger.info(f"Using HF bucket: {self.bucket_path}")
+        logger.info(f"Using HF bucket: {self.bucket_path} (repo: {self.repo_id})")
     
     async def upload(self, key: str, data: bytes, content_type: str) -> str:
         """Upload file to HF bucket."""
@@ -29,20 +31,18 @@ class HFStorageAdapter(StoragePort):
             with open(local_path, 'wb') as f:
                 f.write(data)
             
-            # Upload to HF bucket using hf CLI
-            import subprocess
-            hf_path = f"{self.bucket_path}/{key}"
+            # Upload to HF bucket using huggingface_hub API
+            from huggingface_hub import HfApi
+            api = HfApi()
             
-            result = subprocess.run(
-                ["hf", "upload", hf_path, str(local_path)],
-                capture_output=True,
-                text=True
+            # Upload file to the bucket
+            api.upload_file(
+                path_or_fileobj=str(local_path),
+                path_in_repo=key,
+                repo_id=self.repo_id,
+                repo_type="dataset"
             )
             
-            if result.returncode != 0:
-                logger.error(f"HF upload failed: {result.stderr}")
-                raise Exception(f"Upload failed: {result.stderr}")
-            
             logger.info(f"Uploaded file: {key}")
             return key
         except Exception as e:
@@ -56,19 +56,20 @@ class HFStorageAdapter(StoragePort):
             local_path = self.local_cache / key
             
             if not local_path.exists():
-                # Download from HF bucket
-                import subprocess
-                hf_path = f"{self.bucket_path}/{key}"
+                # Download from HF bucket using huggingface_hub API
+                from huggingface_hub import hf_hub_download
+                
+                local_path.parent.mkdir(parents=True, exist_ok=True)
                 
-                result = subprocess.run(
-                    ["hf", "download", hf_path, "--local-dir", str(self.local_cache)],
-                    capture_output=True,
-                    text=True
+                downloaded_path = hf_hub_download(
+                    repo_id=self.repo_id,
+                    filename=key,
+                    repo_type="dataset",
+                    local_dir=str(self.local_cache),
+                    local_dir_use_symlinks=False
                 )
                 
-                if result.returncode != 0:
-                    logger.error(f"HF download failed: {result.stderr}")
-                    raise Exception(f"Download failed: {result.stderr}")
+                logger.info(f"Downloaded file: {key}")
             
             with open(local_path, 'rb') as f:
                 data = f.read()
@@ -81,20 +82,28 @@ class HFStorageAdapter(StoragePort):
     async def delete(self, key: str) -> None:
         """Delete file from HF bucket."""
         try:
+            # Delete from HF bucket using huggingface_hub API
+            from huggingface_hub import HfApi
+            api = HfApi()
+            
+            api.delete_file(
+                path_in_repo=key,
+                repo_id=self.repo_id,
+                repo_type="dataset"
+            )
+            
             # Delete from local cache
             local_path = self.local_cache / key
             if local_path.exists():
                 local_path.unlink()
             
-            # Note: HF CLI doesn't have direct delete, so we'll just remove from cache
-            # Files in HF buckets can be managed via web interface
-            logger.info(f"Deleted file from cache: {key}")
+            logger.info(f"Deleted file: {key}")
         except Exception as e:
             logger.error(f"Error deleting file: {e}")
             raise
     
     async def get_presigned_url(self, key: str, expires: int = 3600) -> str:
         """Get presigned URL for downloading."""
-        # For HF buckets, return the direct HF URL
-        hf_url = f"https://huggingface.co/datasets/{self.bucket_path.replace('hf://buckets/', '')}/resolve/main/{key}"
+        # For HF datasets, return the direct HF URL
+        hf_url = f"https://huggingface.co/datasets/{self.repo_id}/resolve/main/{key}"
         return hf_url