Spaces:
Running
Running
Pulastya B commited on
Commit ·
9c0055f
1
Parent(s): 1412b09
Align HuggingFace API usage with official docs pattern
Browse files- src/api/app.py +49 -21
- src/storage/huggingface_storage.py +66 -24
src/api/app.py
CHANGED
|
@@ -1416,15 +1416,24 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
|
|
| 1416 |
|
| 1417 |
# Fetch user's HuggingFace token from hf_tokens table (not user_profiles)
|
| 1418 |
logger.info(f"[HF Export] Fetching HF token from hf_tokens table...")
|
| 1419 |
-
|
| 1420 |
-
|
| 1421 |
-
|
| 1422 |
-
|
| 1423 |
-
|
| 1424 |
-
|
| 1425 |
-
|
| 1426 |
-
|
| 1427 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1428 |
|
| 1429 |
if not hf_token:
|
| 1430 |
raise HTTPException(
|
|
@@ -1433,9 +1442,19 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
|
|
| 1433 |
)
|
| 1434 |
|
| 1435 |
# Import HuggingFace storage service
|
| 1436 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1437 |
|
| 1438 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1439 |
|
| 1440 |
# Collect all session assets
|
| 1441 |
uploaded_files = []
|
|
@@ -1496,7 +1515,7 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
|
|
| 1496 |
logger.error(f"[HF Export] Model upload error: {e}")
|
| 1497 |
errors.append(f"Model {Path(model_file).name}: {str(e)}")
|
| 1498 |
|
| 1499 |
-
# Upload visualizations (HTML plots) - use
|
| 1500 |
plot_patterns = [
|
| 1501 |
session_outputs_dir / "*.html",
|
| 1502 |
global_outputs_dir / "*.html",
|
|
@@ -1512,11 +1531,10 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
|
|
| 1512 |
continue
|
| 1513 |
try:
|
| 1514 |
logger.info(f"[HF Export] Uploading HTML plot: {plot_file}")
|
| 1515 |
-
result = hf_service.
|
| 1516 |
-
|
| 1517 |
session_id=request.session_id,
|
| 1518 |
-
|
| 1519 |
-
compress=False # Don't compress HTML plots for direct viewing
|
| 1520 |
)
|
| 1521 |
if result.get("success"):
|
| 1522 |
uploaded_files.append({"type": "plot", "name": Path(plot_file).name, "url": result.get("url")})
|
|
@@ -1526,7 +1544,7 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
|
|
| 1526 |
logger.error(f"[HF Export] Plot upload error: {e}")
|
| 1527 |
errors.append(f"Plot {Path(plot_file).name}: {str(e)}")
|
| 1528 |
|
| 1529 |
-
# Upload PNG images - use
|
| 1530 |
image_patterns = [
|
| 1531 |
session_outputs_dir / "*.png",
|
| 1532 |
global_outputs_dir / "*.png",
|
|
@@ -1539,11 +1557,10 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
|
|
| 1539 |
for image_file in glob.glob(str(pattern)):
|
| 1540 |
try:
|
| 1541 |
logger.info(f"[HF Export] Uploading image: {image_file}")
|
| 1542 |
-
result = hf_service.
|
| 1543 |
-
|
| 1544 |
session_id=request.session_id,
|
| 1545 |
-
|
| 1546 |
-
compress=False
|
| 1547 |
)
|
| 1548 |
if result.get("success"):
|
| 1549 |
uploaded_files.append({"type": "image", "name": Path(image_file).name, "url": result.get("url")})
|
|
@@ -1554,11 +1571,22 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
|
|
| 1554 |
errors.append(f"Image {Path(image_file).name}: {str(e)}")
|
| 1555 |
|
| 1556 |
if not uploaded_files and errors:
|
|
|
|
| 1557 |
raise HTTPException(
|
| 1558 |
status_code=500,
|
| 1559 |
detail=f"Export failed: {'; '.join(errors)}"
|
| 1560 |
)
|
| 1561 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1562 |
return JSONResponse({
|
| 1563 |
"success": True,
|
| 1564 |
"uploaded_files": uploaded_files,
|
|
|
|
| 1416 |
|
| 1417 |
# Fetch user's HuggingFace token from hf_tokens table (not user_profiles)
|
| 1418 |
logger.info(f"[HF Export] Fetching HF token from hf_tokens table...")
|
| 1419 |
+
try:
|
| 1420 |
+
result = supabase.table("hf_tokens").select(
|
| 1421 |
+
"huggingface_token, huggingface_username"
|
| 1422 |
+
).eq("user_id", request.user_id).execute()
|
| 1423 |
+
|
| 1424 |
+
logger.info(f"[HF Export] Query result: {result.data}")
|
| 1425 |
+
|
| 1426 |
+
if not result.data or len(result.data) == 0:
|
| 1427 |
+
raise HTTPException(status_code=404, detail="HuggingFace not connected. Please connect in Settings first.")
|
| 1428 |
+
|
| 1429 |
+
row = result.data[0]
|
| 1430 |
+
hf_token = row.get("huggingface_token")
|
| 1431 |
+
hf_username = row.get("huggingface_username")
|
| 1432 |
+
except HTTPException:
|
| 1433 |
+
raise
|
| 1434 |
+
except Exception as e:
|
| 1435 |
+
logger.error(f"[HF Export] Supabase query error: {e}")
|
| 1436 |
+
raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
|
| 1437 |
|
| 1438 |
if not hf_token:
|
| 1439 |
raise HTTPException(
|
|
|
|
| 1442 |
)
|
| 1443 |
|
| 1444 |
# Import HuggingFace storage service
|
| 1445 |
+
try:
|
| 1446 |
+
from src.storage.huggingface_storage import HuggingFaceStorage
|
| 1447 |
+
logger.info(f"[HF Export] HuggingFaceStorage imported successfully")
|
| 1448 |
+
except ImportError as e:
|
| 1449 |
+
logger.error(f"[HF Export] Failed to import HuggingFaceStorage: {e}")
|
| 1450 |
+
raise HTTPException(status_code=500, detail=f"Server error: {str(e)}")
|
| 1451 |
|
| 1452 |
+
try:
|
| 1453 |
+
hf_service = HuggingFaceStorage(hf_token=hf_token)
|
| 1454 |
+
logger.info(f"[HF Export] HuggingFaceStorage initialized for user: {hf_username}")
|
| 1455 |
+
except Exception as e:
|
| 1456 |
+
logger.error(f"[HF Export] Failed to initialize HuggingFaceStorage: {e}")
|
| 1457 |
+
raise HTTPException(status_code=500, detail=f"HuggingFace error: {str(e)}")
|
| 1458 |
|
| 1459 |
# Collect all session assets
|
| 1460 |
uploaded_files = []
|
|
|
|
| 1515 |
logger.error(f"[HF Export] Model upload error: {e}")
|
| 1516 |
errors.append(f"Model {Path(model_file).name}: {str(e)}")
|
| 1517 |
|
| 1518 |
+
# Upload visualizations (HTML plots) - use generic file upload
|
| 1519 |
plot_patterns = [
|
| 1520 |
session_outputs_dir / "*.html",
|
| 1521 |
global_outputs_dir / "*.html",
|
|
|
|
| 1531 |
continue
|
| 1532 |
try:
|
| 1533 |
logger.info(f"[HF Export] Uploading HTML plot: {plot_file}")
|
| 1534 |
+
result = hf_service.upload_generic_file(
|
| 1535 |
+
file_path=plot_file,
|
| 1536 |
session_id=request.session_id,
|
| 1537 |
+
subfolder="plots"
|
|
|
|
| 1538 |
)
|
| 1539 |
if result.get("success"):
|
| 1540 |
uploaded_files.append({"type": "plot", "name": Path(plot_file).name, "url": result.get("url")})
|
|
|
|
| 1544 |
logger.error(f"[HF Export] Plot upload error: {e}")
|
| 1545 |
errors.append(f"Plot {Path(plot_file).name}: {str(e)}")
|
| 1546 |
|
| 1547 |
+
# Upload PNG images - use generic file upload
|
| 1548 |
image_patterns = [
|
| 1549 |
session_outputs_dir / "*.png",
|
| 1550 |
global_outputs_dir / "*.png",
|
|
|
|
| 1557 |
for image_file in glob.glob(str(pattern)):
|
| 1558 |
try:
|
| 1559 |
logger.info(f"[HF Export] Uploading image: {image_file}")
|
| 1560 |
+
result = hf_service.upload_generic_file(
|
| 1561 |
+
file_path=image_file,
|
| 1562 |
session_id=request.session_id,
|
| 1563 |
+
subfolder="images"
|
|
|
|
| 1564 |
)
|
| 1565 |
if result.get("success"):
|
| 1566 |
uploaded_files.append({"type": "image", "name": Path(image_file).name, "url": result.get("url")})
|
|
|
|
| 1571 |
errors.append(f"Image {Path(image_file).name}: {str(e)}")
|
| 1572 |
|
| 1573 |
if not uploaded_files and errors:
|
| 1574 |
+
logger.error(f"[HF Export] All uploads failed: {errors}")
|
| 1575 |
raise HTTPException(
|
| 1576 |
status_code=500,
|
| 1577 |
detail=f"Export failed: {'; '.join(errors)}"
|
| 1578 |
)
|
| 1579 |
|
| 1580 |
+
if not uploaded_files and not errors:
|
| 1581 |
+
logger.info(f"[HF Export] No files found to export")
|
| 1582 |
+
return JSONResponse({
|
| 1583 |
+
"success": True,
|
| 1584 |
+
"uploaded_files": [],
|
| 1585 |
+
"errors": None,
|
| 1586 |
+
"message": "No files found to export. Run some analysis first to generate outputs."
|
| 1587 |
+
})
|
| 1588 |
+
|
| 1589 |
+
logger.info(f"[HF Export] Export completed: {len(uploaded_files)} files uploaded, {len(errors)} errors")
|
| 1590 |
return JSONResponse({
|
| 1591 |
"success": True,
|
| 1592 |
"uploaded_files": uploaded_files,
|
src/storage/huggingface_storage.py
CHANGED
|
@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
|
|
| 22 |
|
| 23 |
# Optional: huggingface_hub for HF operations
|
| 24 |
try:
|
| 25 |
-
from huggingface_hub import HfApi,
|
| 26 |
from huggingface_hub.utils import RepositoryNotFoundError
|
| 27 |
HF_AVAILABLE = True
|
| 28 |
except ImportError:
|
|
@@ -106,11 +106,11 @@ class HuggingFaceStorage:
|
|
| 106 |
logger.info(f"Repo {repo_id} exists")
|
| 107 |
except RepositoryNotFoundError:
|
| 108 |
logger.info(f"Creating repo {repo_id}")
|
| 109 |
-
create_repo(
|
| 110 |
repo_id=repo_id,
|
| 111 |
repo_type=repo_kind,
|
| 112 |
private=True, # Default to private
|
| 113 |
-
|
| 114 |
)
|
| 115 |
|
| 116 |
return repo_id
|
|
@@ -156,13 +156,12 @@ class HuggingFaceStorage:
|
|
| 156 |
path_in_repo = f"datasets/{session_id}/{file_name}"
|
| 157 |
|
| 158 |
try:
|
| 159 |
-
result = upload_file(
|
| 160 |
path_or_fileobj=upload_path,
|
| 161 |
path_in_repo=path_in_repo,
|
| 162 |
repo_id=repo_id,
|
| 163 |
repo_type="dataset",
|
| 164 |
-
|
| 165 |
-
commit_message=f"Add dataset: {file_name}"
|
| 166 |
)
|
| 167 |
|
| 168 |
# Upload metadata if provided
|
|
@@ -177,13 +176,12 @@ class HuggingFaceStorage:
|
|
| 177 |
}, tmp)
|
| 178 |
tmp.flush()
|
| 179 |
|
| 180 |
-
upload_file(
|
| 181 |
path_or_fileobj=tmp.name,
|
| 182 |
path_in_repo=metadata_path,
|
| 183 |
repo_id=repo_id,
|
| 184 |
repo_type="dataset",
|
| 185 |
-
|
| 186 |
-
commit_message=f"Add metadata for {file_name}"
|
| 187 |
)
|
| 188 |
|
| 189 |
file_size = os.path.getsize(upload_path)
|
|
@@ -244,13 +242,12 @@ class HuggingFaceStorage:
|
|
| 244 |
|
| 245 |
try:
|
| 246 |
# Upload the model file
|
| 247 |
-
upload_file(
|
| 248 |
path_or_fileobj=model_path,
|
| 249 |
path_in_repo=f"{path_in_repo}/{model_file_name}",
|
| 250 |
repo_id=repo_id,
|
| 251 |
repo_type="model",
|
| 252 |
-
|
| 253 |
-
commit_message=f"Add model: {model_name}"
|
| 254 |
)
|
| 255 |
|
| 256 |
# Create and upload model card
|
|
@@ -266,13 +263,12 @@ class HuggingFaceStorage:
|
|
| 266 |
tmp.write(model_card)
|
| 267 |
tmp.flush()
|
| 268 |
|
| 269 |
-
upload_file(
|
| 270 |
path_or_fileobj=tmp.name,
|
| 271 |
path_in_repo=f"{path_in_repo}/README.md",
|
| 272 |
repo_id=repo_id,
|
| 273 |
repo_type="model",
|
| 274 |
-
|
| 275 |
-
commit_message=f"Add model card for {model_name}"
|
| 276 |
)
|
| 277 |
|
| 278 |
# Upload config
|
|
@@ -291,13 +287,12 @@ class HuggingFaceStorage:
|
|
| 291 |
json.dump(config, tmp, indent=2)
|
| 292 |
tmp.flush()
|
| 293 |
|
| 294 |
-
upload_file(
|
| 295 |
path_or_fileobj=tmp.name,
|
| 296 |
path_in_repo=f"{path_in_repo}/config.json",
|
| 297 |
repo_id=repo_id,
|
| 298 |
repo_type="model",
|
| 299 |
-
|
| 300 |
-
commit_message=f"Add config for {model_name}"
|
| 301 |
)
|
| 302 |
|
| 303 |
return {
|
|
@@ -353,13 +348,12 @@ class HuggingFaceStorage:
|
|
| 353 |
tmp.write(plot_json)
|
| 354 |
tmp.flush()
|
| 355 |
|
| 356 |
-
upload_file(
|
| 357 |
path_or_fileobj=tmp.name,
|
| 358 |
path_in_repo=path_in_repo,
|
| 359 |
repo_id=repo_id,
|
| 360 |
repo_type="dataset",
|
| 361 |
-
|
| 362 |
-
commit_message=f"Add plot: {plot_name}"
|
| 363 |
)
|
| 364 |
|
| 365 |
return {
|
|
@@ -416,13 +410,12 @@ class HuggingFaceStorage:
|
|
| 416 |
path_in_repo = f"reports/{session_id}/{file_name}"
|
| 417 |
|
| 418 |
try:
|
| 419 |
-
upload_file(
|
| 420 |
path_or_fileobj=upload_path,
|
| 421 |
path_in_repo=path_in_repo,
|
| 422 |
repo_id=repo_id,
|
| 423 |
repo_type="dataset",
|
| 424 |
-
|
| 425 |
-
commit_message=f"Add report: {report_name}"
|
| 426 |
)
|
| 427 |
|
| 428 |
file_size = os.path.getsize(upload_path)
|
|
@@ -450,6 +443,55 @@ class HuggingFaceStorage:
|
|
| 450 |
except:
|
| 451 |
pass
|
| 452 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 453 |
def list_user_files(
|
| 454 |
self,
|
| 455 |
session_id: Optional[str] = None,
|
|
|
|
| 22 |
|
| 23 |
# Optional: huggingface_hub for HF operations
|
| 24 |
try:
|
| 25 |
+
from huggingface_hub import HfApi, upload_folder
|
| 26 |
from huggingface_hub.utils import RepositoryNotFoundError
|
| 27 |
HF_AVAILABLE = True
|
| 28 |
except ImportError:
|
|
|
|
| 106 |
logger.info(f"Repo {repo_id} exists")
|
| 107 |
except RepositoryNotFoundError:
|
| 108 |
logger.info(f"Creating repo {repo_id}")
|
| 109 |
+
self.api.create_repo(
|
| 110 |
repo_id=repo_id,
|
| 111 |
repo_type=repo_kind,
|
| 112 |
private=True, # Default to private
|
| 113 |
+
exist_ok=True # Don't fail if already exists
|
| 114 |
)
|
| 115 |
|
| 116 |
return repo_id
|
|
|
|
| 156 |
path_in_repo = f"datasets/{session_id}/{file_name}"
|
| 157 |
|
| 158 |
try:
|
| 159 |
+
result = self.api.upload_file(
|
| 160 |
path_or_fileobj=upload_path,
|
| 161 |
path_in_repo=path_in_repo,
|
| 162 |
repo_id=repo_id,
|
| 163 |
repo_type="dataset",
|
| 164 |
+
commit_message=f"Add dataset: {file_name}"
|
|
|
|
| 165 |
)
|
| 166 |
|
| 167 |
# Upload metadata if provided
|
|
|
|
| 176 |
}, tmp)
|
| 177 |
tmp.flush()
|
| 178 |
|
| 179 |
+
self.api.upload_file(
|
| 180 |
path_or_fileobj=tmp.name,
|
| 181 |
path_in_repo=metadata_path,
|
| 182 |
repo_id=repo_id,
|
| 183 |
repo_type="dataset",
|
| 184 |
+
commit_message=f"Add metadata for {file_name}"
|
|
|
|
| 185 |
)
|
| 186 |
|
| 187 |
file_size = os.path.getsize(upload_path)
|
|
|
|
| 242 |
|
| 243 |
try:
|
| 244 |
# Upload the model file
|
| 245 |
+
self.api.upload_file(
|
| 246 |
path_or_fileobj=model_path,
|
| 247 |
path_in_repo=f"{path_in_repo}/{model_file_name}",
|
| 248 |
repo_id=repo_id,
|
| 249 |
repo_type="model",
|
| 250 |
+
commit_message=f"Add model: {model_name}"
|
|
|
|
| 251 |
)
|
| 252 |
|
| 253 |
# Create and upload model card
|
|
|
|
| 263 |
tmp.write(model_card)
|
| 264 |
tmp.flush()
|
| 265 |
|
| 266 |
+
self.api.upload_file(
|
| 267 |
path_or_fileobj=tmp.name,
|
| 268 |
path_in_repo=f"{path_in_repo}/README.md",
|
| 269 |
repo_id=repo_id,
|
| 270 |
repo_type="model",
|
| 271 |
+
commit_message=f"Add model card for {model_name}"
|
|
|
|
| 272 |
)
|
| 273 |
|
| 274 |
# Upload config
|
|
|
|
| 287 |
json.dump(config, tmp, indent=2)
|
| 288 |
tmp.flush()
|
| 289 |
|
| 290 |
+
self.api.upload_file(
|
| 291 |
path_or_fileobj=tmp.name,
|
| 292 |
path_in_repo=f"{path_in_repo}/config.json",
|
| 293 |
repo_id=repo_id,
|
| 294 |
repo_type="model",
|
| 295 |
+
commit_message=f"Add config for {model_name}"
|
|
|
|
| 296 |
)
|
| 297 |
|
| 298 |
return {
|
|
|
|
| 348 |
tmp.write(plot_json)
|
| 349 |
tmp.flush()
|
| 350 |
|
| 351 |
+
self.api.upload_file(
|
| 352 |
path_or_fileobj=tmp.name,
|
| 353 |
path_in_repo=path_in_repo,
|
| 354 |
repo_id=repo_id,
|
| 355 |
repo_type="dataset",
|
| 356 |
+
commit_message=f"Add plot: {plot_name}"
|
|
|
|
| 357 |
)
|
| 358 |
|
| 359 |
return {
|
|
|
|
| 410 |
path_in_repo = f"reports/{session_id}/{file_name}"
|
| 411 |
|
| 412 |
try:
|
| 413 |
+
self.api.upload_file(
|
| 414 |
path_or_fileobj=upload_path,
|
| 415 |
path_in_repo=path_in_repo,
|
| 416 |
repo_id=repo_id,
|
| 417 |
repo_type="dataset",
|
| 418 |
+
commit_message=f"Add report: {report_name}"
|
|
|
|
| 419 |
)
|
| 420 |
|
| 421 |
file_size = os.path.getsize(upload_path)
|
|
|
|
| 443 |
except:
|
| 444 |
pass
|
| 445 |
|
| 446 |
+
def upload_generic_file(
|
| 447 |
+
self,
|
| 448 |
+
file_path: str,
|
| 449 |
+
session_id: str,
|
| 450 |
+
subfolder: str = "files"
|
| 451 |
+
) -> Dict[str, Any]:
|
| 452 |
+
"""
|
| 453 |
+
Upload any file to user's HuggingFace outputs repo.
|
| 454 |
+
|
| 455 |
+
Args:
|
| 456 |
+
file_path: Local path to the file
|
| 457 |
+
session_id: Session ID
|
| 458 |
+
subfolder: Subfolder within outputs (e.g., "plots", "images", "files")
|
| 459 |
+
|
| 460 |
+
Returns:
|
| 461 |
+
Dict with upload info
|
| 462 |
+
"""
|
| 463 |
+
repo_id = self._ensure_repo_exists("outputs", "dataset")
|
| 464 |
+
|
| 465 |
+
file_name = Path(file_path).name
|
| 466 |
+
path_in_repo = f"{subfolder}/{session_id}/{file_name}"
|
| 467 |
+
|
| 468 |
+
try:
|
| 469 |
+
self.api.upload_file(
|
| 470 |
+
path_or_fileobj=file_path,
|
| 471 |
+
path_in_repo=path_in_repo,
|
| 472 |
+
repo_id=repo_id,
|
| 473 |
+
repo_type="dataset",
|
| 474 |
+
commit_message=f"Add {subfolder}: {file_name}"
|
| 475 |
+
)
|
| 476 |
+
|
| 477 |
+
file_size = os.path.getsize(file_path)
|
| 478 |
+
|
| 479 |
+
return {
|
| 480 |
+
"success": True,
|
| 481 |
+
"repo_id": repo_id,
|
| 482 |
+
"path": path_in_repo,
|
| 483 |
+
"url": f"https://huggingface.co/datasets/{repo_id}/blob/main/{path_in_repo}",
|
| 484 |
+
"download_url": f"https://huggingface.co/datasets/{repo_id}/resolve/main/{path_in_repo}",
|
| 485 |
+
"size_bytes": file_size
|
| 486 |
+
}
|
| 487 |
+
|
| 488 |
+
except Exception as e:
|
| 489 |
+
logger.error(f"Failed to upload file: {e}")
|
| 490 |
+
return {
|
| 491 |
+
"success": False,
|
| 492 |
+
"error": str(e)
|
| 493 |
+
}
|
| 494 |
+
|
| 495 |
def list_user_files(
|
| 496 |
self,
|
| 497 |
session_id: Optional[str] = None,
|