Pulastya B commited on
Commit
9c0055f
·
1 Parent(s): 1412b09

Align HuggingFace API usage with official docs pattern

Browse files
Files changed (2) hide show
  1. src/api/app.py +49 -21
  2. src/storage/huggingface_storage.py +66 -24
src/api/app.py CHANGED
@@ -1416,15 +1416,24 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
1416
 
1417
  # Fetch user's HuggingFace token from hf_tokens table (not user_profiles)
1418
  logger.info(f"[HF Export] Fetching HF token from hf_tokens table...")
1419
- result = supabase.table("hf_tokens").select(
1420
- "huggingface_token, huggingface_username"
1421
- ).eq("user_id", request.user_id).single().execute()
1422
-
1423
- if not result.data:
1424
- raise HTTPException(status_code=404, detail="HuggingFace not connected. Please connect in Settings first.")
1425
-
1426
- hf_token = result.data.get("huggingface_token")
1427
- hf_username = result.data.get("huggingface_username")
 
 
 
 
 
 
 
 
 
1428
 
1429
  if not hf_token:
1430
  raise HTTPException(
@@ -1433,9 +1442,19 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
1433
  )
1434
 
1435
  # Import HuggingFace storage service
1436
- from src.storage.huggingface_storage import HuggingFaceStorage
 
 
 
 
 
1437
 
1438
- hf_service = HuggingFaceStorage(hf_token=hf_token)
 
 
 
 
 
1439
 
1440
  # Collect all session assets
1441
  uploaded_files = []
@@ -1496,7 +1515,7 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
1496
  logger.error(f"[HF Export] Model upload error: {e}")
1497
  errors.append(f"Model {Path(model_file).name}: {str(e)}")
1498
 
1499
- # Upload visualizations (HTML plots) - use upload_report for HTML files
1500
  plot_patterns = [
1501
  session_outputs_dir / "*.html",
1502
  global_outputs_dir / "*.html",
@@ -1512,11 +1531,10 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
1512
  continue
1513
  try:
1514
  logger.info(f"[HF Export] Uploading HTML plot: {plot_file}")
1515
- result = hf_service.upload_report(
1516
- report_path=plot_file,
1517
  session_id=request.session_id,
1518
- report_name=Path(plot_file).stem,
1519
- compress=False # Don't compress HTML plots for direct viewing
1520
  )
1521
  if result.get("success"):
1522
  uploaded_files.append({"type": "plot", "name": Path(plot_file).name, "url": result.get("url")})
@@ -1526,7 +1544,7 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
1526
  logger.error(f"[HF Export] Plot upload error: {e}")
1527
  errors.append(f"Plot {Path(plot_file).name}: {str(e)}")
1528
 
1529
- # Upload PNG images - use upload_report with no compression
1530
  image_patterns = [
1531
  session_outputs_dir / "*.png",
1532
  global_outputs_dir / "*.png",
@@ -1539,11 +1557,10 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
1539
  for image_file in glob.glob(str(pattern)):
1540
  try:
1541
  logger.info(f"[HF Export] Uploading image: {image_file}")
1542
- result = hf_service.upload_report(
1543
- report_path=image_file,
1544
  session_id=request.session_id,
1545
- report_name=Path(image_file).stem,
1546
- compress=False
1547
  )
1548
  if result.get("success"):
1549
  uploaded_files.append({"type": "image", "name": Path(image_file).name, "url": result.get("url")})
@@ -1554,11 +1571,22 @@ async def export_to_huggingface(request: HuggingFaceExportRequest):
1554
  errors.append(f"Image {Path(image_file).name}: {str(e)}")
1555
 
1556
  if not uploaded_files and errors:
 
1557
  raise HTTPException(
1558
  status_code=500,
1559
  detail=f"Export failed: {'; '.join(errors)}"
1560
  )
1561
 
 
 
 
 
 
 
 
 
 
 
1562
  return JSONResponse({
1563
  "success": True,
1564
  "uploaded_files": uploaded_files,
 
1416
 
1417
  # Fetch user's HuggingFace token from hf_tokens table (not user_profiles)
1418
  logger.info(f"[HF Export] Fetching HF token from hf_tokens table...")
1419
+ try:
1420
+ result = supabase.table("hf_tokens").select(
1421
+ "huggingface_token, huggingface_username"
1422
+ ).eq("user_id", request.user_id).execute()
1423
+
1424
+ logger.info(f"[HF Export] Query result: {result.data}")
1425
+
1426
+ if not result.data or len(result.data) == 0:
1427
+ raise HTTPException(status_code=404, detail="HuggingFace not connected. Please connect in Settings first.")
1428
+
1429
+ row = result.data[0]
1430
+ hf_token = row.get("huggingface_token")
1431
+ hf_username = row.get("huggingface_username")
1432
+ except HTTPException:
1433
+ raise
1434
+ except Exception as e:
1435
+ logger.error(f"[HF Export] Supabase query error: {e}")
1436
+ raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
1437
 
1438
  if not hf_token:
1439
  raise HTTPException(
 
1442
  )
1443
 
1444
  # Import HuggingFace storage service
1445
+ try:
1446
+ from src.storage.huggingface_storage import HuggingFaceStorage
1447
+ logger.info(f"[HF Export] HuggingFaceStorage imported successfully")
1448
+ except ImportError as e:
1449
+ logger.error(f"[HF Export] Failed to import HuggingFaceStorage: {e}")
1450
+ raise HTTPException(status_code=500, detail=f"Server error: {str(e)}")
1451
 
1452
+ try:
1453
+ hf_service = HuggingFaceStorage(hf_token=hf_token)
1454
+ logger.info(f"[HF Export] HuggingFaceStorage initialized for user: {hf_username}")
1455
+ except Exception as e:
1456
+ logger.error(f"[HF Export] Failed to initialize HuggingFaceStorage: {e}")
1457
+ raise HTTPException(status_code=500, detail=f"HuggingFace error: {str(e)}")
1458
 
1459
  # Collect all session assets
1460
  uploaded_files = []
 
1515
  logger.error(f"[HF Export] Model upload error: {e}")
1516
  errors.append(f"Model {Path(model_file).name}: {str(e)}")
1517
 
1518
+ # Upload visualizations (HTML plots) - use generic file upload
1519
  plot_patterns = [
1520
  session_outputs_dir / "*.html",
1521
  global_outputs_dir / "*.html",
 
1531
  continue
1532
  try:
1533
  logger.info(f"[HF Export] Uploading HTML plot: {plot_file}")
1534
+ result = hf_service.upload_generic_file(
1535
+ file_path=plot_file,
1536
  session_id=request.session_id,
1537
+ subfolder="plots"
 
1538
  )
1539
  if result.get("success"):
1540
  uploaded_files.append({"type": "plot", "name": Path(plot_file).name, "url": result.get("url")})
 
1544
  logger.error(f"[HF Export] Plot upload error: {e}")
1545
  errors.append(f"Plot {Path(plot_file).name}: {str(e)}")
1546
 
1547
+ # Upload PNG images - use generic file upload
1548
  image_patterns = [
1549
  session_outputs_dir / "*.png",
1550
  global_outputs_dir / "*.png",
 
1557
  for image_file in glob.glob(str(pattern)):
1558
  try:
1559
  logger.info(f"[HF Export] Uploading image: {image_file}")
1560
+ result = hf_service.upload_generic_file(
1561
+ file_path=image_file,
1562
  session_id=request.session_id,
1563
+ subfolder="images"
 
1564
  )
1565
  if result.get("success"):
1566
  uploaded_files.append({"type": "image", "name": Path(image_file).name, "url": result.get("url")})
 
1571
  errors.append(f"Image {Path(image_file).name}: {str(e)}")
1572
 
1573
  if not uploaded_files and errors:
1574
+ logger.error(f"[HF Export] All uploads failed: {errors}")
1575
  raise HTTPException(
1576
  status_code=500,
1577
  detail=f"Export failed: {'; '.join(errors)}"
1578
  )
1579
 
1580
+ if not uploaded_files and not errors:
1581
+ logger.info(f"[HF Export] No files found to export")
1582
+ return JSONResponse({
1583
+ "success": True,
1584
+ "uploaded_files": [],
1585
+ "errors": None,
1586
+ "message": "No files found to export. Run some analysis first to generate outputs."
1587
+ })
1588
+
1589
+ logger.info(f"[HF Export] Export completed: {len(uploaded_files)} files uploaded, {len(errors)} errors")
1590
  return JSONResponse({
1591
  "success": True,
1592
  "uploaded_files": uploaded_files,
src/storage/huggingface_storage.py CHANGED
@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
22
 
23
  # Optional: huggingface_hub for HF operations
24
  try:
25
- from huggingface_hub import HfApi, HfFolder, create_repo, upload_file, upload_folder
26
  from huggingface_hub.utils import RepositoryNotFoundError
27
  HF_AVAILABLE = True
28
  except ImportError:
@@ -106,11 +106,11 @@ class HuggingFaceStorage:
106
  logger.info(f"Repo {repo_id} exists")
107
  except RepositoryNotFoundError:
108
  logger.info(f"Creating repo {repo_id}")
109
- create_repo(
110
  repo_id=repo_id,
111
  repo_type=repo_kind,
112
  private=True, # Default to private
113
- token=self.token
114
  )
115
 
116
  return repo_id
@@ -156,13 +156,12 @@ class HuggingFaceStorage:
156
  path_in_repo = f"datasets/{session_id}/{file_name}"
157
 
158
  try:
159
- result = upload_file(
160
  path_or_fileobj=upload_path,
161
  path_in_repo=path_in_repo,
162
  repo_id=repo_id,
163
  repo_type="dataset",
164
- token=self.token,
165
- commit_message=f"Add dataset: {file_name}"
166
  )
167
 
168
  # Upload metadata if provided
@@ -177,13 +176,12 @@ class HuggingFaceStorage:
177
  }, tmp)
178
  tmp.flush()
179
 
180
- upload_file(
181
  path_or_fileobj=tmp.name,
182
  path_in_repo=metadata_path,
183
  repo_id=repo_id,
184
  repo_type="dataset",
185
- token=self.token,
186
- commit_message=f"Add metadata for {file_name}"
187
  )
188
 
189
  file_size = os.path.getsize(upload_path)
@@ -244,13 +242,12 @@ class HuggingFaceStorage:
244
 
245
  try:
246
  # Upload the model file
247
- upload_file(
248
  path_or_fileobj=model_path,
249
  path_in_repo=f"{path_in_repo}/{model_file_name}",
250
  repo_id=repo_id,
251
  repo_type="model",
252
- token=self.token,
253
- commit_message=f"Add model: {model_name}"
254
  )
255
 
256
  # Create and upload model card
@@ -266,13 +263,12 @@ class HuggingFaceStorage:
266
  tmp.write(model_card)
267
  tmp.flush()
268
 
269
- upload_file(
270
  path_or_fileobj=tmp.name,
271
  path_in_repo=f"{path_in_repo}/README.md",
272
  repo_id=repo_id,
273
  repo_type="model",
274
- token=self.token,
275
- commit_message=f"Add model card for {model_name}"
276
  )
277
 
278
  # Upload config
@@ -291,13 +287,12 @@ class HuggingFaceStorage:
291
  json.dump(config, tmp, indent=2)
292
  tmp.flush()
293
 
294
- upload_file(
295
  path_or_fileobj=tmp.name,
296
  path_in_repo=f"{path_in_repo}/config.json",
297
  repo_id=repo_id,
298
  repo_type="model",
299
- token=self.token,
300
- commit_message=f"Add config for {model_name}"
301
  )
302
 
303
  return {
@@ -353,13 +348,12 @@ class HuggingFaceStorage:
353
  tmp.write(plot_json)
354
  tmp.flush()
355
 
356
- upload_file(
357
  path_or_fileobj=tmp.name,
358
  path_in_repo=path_in_repo,
359
  repo_id=repo_id,
360
  repo_type="dataset",
361
- token=self.token,
362
- commit_message=f"Add plot: {plot_name}"
363
  )
364
 
365
  return {
@@ -416,13 +410,12 @@ class HuggingFaceStorage:
416
  path_in_repo = f"reports/{session_id}/{file_name}"
417
 
418
  try:
419
- upload_file(
420
  path_or_fileobj=upload_path,
421
  path_in_repo=path_in_repo,
422
  repo_id=repo_id,
423
  repo_type="dataset",
424
- token=self.token,
425
- commit_message=f"Add report: {report_name}"
426
  )
427
 
428
  file_size = os.path.getsize(upload_path)
@@ -450,6 +443,55 @@ class HuggingFaceStorage:
450
  except:
451
  pass
452
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
453
  def list_user_files(
454
  self,
455
  session_id: Optional[str] = None,
 
22
 
23
  # Optional: huggingface_hub for HF operations
24
  try:
25
+ from huggingface_hub import HfApi, upload_folder
26
  from huggingface_hub.utils import RepositoryNotFoundError
27
  HF_AVAILABLE = True
28
  except ImportError:
 
106
  logger.info(f"Repo {repo_id} exists")
107
  except RepositoryNotFoundError:
108
  logger.info(f"Creating repo {repo_id}")
109
+ self.api.create_repo(
110
  repo_id=repo_id,
111
  repo_type=repo_kind,
112
  private=True, # Default to private
113
+ exist_ok=True # Don't fail if already exists
114
  )
115
 
116
  return repo_id
 
156
  path_in_repo = f"datasets/{session_id}/{file_name}"
157
 
158
  try:
159
+ result = self.api.upload_file(
160
  path_or_fileobj=upload_path,
161
  path_in_repo=path_in_repo,
162
  repo_id=repo_id,
163
  repo_type="dataset",
164
+ commit_message=f"Add dataset: {file_name}"
 
165
  )
166
 
167
  # Upload metadata if provided
 
176
  }, tmp)
177
  tmp.flush()
178
 
179
+ self.api.upload_file(
180
  path_or_fileobj=tmp.name,
181
  path_in_repo=metadata_path,
182
  repo_id=repo_id,
183
  repo_type="dataset",
184
+ commit_message=f"Add metadata for {file_name}"
 
185
  )
186
 
187
  file_size = os.path.getsize(upload_path)
 
242
 
243
  try:
244
  # Upload the model file
245
+ self.api.upload_file(
246
  path_or_fileobj=model_path,
247
  path_in_repo=f"{path_in_repo}/{model_file_name}",
248
  repo_id=repo_id,
249
  repo_type="model",
250
+ commit_message=f"Add model: {model_name}"
 
251
  )
252
 
253
  # Create and upload model card
 
263
  tmp.write(model_card)
264
  tmp.flush()
265
 
266
+ self.api.upload_file(
267
  path_or_fileobj=tmp.name,
268
  path_in_repo=f"{path_in_repo}/README.md",
269
  repo_id=repo_id,
270
  repo_type="model",
271
+ commit_message=f"Add model card for {model_name}"
 
272
  )
273
 
274
  # Upload config
 
287
  json.dump(config, tmp, indent=2)
288
  tmp.flush()
289
 
290
+ self.api.upload_file(
291
  path_or_fileobj=tmp.name,
292
  path_in_repo=f"{path_in_repo}/config.json",
293
  repo_id=repo_id,
294
  repo_type="model",
295
+ commit_message=f"Add config for {model_name}"
 
296
  )
297
 
298
  return {
 
348
  tmp.write(plot_json)
349
  tmp.flush()
350
 
351
+ self.api.upload_file(
352
  path_or_fileobj=tmp.name,
353
  path_in_repo=path_in_repo,
354
  repo_id=repo_id,
355
  repo_type="dataset",
356
+ commit_message=f"Add plot: {plot_name}"
 
357
  )
358
 
359
  return {
 
410
  path_in_repo = f"reports/{session_id}/{file_name}"
411
 
412
  try:
413
+ self.api.upload_file(
414
  path_or_fileobj=upload_path,
415
  path_in_repo=path_in_repo,
416
  repo_id=repo_id,
417
  repo_type="dataset",
418
+ commit_message=f"Add report: {report_name}"
 
419
  )
420
 
421
  file_size = os.path.getsize(upload_path)
 
443
  except:
444
  pass
445
 
446
+ def upload_generic_file(
447
+ self,
448
+ file_path: str,
449
+ session_id: str,
450
+ subfolder: str = "files"
451
+ ) -> Dict[str, Any]:
452
+ """
453
+ Upload any file to user's HuggingFace outputs repo.
454
+
455
+ Args:
456
+ file_path: Local path to the file
457
+ session_id: Session ID
458
+ subfolder: Subfolder within outputs (e.g., "plots", "images", "files")
459
+
460
+ Returns:
461
+ Dict with upload info
462
+ """
463
+ repo_id = self._ensure_repo_exists("outputs", "dataset")
464
+
465
+ file_name = Path(file_path).name
466
+ path_in_repo = f"{subfolder}/{session_id}/{file_name}"
467
+
468
+ try:
469
+ self.api.upload_file(
470
+ path_or_fileobj=file_path,
471
+ path_in_repo=path_in_repo,
472
+ repo_id=repo_id,
473
+ repo_type="dataset",
474
+ commit_message=f"Add {subfolder}: {file_name}"
475
+ )
476
+
477
+ file_size = os.path.getsize(file_path)
478
+
479
+ return {
480
+ "success": True,
481
+ "repo_id": repo_id,
482
+ "path": path_in_repo,
483
+ "url": f"https://huggingface.co/datasets/{repo_id}/blob/main/{path_in_repo}",
484
+ "download_url": f"https://huggingface.co/datasets/{repo_id}/resolve/main/{path_in_repo}",
485
+ "size_bytes": file_size
486
+ }
487
+
488
+ except Exception as e:
489
+ logger.error(f"Failed to upload file: {e}")
490
+ return {
491
+ "success": False,
492
+ "error": str(e)
493
+ }
494
+
495
  def list_user_files(
496
  self,
497
  session_id: Optional[str] = None,