avoigt1121 Claude Opus 4.8 commited on
Commit
ce889e5
·
1 Parent(s): 37c63d2

fix(loader): authenticate h5ad loads in dataset metadata tools

Browse files

dataset_count_metadata_values / dataset_crosstab_metadata_values /
dataset_validate_manifest_against_data treated adata_path as a local path
(Path(url).exists() -> False), so a private pdac-research-data h5ad URL wrongly
returned 'File not found' and never authenticated. These three were missed in
the loader-auth centralization (ce5168d/04b26ba); route them through
src.core.data_io.resolve_to_local_path (HF_TOKEN auth for private
huggingface.co /resolve/ URLs, temp-file cleanup) like the sibling tools.

Surfaced live: the XDI-002 late cross-dataset eval stalled when the agent could
not load paca_au_rnaseq via dataset_count_metadata_values and fell back to an
unauthenticated urllib.urlretrieve that hung on the private repo. Verified the
fixed tool now loads the private paca_au_rnaseq.h5ad and returns real Bailey
subtype counts.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Files changed (1) hide show
  1. src/tools/dataset_tools/metadata.py +48 -7
src/tools/dataset_tools/metadata.py CHANGED
@@ -203,15 +203,30 @@ def dataset_count_metadata_values(
203
  import scanpy as sc
204
  from pathlib import Path
205
  from collections import Counter as _Counter
 
206
 
207
- data_file = Path(adata_path)
208
- if not data_file.exists():
 
 
 
 
 
 
 
 
209
  return {"error": f"File not found: {adata_path}"}
210
 
211
  try:
212
- adata = sc.read_h5ad(adata_path)
213
  except Exception as e:
214
  return {"error": f"Could not load h5ad: {e}"}
 
 
 
 
 
 
215
 
216
  try:
217
  manifest = load_manifest(dataset_id)
@@ -335,15 +350,27 @@ def dataset_crosstab_metadata_values(
335
  import scanpy as sc
336
  import pandas as pd
337
  from pathlib import Path
 
338
 
339
- data_file = Path(adata_path)
340
- if not data_file.exists():
 
 
 
 
 
341
  return {"error": f"File not found: {adata_path}"}
342
 
343
  try:
344
- adata = sc.read_h5ad(adata_path)
345
  except Exception as e:
346
  return {"error": f"Could not load h5ad: {e}"}
 
 
 
 
 
 
347
 
348
  try:
349
  manifest = load_manifest(dataset_id)
@@ -442,11 +469,25 @@ def dataset_validate_manifest_against_data(
442
  import scanpy as sc
443
 
444
  from src.workflows.manifest_data_validation import validate_manifest_against_data
 
 
 
 
 
 
 
 
445
 
446
  try:
447
- adata = sc.read_h5ad(adata_path)
448
  except Exception as e:
449
  return {"error": f"Could not load h5ad at '{adata_path}': {e}"}
 
 
 
 
 
 
450
 
451
  try:
452
  manifest = load_manifest(dataset_id)
 
203
  import scanpy as sc
204
  from pathlib import Path
205
  from collections import Counter as _Counter
206
+ from src.core.data_io import resolve_to_local_path
207
 
208
+ # Resolve local paths AND remote URLs (e.g. private pdac-research-data h5ads)
209
+ # to a readable local file, authenticating via HF_TOKEN for private
210
+ # huggingface.co /resolve/ URLs. Without this, an https URL was treated as a
211
+ # local path (Path.exists() == False) and wrongly reported "File not found".
212
+ try:
213
+ local_path, is_temp = resolve_to_local_path(adata_path)
214
+ except Exception as e:
215
+ return {"error": f"Could not resolve {adata_path}: {e}"}
216
+
217
+ if not is_temp and not Path(local_path).exists():
218
  return {"error": f"File not found: {adata_path}"}
219
 
220
  try:
221
+ adata = sc.read_h5ad(local_path)
222
  except Exception as e:
223
  return {"error": f"Could not load h5ad: {e}"}
224
+ finally:
225
+ if is_temp:
226
+ try:
227
+ Path(local_path).unlink()
228
+ except OSError:
229
+ pass
230
 
231
  try:
232
  manifest = load_manifest(dataset_id)
 
350
  import scanpy as sc
351
  import pandas as pd
352
  from pathlib import Path
353
+ from src.core.data_io import resolve_to_local_path
354
 
355
+ # Resolve local paths AND remote URLs (incl. private HF repos via HF_TOKEN).
356
+ try:
357
+ local_path, is_temp = resolve_to_local_path(adata_path)
358
+ except Exception as e:
359
+ return {"error": f"Could not resolve {adata_path}: {e}"}
360
+
361
+ if not is_temp and not Path(local_path).exists():
362
  return {"error": f"File not found: {adata_path}"}
363
 
364
  try:
365
+ adata = sc.read_h5ad(local_path)
366
  except Exception as e:
367
  return {"error": f"Could not load h5ad: {e}"}
368
+ finally:
369
+ if is_temp:
370
+ try:
371
+ Path(local_path).unlink()
372
+ except OSError:
373
+ pass
374
 
375
  try:
376
  manifest = load_manifest(dataset_id)
 
469
  import scanpy as sc
470
 
471
  from src.workflows.manifest_data_validation import validate_manifest_against_data
472
+ from src.core.data_io import resolve_to_local_path
473
+ from pathlib import Path as _Path
474
+
475
+ # Resolve local paths AND remote URLs (incl. private HF repos via HF_TOKEN).
476
+ try:
477
+ local_path, _is_temp = resolve_to_local_path(adata_path)
478
+ except Exception as e:
479
+ return {"error": f"Could not resolve {adata_path}: {e}"}
480
 
481
  try:
482
+ adata = sc.read_h5ad(local_path)
483
  except Exception as e:
484
  return {"error": f"Could not load h5ad at '{adata_path}': {e}"}
485
+ finally:
486
+ if _is_temp:
487
+ try:
488
+ _Path(local_path).unlink()
489
+ except OSError:
490
+ pass
491
 
492
  try:
493
  manifest = load_manifest(dataset_id)