fix(loader): authenticate h5ad loads in dataset metadata tools
Browse filesdataset_count_metadata_values / dataset_crosstab_metadata_values /
dataset_validate_manifest_against_data treated adata_path as a local path
(Path(url).exists() -> False), so a private pdac-research-data h5ad URL wrongly
returned 'File not found' and never authenticated. These three were missed in
the loader-auth centralization (ce5168d/04b26ba); route them through
src.core.data_io.resolve_to_local_path (HF_TOKEN auth for private
huggingface.co /resolve/ URLs, temp-file cleanup) like the sibling tools.
Surfaced live: the XDI-002 late cross-dataset eval stalled when the agent could
not load paca_au_rnaseq via dataset_count_metadata_values and fell back to an
unauthenticated urllib.urlretrieve that hung on the private repo. Verified the
fixed tool now loads the private paca_au_rnaseq.h5ad and returns real Bailey
subtype counts.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
@@ -203,15 +203,30 @@ def dataset_count_metadata_values(
|
|
| 203 |
import scanpy as sc
|
| 204 |
from pathlib import Path
|
| 205 |
from collections import Counter as _Counter
|
|
|
|
| 206 |
|
| 207 |
-
|
| 208 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
return {"error": f"File not found: {adata_path}"}
|
| 210 |
|
| 211 |
try:
|
| 212 |
-
adata = sc.read_h5ad(
|
| 213 |
except Exception as e:
|
| 214 |
return {"error": f"Could not load h5ad: {e}"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 215 |
|
| 216 |
try:
|
| 217 |
manifest = load_manifest(dataset_id)
|
|
@@ -335,15 +350,27 @@ def dataset_crosstab_metadata_values(
|
|
| 335 |
import scanpy as sc
|
| 336 |
import pandas as pd
|
| 337 |
from pathlib import Path
|
|
|
|
| 338 |
|
| 339 |
-
|
| 340 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 341 |
return {"error": f"File not found: {adata_path}"}
|
| 342 |
|
| 343 |
try:
|
| 344 |
-
adata = sc.read_h5ad(
|
| 345 |
except Exception as e:
|
| 346 |
return {"error": f"Could not load h5ad: {e}"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 347 |
|
| 348 |
try:
|
| 349 |
manifest = load_manifest(dataset_id)
|
|
@@ -442,11 +469,25 @@ def dataset_validate_manifest_against_data(
|
|
| 442 |
import scanpy as sc
|
| 443 |
|
| 444 |
from src.workflows.manifest_data_validation import validate_manifest_against_data
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 445 |
|
| 446 |
try:
|
| 447 |
-
adata = sc.read_h5ad(
|
| 448 |
except Exception as e:
|
| 449 |
return {"error": f"Could not load h5ad at '{adata_path}': {e}"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
|
| 451 |
try:
|
| 452 |
manifest = load_manifest(dataset_id)
|
|
|
|
| 203 |
import scanpy as sc
|
| 204 |
from pathlib import Path
|
| 205 |
from collections import Counter as _Counter
|
| 206 |
+
from src.core.data_io import resolve_to_local_path
|
| 207 |
|
| 208 |
+
# Resolve local paths AND remote URLs (e.g. private pdac-research-data h5ads)
|
| 209 |
+
# to a readable local file, authenticating via HF_TOKEN for private
|
| 210 |
+
# huggingface.co /resolve/ URLs. Without this, an https URL was treated as a
|
| 211 |
+
# local path (Path.exists() == False) and wrongly reported "File not found".
|
| 212 |
+
try:
|
| 213 |
+
local_path, is_temp = resolve_to_local_path(adata_path)
|
| 214 |
+
except Exception as e:
|
| 215 |
+
return {"error": f"Could not resolve {adata_path}: {e}"}
|
| 216 |
+
|
| 217 |
+
if not is_temp and not Path(local_path).exists():
|
| 218 |
return {"error": f"File not found: {adata_path}"}
|
| 219 |
|
| 220 |
try:
|
| 221 |
+
adata = sc.read_h5ad(local_path)
|
| 222 |
except Exception as e:
|
| 223 |
return {"error": f"Could not load h5ad: {e}"}
|
| 224 |
+
finally:
|
| 225 |
+
if is_temp:
|
| 226 |
+
try:
|
| 227 |
+
Path(local_path).unlink()
|
| 228 |
+
except OSError:
|
| 229 |
+
pass
|
| 230 |
|
| 231 |
try:
|
| 232 |
manifest = load_manifest(dataset_id)
|
|
|
|
| 350 |
import scanpy as sc
|
| 351 |
import pandas as pd
|
| 352 |
from pathlib import Path
|
| 353 |
+
from src.core.data_io import resolve_to_local_path
|
| 354 |
|
| 355 |
+
# Resolve local paths AND remote URLs (incl. private HF repos via HF_TOKEN).
|
| 356 |
+
try:
|
| 357 |
+
local_path, is_temp = resolve_to_local_path(adata_path)
|
| 358 |
+
except Exception as e:
|
| 359 |
+
return {"error": f"Could not resolve {adata_path}: {e}"}
|
| 360 |
+
|
| 361 |
+
if not is_temp and not Path(local_path).exists():
|
| 362 |
return {"error": f"File not found: {adata_path}"}
|
| 363 |
|
| 364 |
try:
|
| 365 |
+
adata = sc.read_h5ad(local_path)
|
| 366 |
except Exception as e:
|
| 367 |
return {"error": f"Could not load h5ad: {e}"}
|
| 368 |
+
finally:
|
| 369 |
+
if is_temp:
|
| 370 |
+
try:
|
| 371 |
+
Path(local_path).unlink()
|
| 372 |
+
except OSError:
|
| 373 |
+
pass
|
| 374 |
|
| 375 |
try:
|
| 376 |
manifest = load_manifest(dataset_id)
|
|
|
|
| 469 |
import scanpy as sc
|
| 470 |
|
| 471 |
from src.workflows.manifest_data_validation import validate_manifest_against_data
|
| 472 |
+
from src.core.data_io import resolve_to_local_path
|
| 473 |
+
from pathlib import Path as _Path
|
| 474 |
+
|
| 475 |
+
# Resolve local paths AND remote URLs (incl. private HF repos via HF_TOKEN).
|
| 476 |
+
try:
|
| 477 |
+
local_path, _is_temp = resolve_to_local_path(adata_path)
|
| 478 |
+
except Exception as e:
|
| 479 |
+
return {"error": f"Could not resolve {adata_path}: {e}"}
|
| 480 |
|
| 481 |
try:
|
| 482 |
+
adata = sc.read_h5ad(local_path)
|
| 483 |
except Exception as e:
|
| 484 |
return {"error": f"Could not load h5ad at '{adata_path}': {e}"}
|
| 485 |
+
finally:
|
| 486 |
+
if _is_temp:
|
| 487 |
+
try:
|
| 488 |
+
_Path(local_path).unlink()
|
| 489 |
+
except OSError:
|
| 490 |
+
pass
|
| 491 |
|
| 492 |
try:
|
| 493 |
manifest = load_manifest(dataset_id)
|