feat: Enable authenticated Hugging Face Hub downloads for GAIA using `HF_TOKEN` and enhance the download failure warning message.
Browse files- app copy.py +4 -1
- out.log +0 -0
app copy.py
CHANGED
|
@@ -32,6 +32,8 @@ def file_extract(local_file_path, task_id):
|
|
| 32 |
if not local_file_path:
|
| 33 |
return None
|
| 34 |
|
|
|
|
|
|
|
| 35 |
# GAIA files are usually placed in date-based subdirectories
|
| 36 |
prefixes = ["2023/validation/", "2023/test/", "2023/train/", ""]
|
| 37 |
|
|
@@ -41,12 +43,13 @@ def file_extract(local_file_path, task_id):
|
|
| 41 |
repo_id="gaia-benchmark/GAIA",
|
| 42 |
filename=f"{prefix}{local_file_path}",
|
| 43 |
repo_type="dataset",
|
|
|
|
| 44 |
)
|
| 45 |
return resolved_path
|
| 46 |
except Exception:
|
| 47 |
continue
|
| 48 |
|
| 49 |
-
logger.warning(f"Could not download file '{local_file_path}' for task_id {task_id}")
|
| 50 |
return None
|
| 51 |
|
| 52 |
agent = BasicAgent()
|
|
|
|
| 32 |
if not local_file_path:
|
| 33 |
return None
|
| 34 |
|
| 35 |
+
token = os.getenv("HUGGINGFACEHUB_API_TOKEN") or os.getenv("HF_TOKEN")
|
| 36 |
+
|
| 37 |
# GAIA files are usually placed in date-based subdirectories
|
| 38 |
prefixes = ["2023/validation/", "2023/test/", "2023/train/", ""]
|
| 39 |
|
|
|
|
| 43 |
repo_id="gaia-benchmark/GAIA",
|
| 44 |
filename=f"{prefix}{local_file_path}",
|
| 45 |
repo_type="dataset",
|
| 46 |
+
token=token
|
| 47 |
)
|
| 48 |
return resolved_path
|
| 49 |
except Exception:
|
| 50 |
continue
|
| 51 |
|
| 52 |
+
logger.warning(f"Could not download file '{local_file_path}' for task_id {task_id}. Make sure you accepted GAIA terms on HF and set HF_TOKEN.")
|
| 53 |
return None
|
| 54 |
|
| 55 |
agent = BasicAgent()
|
out.log
ADDED
|
Binary file (8.69 kB). View file
|
|
|