Fix GAIA attachment downloads

#2
by John6666 - opened
Files changed (1) hide show
  1. main.py +13 -6
main.py CHANGED
@@ -107,15 +107,22 @@ def load_questions():
107
 
108
  # 3. Store the file path mapping if file details exist and are valid
109
  if local_file_path and file_name:
110
- # Log if the path from the dataset isn't absolute (might indicate issues)
111
  if not os.path.isabs(local_file_path):
112
- logger.warning(f"Task {task_id}: Path '{local_file_path}' from dataset is not absolute. This might cause issues finding the file on the server.")
113
-
114
-
115
- if os.path.exists(local_file_path) and os.path.isfile(local_file_path):
 
 
 
 
 
 
 
 
116
  task_file_paths[str(task_id)] = local_file_path
117
  logger.debug(f"Stored file path mapping for task_id {task_id}: {local_file_path}")
118
- else:
119
  logger.warning(f"File path '{local_file_path}' for task_id {task_id} does NOT exist or is not a file on server. Mapping skipped.")
120
  elif task_id:
121
 
 
107
 
108
  # 3. Store the file path mapping if file details exist and are valid
109
  if local_file_path and file_name:
 
110
  if not os.path.isabs(local_file_path):
111
+ try:
112
+ local_file_path = hf_hub_download(
113
+ repo_id="gaia-benchmark/GAIA",
114
+ repo_type="dataset",
115
+ filename=local_file_path,
116
+ )
117
+ logger.info(f"Downloaded file for task_id {task_id} to: {local_file_path}")
118
+ except Exception as e:
119
+ logger.warning(f"Could not download file '{local_file_path}' for task_id {task_id}: {e}")
120
+ local_file_path = None
121
+
122
+ if local_file_path and os.path.exists(local_file_path) and os.path.isfile(local_file_path):
123
  task_file_paths[str(task_id)] = local_file_path
124
  logger.debug(f"Stored file path mapping for task_id {task_id}: {local_file_path}")
125
+ elif local_file_path:
126
  logger.warning(f"File path '{local_file_path}' for task_id {task_id} does NOT exist or is not a file on server. Mapping skipped.")
127
  elif task_id:
128