Update utils/utils.py
Browse files- utils/utils.py +5 -7
utils/utils.py
CHANGED
|
@@ -38,23 +38,21 @@ def get_dataset(dataset_type, level):
|
|
| 38 |
username = space_id.split("/")[0]
|
| 39 |
dataset_repo = f"{username}/validation"
|
| 40 |
dataset = load_dataset(dataset_repo, split="validation")
|
|
|
|
| 41 |
df = dataset.to_pandas()
|
| 42 |
|
| 43 |
if dataset_type == DATASET_TYPE_GAIA:
|
| 44 |
-
print(f"filtering for dataset_type={dataset_type}")
|
| 45 |
df = df[df["Level"].isin([1, 2, 3])]
|
| 46 |
elif dataset_type == DATASET_TYPE_HLE:
|
| 47 |
-
print(f"filtering for dataset_type={dataset_type}")
|
| 48 |
df = df[df["Level"] == 0]
|
| 49 |
-
|
| 50 |
df = df[df["Level"] == level]
|
| 51 |
-
|
| 52 |
result=[]
|
| 53 |
-
|
| 54 |
for _, row in df.iterrows():
|
| 55 |
-
print(row)
|
| 56 |
result.append([row["Question"], row["Final answer"], row["file_name"]])
|
| 57 |
-
|
| 58 |
return result
|
| 59 |
|
| 60 |
# File processing
|
|
|
|
| 38 |
username = space_id.split("/")[0]
|
| 39 |
dataset_repo = f"{username}/validation"
|
| 40 |
dataset = load_dataset(dataset_repo, split="validation")
|
| 41 |
+
|
| 42 |
df = dataset.to_pandas()
|
| 43 |
|
| 44 |
if dataset_type == DATASET_TYPE_GAIA:
|
|
|
|
| 45 |
df = df[df["Level"].isin([1, 2, 3])]
|
| 46 |
elif dataset_type == DATASET_TYPE_HLE:
|
|
|
|
| 47 |
df = df[df["Level"] == 0]
|
| 48 |
+
|
| 49 |
df = df[df["Level"] == level]
|
| 50 |
+
|
| 51 |
result=[]
|
| 52 |
+
|
| 53 |
for _, row in df.iterrows():
|
|
|
|
| 54 |
result.append([row["Question"], row["Final answer"], row["file_name"]])
|
| 55 |
+
|
| 56 |
return result
|
| 57 |
|
| 58 |
# File processing
|