Trying to fix loading error. Note that I'm adding trust_remote_code=True; when new datasets are added, should be verified that this is OK.

data_measurements/dataset_utils.py

@@ -17,7 +17,7 @@ from dataclasses import asdict
 from os.path import exists
 
 import pandas as pd
-from datasets import Dataset, get_dataset_infos, load_dataset, load_from_disk
+from datasets import Dataset, get_dataset_infos, load_dataset, load_from_disk, load_dataset_builder, get_dataset_config_names
 
 # treating inf values as NaN as well
 pd.set_option("use_inf_as_na", True)
@@ -110,7 +110,7 @@ def load_truncated_dataset(
     """
     if cache_name is None:
         cache_name = f"{dataset_name}_{config_name}_{split_name}_{num_rows}"
-    if exists(cache_name):
+    if exists(cache_name) and use_cache:
         dataset = load_from_disk(cache_name)
     else:
         if use_streaming and dataset_name in _STREAMABLE_DATASET_LIST:
@@ -130,11 +130,15 @@ def load_truncated_dataset(
                 "temp.jsonl", features=iterable_dataset.features, split=split_name
             )
         else:
+            if dataset_name in _DATASET_LIST:
+                trust_remote_code=True,
+            else:
+                trust_remote_code=False,
             full_dataset = load_dataset(
                 dataset_name,
                 name=config_name,
                 split=split_name,
-                trust_remote_code=True,
+                trust_remote_code=trust_remote_code,
             )
             dataset = full_dataset.select(range(num_rows))
         dataset.save_to_disk(cache_name)
@@ -289,21 +293,62 @@ def dictionarize_info(dset_info):
 def get_dataset_info_dicts(dataset_id=None):
     """
     Creates a dict from dataset configs.
-    Uses the datasets lib's get_dataset_infos
+    Uses the datasets lib's get_dataset_infos or load_dataset_builder
+    with trust_remote_code=True for datasets that require it.
     :return: Dictionary mapping dataset names to their configurations
     """
+    def get_infos_with_trust(ds_id):
+        """Get dataset infos with trust_remote_code=True if needed."""
+        try:
+            # First try without trust_remote_code
+            return get_dataset_infos(ds_id)
+        except (ValueError, RuntimeError) as e:
+            # If it requires trust_remote_code, use load_dataset_builder for each config
+            if ds_id in _DATASET_LIST and ("trust_remote_code" in str(e) or "requires you to execute" in str(e)):
+                try:
+                    # Get all config names
+                    config_names = get_dataset_config_names(ds_id, trust_remote_code=True)
+                    if not config_names:
+                        config_names = ["default"]
+                    
+                    # Load builder for each config and get its info
+                    infos = {}
+                    for config_name in config_names:
+                        try:
+                            builder = load_dataset_builder(ds_id, config_name=config_name, trust_remote_code=True)
+                            if builder.info is not None:
+                                infos[config_name] = builder.info
+                        except Exception:
+                            # If loading with config_name fails, try without it
+                            if config_name == "default":
+                                builder = load_dataset_builder(ds_id, trust_remote_code=True)
+                                if builder.info is not None:
+                                    infos["default"] = builder.info
+                    
+                    if not infos:
+                        raise ValueError(f"Could not load dataset info for {ds_id} with trust_remote_code=True")
+                    return infos
+                except Exception as inner_e:
+                    # Fallback: try get_dataset_infos with config_kwargs if supported
+                    try:
+                        return get_dataset_infos(ds_id, config_kwargs={'trust_remote_code': True})
+                    except Exception:
+                        raise e
+            else:
+                raise e
+    
     if dataset_id != None:
         ds_name_to_conf_dict = {
             dataset_id: {
                 config_name: dictionarize_info(config_info)
-                for config_name, config_info in get_dataset_infos(dataset_id).items()
+                for config_name, config_info in get_infos_with_trust(dataset_id).items()
             }
         }
     else:
         ds_name_to_conf_dict = {
             ds_id: {
                 config_name: dictionarize_info(config_info)
-                for config_name, config_info in get_dataset_infos(ds_id).items()
+                for config_name, config_info in get_infos_with_trust(ds_id).items()
             }
             for ds_id in _DATASET_LIST
         }