scratch/test_dataset_to_dict.py

import datasets as ds
import pandas as pd
import glob

dataset = ds.Dataset.from_dict({})
files = glob.glob("**/*.json", root_dir="data/logs", recursive=True)
for file in files:
    path = "data/logs/" + file
    temp = ds.Dataset.from_json(path)
    dataset = ds.concatenate_datasets([dataset, temp])  # type: ignore
df = dataset.to_pandas()
assert isinstance(df, pd.DataFrame)
print(len(df))
res = df.to_dict(orient="records")
print(res)