Working analysis of size and text/audio derived basic features

.gitignore

@@ -0,0 +1,3 @@
+plots
+reports
+__pycache__

.python-version

@@ -0,0 +1 @@
+streamlit

README.md

@@ -1,10 +1,10 @@
 ---
 title: Amu Bigos Data Dash
-emoji: 🏢
-colorFrom: red
-colorTo: blue
+emoji: 🐨
+colorFrom: indigo
+colorTo: purple
 sdk: streamlit
-sdk_version: 1.34.0
+sdk_version: 1.33.0
 app_file: app.py
 pinned: false
 license: cc-by-nc-nd-4.0

app.py

@@ -0,0 +1,148 @@
+import pandas as pd
+import streamlit as st
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+import os
+import json
+
+from utils import read_reports, dict_to_multindex_df
+#add_test_split_stats_from_secret_dataset, dict_to_multindex_df_all_splits
+from utils import extract_stats_to_agg, extract_stats_all_splits, extract_stats_for_dataset_card
+from constants import BIGOS_INFO, PELCRA_INFO, ABOUT_INFO
+from datasets import get_dataset_config_names
+
+# PL ASR BIGOS analysis
+# PL ASR Diagnostic analysis
+# PELCRA analysis
+# TODO - compare the datasets
+
+st.set_page_config(layout="wide")
+
+about, analysis_bigos, analysis_bigos_pelcra = st.tabs(["About BIGOS datasets", "BIGOS V2 analysis", "PELCRA for BIGOS analysis"])
+#analysis_bigos_diagnostic
+#########################################BIGOS################################################
+with about:
+    
+    st.title("About BIGOS project")
+    st.markdown(ABOUT_INFO, unsafe_allow_html=True)
+    # TODO - load and display about BIGOS benchmark
+
+with analysis_bigos:
+    dataset_name = "amu-cai/pl-asr-bigos-v2"
+    #dataset_secret = "amu-cai/pl-asr-bigos-v2-secret"
+    dataset_short_name = "BIGOS"
+    dataset_version = "V2"
+    
+    dataset_configs = get_dataset_config_names(dataset_name,trust_remote_code=True)
+    # remove "all" subset, which is always the last config type
+    dataset_configs.pop()
+    print(dataset_configs)
+    # read the reports for public and secret datasets
+    [stats_dict_public, contents_dict_public] = read_reports(dataset_name)
+
+    # update the metrics for test split with the secret dataset statistics
+    #stats_dict_public = add_test_split_stats_from_secret_dataset(stats_dict_public, stats_dict_secret)
+    df_multindex_for_agg = dict_to_multindex_df(stats_dict_public, all_splits=False)
+    df_multindex_all_splits = dict_to_multindex_df(stats_dict_public, all_splits=True)
+
+    # extract metrics from dictionary and convert to various dataframes for easier analysis and visualization
+    #st.header("Summary statistics")
+
+
+    st.header("Dataset level metrics")
+    metrics_size = ["samples", "audio[h]", "speakers", "words",  "chars"]
+    df_sum_stats_agg = extract_stats_to_agg(df_multindex_for_agg, metrics_size)
+
+    # split dataframe into separate dataframes for easier analysis and visualization
+    st.subheader("Dataset size (audio)")
+    df_sum_stats_audio = df_sum_stats_agg[["audio[h]", "samples", "speakers"]]
+    st.dataframe(df_sum_stats_audio)
+    
+    st.subheader("Dataset size (text)")
+    df_sum_stats_text = df_sum_stats_agg[["samples", "words", "chars"]]
+    st.dataframe(df_sum_stats_text)
+
+
+    metrics_features = ["utts_unique", "words_unique", "chars_unique", "words_per_sec", "chars_per_sec"]
+
+    df_sum_stats_all_splits = extract_stats_all_splits(df_multindex_all_splits, metrics_features)
+
+    st.subheader("Dataset features (text)")
+    df_sum_stats_feats_text = df_sum_stats_all_splits[metrics_features[0:2]]
+    st.dataframe(df_sum_stats_feats_text)
+
+    st.subheader("Dataset features (audio)")
+    df_sum_stats_feats_audio = df_sum_stats_all_splits[metrics_features[3:]]
+    st.dataframe(df_sum_stats_feats_audio)
+
+    st.header("BIGOS subsets (source datasets) cards")
+    for subset in dataset_configs:
+        st.subheader("Dataset card for: {}".format(subset))
+        df_metrics_subset_size = extract_stats_for_dataset_card(df_multindex_for_agg, subset, metrics_size, add_total=True)
+        st.dataframe(df_metrics_subset_size)
+        df_metrics_subset_features = extract_stats_for_dataset_card(df_multindex_for_agg, subset, metrics_features, add_total=False)
+        st.dataframe(df_metrics_subset_features)
+
+
+    
+#########################################PELCRA################################################
+with analysis_bigos_pelcra:
+
+    dataset_name = "pelcra/pl-asr-pelcra-for-bigos"
+    #dataset_secret = "pelcra/pl-asr-pelcra-for-bigos-secret"
+
+    dataset_short_name = "PELCRA"
+
+    dataset_configs = get_dataset_config_names(dataset_name,trust_remote_code=True)
+    # remove "all" subset, which is always the last config type
+    dataset_configs.pop()
+    print(dataset_configs)
+    # read the reports for public and secret datasets
+    [stats_dict_public, contents_dict_public] = read_reports(dataset_name)
+
+    # update the metrics for test split with the secret dataset statistics
+    #stats_dict_public = add_test_split_stats_from_secret_dataset(stats_dict_public, stats_dict_secret)
+    df_multindex_for_agg = dict_to_multindex_df(stats_dict_public, all_splits=False)
+    df_multindex_all_splits = dict_to_multindex_df(stats_dict_public, all_splits=True)
+
+    # extract metrics from dictionary and convert to various dataframes for easier analysis and visualization
+    #st.header("Summary statistics")
+
+
+    st.header("Dataset level metrics")
+    metrics_size = ["samples", "audio[h]", "speakers", "words",  "chars"]
+    df_sum_stats_agg = extract_stats_to_agg(df_multindex_for_agg, metrics_size)
+
+    #st.dataframe(df_sum_stats_agg)
+    #print(df_sum_stats.columns)
+    
+    # split dataframe into separate dataframes for easier analysis and visualization
+    st.subheader("Dataset size (audio)")
+    df_sum_stats_audio = df_sum_stats_agg[["audio[h]", "samples", "speakers"]]
+    st.dataframe(df_sum_stats_audio)
+    
+    st.subheader("Dataset size (text)")
+    df_sum_stats_text = df_sum_stats_agg[["samples", "words", "chars"]]
+    st.dataframe(df_sum_stats_text)
+
+
+    metrics_features = ["utts_unique", "words_unique", "chars_unique", "words_per_sec", "chars_per_sec"]
+
+    df_sum_stats_all_splits = extract_stats_all_splits(df_multindex_all_splits, metrics_features)
+
+    st.subheader("Dataset features (text)")
+    df_sum_stats_feats_text = df_sum_stats_all_splits[metrics_features[0:2]]
+    st.dataframe(df_sum_stats_feats_text)
+
+    st.subheader("Dataset features (audio)")
+    df_sum_stats_feats_audio = df_sum_stats_all_splits[metrics_features[3:]]
+    st.dataframe(df_sum_stats_feats_audio)
+
+    st.header("BIGOS subsets (source datasets) cards")
+    for subset in dataset_configs:
+        st.subheader("Dataset card for: {}".format(subset))
+        df_metrics_subset_size = extract_stats_for_dataset_card(df_multindex_for_agg, subset, metrics_size, add_total=True)
+        st.dataframe(df_metrics_subset_size)
+        df_metrics_subset_features = extract_stats_for_dataset_card(df_multindex_for_agg, subset, metrics_features, add_total=False)
+        st.dataframe(df_metrics_subset_features)

constants.py

@@ -0,0 +1,19 @@
+ABOUT_INFO = "Welcome to the BIGOS (Benchmark Intended Grouping of Open Speech) dataset analysis dashboard! <br> \
+Here you can learn more about the contents of BIGOS speech datasets for Polish: <br> \
+* **BIGOS V2 dataset** [HF datasets hub](https://huggingface.co/datasets/amu-cai/pl-asr-bigos-v2) <br> \
+* **BIGOS V2 diagnostic dataset** [HF datasets hub](https://huggingface.co/datasets/amu-cai/pl-asr-bigos-v2-diagnostic) <br> \
+* **PELCRA for BIGOS dataset** [HF datasets hub](https://huggingface.co/datasets/pelcra/pl-asr-pelcra-for-bigos) <br> \
+Please visit respective tab to learn more about the contents of specific dataset. <br><br> \
+Survey of Polish ASR speech datasets and benchmarks is available here: [Polish ASR survey](https://huggingface.co/spaces/amu-cai/pl-asr-survey) <br><br>\
+The latest and most comprehensive ASR benchmarks for Polish is available here: [BIGOS/PELCRA ASR leaderboard](https://huggingface.co/spaces/amu-cai/pl-asr-bigos-bench-dash). <br><br> \
+You can also contact the author via [email](mailto:michal.junczyk@amu.edu.pl) or [LinkedIn](https://www.linkedin.com/in/michaljunczyk/).<br>"
+
+BIGOS_INFO = "TODO"
+PELCRA_INFO = "TODO"
+
+CITATION_MAIN = "@misc{junczyk-2024-pl-asr-bigos-dataset-analysis <br> \
+  title = {Analysis of BIGOS Dataset for Polish ASR evaluation.}, <br> \
+  author = {Michał Junczyk}, <br> \
+  year = {2024}, <br> \
+  publisher = {Hugging Face}, <br> \
+  url = {https://huggingface.co/spaces/amu-cai/amu-bigos-data-dash} }"

requirements.txt

@@ -0,0 +1,6 @@
+datasets
+pandas
+streamlit
+seaborn
+matplotlib
+librosa

run-analysis.py

@@ -0,0 +1,101 @@
+import os
+import json
+from datasets import load_dataset, get_dataset_config_names, Features, Value
+from utils import num_of_samples_per_split, uniq_utts_per_split, words_per_split, uniq_words_per_split, chars_per_split, uniq_chars_per_split
+from utils import audio_duration_per_split, speakers_per_split, meta_cov_per_split
+#, uniq_utts_per_speaker
+from utils import meta_distribution_text, meta_distribution_violin_plot, recordings_per_speaker, speech_rate_words_per_split, speech_rate_chars_per_split
+import argparse
+# move to constants
+output_dir_plots = "./plots"
+output_dir_reports = "./reports"
+os.makedirs(output_dir_plots, exist_ok=True)
+os.makedirs(output_dir_plots, exist_ok=True)
+
+# get as cmd line args
+# read from command line argument
+parser = argparse.ArgumentParser()
+parser.add_argument("--dataset", type=str, required=True, help="Name of the dataset to generate reports for")
+parser.add_argument("--secret_test_split", default=True, type=bool, help="Should references for test split be retrieved from the secret distribution?")
+
+args = parser.parse_args()
+
+dataset_name = args.dataset
+print("Generating reports for dataset: {}".format(dataset_name))
+if (args.secret_test_split):
+    dataset_name_secret = str.join("-", [dataset_name, "secret"])
+    # check if secret repo exists
+    print(dataset_name_secret)
+    try:
+        dataset_configs_secret = get_dataset_config_names(dataset_name_secret)
+    except:
+        print("Config for secret dataset {} cannot be retrieved!".format(dataset_name_secret))
+
+#dataset_name = "amu-cai/pl-asr-bigos-v2"
+output_dir_reports_dataset = os.path.join(output_dir_reports, dataset_name)
+os.makedirs(output_dir_reports_dataset, exist_ok=True)
+
+# get dataset config names
+dataset_configs = get_dataset_config_names(dataset_name)
+
+# initialize output structures
+dataset_statistics = {}
+output_fn_stats = os.path.join(output_dir_reports_dataset, "dataset_statistics.json")
+
+dataset_contents = {}
+output_fn_contents = os.path.join(output_dir_reports_dataset, "dataset_contents.json")
+
+# specify features to load. Skip loading of audio data
+features_to_load = Features({'audioname': Value(dtype='string', id=None), 'split': Value(dtype='string', id=None), 'dataset': Value(dtype='string', id=None), 'speaker_id': Value(dtype='string', id=None), 'ref_orig': Value(dtype='string', id=None), 'audio_duration_samples': Value(dtype='int32', id=None), 'audio_duration_seconds': Value(dtype='float32', id=None), 'samplingrate_orig': Value(dtype='int32', id=None), 'sampling_rate': Value(dtype='int32', id=None), 'audiopath_bigos': Value(dtype='string', id=None), 'audiopath_local': Value(dtype='string', id=None), 'speaker_age': Value(dtype='string', id=None), 'speaker_sex': Value(dtype='string', id=None)})
+
+for config_name in dataset_configs:
+    print("Generating stats for {}".format(config_name))
+    
+    dataset_statistics[config_name] = {}
+    dataset_contents[config_name] = {}
+
+    dataset_hf_subset = load_dataset(dataset_name, config_name, features=features_to_load, trust_remote_code=True)
+    if(args.secret_test_split):
+        dataset_hf_subset_secret = load_dataset(dataset_name_secret, config_name, features=features_to_load, trust_remote_code=True)
+
+    dataset_statistics[config_name]["samples"] = num_of_samples_per_split(dataset_hf_subset)
+    dataset_statistics[config_name]["audio[h]"] = audio_duration_per_split(dataset_hf_subset)
+    dataset_statistics[config_name]["speakers"] = speakers_per_split(dataset_hf_subset)
+
+    # metrics based on transcriptions (references) - requires reading secret repo for test split
+    dataset_statistics[config_name]["utts_unique"], dataset_contents[config_name]["unique_utts"] = uniq_utts_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+    dataset_statistics[config_name]["words"] = words_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+    dataset_statistics[config_name]["words_unique"], dataset_contents[config_name]["unique_words"] = uniq_words_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+    dataset_statistics[config_name]["chars"] = chars_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+    dataset_statistics[config_name]["chars_unique"], dataset_contents[config_name]["unique_chars"] = uniq_chars_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+    dataset_statistics[config_name]["words_per_sec"] = speech_rate_words_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+    dataset_statistics[config_name]["chars_per_sec"] = speech_rate_chars_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+
+    # metadata coverage per subset in percent - speaker accent
+    dataset_statistics[config_name]["meta_cov_sex"] = meta_cov_per_split(dataset_hf_subset, 'speaker_sex')
+    dataset_statistics[config_name]["meta_cov_age"] = meta_cov_per_split(dataset_hf_subset, 'speaker_age')
+
+    # speech rate per subset
+    dataset_statistics[config_name]["meta_dist_sex"] = meta_distribution_text(dataset_hf_subset, 'speaker_sex')
+    dataset_statistics[config_name]["meta_dist_age"] = meta_distribution_text(dataset_hf_subset, 'speaker_age')
+
+    dataset_statistics[config_name]["samples_per_spk"], dataset_contents[config_name]["samples_per_spk"]  = recordings_per_speaker(dataset_hf_subset)
+    # dataset_statistics[config_name] = uniq_utts_per_speaker(dataset_hf_subset)
+    # number of words per speaker (min, max, med, avg, std)
+
+    # distribution of audio duration per subset
+    output_dir_plots_subset = os.path.join(output_dir_plots, config_name)
+    meta_distribution_violin_plot(dataset_hf_subset, output_dir_plots_subset, 'audio_duration_seconds', 'speaker_sex')
+    
+    # distribution of audio duration per age
+    meta_distribution_violin_plot(dataset_hf_subset, output_dir_plots_subset, 'audio_duration_seconds', 'speaker_age')
+
+    
+# save datasets statistics dict to storage as JSON file
+with open(output_fn_stats, 'w') as f:
+    json.dump(dataset_statistics, f)
+
+# save dataset content analysis to storage
+with open(output_fn_contents, 'w') as f:
+    json.dump(dataset_contents, f)
+    

utils.py

@@ -0,0 +1,601 @@
+import json
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+import pandas as pd
+import os
+import numpy as np
+
+# move to consts
+buckets_age=['teens','twenties', 'thirties', 'fourties', 'fifties', 'sixties', 'seventies', 'eighties', 'nineties']
+buckets_sex=["male", "female"]
+
+def load_bigos_analyzer_report(fp:str)->dict:
+    with open(fp, 'r') as f:
+        data = json.load(f)
+    return data
+
+def num_of_samples_per_split(dataset_hf):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about number of samples per split
+    out_dict = {}
+    # number of samples per subset and split
+    metric = "samples"
+    print("Calculating {}".format(metric))
+
+    for split in dataset_hf.keys():
+        samples = dataset_hf[split].num_rows
+        ##print(split, samples)
+        out_dict[split] = samples
+    # add number of samples for all splits
+    out_dict["all_splits"] = sum(out_dict.values())
+
+    return out_dict
+
+def audio_duration_per_split(dataset_hf):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict = {}
+    metric = "audio[h]"
+    print("Calculating {}".format(metric))
+
+    
+    for split in dataset_hf.keys():
+        #sampling_rate = dataset_hf[split]["sampling_rate"][0]
+        #audio_total_length_samples = 0
+        #audio_total_length_samples = sum(len(audio_file["array"]) for audio_file in dataset_hf["test"]["audio"])
+        audio_total_length_seconds = sum(dataset_hf[split]["audio_duration_seconds"])
+        audio_total_length_hours = round(audio_total_length_seconds / 3600,2)
+        out_dict[split] = audio_total_length_hours
+        #print(split, audio_total_length_hours)
+    # add number of samples for all splits
+    out_dict["all_splits"] = sum(out_dict.values())
+    return out_dict
+
+def speakers_per_split(dataset_hf):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict = {}
+    metric = "speakers"
+    print("Calculating {}".format(metric))
+
+    
+    for split in dataset_hf.keys():
+        # extract speakers from file_id 
+        speakers_ids_all = [str(fileid).split("-")[4] for fileid in dataset_hf[split]["audioname"]]
+        speakers_ids_uniq = list(set(speakers_ids_all))
+        speakers_count = len(speakers_ids_uniq)
+        #print(split, speakers_count)
+        out_dict[split] = speakers_count
+    # add number of samples for all splits
+    out_dict["all_splits"] = sum(out_dict.values())
+    return out_dict
+
+
+def uniq_utts_per_split(dataset_hf, dataset_hf_secret):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict = {}
+    metric = "utts_unique"
+    print("Calculating {}".format(metric))
+    utts_all = []
+    for split in dataset_hf.keys():
+        # extract speakers from file_id
+        if (split == "test"):
+            utts_split = dataset_hf_secret[split]["ref_orig"]
+        else:
+            utts_split = dataset_hf[split]["ref_orig"]
+        utts_all = utts_all + utts_split
+        utts_uniq = list(set(utts_split))
+        utts_uniq_count = len(utts_uniq)
+        #print(split, utts_uniq_count)
+        out_dict[split] = utts_uniq_count
+    # add number of samples for all splits
+    out_dict["all_splits"] = len(list(set(utts_all)))
+    return out_dict,utts_all
+
+
+def words_per_split(dataset_hf, dataset_hf_secret):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict = {}
+    metric = "words"
+    print("Calculating {}".format(metric))
+
+    for split in dataset_hf.keys():
+        # extract speakers from file_id 
+        if (split == "test"):
+            utts_all = dataset_hf_secret[split]["ref_orig"]
+        else:
+            utts_all = dataset_hf[split]["ref_orig"]
+        utts_lenghts = [len(utt.split(" ")) for utt in utts_all]
+        words_all_count = sum(utts_lenghts)
+        #print(split, words_all_count)
+        out_dict[split] = words_all_count
+    # add number of samples for all splits
+    out_dict["all_splits"] = sum(out_dict.values())
+    return out_dict
+
+
+def uniq_words_per_split(dataset_hf, dataset_hf_secret):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict = {}
+    out_words_list = []
+    metric = "words_unique"
+    print("Calculating {}".format(metric))
+
+    
+    for split in dataset_hf.keys():
+        # extract speakers from file_id 
+        if (split == "test"):
+            utts_all = dataset_hf_secret[split]["ref_orig"]
+        else:
+            utts_all = dataset_hf[split]["ref_orig"]
+
+        words_all = " ".join(utts_all).split(" ")
+        words_uniq = list(set(words_all))
+        out_words_list = out_words_list + words_uniq
+        words_uniq_count = len(words_uniq)
+        #print(split, words_uniq_count)
+        out_dict[split] = words_uniq_count
+
+    # add number of samples for all splits
+    out_words_uniq = list(set((out_words_list)))
+    out_words_uniq_count = len(out_words_uniq)
+    out_dict["all_splits"] = out_words_uniq_count
+    #print("all", out_words_uniq_count)
+
+    return out_dict, out_words_uniq
+
+
+def chars_per_split(dataset_hf, dataset_hf_secret):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict = {}
+
+    metric = "chars"
+    print("Calculating {}".format(metric))
+
+    
+    for split in dataset_hf.keys():
+        # extract speakers from file_id 
+        if (split=="test"):
+            utts_all = dataset_hf_secret[split]["ref_orig"]
+        else:
+            utts_all = dataset_hf[split]["ref_orig"]
+        words_all = " ".join(utts_all).split(" ")
+        chars_all = " ".join(words_all)
+        chars_all_count = len(chars_all)
+        #print(split, chars_all_count)
+        out_dict[split] = chars_all_count
+    # add number of samples for all splits
+    out_dict["all_splits"] = sum(out_dict.values())
+    return out_dict
+
+
+def uniq_chars_per_split(dataset_hf, dataset_hf_secret):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict = {}
+    out_chars_list = []
+    metric = "chars_unique"
+    print("Calculating {}".format(metric))
+
+    
+    for split in dataset_hf.keys():
+        # extract speakers from file_id 
+        if(split == "test"):
+            utts_all = dataset_hf_secret[split]["ref_orig"]        
+        else:
+            utts_all = dataset_hf[split]["ref_orig"]
+        words_all = " ".join(utts_all).split(" ")
+        words_uniq = list(set(words_all))
+        chars_uniq = list(set("".join(words_uniq)))
+        chars_uniq_count = len(chars_uniq)
+        #print(split, chars_uniq_count)
+        out_dict[split] = chars_uniq_count
+        out_chars_list = out_chars_list + chars_uniq
+    # add number of samples for all splits
+    out_chars_uniq = list(set((out_chars_list)))
+    out_chars_uniq_count = len(out_chars_uniq)
+    out_dict["all_splits"] = out_chars_uniq_count
+    #print("all", out_chars_uniq_count)
+
+    return out_dict, out_chars_uniq
+
+def meta_cov_per_split(dataset_hf, meta_field):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    no_meta=False
+    # TODO move to config
+    if meta_field == 'speaker_age':
+        buckets = buckets_age
+    if meta_field == 'speaker_sex':
+        buckets = buckets_sex
+    out_dict = {}
+    metric = "meta_cov_" + meta_field
+    print("Calculating {}".format(metric))
+
+    meta_info_all = 0
+    meta_info_not_null_all = 0
+    for split in dataset_hf.keys():
+        
+        # extract speakers from file_id
+        meta_info = dataset_hf[split][meta_field]
+        meta_info_count = len(meta_info)
+        meta_info_all += meta_info_count
+        # calculate coverage
+        meta_info_not_null_count = len([x for x in meta_info if x != "N/A"])
+        if meta_info_not_null_count == 0:
+            out_dict[split] = "N/A"
+            continue
+        meta_info_not_null_all += meta_info_not_null_count
+        meta_info_coverage = round(meta_info_not_null_count / meta_info_count, 2)
+        #print(split, meta_info_coverage)
+
+        # add number of samples for all splits
+        out_dict[split] = meta_info_coverage
+
+    # add number of samples for all splits
+    if (meta_info_not_null_all == 0):
+        out_dict["all_splits"] = "N/A"
+    else:
+        out_dict["all_splits"] = round(meta_info_not_null_all/meta_info_all,2 )
+    return out_dict
+
+
+def speech_rate_words_per_split(dataset_hf, dataset_hf_secret):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict = {}
+    metric = "words_per_second"
+    print("Calculating {}".format(metric))
+
+    words_all_count = 0
+    audio_total_length_seconds = 0
+
+    for split in dataset_hf.keys():
+        # extract speakers from file_id 
+        if (split == "test"):
+            utts_split = dataset_hf_secret[split]["ref_orig"]
+        else:
+            utts_split = dataset_hf[split]["ref_orig"]
+        words_split = " ".join(utts_split).split(" ")
+        words_split_count = len(words_split)
+        words_all_count += words_split_count
+        audio_split_length_seconds = sum(dataset_hf[split]["audio_duration_seconds"])
+        audio_total_length_seconds += audio_split_length_seconds
+        speech_rate = round(words_split_count / audio_split_length_seconds, 2)
+        #print(split, speech_rate)
+        out_dict[split] = speech_rate
+    # add number of samples for all splits
+    out_dict["all_splits"] = round(words_all_count / audio_total_length_seconds, 2)
+    return out_dict
+
+def speech_rate_chars_per_split(dataset_hf, dataset_hf_secret):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict = {}
+    metric = "chars_per_second"
+    print("Calculating {}".format(metric))
+
+    chars_all_count = 0
+    audio_total_length_seconds = 0
+
+    for split in dataset_hf.keys():
+        # extract speakers from file_id 
+        if (split == "test"):
+            utts_split = dataset_hf_secret[split]["ref_orig"]
+        else:
+            utts_split = dataset_hf[split]["ref_orig"]
+        words_split = " ".join(utts_split).split(" ")
+        chars_split_count = len("".join(words_split))
+        chars_all_count += chars_split_count
+        audio_split_length_seconds = sum(dataset_hf[split]["audio_duration_seconds"])
+        audio_total_length_seconds += audio_split_length_seconds
+        speech_rate = round(chars_split_count / audio_split_length_seconds, 2)
+        #print(split, speech_rate)
+        out_dict[split] = speech_rate
+    # add number of samples for all splits
+    out_dict["all_splits"] = round(chars_all_count / audio_total_length_seconds, 2)
+    return out_dict
+
+
+# distribution of speaker age
+def meta_distribution_text(dataset_hf, meta_field):
+    no_meta=False
+    if meta_field == 'speaker_age':
+        buckets = buckets_age
+    if meta_field == 'speaker_sex':
+        buckets = buckets_sex
+    
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict = {}
+    metric = "distribution_" + meta_field 
+    print("Calculating {}".format(metric))
+
+    
+    values_count_total = {}
+    for bucket in buckets:
+        values_count_total[bucket]=0
+
+    for split in dataset_hf.keys():
+        out_dict[split] = {}
+        # extract speakers from file_id
+        meta_info = dataset_hf[split][meta_field]
+        meta_info_not_null = [x for x in meta_info if x != "N/A"]
+
+        if len(meta_info_not_null) == 0:
+            out_dict[split]="N/A"
+            no_meta=True
+            continue
+        for bucket in buckets:
+            values_count = meta_info_not_null.count(bucket)
+            values_count_total[bucket] += values_count
+            out_dict[split][bucket] = round(values_count/len(meta_info_not_null),2)
+        #print(split, out_dict[split])
+    
+    # add number of samples for all splits
+    if (no_meta):
+        out_dict["all_splits"] = "N/A"
+        return out_dict
+    
+    out_dict["all_splits"] = {}
+    # calculate total number of samples in values_count_total
+    for bucket in buckets:
+        total_samples = sum(values_count_total.values())
+        out_dict["all_splits"][bucket] = round(values_count_total[bucket]/total_samples,2)
+    return out_dict
+
+
+
+def recordings_per_speaker(dataset_hf):
+    recordings_per_speaker_stats_dict = {}
+
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict_stats = {}
+    out_dict_contents = {}
+
+    metric = "recordings_per_speaker"
+    print("Calculating {}".format(metric))
+    
+    recordings_per_speaker_stats_dict_all = {}
+    recordings_total=0
+
+    speakers_total = 0
+
+    for split in dataset_hf.keys():
+        # extract speakers from file_id 
+        audiopaths = dataset_hf[split]["audioname"]
+        speaker_prefixes = [str(fileid).split("-")[0:5] for fileid in audiopaths]
+
+        speakers_dict_split = {}
+        # create dictionary with list of audio paths matching speaker prefix
+
+        # Create initial dictionary keys from speaker prefixes
+        for speaker_prefix in speaker_prefixes:
+            speaker_prefix_str = "-".join(speaker_prefix)
+            speakers_dict_split[speaker_prefix_str] = []
+
+        # Populate the dictionary with matching audio paths
+        for audio_path in audiopaths:
+            for speaker_prefix_str in speakers_dict_split.keys():
+                if speaker_prefix_str in audio_path:
+                    speakers_dict_split[speaker_prefix_str].append(audio_path)
+
+
+        # iterate of speaker_dict prefixes and calculate number of recordings per speaker.
+        recordings_per_speaker_stats_dict_split = {}
+        for speaker_prefix_str in speakers_dict_split.keys():
+            recordings_per_speaker_stats_dict_split[speaker_prefix_str] = len(speakers_dict_split[speaker_prefix_str])
+        
+        out_dict_contents[split] = {}
+        out_dict_contents[split] = recordings_per_speaker_stats_dict_split    
+        
+        # use recordings_per_speaker_stats to calculate statistics like min, max, avg, median, std
+        out_dict_stats[split] = {}
+        speakers_split = len(list(recordings_per_speaker_stats_dict_split.keys()))
+        speakers_total += speakers_split
+        
+        recordings_split  = len(audiopaths)
+        recordings_total += recordings_split
+
+        average_recordings_per_speaker = round( recordings_split / speakers_split,2)
+        out_dict_stats[split]["average"] = average_recordings_per_speaker
+        out_dict_stats[split]["std"] = round(np.std(list(recordings_per_speaker_stats_dict_split.values())),2)
+        out_dict_stats[split]["median"] = np.median(list(recordings_per_speaker_stats_dict_split.values()))
+        out_dict_stats[split]["min"] = min(recordings_per_speaker_stats_dict_split.values())
+        out_dict_stats[split]["max"] = max(recordings_per_speaker_stats_dict_split.values())
+
+        recordings_per_speaker_stats_dict_all = recordings_per_speaker_stats_dict_all |  recordings_per_speaker_stats_dict_split
+         # add number of samples for all splits
+    
+    average_recordings_per_speaker_all = round( recordings_total / speakers_total , 2)
+    out_dict_stats["all_splits"] = {}
+    out_dict_stats["all_splits"]["average"] = average_recordings_per_speaker_all
+    out_dict_stats["all_splits"]["std"] = round(np.std(list(recordings_per_speaker_stats_dict_all.values())),2)
+    out_dict_stats["all_splits"]["median"] = np.median(list(recordings_per_speaker_stats_dict_all.values()))
+    out_dict_stats["all_splits"]["min"] = min(recordings_per_speaker_stats_dict_all.values())
+    out_dict_stats["all_splits"]["max"] = max(recordings_per_speaker_stats_dict_all.values())
+    out_dict_contents["all_splits"] = recordings_per_speaker_stats_dict_all
+    return out_dict_stats, out_dict_contents
+
+
+def meta_distribution_bar_plot(dataset_hf, output_dir, dimension = "speaker_sex"):
+    pass
+
+def meta_distribution_violin_plot(dataset_hf, output_dir, metric = "audio_duration_seconds",  dimension = "speaker_sex"):
+    # input - huggingface dataset object
+    # output - figure with distribution of audio duration per sex
+    out_dict = {}
+
+    print("Generating violin plat for metric {} for dimension {}".format(metric, dimension))
+    
+    # drop samples for which dimension column values are equal to "N/A"
+    for split in dataset_hf.keys():
+        df_dataset = pd.DataFrame(dataset_hf[split])
+        
+        # remove values equal to "N/A" for column dimension
+        df_filtered = df_dataset[df_dataset[dimension] != "N/A"]  
+        df_filtered = df_filtered[df_filtered[dimension] != "other"]
+        df_filtered = df_filtered[df_filtered[dimension] != "unknown"]
+        if df_filtered.empty:
+            print("No data for split {} and dimension {}".format(split, dimension))
+            continue
+
+        if (len(df_filtered)>=5000):
+            sample_size = 5000
+            print("Selecting sample of size {}".format(sample_size))
+        else:
+            sample_size = len(df_filtered)
+            print("Selecting full split of size {}".format(sample_size))
+        
+        df = df_filtered.sample(sample_size)
+        # if df_filtered is empty, skip violin plot generation for this split and dimension
+
+        print("Generating plot")
+        plt.figure(figsize=(20, 15))
+        plot = sns.violinplot(data = df, hue=dimension, x='dataset', y=metric, split=True, fill = False,inner = 'quart', legend='auto', common_norm=True)
+        plot.set_xticklabels(plot.get_xticklabels(), rotation = 30, horizontalalignment = 'right')
+
+        plt.title('Violin plot of {} by {} for split {}'.format(metric, dimension, split))
+        plt.xlabel(dimension)
+        plt.ylabel(metric)
+        
+        
+        #plt.show(
+        # save figure to file
+        os.makedirs(output_dir, exist_ok=True)
+        output_fn = os.path.join(output_dir, metric + "-" + dimension + "-" + split + ".png") 
+        plt.savefig(output_fn)
+        print("Plot generation completed")
+
+def read_reports(dataset_name):
+    
+    json_contents = "./reports/{}/dataset_contents.json".format(dataset_name)
+    json_stats = "reports/{}/dataset_statistics.json".format(dataset_name)
+
+    with open(json_contents, 'r') as file:
+        contents_dict = json.load(file)
+
+    with open(json_stats, 'r') as file:
+        stats_dict = json.load(file)
+
+    return(stats_dict, contents_dict)
+
+
+def add_test_split_stats_from_secret_dataset(stats_dict_public, stats_dict_secret):
+    # merge contents if dictionaries for fields utts, words, words_unique, chars, chars_unique and speech_rate
+    for dataset in stats_dict_public.keys():
+        print(dataset)
+        for metric in stats_dict_secret[dataset].keys():
+            for split in stats_dict_secret[dataset][metric].keys():
+                if split == "test":
+                    stats_dict_public[dataset][metric][split] = stats_dict_secret[dataset][metric][split]
+
+    return(stats_dict_public)
+
+def dict_to_multindex_df(dict_in, all_splits=False):
+    # Creating a MultiIndex DataFrame
+    rows = []
+    for dataset, metrics in dict_in.items():
+        if (dataset == "all"):
+            continue
+        for metric, splits in metrics.items():
+            for split, value in splits.items():
+                if (all_splits):
+                    if (split == "all_splits"):
+                        rows.append((dataset, metric, split, value))
+                else:
+                    if (split == "all_splits"):
+                        continue
+                    rows.append((dataset, metric, split, value))
+
+    # Convert to DataFrame
+    df = pd.DataFrame(rows, columns=['dataset', 'metric', 'split', 'value'])
+    df.set_index(['dataset', 'metric', 'split'], inplace=True)
+
+    return(df)
+
+
+def dict_to_multindex_df_all_splits(dict_in):
+    # Creating a MultiIndex DataFrame
+    rows = []
+    for dataset, metrics in dict_in.items():
+        if (dataset == "all"):
+            continue
+        for metric, splits in metrics.items():
+            for split, value in splits.items():
+                if (split == "all_splits"):
+                    rows.append((dataset, metric, split, value))
+
+    # Convert to DataFrame
+    df = pd.DataFrame(rows, columns=['dataset', 'metric', 'split', 'value'])
+    df.set_index(['dataset', 'metric', 'split'], inplace=True)
+
+    return(df)
+
+
+def extract_stats_to_agg(df_multindex_per_split, metrics):
+    # input - multiindex dataframe has three indexes - dataset, metric, split 
+    
+    # select only relevant metrics
+    df_agg_splits = df_multindex_per_split.loc[(slice(None), metrics), :]
+    
+    # unstack - move rows per split to columns
+    df_agg_splits = df_agg_splits.unstack(level ='split')
+
+    # aggregate values for all splits
+    df_agg_splits['value', 'total'] = df_agg_splits['value'].sum(axis=1)
+    # drop columns with splits
+    df_agg_splits.columns = df_agg_splits.columns.droplevel(0)
+    columns_to_drop = ['test', 'train', 'validation']
+    df_agg_splits.drop(columns = columns_to_drop, inplace = True)
+
+    # move rows corresponding to specific metrics into specific columns
+    df_agg_splits = df_agg_splits.unstack(level ='metric')
+    df_agg_splits.columns = df_agg_splits.columns.droplevel(0)
+           
+    return(df_agg_splits)
+
+
+
+def extract_stats_all_splits(df_multiindex_all_splits, metrics):
+    
+    df_all_splits = df_multiindex_all_splits.loc[(slice(None), metrics), :]
+
+    df_all_splits = df_all_splits.unstack(level ='metric')
+    df_all_splits.columns = df_all_splits.columns.droplevel(0)
+
+    #print(df_all_splits)
+    df_all_splits = df_all_splits.droplevel('split', axis=0)
+
+    return(df_all_splits)
+
+def extract_stats_for_dataset_card(df_multindex_per_split, subset, metrics, add_total=False):
+    
+    print(df_multindex_per_split)
+    df_metrics_subset = df_multindex_per_split
+    
+    df_metrics_subset = df_metrics_subset.unstack(level ='split')
+    df_metrics_subset.columns = df_metrics_subset.columns.droplevel(0)
+    
+    df_metrics_subset = df_metrics_subset.loc[(slice(None), metrics), :]
+    
+    df_metrics_subset = df_metrics_subset.query("dataset == '{}'".format(subset))
+    # change order of columns to train validation test
+    df_metrics_subset.reset_index(inplace=True)
+    if (add_total):
+        new_columns = ['metric', 'train', 'validation', 'test', 'total']
+        total = df_metrics_subset[['train', 'validation','test']].sum(axis=1)
+        df_metrics_subset['total'] = total
+    else:
+        new_columns = ['metric', 'train', 'validation', 'test']
+   
+    df_metrics_subset = df_metrics_subset.reindex(columns=new_columns)
+    df_metrics_subset.set_index('metric', inplace=True)
+    
+    return(df_metrics_subset)