Added average audio utterance calculation

app.py

@@ -64,7 +64,7 @@ with analysis_bigos:
     st.dataframe(df_sum_stats_text)
 
 
-    metrics_features = ["utts_unique", "words_unique", "chars_unique", "words_per_sec", "chars_per_sec"]
+    metrics_features = ["utts_unique", "words_unique", "chars_unique", "words_per_sec", "chars_per_sec", "average_audio_duration[s]"]
 
     df_sum_stats_all_splits = extract_stats_all_splits(df_multindex_all_splits, metrics_features)
 
@@ -131,12 +131,12 @@ with analysis_bigos_pelcra:
     st.dataframe(df_sum_stats_text)
 
 
-    metrics_features = ["utts_unique", "words_unique", "chars_unique", "words_per_sec", "chars_per_sec"]
+    metrics_features = ["utts_unique", "words_unique", "chars_unique", "words_per_sec", "chars_per_sec", "average_audio_duration[s]"]
 
     df_sum_stats_all_splits = extract_stats_all_splits(df_multindex_all_splits, metrics_features)
 
     st.subheader("Dataset features (text)")
-    df_sum_stats_feats_text = df_sum_stats_all_splits[metrics_features[0:2]]
+    df_sum_stats_feats_text = df_sum_stats_all_splits[metrics_features[0:3]]
     st.dataframe(df_sum_stats_feats_text)
 
     st.subheader("Dataset features (audio)")

reports/amu-cai/pl-asr-bigos-v2/dataset_contents.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9d906a6794c9928df18c639c5b6095f304a64e3b1aa89a7a7ae62ec6e5ea398
+oid sha256:43e808b081d9b692c2469396565fb967105fd815894a7eaded34e89969dbc890
 size 46668863

reports/amu-cai/pl-asr-bigos-v2/dataset_statistics.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c6f9fbd2061fe698519347857bd047c5d756d9358d988d86f8684d7d533c390
-size 22627
+oid sha256:0acb30a9a215f9c96b567b8753f565f400eac2366df6dba6248ccba859e190e3
+size 23940

reports/pelcra/pl-asr-pelcra-for-bigos/dataset_contents.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:341199e4818ae3327eb100d85d0e2ed4d7a3bf81d0f70914d3c434ad763eb30c
+oid sha256:9cea38447dc7485c0f628eba6e52f45e24d1d467fbe23c065162d6b36455ab1d
 size 95274266

reports/pelcra/pl-asr-pelcra-for-bigos/dataset_statistics.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65f591c54e5a1daedc640482f88fd65acd0b2fb193e01f4ed58fb81c54d70cea
-size 29098
+oid sha256:0ab97523e5f4776bb566ed57c38126004bfac43f64bb3177e9ae39f1ee6e51d5
+size 30399

run-analysis.py

@@ -2,7 +2,7 @@ import os
 import json
 from datasets import load_dataset, get_dataset_config_names, Features, Value
 from utils import num_of_samples_per_split, uniq_utts_per_split, words_per_split, uniq_words_per_split, chars_per_split, uniq_chars_per_split
-from utils import audio_duration_per_split, speakers_per_split, meta_cov_per_split
+from utils import total_audio_duration_per_split, average_audio_duration_per_split, speakers_per_split, meta_cov_per_split
 #, uniq_utts_per_speaker
 from utils import meta_distribution_text, meta_distribution_violin_plot, recordings_per_speaker, speech_rate_words_per_split, speech_rate_chars_per_split
 import argparse
@@ -58,19 +58,26 @@ for config_name in dataset_configs:
     if(args.secret_test_split):
         dataset_hf_subset_secret = load_dataset(dataset_name_secret, config_name, features=features_to_load, trust_remote_code=True)
 
+    #audio content size
     dataset_statistics[config_name]["samples"] = num_of_samples_per_split(dataset_hf_subset)
-    dataset_statistics[config_name]["audio[h]"] = audio_duration_per_split(dataset_hf_subset)
+    dataset_statistics[config_name]["audio[h]"] = total_audio_duration_per_split(dataset_hf_subset)
     dataset_statistics[config_name]["speakers"] = speakers_per_split(dataset_hf_subset)
 
+    # text content size
     # metrics based on transcriptions (references) - requires reading secret repo for test split
-    dataset_statistics[config_name]["utts_unique"], dataset_contents[config_name]["unique_utts"] = uniq_utts_per_split(dataset_hf_subset, dataset_hf_subset_secret)
     dataset_statistics[config_name]["words"] = words_per_split(dataset_hf_subset, dataset_hf_subset_secret)
-    dataset_statistics[config_name]["words_unique"], dataset_contents[config_name]["unique_words"] = uniq_words_per_split(dataset_hf_subset, dataset_hf_subset_secret)
     dataset_statistics[config_name]["chars"] = chars_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+
+    # text content derived features
+    dataset_statistics[config_name]["utts_unique"], dataset_contents[config_name]["unique_utts"] = uniq_utts_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+    dataset_statistics[config_name]["words_unique"], dataset_contents[config_name]["unique_words"] = uniq_words_per_split(dataset_hf_subset, dataset_hf_subset_secret)
     dataset_statistics[config_name]["chars_unique"], dataset_contents[config_name]["unique_chars"] = uniq_chars_per_split(dataset_hf_subset, dataset_hf_subset_secret)
+    
+    # audio content derived features
     dataset_statistics[config_name]["words_per_sec"] = speech_rate_words_per_split(dataset_hf_subset, dataset_hf_subset_secret)
     dataset_statistics[config_name]["chars_per_sec"] = speech_rate_chars_per_split(dataset_hf_subset, dataset_hf_subset_secret)
-
+    dataset_statistics[config_name]["average_audio_duration[s]"] = average_audio_duration_per_split(dataset_hf_subset)
+    
     # metadata coverage per subset in percent - speaker accent
     dataset_statistics[config_name]["meta_cov_sex"] = meta_cov_per_split(dataset_hf_subset, 'speaker_sex')
     dataset_statistics[config_name]["meta_cov_age"] = meta_cov_per_split(dataset_hf_subset, 'speaker_age')

utils.py

@@ -32,7 +32,7 @@ def num_of_samples_per_split(dataset_hf):
 
     return out_dict
 
-def audio_duration_per_split(dataset_hf):
+def total_audio_duration_per_split(dataset_hf):
     # input - huggingface dataset object
     # output - dictionary with statistics about audio duration per split
     out_dict = {}
@@ -52,6 +52,31 @@ def audio_duration_per_split(dataset_hf):
     out_dict["all_splits"] = sum(out_dict.values())
     return out_dict
 
+
+def average_audio_duration_per_split(dataset_hf):
+    # input - huggingface dataset object
+    # output - dictionary with statistics about audio duration per split
+    out_dict = {}
+    metric = "average_audio_duration[s]"
+    print("Calculating {}".format(metric))
+    samples_all=0
+    audio_length_total_seconds=0
+    for split in dataset_hf.keys():
+        #sampling_rate = dataset_hf[split]["sampling_rate"][0]
+        #audio_total_length_samples = 0
+        #audio_total_length_samples = sum(len(audio_file["array"]) for audio_file in dataset_hf["test"]["audio"])
+        audio_length_split_seconds = sum(dataset_hf[split]["audio_duration_seconds"])
+        audio_length_total_seconds += audio_length_split_seconds
+
+        samples_split = len(dataset_hf[split]["audio_duration_seconds"])
+        samples_all += samples_split
+        audio_average_length_seconds = round(audio_length_split_seconds / samples_split,2)
+        out_dict[split] = audio_average_length_seconds
+        #print(split, audio_total_length_hours)
+    # add number of samples for all splits
+    out_dict["all_splits"] = round(audio_length_total_seconds / samples_all,2)
+    return out_dict
+
 def speakers_per_split(dataset_hf):
     # input - huggingface dataset object
     # output - dictionary with statistics about audio duration per split
@@ -350,7 +375,6 @@ def meta_distribution_text(dataset_hf, meta_field):
     return out_dict
 
 
-
 def recordings_per_speaker(dataset_hf):
     recordings_per_speaker_stats_dict = {}