Fixed filtering of freely and commercially available datasets

app.py

@@ -1,5 +1,7 @@
 import pandas as pd
 import streamlit as st
+import matplotlib.pyplot as plt
+import seaborn as sns
 
 from app_utils import filter_dataframe, calculate_height_to_display
 from contants import INFO_CATALOG, CITATION_CATALOG, HOWTO_CATALOG,INFO_BENCHMARK, CITATION_BENCHMARK, HOWTO_BENCHMARK, INFO_MAIN, CITATION_MAIN, HOWTO_TAXONOMY_CAT
@@ -8,10 +10,6 @@ from utils import  load_data_catalog, load_data_taxonomy, load_bench_catalog, lo
 from utils import datasets_count_and_size, datasets_count_and_size_standard, metadata_coverage, catalog_summary_statistics
 from utils import left_align, right_align
 
-import matplotlib.pyplot as plt
-import seaborn as sns
-
-
 st.set_page_config(layout="wide")
 
 
@@ -23,10 +21,10 @@ df_data_tax = load_data_taxonomy()
 # Filter out non available datasets
 df_data_cat_available = df_data_cat[df_data_cat['Available online'] == 'yes']
 # Available and free
-df_data_cat_available_free = df_data_cat[(df_data_cat['Available online'] == 'yes') & (df_data_cat['Price - non-commercial usage'] == 'free')]
+df_data_cat_available_free = df_data_cat[(df_data_cat['Available online'] == 'yes') & (df_data_cat['Price - non-commercial usage'] == '0')]
 
 # Available and paid    
-df_data_cat_available_paid = df_data_cat[(df_data_cat['Available online'] == 'yes') & (df_data_cat['Price - non-commercial usage'] != 'free')]
+df_data_cat_available_paid = df_data_cat[(df_data_cat['Available online'] == 'yes') & (df_data_cat['Price - non-commercial usage'] != '0')]
 
 
 # Load PL ASR benchmarks survey data
@@ -86,11 +84,17 @@ with data_survey:
 
     st.dataframe(df_datasets_per_year, use_container_width=False)
 
-    st.header("Institutions contributing Polish ASR speech dataset")
+    st.header("Institutions contributing Polish ASR speech datasets")
     col_groupby = ['Publisher']
     df_datasets_per_publisher = datasets_count_and_size(df_data_cat, col_groupby, col_sort='Count Dataset ID', col_percent=None, col_sum=['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID'])
     st.dataframe(df_datasets_per_publisher, use_container_width=False)
 
+    st.header("Institutions contributing freely available Polish ASR speech datasets")
+    col_groupby = ['Publisher']
+    df_datasets_per_publisher_free = datasets_count_and_size(df_data_cat_available_free, col_groupby, col_sort='Count Dataset ID', col_percent=None, col_sum=['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID'])
+    st.dataframe(df_datasets_per_publisher_free, use_container_width=False)
+
+
     st.header("Repositories hosting Polish ASR speech datasets")
     col_groupby = ['Repository']
     df_datasets_per_repo = datasets_count_and_size(df_data_cat, col_groupby, col_sort='Count Dataset ID', col_percent=None, col_sum=['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID'])
@@ -114,19 +118,43 @@ with data_survey:
     st.header("Datasets per speech type")
     col_groupby = ['Speech type']
     df_datasets_per_speech_type = datasets_count_and_size(df_data_cat, col_groupby, col_sort=col_groupby, col_percent = ['Size audio transcribed [hours]'], col_sum = ['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID'])
+    # sort by the size of audio transcribed
+    df_datasets_per_speech_type = df_datasets_per_speech_type.sort_values(by='Size audio transcribed [hours]', ascending=False)
     st.dataframe(df_datasets_per_speech_type, use_container_width=False)
 
+
+    # Display distribution of datasets for various speech types
+    st.header("Distribution of available speech data per audio device - All available datasets")
+    col_groupby = ['Audio device']
+    df_datasets_per_device_all = datasets_count_and_size(df_data_cat, col_groupby, col_sort=col_groupby, col_percent = ['Size audio transcribed [hours]'], col_sum = ['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID'])
+    # sort by the size of audio transcribed
+    df_datasets_per_device_all = df_datasets_per_device_all.sort_values(by='Size audio transcribed [hours]', ascending=False)
+    st.dataframe(df_datasets_per_device_all, use_container_width=False)
+
     # Display distribution of datasets for various speech types
     st.header("Distribution of available speech data per audio device - Public domain datasets")
     col_groupby = ['Audio device']
-    df_datasets_per_device = datasets_count_and_size(df_data_cat_available_free, col_groupby, col_sort=col_groupby, col_percent = ['Size audio transcribed [hours]'], col_sum = ['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID'])
-    st.dataframe(df_datasets_per_device, use_container_width=False)
+    df_datasets_per_device_free = datasets_count_and_size(df_data_cat_available_free, col_groupby, col_sort=col_groupby, col_percent = ['Size audio transcribed [hours]'], col_sum = ['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID'])
+    # sort by the size of audio transcribed
+    df_datasets_per_device_free = df_datasets_per_device_free.sort_values(by='Size audio transcribed [hours]', ascending=False)
+    st.dataframe(df_datasets_per_device_free, use_container_width=False)
 
     # Display distribution of datasets for various speech types
     st.header("Distribution of available speech data per audio device - Commercial datasets")
     col_groupby = ['Audio device']
-    df_datasets_per_device = datasets_count_and_size(df_data_cat_available_paid, col_groupby, col_sort=col_groupby, col_percent = ['Size audio transcribed [hours]'], col_sum = ['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID'])
-    st.dataframe(df_datasets_per_device, use_container_width=False)
+    df_datasets_per_device_paid = datasets_count_and_size(df_data_cat_available_paid, col_groupby, col_sort=col_groupby, col_percent = ['Size audio transcribed [hours]'], col_sum = ['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID'])
+    # sort by the size of audio transcribed
+    df_datasets_per_device_paid = df_datasets_per_device_paid.sort_values(by='Size audio transcribed [hours]', ascending=False)
+    st.dataframe(df_datasets_per_device_paid, use_container_width=False)
+
+    # Display distribution of datasets for various speech types
+    st.header("Datasets per sampling rate")
+    col_groupby = ['Sampling rate [Hz]']
+    df_datasets_per_sr = datasets_count_and_size(df_data_cat, col_groupby, col_sort=col_groupby, col_percent = ['Size audio transcribed [hours]'], col_sum = ['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID'])
+    # sort by the size of audio transcribed
+    df_datasets_per_sr = df_datasets_per_sr.sort_values(by='Size audio transcribed [hours]', ascending=False)
+    st.dataframe(df_datasets_per_sr, use_container_width=False)
+
 
 with data_taxonomy:
     st.title("Polish ASR Speech Data Taxonomy")

contants.py

@@ -1,5 +1,5 @@
 INFO_MAIN= " Welcome to the Polish ASR Survey dashboard! <br> \
-You can use this dashboard to learn about the state of Polish ASR speech data and benchmarks. <br> \
+You can use it to learn about the state of Polish ASR speech data and benchmarks. <br> \
 The dashboard is built upon the [*Polish ASR Speech Datasets Catalog*](https://github.com/goodmike31/pl-asr-speech-data-survey) and [*Polish ASR Benchmarks Catalog*](https://docs.google.com/spreadsheets/d/1fVsE98Ulmt-EIEe4wx8sUdo7RLigDdAVjQxNpAJIrH8/edit?usp=sharing). <br><br> \
 The dashboard is divided into the following tabs: <br> \
 * **About Polish ASR Survey** - general information about the survey, references, and contact points <br> \
@@ -11,7 +11,7 @@ The dashboard is divided into the following tabs: <br> \
 * **ASR Benchmarks Taxonomy** - explanation of the columns in the *Polish ASR Benchmarks Catalog* <br> \
 Please visit respective tab to learn how to use it and provide feedback. <br><br> \
 If you want to share your feedback regarding the Speech Data catalog, please use this [FORM](https://forms.gle/EWJ6YfbJJTyEzQs66). <br><br> \
-If you are looking for the latest ASR benchmarks for Polish, please visit the [BIGOS/PELCRA ASR leaderboard](https://huggingface.co/spaces/amu-cai/pl-asr-bigos-bench-dash). <br><br> \
+If you are looking for the latest ASR benchmarks for Polish, please visit the [AMU ASR leaderboard](https://huggingface.co/spaces/amu-cai/pl-asr-leaderboard). <br><br> \
 You can also contact the author via [email](mailto:michal.junczyk@amu.edu.pl) or [LinkedIn](https://www.linkedin.com/in/michaljunczyk/).<br>"
 
 CITATION_MAIN = "@misc{junczyk-2024-pl-asr-survey <br> \