add prop of non-ascii

app.py

@@ -1,10 +1,13 @@
 import requests
 from collections import Counter
 from requests.adapters import HTTPAdapter, Retry
+import multiprocessing
 
 import gradio as gr
 import pandas as pd
 import polars as pl
+import numpy as np
+import matplotlib.pyplot as plt
 import spaces
 from gradio_huggingfacehub_search import HuggingfaceHubSearch
 from huggingface_hub import PyTorchModelHubMixin
@@ -19,6 +22,21 @@ retries = Retry(total=5, backoff_factor=1, status_forcelist=[502, 503, 504])
 session.mount('http://', HTTPAdapter(max_retries=retries))
 
 
+def proportion_non_ascii(s):
+    """
+    Compute the proportion of non-ASCII characters in a string.
+
+    Parameters:
+    s (str): The input string.
+
+    Returns:
+    float: The proportion of non-ASCII characters in the string.
+    """
+    non_ascii_count = sum(1 for c in s if ord(c) > 127)
+    total_chars = len(s)
+    return non_ascii_count / total_chars if total_chars > 0 else 0.0
+
+
 class QualityModel(nn.Module, PyTorchModelHubMixin):
     def __init__(self, config):
         super(QualityModel, self).__init__()
@@ -76,7 +94,7 @@ def run_quality_check(dataset, column, batch_size, num_examples):
     # config = "default"
     info_resp = session.get(f"https://datasets-server.huggingface.co/info?dataset={dataset}", timeout=3).json()
     if "error" in info_resp:
-        yield "❌ " + info_resp["error"], gr.BarPlot(), pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
+        yield "❌ " + info_resp["error"], gr.BarPlot(), pd.DataFrame(), pd.DataFrame(), pd.DataFrame(), plt.Figure()
         return
     config = "default" if "default" in info_resp["dataset_info"] else next(iter(info_resp["dataset_info"]))
     split = "train" if "train" in info_resp["dataset_info"][config]["splits"] else next(
@@ -87,7 +105,7 @@ def run_quality_check(dataset, column, batch_size, num_examples):
         try:
             data = pl.read_parquet(f"hf://datasets/{dataset}@~parquet/{config}/partial-{split}/0000.parquet", columns=[column])
         except Exception as error:
-            yield f"❌ {error}", gr.BarPlot(), pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
+            yield f"❌ {error}", gr.BarPlot(), pd.DataFrame(), pd.DataFrame(), pd.DataFrame(), plt.Figure()
             return
     texts = data[column].to_list()
     # batch_size = 100
@@ -98,8 +116,18 @@ def run_quality_check(dataset, column, batch_size, num_examples):
         batch_predictions = predict(batch_texts)
         predictions.extend(batch_predictions)
         texts_processed.extend(batch_texts)
-        yield {"check in progress...": (i+batch_size) / num_examples}, *plot_and_df(texts_processed, predictions)
-    yield {"finished": 1.}, *plot_and_df(texts_processed, predictions)
+        yield {"check in progress...": (i+batch_size) / num_examples}, *plot_and_df(texts_processed, predictions), plt.Figure()
+
+    with multiprocessing.Pool(processes=8) as pool:
+        props = pool.map(proportion_non_ascii, texts)
+
+    # non_ascii_df = pd.DataFrame.from_dict({"prop_non_ascii": props, "text": texts})
+    plt.hist(props, bins=20, range=(0., 1.))
+    plt.title('Histogram of proportion of non-ASCII characters')
+    plt.xlabel('Proportion of non-ASCII characters')
+    plt.ylabel('Number of texts')
+
+    yield {"finished": 1.}, *plot_and_df(texts_processed, predictions), plt.gcf()
 
 with gr.Blocks() as demo:
     gr.Markdown(
@@ -135,6 +163,7 @@ with gr.Blocks() as demo:
     progress_bar = gr.Label(show_label=False)
     plot = gr.BarPlot()
 
+
     with gr.Accordion("Explore some individual examples for each class", open=False):
         gr.Markdown("### Low")
         df_low = gr.DataFrame()
@@ -142,6 +171,9 @@ with gr.Blocks() as demo:
         df_medium = gr.DataFrame()
         gr.Markdown("### High")
         df_high = gr.DataFrame()
-    gr_check_btn.click(run_quality_check, inputs=[dataset_name, text_column, batch_size, num_examples], outputs=[progress_bar, plot, df_low, df_medium, df_high])
+
+    # non_ascii_hist = gr.DataFrame(visible=False)
+    non_ascii_hist = gr.Plot()
+    gr_check_btn.click(run_quality_check, inputs=[dataset_name, text_column, batch_size, num_examples], outputs=[progress_bar, plot, df_low, df_medium, df_high, non_ascii_hist])
 
 demo.launch()