app.py

"""
Dashboard for showcasing extraction of text metrics with textdescriptives.

"""

from io import StringIO

import pandas as pd
import streamlit as st
import textdescriptives as td

from data_viewer import DataViewer
from process_text import text_to_metrics
from options import (
    all_model_size_options_pretty_to_short,
    available_model_size_options,
    language_options,
    metrics_options,
)

################
# Introduction #
################


col1, col2 = st.columns([9, 2])
with col1:
    st.title("Extract Text Statistics")
with col2:
    st.image(
        "https://github.com/HLasse/TextDescriptives/raw/main/docs/_static/icon.png",
        width=125,
    )

st.write(
    "Calculate a large variety of statistics from text via the "
    "[**TextDescriptives**](https://github.com/HLasse/TextDescriptives) python package "
    f"(v/{td.__version__}) and download the results as a .csv file. "
    "Includes descriptive statistics and metrics related to readability, "
    "information theory, text coherence and text quality."
)

st.write(
    "The source code for this application can be found on [**GitHub**](https://github.com/HLasse/TextDescriptives_app). "
    "If you have feedback, please open an [issue](https://github.com/HLasse/textdescriptives_app/issues)."
)

st.caption(
    "Hansen, L., Olsen, L. R., & Enevoldsen, K. (2023). TextDescriptives: A Python package for "
    "calculating a large variety of metrics from text. [Journal of Open Source Software, 8(84), "
    "5153, https://doi.org/10.21105/joss.05153](https://doi.org/10.21105/joss.05153)"
)


############
# Settings #
############


input_choice = st.radio(
    label="Input", options=["Enter text", "Upload file(s)"], index=0, horizontal=True
)

with st.form(key="settings_form"):
    split_by_line = st.checkbox(label="Split by newline", value=True)

    file_name_to_text_string = {}

    if input_choice == "Upload file(s)":
        uploaded_files = st.file_uploader(
            label="Choose a .txt file", type=["txt"], accept_multiple_files=True
        )

        if uploaded_files is not None and len(uploaded_files) > 0:
            # To convert to a string based IO:
            file_name_to_text_string = {
                file.name: StringIO(file.getvalue().decode("utf-8")).read()
                for file in uploaded_files
            }

    else:
        default_text = """Hello, morning dew. The grass whispers low.
I'm here to dance. The gentle breeze does show.
Good morning, world. The birds sing in delight.
Let's spread our wings. The butterflies take flight.
Nature's chorus sings, a symphony of light."""

        file_name_to_text_string = {
            "input": st.text_area(
                label="Enter text", value=default_text, height=145, max_chars=None
            )
        }

    # Row of selectors
    col1, col2 = st.columns([1, 1])

    with col1:
        # Selection of language
        language_pretty = st.selectbox(
            label="Language",
            options=list(language_options().keys()),
            index=5,
            key="language_selector",
        )

        language_short = language_options()[language_pretty]

    with col2:
        # Selection of model size
        model_size_pretty = st.selectbox(
            label="Model Size",
            options=available_model_size_options(lang="all"),
            index=0,
            key="size_selector",
        )

        model_size_short = all_model_size_options_pretty_to_short()[model_size_pretty]

    # Multiselection of metrics
    metrics = st.multiselect(
        label="Metrics", options=metrics_options(), default=metrics_options()
    )

    st.write(
        "See the [**documentation**](https://hlasse.github.io/TextDescriptives/) for "
        "information on the available metrics."
    )

    # This shouldn't happen but better safe than sorry
    if isinstance(metrics, list) and not metrics:
        metrics = None

    apply_settings_button = st.form_submit_button(label="Apply")


#############
# Apply NLP #
#############


if apply_settings_button and len(file_name_to_text_string) > 0:
    if model_size_pretty not in available_model_size_options(lang=language_short):
        st.write(
            "**Sorry!** The chosen *model size* is not available in this language. Please try another."
        )
    else:
        # Extract metrics for each text
        output_df = pd.concat(
            [
                text_to_metrics(
                    string=string,
                    language_short=language_short,
                    model_size_short=model_size_short,
                    metrics=metrics,
                    split_by_line=split_by_line,
                    filename=filename if "Upload" in input_choice else None,
                )
                for filename, string in file_name_to_text_string.items()
            ],
            ignore_index=True,
        )

        ###################
        # Present Results #
        ###################

        # Create 2 columns with 1) the output header
        # and 2) a download button
        DataViewer()._header_and_download(
            header="The calculated metrics",
            data=output_df,
            file_name="text_metrics.csv",
        )

        st.write("**Note**: This data frame has been transposed for readability.")
        output_df = output_df.transpose().reset_index()
        output_df.columns = ["Metric"] + [str(c) for c in list(output_df.columns)[1:]]
        st.dataframe(data=output_df, use_container_width=True)


############################
# Code For Reproducibility #
############################


with st.expander("See python code"):
    st.code(
        """
# Note: This is the code for a single text file
# The actual code is slightly more complex
# to allow processing multiple files at once

import textdescriptives as td

# Given a string of text and the settings
text = "..."
language = "..."
model_size = "..."
metrics = [...]
split_by_newline = True

# Remove whitespace from both ends of the string
text = text.strip()

# When asked, split by newlines
if split_by_newline:
    lines = text.split("\\n")
else:
    lines = [text]

# Remove empty lines
# E.g. due to consecutive newlines
lines = [l for l in lines if l]

# Extract metrics for each line
extracted_metrics = td.extract_metrics(
    text=lines,
    lang=language,
    spacy_model_size=model_size,
    metrics=metrics
)

""",
        language="python",
    )

#######
# FAQ #
#######

st.subheader("Frequently Asked Questions (FAQ)")

with st.expander("What does the 'Split by newline' option do?"):
    st.write(
        """
    When the `Split by newline` option is `enabled`, the metrics calculation is 
    performed separately for each paragraph. I.e. whenever there's a line break, 
    we split the text.

    When this option is `disabled`, the entire text is processed at once.
    """
    )

with st.expander(
    "Why do I get a warning/error message for certain languages or model sizes?"
):
    st.write(
        """
    Some combinations of languages, model sizes, and metrics are not currently supported in the app. 
    While we *are* working on this, you may currently see a red box
    with an error message after clicking `Apply`. 

    If you need this language and/or model size to work for your project, 
    please open an [issue](https://github.com/HLasse/textdescriptives_app/issues). 
    This may cause us to prioritize supporting your use case.
    """
    )