black

posts/conclusion.py

@@ -19,40 +19,52 @@ Next, please answer the following questions about the information presented in t
 
 def run_article():
     st.markdown(__KEY_TAKEAWAYS)
-    st.text_area(
+    res = {}
+    res["used_links"] = st.text_area(
         "Did you click on any of the links provided in the **Hate Speech in ACM** page? If so, which one did you find most surprising?"
     )
-    st.text_area(
+    res["dataset_feedback"] = st.text_area(
         "Of the datasets presented in the **Dataset Exploration** page, which one did you think best represented content that should be moderated? Which worst?"
     )
-    st.text_area(
+    res["model_feedback"] = st.text_area(
         "Of the models presented in the **Model Exploration** page, which one did you think performed best? Which worst?"
     )
-    st.text_area(
+    res["additional_material"] = st.text_area(
         "Any additional comments about the materials?"
     )
     # from paper
-    st.text_area(
+    res["role"] = st.text_area(
         "How would you describe your role? E.g. model developer, dataset developer, domain expert, policy maker, platform manager, community advocate, platform user, student"
     )
-    st.text_area(
-        "Why are you interested in content moderation?"    
+    res["interest"] = st.text_area("Why are you interested in content moderation?")
+    res["modules_used"] = st.multiselect(
+        "Which modules did you use the most?",
+        options=[
+            "Welcome - Introduction",
+            "Hate Speech in ACM",
+            "Dataset Exploration",
+            "Model Exploration",
+        ],
     )
-    st.text_area(
-        "Which modules did you use the most?"    
+    res["modules_informative"] = st.selectbox(
+        "Which module did you find the most informative?",
+        options=[
+            "Welcome - Introduction",
+            "Hate Speech in ACM",
+            "Dataset Exploration",
+            "Model Exploration",
+        ],
     )
-    st.text_area(
-        "Which module did you find the most informative?"    
+    res["application)interest"] = st.text_area(
+        "Which application were you most interested in learning more about?"
     )
-    st.text_area(
-        "Which application were you most interested in learning more about?"    
+    res["dataset_surprise"] = st.text_area(
+        "What surprised you most about the datasets?"
     )
-    st.text_area(
-        "What surprised you most about the datasets?"    
+    res["model_concern"] = st.text_area(
+        "Which models are you most concerned about as a user?"
     )
-    st.text_area(
-        "Which models are you most concerned about as a user?"    
+    res["comments_suggestions"] = st.text_area(
+        "Do you have any comments or suggestions?"
     )
-    st.text_area(
-        "Do you have any comments or suggestions?"    
-    )
+    st.write(res)

posts/context.py

@@ -86,6 +86,7 @@ __CRITIC_EXAMPLES = """
 - [Reddit Self Reflection on Lack of Content Policy](https://www.reddit.com/r/announcements/comments/gxas21/upcoming_changes_to_our_content_policy_our_board/)
 """
 
+
 def run_article():
     st.markdown("## Automatic Content Moderation (ACM)")
     with st.expander("ACM definition", expanded=False):

posts/dataset_exploration.py

@@ -58,35 +58,82 @@ the labels for those examples, and the distribution of labels within the
 cluster. Note that cluster 0 will always be the full dataset.
 """
 
-DSET_OPTIONS = {'classla/FRENK-hate-en': {'binary': {'train': {('text',): {'label': {100000: {
-       'sentence-transformers/all-mpnet-base-v2': {'tree': {'dataset_name': 'classla/FRENK-hate-en',
-         'config_name': 'binary',
-         'split_name': 'train',
-         'input_field_path': ('text',),
-         'label_name': 'label',
-         'num_rows': 100000,
-         'model_name': 'sentence-transformers/all-mpnet-base-v2',
-         'file_name': 'tree'}}}}}}}},
- 'tweets_hate_speech_detection': {'default': {'train': {('tweet',): {'label': {100000: {
-       'sentence-transformers/all-mpnet-base-v2': {'tree': {'dataset_name': 'tweets_hate_speech_detection',
-         'config_name': 'default',
-         'split_name': 'train',
-         'input_field_path': ('tweet',),
-         'label_name': 'label',
-         'num_rows': 100000,
-         'model_name': 'sentence-transformers/all-mpnet-base-v2',
-         'file_name': 'tree'}}}}}}}},
- 'ucberkeley-dlab/measuring-hate-speech': {'default': {'train': {('text',): {'hatespeech': {100000: {
-       'sentence-transformers/all-mpnet-base-v2': {'tree': {'dataset_name': 'ucberkeley-dlab/measuring-hate-speech',
-         'config_name': 'default',
-         'split_name': 'train',
-         'input_field_path': ('text',),
-         'label_name': 'hatespeech',
-         'num_rows': 100000,
-         'model_name': 'sentence-transformers/all-mpnet-base-v2',
-         'file_name': 'tree'}}}}}}}},
+DSET_OPTIONS = {
+    "classla/FRENK-hate-en": {
+        "binary": {
+            "train": {
+                ("text",): {
+                    "label": {
+                        100000: {
+                            "sentence-transformers/all-mpnet-base-v2": {
+                                "tree": {
+                                    "dataset_name": "classla/FRENK-hate-en",
+                                    "config_name": "binary",
+                                    "split_name": "train",
+                                    "input_field_path": ("text",),
+                                    "label_name": "label",
+                                    "num_rows": 100000,
+                                    "model_name": "sentence-transformers/all-mpnet-base-v2",
+                                    "file_name": "tree",
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "tweets_hate_speech_detection": {
+        "default": {
+            "train": {
+                ("tweet",): {
+                    "label": {
+                        100000: {
+                            "sentence-transformers/all-mpnet-base-v2": {
+                                "tree": {
+                                    "dataset_name": "tweets_hate_speech_detection",
+                                    "config_name": "default",
+                                    "split_name": "train",
+                                    "input_field_path": ("tweet",),
+                                    "label_name": "label",
+                                    "num_rows": 100000,
+                                    "model_name": "sentence-transformers/all-mpnet-base-v2",
+                                    "file_name": "tree",
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "ucberkeley-dlab/measuring-hate-speech": {
+        "default": {
+            "train": {
+                ("text",): {
+                    "hatespeech": {
+                        100000: {
+                            "sentence-transformers/all-mpnet-base-v2": {
+                                "tree": {
+                                    "dataset_name": "ucberkeley-dlab/measuring-hate-speech",
+                                    "config_name": "default",
+                                    "split_name": "train",
+                                    "input_field_path": ("text",),
+                                    "label_name": "hatespeech",
+                                    "num_rows": 100000,
+                                    "model_name": "sentence-transformers/all-mpnet-base-v2",
+                                    "file_name": "tree",
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    },
 }
 
+
 @st.cache(allow_output_mutation=True)
 def download_tree(args):
     clusters = Clustering(**args)
@@ -115,7 +162,7 @@ def run_article():
 
     pre_args = DSET_OPTIONS[choose_dset]
     args = pre_args
-    while not 'dataset_name' in args:
+    while not "dataset_name" in args:
         args = list(args.values())[0]
 
     clustering = download_tree(args)
@@ -130,8 +177,12 @@ def run_article():
         "Visualize cluster node:",
         range(len(clustering.node_list)),
     )
-    st.markdown(f"Node {show_node} has {clustering.node_list[show_node]['weight']} examples.")
-    st.markdown(f"Node {show_node} was merged at {clustering.node_list[show_node]['merged_at']:.2f}.")
+    st.markdown(
+        f"Node {show_node} has {clustering.node_list[show_node]['weight']} examples."
+    )
+    st.markdown(
+        f"Node {show_node} was merged at {clustering.node_list[show_node]['merged_at']:.2f}."
+    )
     examplars = clustering.get_node_examplars(show_node)
     st.markdown("---\n")
 

posts/model_exploration.py

@@ -1,10 +1,10 @@
-import streamlit as st
 import json
 import random
 import sys
 
 import numpy as np
 import pandas as pd
+import streamlit as st
 # from transformers import AutoTokenizer, AutoModelForSequenceClassification
 from transformers import pipeline
 
@@ -95,31 +95,31 @@ sentence using the tool below.
 """
 
 __MODELS = [
-        "classla/roberta-base-frenk-hate",
-        "cardiffnlp/twitter-roberta-base-hate",
-        "Hate-speech-CNERG/dehatebert-mono-english",
-        "facebook/roberta-hate-speech-dynabench-r1-target",
-        "facebook/roberta-hate-speech-dynabench-r2-target",
-        "facebook/roberta-hate-speech-dynabench-r3-target",
-        "facebook/roberta-hate-speech-dynabench-r4-target"
-    ]
+    "classla/roberta-base-frenk-hate",
+    "cardiffnlp/twitter-roberta-base-hate",
+    "Hate-speech-CNERG/dehatebert-mono-english",
+    "facebook/roberta-hate-speech-dynabench-r1-target",
+    "facebook/roberta-hate-speech-dynabench-r2-target",
+    "facebook/roberta-hate-speech-dynabench-r3-target",
+    "facebook/roberta-hate-speech-dynabench-r4-target",
+]
 
 autopop_ranking = [
     [
         "I don't hate black people.",
         "Black people should die.",
-        "My friend likes pancakes."
+        "My friend likes pancakes.",
     ],
     [
         "Those dirty redskins are all drunks.",
         "I can't stand Chinese food. It's too spicy.",
-        "I lived near many Chinese people and I hated every single one of them."
+        "I lived near many Chinese people and I hated every single one of them.",
     ],
     [
         "Stop saying that Mexicans don't belong in our country!",
         "We walked along the dyke down the road.",
-        "Ah shit, I fucked up."
-    ]
+        "Ah shit, I fucked up.",
+    ],
 ]
 
 # Creates the forms for receiving multiple inputs to compare for a single
@@ -130,22 +130,13 @@ def run_article():
         st.markdown(__HATE_DETECTION)
     hc_path = "posts/resources/"
     hc_pholders = json.load(
-        open(
-            hc_path + "template_placeholders.json",
-            encoding="utf-8"
-        )
+        open(hc_path + "template_placeholders.json", encoding="utf-8")
     )
     hc_templates = json.load(
-        open(
-            hc_path + "hatecheck_category_templates.json",
-            encoding="utf-8"
-        )
+        open(hc_path + "hatecheck_category_templates.json", encoding="utf-8")
     )
     hc_info = json.load(
-        open(
-            hc_path + "hatecheck_category_info.json",
-            encoding="utf-8"
-        )
+        open(hc_path + "hatecheck_category_info.json", encoding="utf-8")
     )
     hc_cats = [""] + list(hc_info.keys())
 
@@ -153,10 +144,8 @@ def run_article():
     with st.expander("HateCheck Examples", expanded=False):
         st.markdown(__HATECHECK)
         category = st.selectbox(
-                "Select a category of examples from HateCheck",
-                hc_cats,
-                key="hc_cat_select"
-            )
+            "Select a category of examples from HateCheck", hc_cats, key="hc_cat_select"
+        )
         if category:
             with st.form(key="hate_check"):
                 hc_cat = hc_info[category]
@@ -166,24 +155,22 @@ def run_article():
                     templates.append(hc_temp)
                     names.append(hc_cat[hc_temp]["name"])
                 selected_names = st.multiselect(
-                        "Select one or more HateCheck templates to generate examples for",
-                        names,
-                        key="hc_temp_multiselect"
+                    "Select one or more HateCheck templates to generate examples for",
+                    names,
+                    key="hc_temp_multiselect",
                 )
                 num_exs = st.number_input(
-                        "Select a number of examples to generate for each selected template",
-                        min_value = 1,
-                        max_value = 5,
-                        value = 3
-                        )
+                    "Select a number of examples to generate for each selected template",
+                    min_value=1,
+                    max_value=5,
+                    value=3,
+                )
                 if st.form_submit_button(label="Generate Examples"):
                     for name in selected_names:
                         index = names.index(name)
                         template = templates[index]
                         examples = generate_hc_ex(
-                            hc_templates[template],
-                            hc_pholders,
-                            num_exs
+                            hc_templates[template], hc_pholders, num_exs
                         )
                         st.header(name)
                         st.subheader("Label: " + hc_cat[template]["value"])
@@ -215,12 +202,12 @@ def run_article():
             input_3 = st.text_input(
                 "Input 3",
                 help="Try a phrase like 'Good morning'",
-                #placeholder="Good morning."
+                # placeholder="Good morning."
             )
             autopop = st.checkbox(
-                'Choose examples for me',
+                "Choose examples for me",
                 key="rank_autopop_ckbx",
-                help="Check this box to run the model with 3 preselected sentences."
+                help="Check this box to run the model with 3 preselected sentences.",
             )
             if st.form_submit_button(label="Rank inputs"):
                 if autopop:
@@ -246,9 +233,9 @@ def run_article():
                 key="compare_model_2",
             )
             autopop = st.checkbox(
-                'Choose an example for me',
+                "Choose an example for me",
                 key="comp_autopop_ckbx",
-                help="Check this box to compare the models with a preselected sentence."
+                help="Check this box to compare the models with a preselected sentence.",
             )
             input_text = st.text_input("Comparison input")
             if st.form_submit_button(label="Compare models"):
@@ -257,16 +244,11 @@ def run_article():
                 results = run_compare(model_name_1, model_name_2, input_text)
                 st.write("### Showing results for: " + input_text)
                 st.dataframe(results)
-                outside_ds = [
-                              "hatecheck",
-                              "dynabench",
-                              "hatefulmemes",
-                              "opensubtitles"
-                       ]
+                outside_ds = ["hatecheck", "dynabench", "hatefulmemes", "opensubtitles"]
                 name_1_short = model_name_1.split("/")[1]
                 name_2_short = model_name_2.split("/")[1]
                 for calib_ds in outside_ds:
-                    ds_loc = "posts/resources/charts/" + calib_ds + "/" 
+                    ds_loc = "posts/resources/charts/" + calib_ds + "/"
                     images, captions = [], []
                     for model in [name_1_short, name_2_short]:
                         images.append(ds_loc + model + "_" + calib_ds + ".png")
@@ -274,6 +256,7 @@ def run_article():
                     st.write("#### Model performance comparison on " + calib_ds)
                     st.image(images, captions)
 
+
 #                if model_name_1 == "Hate-speech-CNERG/dehatebert-mono-english":
 #                    st.image("posts/resources/dehatebert-mono-english_calibration.png")
 #                elif model_name_1 == "cardiffnlp/twitter-roberta-base-hate":
@@ -303,11 +286,7 @@ def generate_hc_ex(template, placeholders, gen_num):
 # Runs the received input strings through the given model and returns the
 # all scores for all possible labels as a DataFrame
 def run_ranked(model, input_list):
-    classifier = pipeline(
-        "text-classification",
-        model=model,
-        return_all_scores=True
-    )
+    classifier = pipeline("text-classification", model=model, return_all_scores=True)
     output = {}
     results = classifier(input_list)
     for result in results:

posts/welcome.py

@@ -55,6 +55,7 @@ __MODEL_LIST = """
 - [RoBERTa trained on 11 English hate speech datasets and Rounds 1, 2, and 3 of the Dynamically Generated Hate Speech Dataset](https://huggingface.co/facebook/roberta-hate-speech-dynabench-r4-target)
 """
 
+
 def run_article():
     st.markdown("# Welcome!")
     st.markdown(__INTRO_TEXT)