Displaying target_text with [MASK]. More doc improvements. Formatting.

app.py

@@ -22,7 +22,8 @@ DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 MAX_TOKEN_LENGTH = 128
 NON_LOSS_TOKEN_ID = -100
 NON_GENDERED_TOKEN_ID = 30  # Picked an int that will pop out visually
-LABEL_DICT = {"female": 9, "male": -9}  # Picked an int that will pop out visually
+# Picked an int that will pop out visually
+LABEL_DICT = {"female": 9, "male": -9}
 CLASSES = list(LABEL_DICT.keys())
 
 
@@ -48,12 +49,14 @@ for var in CONDITIONING_VARIABLES:
         )
 for bert_like in BERT_LIKE_MODELS:
     models_paths[(bert_like,)] = f"{bert_like}-base-uncased"
-    models[(bert_like,)] = pipeline("fill-mask", model=models_paths[(bert_like,)])
+    models[(bert_like,)] = pipeline(
+        "fill-mask", model=models_paths[(bert_like,)])
 
 
 # Tokenizers same for each model, so just grabbing one of them
 tokenizer = AutoTokenizer.from_pretrained(
-    models_paths[(CONDITIONING_VARIABLES[0], FEMALE_WEIGHTS[0])], add_prefix_space=True
+    models_paths[(CONDITIONING_VARIABLES[0], FEMALE_WEIGHTS[0])
+                 ], add_prefix_space=True
 )
 MASK_TOKEN_ID = tokenizer.mask_token_id
 
@@ -72,7 +75,8 @@ male_gendered_tokens = [list[0] for list in gendered_lists]
 female_gendered_tokens = [list[1] for list in gendered_lists]
 
 male_gendered_token_ids = tokenizer.convert_tokens_to_ids(male_gendered_tokens)
-female_gendered_token_ids = tokenizer.convert_tokens_to_ids(female_gendered_tokens)
+female_gendered_token_ids = tokenizer.convert_tokens_to_ids(
+    female_gendered_tokens)
 
 assert tokenizer.unk_token_id not in male_gendered_token_ids
 assert tokenizer.unk_token_id not in female_gendered_token_ids
@@ -122,7 +126,8 @@ def tokenize_and_append_metadata(text, tokenizer):
         male_tags == LABEL_DICT["male"], label2id[LABEL_DICT["male"]], labels
     )
     masked_token_ids = torch.where(
-        female_tags == LABEL_DICT["female"], MASK_TOKEN_ID, torch.tensor(token_ids)
+        female_tags == LABEL_DICT["female"], MASK_TOKEN_ID, torch.tensor(
+            token_ids)
     )
     masked_token_ids = torch.where(
         male_tags == LABEL_DICT["male"], MASK_TOKEN_ID, masked_token_ids
@@ -138,22 +143,23 @@ def get_tokenized_text_with_years(years, input_text):
     """Construct dict of tokenized texts with each year injected into the text."""
     text_portions = input_text.split(SPLIT_KEY)
 
-    tokenized_w_year = {'ids':[], 'atten_mask':[], 'toks':[], 'labels':[]}
+    tokenized_w_year = {'ids': [], 'atten_mask': [], 'toks': [], 'labels': []}
     for b_date in years:
 
         target_text = f"{b_date}".join(text_portions)
         tokenized_sample = tokenize_and_append_metadata(
             target_text,
-            tokenizer=tokenizer,    
+            tokenizer=tokenizer,
         )
 
         tokenized_w_year['ids'].append(tokenized_sample["input_ids"])
-        tokenized_w_year['atten_mask'].append(torch.tensor(tokenized_sample["attention_mask"]))
-        tokenized_w_year['toks'].append(tokenizer.convert_ids_to_tokens(tokenized_sample["input_ids"]))
+        tokenized_w_year['atten_mask'].append(
+            torch.tensor(tokenized_sample["attention_mask"]))
+        tokenized_w_year['toks'].append(
+            tokenizer.convert_ids_to_tokens(tokenized_sample["input_ids"]))
         tokenized_w_year['labels'].append(tokenized_sample["labels"])
 
-    # Also returning last `target_text`` to display as example text
-    return tokenized_w_year, target_text  
+    return tokenized_w_year
 
 
 def predict_gender_pronouns(
@@ -165,7 +171,8 @@ def predict_gender_pronouns(
 
     years = np.linspace(START_YEAR, STOP_YEAR, int(num_points)).astype(int)
 
-    tokenized, target_text = get_tokenized_text_with_years(years, input_text)
+    tokenized = get_tokenized_text_with_years(years, input_text)
+
     is_masked = tokenized['ids'][0] == MASK_TOKEN_ID
     num_preds = torch.sum(is_masked).item()
 
@@ -184,17 +191,20 @@ def predict_gender_pronouns(
                 labels = tokenized["labels"][year_idx]
 
                 with torch.no_grad():
-                    outputs = model(ids.unsqueeze(dim=0), atten_mask.unsqueeze(dim=0))
+                    outputs = model(ids.unsqueeze(dim=0),
+                                    atten_mask.unsqueeze(dim=0))
                     preds = torch.argmax(outputs[0][0].cpu(), dim=1)
 
                     #was_masked = labels.cpu() != -100
                     preds = torch.where(is_masked, preds, -100)
 
-                    p_female.append(len(torch.where(preds == 0)[0]) / num_preds * 100)
-                    p_male.append(len(torch.where(preds == 1)[0]) / num_preds * 100)
-
-            dfs.append(pd.DataFrame({f"%f_{prefix}": p_female, f"%m_{prefix}": p_male}))
+                    p_female.append(
+                        len(torch.where(preds == 0)[0]) / num_preds * 100)
+                    p_male.append(
+                        len(torch.where(preds == 1)[0]) / num_preds * 100)
 
+            dfs.append(pd.DataFrame(
+                {f"%f_{prefix}": p_female, f"%m_{prefix}": p_male}))
 
     for bert_like in bert_like_models:
 
@@ -202,13 +212,13 @@ def predict_gender_pronouns(
         p_male = []
         for year_idx in range(len(tokenized['ids'])):
             toks = tokenized["toks"][year_idx]
-            target_text_for_bert = ' '.join(toks[1:-1] ) # Removing [CLS] and [SEP]
+            target_text_for_bert = ' '.join(
+                toks[1:-1])  # Removing [CLS] and [SEP]
 
             prefix = bert_like
             model = models[(bert_like,)]
 
-
-            mask_filled_text =  model(target_text_for_bert)
+            mask_filled_text = model(target_text_for_bert)
 
             female_pronouns = [
                 1 if pronoun[0]["token_str"] in female_gendered_tokens else 0
@@ -222,8 +232,12 @@ def predict_gender_pronouns(
             p_female.append(sum(female_pronouns) / num_preds * 100)
             p_male.append(sum(male_pronouns) / num_preds * 100)
 
-        dfs.append(pd.DataFrame({f"%f_{prefix}": p_female, f"%m_{prefix}": p_male}))
+        dfs.append(pd.DataFrame(
+            {f"%f_{prefix}": p_female, f"%m_{prefix}": p_male}))
 
+    # To display to user as an example
+    toks = tokenized["toks"][0]
+    target_text_w_masks = ' '.join(toks[1:-1])
 
     results = pd.concat(dfs, axis=1).set_index("year")
 
@@ -238,7 +252,7 @@ def predict_gender_pronouns(
     )  # Gradio timeseries requires x-axis as column?
 
     return (
-        target_text,
+        target_text_w_masks,
         female_df_for_plot,
         female_df,
         male_df_for_plot,
@@ -249,15 +263,16 @@ def predict_gender_pronouns(
 title = "Changing Gender Pronouns"
 description = """
 <h2> Intro </h2>
-This is a demo for a project exploring possible spurious correlations that have been learned by our models. We can examine the training datasets and learning tasks to hypothesize what spurious correlations may exist, then condition on these variables to determine if we can achieve alternative outcomes. 
+This is a demo for a project exploring possible spurious correlations that have been learned by our models. We can examine the training datasets and learning tasks to hypothesize what spurious correlations may exist, then condition on these variables to determine if we can achieve alternative outcomes.
 
 Specially in this demo: In a user provided sentence, with at least one reference to a `DATE` and one gender pronoun, we will see how sweeping through a range of `DATE` values can change the predicted pronouns. This effect can be observed in BERT base models and in our fine-tuned models (with a specific pronoun predicting task on the [wiki-bio](https://huggingface.co/datasets/wiki_bio) dataset).
 
-One way to explain this phenomena is by looking at a likely data generating process for biographical-like data in both the main BERT training dataset as well as the `wiki_bio` dataset, in the form of a causal DAG. 
+One way to explain this phenomena is by looking at a likely data generating process for biographical-like data in both the main BERT training dataset as well as the `wiki_bio` dataset, in the form of a causal DAG.
  
 <h2> Causal DAG </h2> 
 In the DAG, we can see that `birth_place`, `birth_date` and `gender` are all independent elements that have no common cause with the other covariates in the DAG. However `birth_place`, `birth_date` and `gender` may all have a role in causing one's `access_to_resources`, with the general trend that `access_to_resources` has become less gender-dependent over time, but not in every `birth_place`, with recent events in Afghanistan providing a stark counterexample to this trend. Importantly, `access_to_resources` determines how, **if at all**, you may appear in the dataset's `context_words`.
- 
+
+
 We argue that although there are complex causal interactions between each words in any given sentence, the `context_words` are more likely to cause the `gender_pronouns`, rather than vice versa. For example, if the subject is a famous doctor and the object is her wealthy father, these context words will determine which person is being referred to, and thus which gendered-pronoun to use.
  
  
@@ -298,9 +313,10 @@ In the resulting plots, we can look for a dose-response relationship between:
 - our treatment: the sample text,
 - and our outcome: the predicted gender of pronouns in the text.
  
-Specifically we are seeing if making larger magnitude intervention: an older `DATE` in the text will result in a larger magnitude effect in the outcome: higher percentage of predicted female pronouns.
+Specifically we are seeing if 1) making larger magnitude intervention: an older `DATE` in the text will, 2) result in a larger magnitude effect in the outcome: higher percentage of predicted female pronouns.
 
-- One trend that appears is: conditioning on `birth_date` metadata in both training and inference text has the largest dose-response relationship. This seems reasonable, as the fine-tuned model is able to 'stratify' a learned relationship between gender pronouns and dates, when both are present in the text.
+Some trends that appear in the test sentences I have tried:
+- Conditioning on `birth_date` metadata in both training and inference text has the largest dose-response relationship. This seems reasonable, as the fine-tuned model is able to 'stratify' a learned relationship between gender pronouns and dates, when both are present in the text.
 - While conditioning on either no metadata or `birth_place` data training, have similar middle-ground effects for this inference task. 
 - Finally,  conditioning on `name` metadata in training, (while again conditioning on `date` in inference) has almost no dose-response relationship. It appears the learning of a `name —> gender pronouns` relationship was sufficiently successful to overwhelm any potential more nuanced learning, such as that driven by `birth_date` or `place`. 
 
@@ -339,11 +355,12 @@ gr.Interface(
         gr.inputs.Textbox(
             lines=7,
             label="Input Text: Include one of more instance of the word 'DATE' below (to be replace with a range of dates in demo), and one of more gender pronoun (to be masked for prediction).",
-            default="Born DATE, she was a computer scientist. Her work was greatly respected, and she was well-regarded in her field.",
+            default="Born in DATE, she was a computer scientist. Her work was greatly respected, and she was well-regarded in her field.",
         ),
     ],
     outputs=[
-        gr.outputs.Textbox(type="auto", label="Sample target text fed to model"),
+        gr.outputs.Textbox(
+            type="auto", label="Sample target text fed to model"),
         gr.outputs.Timeseries(
             x="year",
             label="Precent pred female pronoun vs year, per model trained with conditioning and with weight for female preds",
@@ -364,5 +381,4 @@ gr.Interface(
     title=title,
     description=description,
     article=article,
-).launch()
- 
+).launch()