Remove subtoken indicators ('##') in token attribution

app.py

@@ -62,11 +62,17 @@ toxicity_pipeline, cls_explainer = load_pipeline()
 # Auxiliary functions
 def format_explainer_html(html_string):
     """Extract tokens with attribution-based background color."""
+    inside_token_prefix = '##'
     soup = BeautifulSoup(html_string, 'html.parser')
     p = soup.new_tag('p')
+    p.append(soup.new_tag('font', attrs={'color': 'black'}))
     # Select token elements and remove model specific tokens
     for token in soup.find_all('td')[-1].find_all('mark')[1:-1]:
-        p.append(token)
+        text = token.font.text.strip()
+        if text.startswith(inside_token_prefix):
+            text = text[len(inside_token_prefix):]
+        token.string = text
+        p.font.append(token)
     return p.prettify()