Add cumulative token count

tapas_visualizer.py

@@ -115,7 +115,7 @@ class TapasVisualizer:
             token_text = self.tokenizer._convert_id_to_token(input_id)
             cell_tokens[(row_id, col_id)].append(token_text)
 
-        # token_df = pd.DataFrame(token_data, columns=['id', 'token', 'segment_id', 'column_id', 'row_id'])
+        cumulative_cnt = 0
         header_row_html = ""
         header_row_token_cnt = 0
         for col_id, col in enumerate(table.columns, start=1):
@@ -124,7 +124,9 @@ class TapasVisualizer:
             cell_html = "".join(span_htmls)
             header_row_html += f"<th>{cell_html}</th>"
             header_row_token_cnt += len(cur_cell_tokens)
-        header_row_html += f'<th style="border: none;">{self.style_span(header_row_token_cnt, ["non-token", "count"])}</th>'
+        cumulative_cnt += header_row_token_cnt
+        cnt_str = f'{header_row_token_cnt} | {cumulative_cnt}'
+        header_row_html += f'<th style="border: none;">{self.style_span(cnt_str, ["non-token", "count"])}</th>'
         header_row_html = f'<tr>{header_row_html}</tr>'
 
         table_vals = table.values
@@ -140,7 +142,9 @@ class TapasVisualizer:
                 cell_html = "".join(span_htmls)
                 row_html += f"<td>{cell_html}</td>"
                 row_token_cnt += len(cur_cell_tokens)
-            row_html += f'<td style="border: none;">{self.style_span(row_token_cnt, ["non-token", "count"])}</td>'
+            cumulative_cnt += row_token_cnt
+            cnt_str = f'{row_token_cnt} | {cumulative_cnt}'
+            row_html += f'<td style="border: none;">{self.style_span(cnt_str, ["non-token", "count"])}</td>'
             table_html += f'<tr>{row_html}</tr>'
 
         table_html = f'<table>{table_html}</table>'