Spaces:
Runtime error
Runtime error
Commit
·
5008e92
1
Parent(s):
588a02c
Add title and preamble
Browse files
app.py
CHANGED
|
@@ -26,6 +26,18 @@ def main():
|
|
| 26 |
tokenizer = AutoTokenizer.from_pretrained(selected_model)
|
| 27 |
return tokenizer
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
col1, col2 = st.columns([1, 2])
|
| 30 |
with col1:
|
| 31 |
selected_model = st.selectbox("Select a tokenizer", models, key=1)
|
|
|
|
| 26 |
tokenizer = AutoTokenizer.from_pretrained(selected_model)
|
| 27 |
return tokenizer
|
| 28 |
|
| 29 |
+
st.markdown(
|
| 30 |
+
"""
|
| 31 |
+
## TAPAS Tokenization Visualization
|
| 32 |
+
|
| 33 |
+
[TAPAS](https://huggingface.co/docs/transformers/model_doc/tapas) models work on Tables.
|
| 34 |
+
The tool below is to help visualize how the table is tokenized and give total (+ row-wise)
|
| 35 |
+
token counts.
|
| 36 |
+
|
| 37 |
+
Implementation adapted from `tokenizers.tools.EncodingVisualizer`.
|
| 38 |
+
"""
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
col1, col2 = st.columns([1, 2])
|
| 42 |
with col1:
|
| 43 |
selected_model = st.selectbox("Select a tokenizer", models, key=1)
|