Spaces:
Runtime error
Runtime error
Commit ·
4f9d18b
1
Parent(s): 2217075
add example image
Browse files- .gitignore +1 -0
- app.py +15 -3
- spec_metric_result.png +0 -0
.gitignore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
venv_unc*
|
app.py
CHANGED
|
@@ -210,6 +210,16 @@ demo = gr.Blocks()
|
|
| 210 |
with demo:
|
| 211 |
input_texts = gr.Variable([])
|
| 212 |
gr.Markdown("**Detect Task Specification at Inference-time.**")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 213 |
|
| 214 |
gr.Markdown("**Follow the numbered steps below to test one of the pre-loaded options.** Once you get the hang of it, you can load a new model and/or provide your own input texts.")
|
| 215 |
gr.Markdown(f"""1) Pick a preloaded BERT-like model.
|
|
@@ -249,7 +259,7 @@ with demo:
|
|
| 249 |
)
|
| 250 |
|
| 251 |
with gr.Row():
|
| 252 |
-
get_text_btn = gr.Button("3) Click to load input texts.
|
| 253 |
|
| 254 |
get_text_btn.click(
|
| 255 |
fn=display_input_texts,
|
|
@@ -262,7 +272,9 @@ with demo:
|
|
| 262 |
with gr.Row():
|
| 263 |
uncertain_btn = gr.Button("4) Click to get Task Specification Metric results!")
|
| 264 |
gr.Markdown(
|
| 265 |
-
"
|
|
|
|
|
|
|
| 266 |
|
| 267 |
with gr.Row():
|
| 268 |
female_fig = gr.Plot(type="auto")
|
|
@@ -270,7 +282,7 @@ with demo:
|
|
| 270 |
female_df = gr.Dataframe()
|
| 271 |
with gr.Row():
|
| 272 |
display_text = gr.Textbox(
|
| 273 |
-
type="
|
| 274 |
|
| 275 |
uncertain_btn.click(
|
| 276 |
fn=predict_gender_pronouns,
|
|
|
|
| 210 |
with demo:
|
| 211 |
input_texts = gr.Variable([])
|
| 212 |
gr.Markdown("**Detect Task Specification at Inference-time.**")
|
| 213 |
+
gr.Markdown("""Well-specified tasks should have a lower specification metric value.
|
| 214 |
+
For example, with a close read, you can see that only Winogender schema sentence numbers (3) and (4) are well-specified:
|
| 215 |
+
the masked pronoun is coreferent with the `man` or `woman`, for the gendered pronoun resolution task, but the remainder are unspecfied.
|
| 216 |
+
|
| 217 |
+
In this example we have 100\% accurate detection with the specification metric near zero for only sentence (3) and (4).
|
| 218 |
+
<p align="center">
|
| 219 |
+
<img src="file/spec_metric_result.png" alt="results" width="500"/>
|
| 220 |
+
</p>
|
| 221 |
+
""")
|
| 222 |
+
|
| 223 |
|
| 224 |
gr.Markdown("**Follow the numbered steps below to test one of the pre-loaded options.** Once you get the hang of it, you can load a new model and/or provide your own input texts.")
|
| 225 |
gr.Markdown(f"""1) Pick a preloaded BERT-like model.
|
|
|
|
| 259 |
)
|
| 260 |
|
| 261 |
with gr.Row():
|
| 262 |
+
get_text_btn = gr.Button("3) Click to load input texts.")
|
| 263 |
|
| 264 |
get_text_btn.click(
|
| 265 |
fn=display_input_texts,
|
|
|
|
| 272 |
with gr.Row():
|
| 273 |
uncertain_btn = gr.Button("4) Click to get Task Specification Metric results!")
|
| 274 |
gr.Markdown(
|
| 275 |
+
"""We expect a lower specification metric for well-specified tasks.
|
| 276 |
+
|
| 277 |
+
Note: If there is an * by a sentence number, then at least one top prediction for that sentence was non-gendered.""")
|
| 278 |
|
| 279 |
with gr.Row():
|
| 280 |
female_fig = gr.Plot(type="auto")
|
|
|
|
| 282 |
female_df = gr.Dataframe()
|
| 283 |
with gr.Row():
|
| 284 |
display_text = gr.Textbox(
|
| 285 |
+
type="text", label="Sample of text fed to model")
|
| 286 |
|
| 287 |
uncertain_btn.click(
|
| 288 |
fn=predict_gender_pronouns,
|
spec_metric_result.png
ADDED
|