Spaces:
Runtime error
Runtime error
Sebastian Gehrmann
commited on
Commit
·
ef1f338
1
Parent(s):
4081c39
added structured annotation section
Browse files- datacards/curation.py +71 -1
datacards/curation.py
CHANGED
|
@@ -12,7 +12,7 @@ from .streamlit_utils import (
|
|
| 12 |
|
| 13 |
N_FIELDS_ORIGINAL = 4
|
| 14 |
N_FIELDS_LANGUAGE = 12
|
| 15 |
-
N_FIELDS_ANNOTATIONS =
|
| 16 |
N_FIELDS_CONSENT = 0
|
| 17 |
N_FIELDS_PII = 0
|
| 18 |
N_FIELDS_MAINTENANCE = 0
|
|
@@ -148,11 +148,81 @@ def curation_page():
|
|
| 148 |
key_list=key_pref + ["filtered-criteria"],
|
| 149 |
help="Describe the process for selecting instances to include in the dataset, including any tools used. If no selection was done, enter N/A.",
|
| 150 |
)
|
|
|
|
| 151 |
with st.expander("Structured Annotations", expanded=False):
|
| 152 |
key_pref = ["curation", "annotations"]
|
| 153 |
st.session_state.card_dict["curation"][
|
| 154 |
"annotations"
|
| 155 |
] = st.session_state.card_dict["curation"].get("annotations", {})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
with st.expander("Consent", expanded=False):
|
| 157 |
key_pref = ["curation", "consent"]
|
| 158 |
st.session_state.card_dict["curation"]["consent"] = st.session_state.card_dict[
|
|
|
|
| 12 |
|
| 13 |
N_FIELDS_ORIGINAL = 4
|
| 14 |
N_FIELDS_LANGUAGE = 12
|
| 15 |
+
N_FIELDS_ANNOTATIONS = 10
|
| 16 |
N_FIELDS_CONSENT = 0
|
| 17 |
N_FIELDS_PII = 0
|
| 18 |
N_FIELDS_MAINTENANCE = 0
|
|
|
|
| 148 |
key_list=key_pref + ["filtered-criteria"],
|
| 149 |
help="Describe the process for selecting instances to include in the dataset, including any tools used. If no selection was done, enter N/A.",
|
| 150 |
)
|
| 151 |
+
|
| 152 |
with st.expander("Structured Annotations", expanded=False):
|
| 153 |
key_pref = ["curation", "annotations"]
|
| 154 |
st.session_state.card_dict["curation"][
|
| 155 |
"annotations"
|
| 156 |
] = st.session_state.card_dict["curation"].get("annotations", {})
|
| 157 |
+
|
| 158 |
+
make_radio(
|
| 159 |
+
label="Does the dataset have additional annotations for each instance?",
|
| 160 |
+
options=["none", "found", "automatically created", "expert created", "crowd-sourced"],
|
| 161 |
+
key_list=key_pref + ["origin"],
|
| 162 |
+
help="Was any additional data collected?",
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
# TODO: If yes....
|
| 166 |
+
# If expert or crowdsourced, this branch
|
| 167 |
+
make_radio(
|
| 168 |
+
label="What is the number of raters ",
|
| 169 |
+
options=["unknown", "1", "2<n<10", "11<n<50", "51<n<100", "n>100"],
|
| 170 |
+
key_list=key_pref + ["rater-number"],
|
| 171 |
+
help="How many raters were used to create the additional annotations?",
|
| 172 |
+
)
|
| 173 |
+
make_text_area(
|
| 174 |
+
label="Describe the qualifications required of an annotator.",
|
| 175 |
+
key_list=key_pref + ["rater-qualifications"],
|
| 176 |
+
help="e.g., languages or dialects they speak, education requirements, number of HITs (if MTurk).",
|
| 177 |
+
)
|
| 178 |
+
make_radio(
|
| 179 |
+
label="How many annotators saw each training example?",
|
| 180 |
+
options=["0", "1", "2", "3", "4", "5", ">5"],
|
| 181 |
+
key_list=key_pref + ["rater-training-num"],
|
| 182 |
+
help="",
|
| 183 |
+
)
|
| 184 |
+
make_radio(
|
| 185 |
+
label="How many annotators saw each test example?",
|
| 186 |
+
options=["0", "1", "2", "3", "4", "5", ">5"],
|
| 187 |
+
key_list=key_pref + ["rater-test-num"],
|
| 188 |
+
help="",
|
| 189 |
+
)
|
| 190 |
+
make_radio(
|
| 191 |
+
label="Was an annotation service used?",
|
| 192 |
+
options=["yes", "no", "unknown"],
|
| 193 |
+
key_list=key_pref + ["rater-annotation-service-bool"],
|
| 194 |
+
help="",
|
| 195 |
+
)
|
| 196 |
+
# TODO if yes
|
| 197 |
+
make_multiselect(
|
| 198 |
+
label="Which annotation services were used?",
|
| 199 |
+
options=[
|
| 200 |
+
"Amazon Mechanical Turk", "Prolific Academic",
|
| 201 |
+
"Upwork", "Appen", "Crowdflower", "other"
|
| 202 |
+
],
|
| 203 |
+
key_list=key_pref + ["rater-annotation-service"],
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
make_text_area(
|
| 208 |
+
label="Purpose and values for each annoation",
|
| 209 |
+
key_list=key_pref + ["values"],
|
| 210 |
+
help="Describe the purpose and possible values for each kind of annotation.",
|
| 211 |
+
)
|
| 212 |
+
make_multiselect(
|
| 213 |
+
label="Quality control measures?",
|
| 214 |
+
options=["none", "unknown", "validated by another rater", "validated by data curators", "validated through automated script", "other"],
|
| 215 |
+
key_list=key_pref + ["quality-control"],
|
| 216 |
+
help="How was annotation quality controlled for / what control measures were put in place to ensure annotation quality?",
|
| 217 |
+
)
|
| 218 |
+
# TODO: If not none / unknown
|
| 219 |
+
make_text_area(
|
| 220 |
+
label="Describe the quality control measures that were taken.",
|
| 221 |
+
key_list=key_pref + ["quality-control-details"],
|
| 222 |
+
help="Describe how quality was ensured in the data curation process.",
|
| 223 |
+
)
|
| 224 |
+
|
| 225 |
+
|
| 226 |
with st.expander("Consent", expanded=False):
|
| 227 |
key_pref = ["curation", "consent"]
|
| 228 |
st.session_state.card_dict["curation"]["consent"] = st.session_state.card_dict[
|