{
"overall": {
"stage3_f1": 0.4784688995215311,
"stage4_f1": 0.5849056603773586,
"f1_delta": 0.10643676085582748,
"stage3_errors": {
"boundary": 18,
"false_positive": 22,
"missed_entity": 9,
"wrong_label": 21
},
"stage4_errors": {
"boundary": 18,
"false_positive": 20,
"missed_entity": 4,
"wrong_label": 14
}
},
"status_counts": {
"improved": 7,
"regressed": 3,
"unchanged": 0,
"unresolved": 9
},
"patterns": {
"institution_context": {
"description": "The same institution name changes between ORGANIZATION and physical LOCATION by context.",
"status": "improved",
"stage3_f1": 0.4761904761904762,
"stage4_f1": 0.7000000000000001,
"f1_delta": 0.22380952380952385,
"stage3_error_counts": {
"boundary": 1,
"false_positive": 4,
"missed_entity": 1,
"wrong_label": 2
},
"stage4_error_counts": {
"boundary": 1,
"false_positive": 2,
"wrong_label": 1
},
"unresolved": true,
"stage4_records_with_errors": 3,
"representative": {
"unique_index": "wild-institution_context-4",
"gold_xml": "Northstar Bank approved the loan at its Northstar Bank downtown branch.",
"stage3_predicted_xml": "Northstar Bank approved the loan at its Northstar Bank downtown branch.",
"stage4_predicted_xml": "Northstar Bank approved the loan at its Northstar Bank downtown branch.",
"stage3_errors": [
{
"type": "wrong_label",
"gold": {
"start": 40,
"end": 54,
"label": "LOCATION",
"value": "Northstar Bank"
},
"predicted": {
"start": 40,
"end": 49,
"label": "ORGANIZATION",
"value": "Northstar"
}
},
{
"type": "false_positive",
"gold": null,
"predicted": {
"start": 50,
"end": 54,
"label": "LOCATION",
"value": "Bank"
}
},
{
"type": "false_positive",
"gold": null,
"predicted": {
"start": 55,
"end": 70,
"label": "ORGANIZATION",
"value": "downtown branch"
}
}
],
"stage4_errors": [
{
"type": "wrong_label",
"gold": {
"start": 40,
"end": 54,
"label": "LOCATION",
"value": "Northstar Bank"
},
"predicted": {
"start": 40,
"end": 70,
"label": "ORGANIZATION",
"value": "Northstar Bank downtown branch"
}
}
]
}
},
"company_product": {
"description": "Company names and their branded products are confused or fragmented.",
"status": "improved",
"stage3_f1": 0.10526315789473685,
"stage4_f1": 0.4000000000000001,
"f1_delta": 0.29473684210526324,
"stage3_error_counts": {
"boundary": 2,
"false_positive": 2,
"missed_entity": 3,
"wrong_label": 4
},
"stage4_error_counts": {
"boundary": 1,
"false_positive": 1,
"missed_entity": 1,
"wrong_label": 4
},
"unresolved": true,
"stage4_records_with_errors": 4,
"representative": {
"unique_index": "wild-company_product-3",
"gold_xml": "Tesla recalled 50 Model Y vehicles.",
"stage3_predicted_xml": "Tesla recalled 50 Model Y vehicles.",
"stage4_predicted_xml": "Tesla recalled 50 Model Y vehicles.",
"stage3_errors": [
{
"type": "wrong_label",
"gold": {
"start": 0,
"end": 5,
"label": "ORGANIZATION",
"value": "Tesla"
},
"predicted": {
"start": 0,
"end": 14,
"label": "PRODUCT",
"value": "Tesla recalled"
}
},
{
"type": "wrong_label",
"gold": {
"start": 15,
"end": 17,
"label": "AMOUNT",
"value": "50"
},
"predicted": {
"start": 15,
"end": 25,
"label": "PRODUCT",
"value": "50 Model Y"
}
},
{
"type": "missed_entity",
"gold": {
"start": 18,
"end": 25,
"label": "PRODUCT",
"value": "Model Y"
},
"predicted": null
}
],
"stage4_errors": [
{
"type": "wrong_label",
"gold": {
"start": 0,
"end": 5,
"label": "ORGANIZATION",
"value": "Tesla"
},
"predicted": {
"start": 0,
"end": 5,
"label": "PRODUCT",
"value": "Tesla"
}
}
]
}
},
"product_work": {
"description": "Named games/products and creative works receive each other's labels.",
"status": "regressed",
"stage3_f1": 0.1904761904761905,
"stage4_f1": 0.15384615384615383,
"f1_delta": -0.036630036630036666,
"stage3_error_counts": {
"boundary": 1,
"false_positive": 4,
"missed_entity": 1,
"wrong_label": 5
},
"stage4_error_counts": {
"boundary": 2,
"false_positive": 9,
"missed_entity": 1,
"wrong_label": 4
},
"unresolved": true,
"stage4_records_with_errors": 4,
"representative": {
"unique_index": "wild-product_work-4",
"gold_xml": "Kind of Blue played while he used the Kindle Paperwhite.",
"stage3_predicted_xml": "Kind of Blue played while he used the Kindle Paperwhite.",
"stage4_predicted_xml": "Kind of Blue played while he used the Kindle Paperwhite.",
"stage3_errors": [
{
"type": "wrong_label",
"gold": {
"start": 0,
"end": 12,
"label": "WORKOFART",
"value": "Kind of Blue"
},
"predicted": {
"start": 8,
"end": 12,
"label": "ORGANIZATION",
"value": "Blue"
}
},
{
"type": "wrong_label",
"gold": {
"start": 38,
"end": 55,
"label": "PRODUCT",
"value": "Kindle Paperwhite"
},
"predicted": {
"start": 51,
"end": 54,
"label": "ORGANIZATION",
"value": "hit"
}
}
],
"stage4_errors": [
{
"type": "wrong_label",
"gold": {
"start": 0,
"end": 12,
"label": "WORKOFART",
"value": "Kind of Blue"
},
"predicted": {
"start": 8,
"end": 12,
"label": "ORGANIZATION",
"value": "Blue"
}
},
{
"type": "boundary",
"gold": {
"start": 38,
"end": 55,
"label": "PRODUCT",
"value": "Kindle Paperwhite"
},
"predicted": {
"start": 38,
"end": 42,
"label": "PRODUCT",
"value": "Kind"
}
},
{
"type": "false_positive",
"gold": null,
"predicted": {
"start": 42,
"end": 44,
"label": "ORGANIZATION",
"value": "le"
}
},
{
"type": "false_positive",
"gold": null,
"predicted": {
"start": 50,
"end": 55,
"label": "ORGANIZATION",
"value": "white"
}
}
]
}
},
"work_title_boundary": {
"description": "Multiword titles, function words, punctuation, and subtitles fragment.",
"status": "regressed",
"stage3_f1": 0.375,
"stage4_f1": 0.25,
"f1_delta": -0.125,
"stage3_error_counts": {
"boundary": 4,
"false_positive": 1,
"missed_entity": 1
},
"stage4_error_counts": {
"boundary": 5,
"false_positive": 1,
"missed_entity": 1
},
"unresolved": true,
"stage4_records_with_errors": 4,
"representative": {
"unique_index": "wild-work_title_boundary-1",
"gold_xml": "They reviewed Spider-Man: Brand New Day yesterday.",
"stage3_predicted_xml": "They reviewed Spider-Man: Brand New Day yesterday.",
"stage4_predicted_xml": "They reviewed Spider-Man: Brand New Day yesterday.",
"stage3_errors": [],
"stage4_errors": [
{
"type": "boundary",
"gold": {
"start": 14,
"end": 39,
"label": "WORKOFART",
"value": "Spider-Man: Brand New Day"
},
"predicted": {
"start": 14,
"end": 49,
"label": "WORKOFART",
"value": "Spider-Man: Brand New Day yesterday"
}
},
{
"type": "missed_entity",
"gold": {
"start": 40,
"end": 49,
"label": "TIMEDATE",
"value": "yesterday"
},
"predicted": null
}
]
}
},
"software_namedness": {
"description": "CamelCase and lowercase software names fragment or become other entity types.",
"status": "improved",
"stage3_f1": 0.380952380952381,
"stage4_f1": 0.5714285714285713,
"f1_delta": 0.1904761904761903,
"stage3_error_counts": {
"boundary": 2,
"false_positive": 2,
"missed_entity": 1,
"wrong_label": 3
},
"stage4_error_counts": {
"boundary": 1,
"false_positive": 2,
"missed_entity": 1,
"wrong_label": 2
},
"unresolved": true,
"stage4_records_with_errors": 3,
"representative": {
"unique_index": "wild-software_namedness-4",
"gold_xml": "We deployed nginx beside Redis at 08:30.",
"stage3_predicted_xml": "We deployed nginx beside Redis at 08:30.",
"stage4_predicted_xml": "We deployed nginx beside Redis at 08:30.",
"stage3_errors": [
{
"type": "boundary",
"gold": {
"start": 12,
"end": 17,
"label": "PRODUCT",
"value": "nginx"
},
"predicted": {
"start": 12,
"end": 16,
"label": "PRODUCT",
"value": "ngin"
}
},
{
"type": "wrong_label",
"gold": {
"start": 25,
"end": 30,
"label": "PRODUCT",
"value": "Redis"
},
"predicted": {
"start": 25,
"end": 28,
"label": "WORKOFART",
"value": "Red"
}
},
{
"type": "boundary",
"gold": {
"start": 34,
"end": 39,
"label": "TIMEDATE",
"value": "08:30"
},
"predicted": {
"start": 34,
"end": 37,
"label": "TIMEDATE",
"value": "08:"
}
},
{
"type": "false_positive",
"gold": null,
"predicted": {
"start": 28,
"end": 33,
"label": "PRODUCT",
"value": "is at"
}
},
{
"type": "false_positive",
"gold": null,
"predicted": {
"start": 37,
"end": 39,
"label": "WORKOFART",
"value": "30"
}
}
],
"stage4_errors": [
{
"type": "wrong_label",
"gold": {
"start": 34,
"end": 39,
"label": "TIMEDATE",
"value": "08:30"
},
"predicted": {
"start": 34,
"end": 39,
"label": "PRODUCT",
"value": "08:30"
}
},
{
"type": "false_positive",
"gold": null,
"predicted": {
"start": 0,
"end": 2,
"label": "PRODUCT",
"value": "We"
}
}
]
}
},
"person_alias_handle": {
"description": "Aliases, handles, and repeated person mentions are missed or mislabeled.",
"status": "regressed",
"stage3_f1": 0.8333333333333334,
"stage4_f1": 0.75,
"f1_delta": -0.08333333333333337,
"stage3_error_counts": {
"false_positive": 1,
"missed_entity": 1,
"wrong_label": 1
},
"stage4_error_counts": {
"boundary": 1,
"wrong_label": 2
},
"unresolved": true,
"stage4_records_with_errors": 2,
"representative": {
"unique_index": "wild-person_alias_handle-3",
"gold_xml": "Dr. Lin met Person1 and later called Lin.",
"stage3_predicted_xml": "Dr. Lin met Person1 and later called Lin.",
"stage4_predicted_xml": "Dr. Lin met Person1 and later called Lin.",
"stage3_errors": [
{
"type": "false_positive",
"gold": null,
"predicted": {
"start": 0,
"end": 2,
"label": "PERSON",
"value": "Dr"
}
}
],
"stage4_errors": [
{
"type": "boundary",
"gold": {
"start": 4,
"end": 7,
"label": "PERSON",
"value": "Lin"
},
"predicted": {
"start": 0,
"end": 7,
"label": "PERSON",
"value": "Dr. Lin"
}
},
{
"type": "wrong_label",
"gold": {
"start": 37,
"end": 40,
"label": "PERSON",
"value": "Lin"
},
"predicted": {
"start": 37,
"end": 40,
"label": "PRODUCT",
"value": "Lin"
}
}
]
}
},
"job_boundary": {
"description": "Seniority and specialization fall outside a JOB span or absorb grammar.",
"status": "improved",
"stage3_f1": 0.42857142857142855,
"stage4_f1": 0.5517241379310345,
"f1_delta": 0.12315270935960593,
"stage3_error_counts": {
"boundary": 3,
"false_positive": 3,
"missed_entity": 1,
"wrong_label": 3
},
"stage4_error_counts": {
"boundary": 5,
"false_positive": 3
},
"unresolved": true,
"stage4_records_with_errors": 3,
"representative": {
"unique_index": "wild-job_boundary-2",
"gold_xml": "The chief financial officer Maria Lopez resigned from Northstar Bank.",
"stage3_predicted_xml": "The chief financial officer Maria Lopez resigned from Northstar Bank.",
"stage4_predicted_xml": "The chief financial officer Maria Lopez resigned from Northstar Bank.",
"stage3_errors": [
{
"type": "boundary",
"gold": {
"start": 4,
"end": 27,
"label": "JOB",
"value": "chief financial officer"
},
"predicted": {
"start": 10,
"end": 19,
"label": "JOB",
"value": "financial"
}
}
],
"stage4_errors": []
}
},
"amount_boundary": {
"description": "Units, symbols, ranges, and counted quantities have incomplete boundaries.",
"status": "improved",
"stage3_f1": 0.47058823529411764,
"stage4_f1": 1.0,
"f1_delta": 0.5294117647058824,
"stage3_error_counts": {
"boundary": 4,
"false_positive": 1
},
"stage4_error_counts": {},
"unresolved": false,
"stage4_records_with_errors": 0,
"representative": {
"unique_index": "wild-amount_boundary-2",
"gold_xml": "The tower is 3 feet 1 inch tall and 20 meters wide.",
"stage3_predicted_xml": "The tower is 3 feet 1 inch tall and 20 meters wide.",
"stage4_predicted_xml": "The tower is 3 feet 1 inch tall and 20 meters wide.",
"stage3_errors": [
{
"type": "boundary",
"gold": {
"start": 13,
"end": 26,
"label": "AMOUNT",
"value": "3 feet 1 inch"
},
"predicted": {
"start": 13,
"end": 31,
"label": "AMOUNT",
"value": "3 feet 1 inch tall"
}
},
{
"type": "boundary",
"gold": {
"start": 36,
"end": 45,
"label": "AMOUNT",
"value": "20 meters"
},
"predicted": {
"start": 36,
"end": 38,
"label": "AMOUNT",
"value": "20"
}
},
{
"type": "false_positive",
"gold": null,
"predicted": {
"start": 4,
"end": 9,
"label": "LOCATION",
"value": "tower"
}
}
],
"stage4_errors": []
}
},
"timedate_boundary": {
"description": "Dates, year ranges, ages, and durations fragment across subwords.",
"status": "improved",
"stage3_f1": 0.6956521739130435,
"stage4_f1": 0.7272727272727272,
"f1_delta": 0.03162055335968372,
"stage3_error_counts": {
"boundary": 1,
"false_positive": 3,
"wrong_label": 1
},
"stage4_error_counts": {
"boundary": 2,
"false_positive": 2
},
"unresolved": true,
"stage4_records_with_errors": 2,
"representative": {
"unique_index": "wild-timedate_boundary-3",
"gold_xml": "At 08:30, the 36-year-old doctor began a 50-minute procedure.",
"stage3_predicted_xml": "At 08:30, the 36-year-old doctor began a 50-minute procedure.",
"stage4_predicted_xml": "At 08:30, the 36-year-old doctor began a 50-minute procedure.",
"stage3_errors": [
{
"type": "wrong_label",
"gold": {
"start": 41,
"end": 50,
"label": "TIMEDATE",
"value": "50-minute"
},
"predicted": {
"start": 41,
"end": 43,
"label": "AMOUNT",
"value": "50"
}
},
{
"type": "false_positive",
"gold": null,
"predicted": {
"start": 43,
"end": 50,
"label": "TIMEDATE",
"value": "-minute"
}
}
],
"stage4_errors": [
{
"type": "boundary",
"gold": {
"start": 3,
"end": 8,
"label": "TIMEDATE",
"value": "08:30"
},
"predicted": {
"start": 3,
"end": 9,
"label": "TIMEDATE",
"value": "08:30,"
}
}
]
}
},
"namedness_repetition": {
"description": "Demonyms become entities and repeated named mentions are inconsistently labeled.",
"status": "improved",
"stage3_f1": 0.7368421052631577,
"stage4_f1": 0.8888888888888888,
"f1_delta": 0.1520467836257311,
"stage3_error_counts": {
"false_positive": 1,
"wrong_label": 2
},
"stage4_error_counts": {
"wrong_label": 1
},
"unresolved": true,
"stage4_records_with_errors": 1,
"representative": {
"unique_index": "wild-namedness_repetition-1",
"gold_xml": "American researchers met in America.",
"stage3_predicted_xml": "American researchers met in America.",
"stage4_predicted_xml": "American researchers met in America.",
"stage3_errors": [
{
"type": "false_positive",
"gold": null,
"predicted": {
"start": 0,
"end": 8,
"label": "JOB",
"value": "American"
}
}
],
"stage4_errors": []
}
}
}
}