| { |
| "overall": { |
| "stage3_f1": 0.4784688995215311, |
| "stage4_f1": 0.5849056603773586, |
| "f1_delta": 0.10643676085582748, |
| "stage3_errors": { |
| "boundary": 18, |
| "false_positive": 22, |
| "missed_entity": 9, |
| "wrong_label": 21 |
| }, |
| "stage4_errors": { |
| "boundary": 18, |
| "false_positive": 20, |
| "missed_entity": 4, |
| "wrong_label": 14 |
| } |
| }, |
| "status_counts": { |
| "improved": 7, |
| "regressed": 3, |
| "unchanged": 0, |
| "unresolved": 9 |
| }, |
| "patterns": { |
| "institution_context": { |
| "description": "The same institution name changes between ORGANIZATION and physical LOCATION by context.", |
| "status": "improved", |
| "stage3_f1": 0.4761904761904762, |
| "stage4_f1": 0.7000000000000001, |
| "f1_delta": 0.22380952380952385, |
| "stage3_error_counts": { |
| "boundary": 1, |
| "false_positive": 4, |
| "missed_entity": 1, |
| "wrong_label": 2 |
| }, |
| "stage4_error_counts": { |
| "boundary": 1, |
| "false_positive": 2, |
| "wrong_label": 1 |
| }, |
| "unresolved": true, |
| "stage4_records_with_errors": 3, |
| "representative": { |
| "unique_index": "wild-institution_context-4", |
| "gold_xml": "<ORGANIZATION>Northstar Bank</ORGANIZATION> approved the loan at its <LOCATION>Northstar Bank</LOCATION> downtown branch.", |
| "stage3_predicted_xml": "<ORGANIZATION>Northstar Bank</ORGANIZATION> approved the loan at its <ORGANIZATION>Northstar</ORGANIZATION> <LOCATION>Bank</LOCATION> <ORGANIZATION>downtown branch</ORGANIZATION>.", |
| "stage4_predicted_xml": "<ORGANIZATION>Northstar Bank</ORGANIZATION> approved the loan at its <ORGANIZATION>Northstar Bank downtown branch</ORGANIZATION>.", |
| "stage3_errors": [ |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 40, |
| "end": 54, |
| "label": "LOCATION", |
| "value": "Northstar Bank" |
| }, |
| "predicted": { |
| "start": 40, |
| "end": 49, |
| "label": "ORGANIZATION", |
| "value": "Northstar" |
| } |
| }, |
| { |
| "type": "false_positive", |
| "gold": null, |
| "predicted": { |
| "start": 50, |
| "end": 54, |
| "label": "LOCATION", |
| "value": "Bank" |
| } |
| }, |
| { |
| "type": "false_positive", |
| "gold": null, |
| "predicted": { |
| "start": 55, |
| "end": 70, |
| "label": "ORGANIZATION", |
| "value": "downtown branch" |
| } |
| } |
| ], |
| "stage4_errors": [ |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 40, |
| "end": 54, |
| "label": "LOCATION", |
| "value": "Northstar Bank" |
| }, |
| "predicted": { |
| "start": 40, |
| "end": 70, |
| "label": "ORGANIZATION", |
| "value": "Northstar Bank downtown branch" |
| } |
| } |
| ] |
| } |
| }, |
| "company_product": { |
| "description": "Company names and their branded products are confused or fragmented.", |
| "status": "improved", |
| "stage3_f1": 0.10526315789473685, |
| "stage4_f1": 0.4000000000000001, |
| "f1_delta": 0.29473684210526324, |
| "stage3_error_counts": { |
| "boundary": 2, |
| "false_positive": 2, |
| "missed_entity": 3, |
| "wrong_label": 4 |
| }, |
| "stage4_error_counts": { |
| "boundary": 1, |
| "false_positive": 1, |
| "missed_entity": 1, |
| "wrong_label": 4 |
| }, |
| "unresolved": true, |
| "stage4_records_with_errors": 4, |
| "representative": { |
| "unique_index": "wild-company_product-3", |
| "gold_xml": "<ORGANIZATION>Tesla</ORGANIZATION> recalled <AMOUNT>50</AMOUNT> <PRODUCT>Model Y</PRODUCT> vehicles.", |
| "stage3_predicted_xml": "<PRODUCT>Tesla recalled</PRODUCT> <PRODUCT>50 Model Y</PRODUCT> vehicles.", |
| "stage4_predicted_xml": "<PRODUCT>Tesla</PRODUCT> recalled <AMOUNT>50</AMOUNT> <PRODUCT>Model Y</PRODUCT> vehicles.", |
| "stage3_errors": [ |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 0, |
| "end": 5, |
| "label": "ORGANIZATION", |
| "value": "Tesla" |
| }, |
| "predicted": { |
| "start": 0, |
| "end": 14, |
| "label": "PRODUCT", |
| "value": "Tesla recalled" |
| } |
| }, |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 15, |
| "end": 17, |
| "label": "AMOUNT", |
| "value": "50" |
| }, |
| "predicted": { |
| "start": 15, |
| "end": 25, |
| "label": "PRODUCT", |
| "value": "50 Model Y" |
| } |
| }, |
| { |
| "type": "missed_entity", |
| "gold": { |
| "start": 18, |
| "end": 25, |
| "label": "PRODUCT", |
| "value": "Model Y" |
| }, |
| "predicted": null |
| } |
| ], |
| "stage4_errors": [ |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 0, |
| "end": 5, |
| "label": "ORGANIZATION", |
| "value": "Tesla" |
| }, |
| "predicted": { |
| "start": 0, |
| "end": 5, |
| "label": "PRODUCT", |
| "value": "Tesla" |
| } |
| } |
| ] |
| } |
| }, |
| "product_work": { |
| "description": "Named games/products and creative works receive each other's labels.", |
| "status": "regressed", |
| "stage3_f1": 0.1904761904761905, |
| "stage4_f1": 0.15384615384615383, |
| "f1_delta": -0.036630036630036666, |
| "stage3_error_counts": { |
| "boundary": 1, |
| "false_positive": 4, |
| "missed_entity": 1, |
| "wrong_label": 5 |
| }, |
| "stage4_error_counts": { |
| "boundary": 2, |
| "false_positive": 9, |
| "missed_entity": 1, |
| "wrong_label": 4 |
| }, |
| "unresolved": true, |
| "stage4_records_with_errors": 4, |
| "representative": { |
| "unique_index": "wild-product_work-4", |
| "gold_xml": "<WORKOFART>Kind of Blue</WORKOFART> played while he used the <PRODUCT>Kindle Paperwhite</PRODUCT>.", |
| "stage3_predicted_xml": "Kind of <ORGANIZATION>Blue</ORGANIZATION> played while he used the Kindle Paperw<ORGANIZATION>hit</ORGANIZATION>e.", |
| "stage4_predicted_xml": "Kind of <ORGANIZATION>Blue</ORGANIZATION> played while he used the <PRODUCT>Kind</PRODUCT><ORGANIZATION>le</ORGANIZATION> Paper<ORGANIZATION>white</ORGANIZATION>.", |
| "stage3_errors": [ |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 0, |
| "end": 12, |
| "label": "WORKOFART", |
| "value": "Kind of Blue" |
| }, |
| "predicted": { |
| "start": 8, |
| "end": 12, |
| "label": "ORGANIZATION", |
| "value": "Blue" |
| } |
| }, |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 38, |
| "end": 55, |
| "label": "PRODUCT", |
| "value": "Kindle Paperwhite" |
| }, |
| "predicted": { |
| "start": 51, |
| "end": 54, |
| "label": "ORGANIZATION", |
| "value": "hit" |
| } |
| } |
| ], |
| "stage4_errors": [ |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 0, |
| "end": 12, |
| "label": "WORKOFART", |
| "value": "Kind of Blue" |
| }, |
| "predicted": { |
| "start": 8, |
| "end": 12, |
| "label": "ORGANIZATION", |
| "value": "Blue" |
| } |
| }, |
| { |
| "type": "boundary", |
| "gold": { |
| "start": 38, |
| "end": 55, |
| "label": "PRODUCT", |
| "value": "Kindle Paperwhite" |
| }, |
| "predicted": { |
| "start": 38, |
| "end": 42, |
| "label": "PRODUCT", |
| "value": "Kind" |
| } |
| }, |
| { |
| "type": "false_positive", |
| "gold": null, |
| "predicted": { |
| "start": 42, |
| "end": 44, |
| "label": "ORGANIZATION", |
| "value": "le" |
| } |
| }, |
| { |
| "type": "false_positive", |
| "gold": null, |
| "predicted": { |
| "start": 50, |
| "end": 55, |
| "label": "ORGANIZATION", |
| "value": "white" |
| } |
| } |
| ] |
| } |
| }, |
| "work_title_boundary": { |
| "description": "Multiword titles, function words, punctuation, and subtitles fragment.", |
| "status": "regressed", |
| "stage3_f1": 0.375, |
| "stage4_f1": 0.25, |
| "f1_delta": -0.125, |
| "stage3_error_counts": { |
| "boundary": 4, |
| "false_positive": 1, |
| "missed_entity": 1 |
| }, |
| "stage4_error_counts": { |
| "boundary": 5, |
| "false_positive": 1, |
| "missed_entity": 1 |
| }, |
| "unresolved": true, |
| "stage4_records_with_errors": 4, |
| "representative": { |
| "unique_index": "wild-work_title_boundary-1", |
| "gold_xml": "They reviewed <WORKOFART>Spider-Man: Brand New Day</WORKOFART> <TIMEDATE>yesterday</TIMEDATE>.", |
| "stage3_predicted_xml": "They reviewed <WORKOFART>Spider-Man: Brand New Day</WORKOFART> <TIMEDATE>yesterday</TIMEDATE>.", |
| "stage4_predicted_xml": "They reviewed <WORKOFART>Spider-Man: Brand New Day yesterday</WORKOFART>.", |
| "stage3_errors": [], |
| "stage4_errors": [ |
| { |
| "type": "boundary", |
| "gold": { |
| "start": 14, |
| "end": 39, |
| "label": "WORKOFART", |
| "value": "Spider-Man: Brand New Day" |
| }, |
| "predicted": { |
| "start": 14, |
| "end": 49, |
| "label": "WORKOFART", |
| "value": "Spider-Man: Brand New Day yesterday" |
| } |
| }, |
| { |
| "type": "missed_entity", |
| "gold": { |
| "start": 40, |
| "end": 49, |
| "label": "TIMEDATE", |
| "value": "yesterday" |
| }, |
| "predicted": null |
| } |
| ] |
| } |
| }, |
| "software_namedness": { |
| "description": "CamelCase and lowercase software names fragment or become other entity types.", |
| "status": "improved", |
| "stage3_f1": 0.380952380952381, |
| "stage4_f1": 0.5714285714285713, |
| "f1_delta": 0.1904761904761903, |
| "stage3_error_counts": { |
| "boundary": 2, |
| "false_positive": 2, |
| "missed_entity": 1, |
| "wrong_label": 3 |
| }, |
| "stage4_error_counts": { |
| "boundary": 1, |
| "false_positive": 2, |
| "missed_entity": 1, |
| "wrong_label": 2 |
| }, |
| "unresolved": true, |
| "stage4_records_with_errors": 3, |
| "representative": { |
| "unique_index": "wild-software_namedness-4", |
| "gold_xml": "We deployed <PRODUCT>nginx</PRODUCT> beside <PRODUCT>Redis</PRODUCT> at <TIMEDATE>08:30</TIMEDATE>.", |
| "stage3_predicted_xml": "We deployed <PRODUCT>ngin</PRODUCT>x beside <WORKOFART>Red</WORKOFART><PRODUCT>is at</PRODUCT> <TIMEDATE>08:</TIMEDATE><WORKOFART>30</WORKOFART>.", |
| "stage4_predicted_xml": "<PRODUCT>We</PRODUCT> deployed <PRODUCT>nginx</PRODUCT> beside <PRODUCT>Redis</PRODUCT> at <PRODUCT>08:30</PRODUCT>.", |
| "stage3_errors": [ |
| { |
| "type": "boundary", |
| "gold": { |
| "start": 12, |
| "end": 17, |
| "label": "PRODUCT", |
| "value": "nginx" |
| }, |
| "predicted": { |
| "start": 12, |
| "end": 16, |
| "label": "PRODUCT", |
| "value": "ngin" |
| } |
| }, |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 25, |
| "end": 30, |
| "label": "PRODUCT", |
| "value": "Redis" |
| }, |
| "predicted": { |
| "start": 25, |
| "end": 28, |
| "label": "WORKOFART", |
| "value": "Red" |
| } |
| }, |
| { |
| "type": "boundary", |
| "gold": { |
| "start": 34, |
| "end": 39, |
| "label": "TIMEDATE", |
| "value": "08:30" |
| }, |
| "predicted": { |
| "start": 34, |
| "end": 37, |
| "label": "TIMEDATE", |
| "value": "08:" |
| } |
| }, |
| { |
| "type": "false_positive", |
| "gold": null, |
| "predicted": { |
| "start": 28, |
| "end": 33, |
| "label": "PRODUCT", |
| "value": "is at" |
| } |
| }, |
| { |
| "type": "false_positive", |
| "gold": null, |
| "predicted": { |
| "start": 37, |
| "end": 39, |
| "label": "WORKOFART", |
| "value": "30" |
| } |
| } |
| ], |
| "stage4_errors": [ |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 34, |
| "end": 39, |
| "label": "TIMEDATE", |
| "value": "08:30" |
| }, |
| "predicted": { |
| "start": 34, |
| "end": 39, |
| "label": "PRODUCT", |
| "value": "08:30" |
| } |
| }, |
| { |
| "type": "false_positive", |
| "gold": null, |
| "predicted": { |
| "start": 0, |
| "end": 2, |
| "label": "PRODUCT", |
| "value": "We" |
| } |
| } |
| ] |
| } |
| }, |
| "person_alias_handle": { |
| "description": "Aliases, handles, and repeated person mentions are missed or mislabeled.", |
| "status": "regressed", |
| "stage3_f1": 0.8333333333333334, |
| "stage4_f1": 0.75, |
| "f1_delta": -0.08333333333333337, |
| "stage3_error_counts": { |
| "false_positive": 1, |
| "missed_entity": 1, |
| "wrong_label": 1 |
| }, |
| "stage4_error_counts": { |
| "boundary": 1, |
| "wrong_label": 2 |
| }, |
| "unresolved": true, |
| "stage4_records_with_errors": 2, |
| "representative": { |
| "unique_index": "wild-person_alias_handle-3", |
| "gold_xml": "Dr. <PERSON>Lin</PERSON> met <PERSON>Person1</PERSON> and later called <PERSON>Lin</PERSON>.", |
| "stage3_predicted_xml": "<PERSON>Dr</PERSON>. <PERSON>Lin</PERSON> met <PERSON>Person1</PERSON> and later called <PERSON>Lin</PERSON>.", |
| "stage4_predicted_xml": "<PERSON>Dr. Lin</PERSON> met <PERSON>Person1</PERSON> and later called <PRODUCT>Lin</PRODUCT>.", |
| "stage3_errors": [ |
| { |
| "type": "false_positive", |
| "gold": null, |
| "predicted": { |
| "start": 0, |
| "end": 2, |
| "label": "PERSON", |
| "value": "Dr" |
| } |
| } |
| ], |
| "stage4_errors": [ |
| { |
| "type": "boundary", |
| "gold": { |
| "start": 4, |
| "end": 7, |
| "label": "PERSON", |
| "value": "Lin" |
| }, |
| "predicted": { |
| "start": 0, |
| "end": 7, |
| "label": "PERSON", |
| "value": "Dr. Lin" |
| } |
| }, |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 37, |
| "end": 40, |
| "label": "PERSON", |
| "value": "Lin" |
| }, |
| "predicted": { |
| "start": 37, |
| "end": 40, |
| "label": "PRODUCT", |
| "value": "Lin" |
| } |
| } |
| ] |
| } |
| }, |
| "job_boundary": { |
| "description": "Seniority and specialization fall outside a JOB span or absorb grammar.", |
| "status": "improved", |
| "stage3_f1": 0.42857142857142855, |
| "stage4_f1": 0.5517241379310345, |
| "f1_delta": 0.12315270935960593, |
| "stage3_error_counts": { |
| "boundary": 3, |
| "false_positive": 3, |
| "missed_entity": 1, |
| "wrong_label": 3 |
| }, |
| "stage4_error_counts": { |
| "boundary": 5, |
| "false_positive": 3 |
| }, |
| "unresolved": true, |
| "stage4_records_with_errors": 3, |
| "representative": { |
| "unique_index": "wild-job_boundary-2", |
| "gold_xml": "The <JOB>chief financial officer</JOB> <PERSON>Maria Lopez</PERSON> resigned from <ORGANIZATION>Northstar Bank</ORGANIZATION>.", |
| "stage3_predicted_xml": "The chief <JOB>financial</JOB> officer <PERSON>Maria Lopez</PERSON> resigned from <ORGANIZATION>Northstar Bank</ORGANIZATION>.", |
| "stage4_predicted_xml": "The <JOB>chief financial officer</JOB> <PERSON>Maria Lopez</PERSON> resigned from <ORGANIZATION>Northstar Bank</ORGANIZATION>.", |
| "stage3_errors": [ |
| { |
| "type": "boundary", |
| "gold": { |
| "start": 4, |
| "end": 27, |
| "label": "JOB", |
| "value": "chief financial officer" |
| }, |
| "predicted": { |
| "start": 10, |
| "end": 19, |
| "label": "JOB", |
| "value": "financial" |
| } |
| } |
| ], |
| "stage4_errors": [] |
| } |
| }, |
| "amount_boundary": { |
| "description": "Units, symbols, ranges, and counted quantities have incomplete boundaries.", |
| "status": "improved", |
| "stage3_f1": 0.47058823529411764, |
| "stage4_f1": 1.0, |
| "f1_delta": 0.5294117647058824, |
| "stage3_error_counts": { |
| "boundary": 4, |
| "false_positive": 1 |
| }, |
| "stage4_error_counts": {}, |
| "unresolved": false, |
| "stage4_records_with_errors": 0, |
| "representative": { |
| "unique_index": "wild-amount_boundary-2", |
| "gold_xml": "The tower is <AMOUNT>3 feet 1 inch</AMOUNT> tall and <AMOUNT>20 meters</AMOUNT> wide.", |
| "stage3_predicted_xml": "The <LOCATION>tower</LOCATION> is <AMOUNT>3 feet 1 inch tall</AMOUNT> and <AMOUNT>20</AMOUNT> meters wide.", |
| "stage4_predicted_xml": "The tower is <AMOUNT>3 feet 1 inch</AMOUNT> tall and <AMOUNT>20 meters</AMOUNT> wide.", |
| "stage3_errors": [ |
| { |
| "type": "boundary", |
| "gold": { |
| "start": 13, |
| "end": 26, |
| "label": "AMOUNT", |
| "value": "3 feet 1 inch" |
| }, |
| "predicted": { |
| "start": 13, |
| "end": 31, |
| "label": "AMOUNT", |
| "value": "3 feet 1 inch tall" |
| } |
| }, |
| { |
| "type": "boundary", |
| "gold": { |
| "start": 36, |
| "end": 45, |
| "label": "AMOUNT", |
| "value": "20 meters" |
| }, |
| "predicted": { |
| "start": 36, |
| "end": 38, |
| "label": "AMOUNT", |
| "value": "20" |
| } |
| }, |
| { |
| "type": "false_positive", |
| "gold": null, |
| "predicted": { |
| "start": 4, |
| "end": 9, |
| "label": "LOCATION", |
| "value": "tower" |
| } |
| } |
| ], |
| "stage4_errors": [] |
| } |
| }, |
| "timedate_boundary": { |
| "description": "Dates, year ranges, ages, and durations fragment across subwords.", |
| "status": "improved", |
| "stage3_f1": 0.6956521739130435, |
| "stage4_f1": 0.7272727272727272, |
| "f1_delta": 0.03162055335968372, |
| "stage3_error_counts": { |
| "boundary": 1, |
| "false_positive": 3, |
| "wrong_label": 1 |
| }, |
| "stage4_error_counts": { |
| "boundary": 2, |
| "false_positive": 2 |
| }, |
| "unresolved": true, |
| "stage4_records_with_errors": 2, |
| "representative": { |
| "unique_index": "wild-timedate_boundary-3", |
| "gold_xml": "At <TIMEDATE>08:30</TIMEDATE>, the <TIMEDATE>36-year-old</TIMEDATE> <JOB>doctor</JOB> began a <TIMEDATE>50-minute</TIMEDATE> procedure.", |
| "stage3_predicted_xml": "At <TIMEDATE>08:30</TIMEDATE>, the <TIMEDATE>36-year-old</TIMEDATE> <JOB>doctor</JOB> began a <AMOUNT>50</AMOUNT><TIMEDATE>-minute</TIMEDATE> procedure.", |
| "stage4_predicted_xml": "At <TIMEDATE>08:30,</TIMEDATE> the <TIMEDATE>36-year-old</TIMEDATE> <JOB>doctor</JOB> began a <TIMEDATE>50-minute</TIMEDATE> procedure.", |
| "stage3_errors": [ |
| { |
| "type": "wrong_label", |
| "gold": { |
| "start": 41, |
| "end": 50, |
| "label": "TIMEDATE", |
| "value": "50-minute" |
| }, |
| "predicted": { |
| "start": 41, |
| "end": 43, |
| "label": "AMOUNT", |
| "value": "50" |
| } |
| }, |
| { |
| "type": "false_positive", |
| "gold": null, |
| "predicted": { |
| "start": 43, |
| "end": 50, |
| "label": "TIMEDATE", |
| "value": "-minute" |
| } |
| } |
| ], |
| "stage4_errors": [ |
| { |
| "type": "boundary", |
| "gold": { |
| "start": 3, |
| "end": 8, |
| "label": "TIMEDATE", |
| "value": "08:30" |
| }, |
| "predicted": { |
| "start": 3, |
| "end": 9, |
| "label": "TIMEDATE", |
| "value": "08:30," |
| } |
| } |
| ] |
| } |
| }, |
| "namedness_repetition": { |
| "description": "Demonyms become entities and repeated named mentions are inconsistently labeled.", |
| "status": "improved", |
| "stage3_f1": 0.7368421052631577, |
| "stage4_f1": 0.8888888888888888, |
| "f1_delta": 0.1520467836257311, |
| "stage3_error_counts": { |
| "false_positive": 1, |
| "wrong_label": 2 |
| }, |
| "stage4_error_counts": { |
| "wrong_label": 1 |
| }, |
| "unresolved": true, |
| "stage4_records_with_errors": 1, |
| "representative": { |
| "unique_index": "wild-namedness_repetition-1", |
| "gold_xml": "American <JOB>researchers</JOB> met in <LOCATION>America</LOCATION>.", |
| "stage3_predicted_xml": "<JOB>American</JOB> <JOB>researchers</JOB> met in <LOCATION>America</LOCATION>.", |
| "stage4_predicted_xml": "American <JOB>researchers</JOB> met in <LOCATION>America</LOCATION>.", |
| "stage3_errors": [ |
| { |
| "type": "false_positive", |
| "gold": null, |
| "predicted": { |
| "start": 0, |
| "end": 8, |
| "label": "JOB", |
| "value": "American" |
| } |
| } |
| ], |
| "stage4_errors": [] |
| } |
| } |
| } |
| } |
|
|
