scripts/select_error_cases.py

'''
Modest script to help selecting "interesting" error cases in a set of Grobid processed JATS/PDF pairs (e.g. PMC, bioRxiv, etc.).
For example:
- we can select "hard-failed" results without title, authors, affiliation, abstract, without full text and/or without bibliographical references,
- we can select results with interesting patterns to be better cover, like availability statements,
- we can select error extracted metadata when comparing with JATS encoding.

The script offers a base for different selection scenario, to be refined at desired. 

Before running this script, it is assumed that the following resources are available:
- a repository of JATS and PDF pair files is available, for instance from bioRxiv,
e.g. https://zenodo.org/record/3873702 or PMC 
(see https://grobid.readthedocs.io/en/latest/End-to-end-evaluation/#directory-structure)
- a processing of the PDF files by Grobid with resulting TEI XML files
- the batch createTraining has been run on all the PDF
https://grobid.readthedocs.io/en/latest/Grobid-batch/#createtraining

Then the present script can be run as follow:

---
usage: select_error_cases.py [-h] --grobid-tei GROBID_TEI --jats JATS
                             [--grobid-training GROBID_TRAINING] [--out OUT]

Select ineresting error cases from biorxiv for Grobid training data

optional arguments:
  -h, --help            show this help message and exit
  --grobid-tei GROBID_TEI
                        path to the generated Grobid TEI result directory
  --jats JATS           path to the bioRxiv JATS directory
  --grobid-training GROBID_TRAINING
                        path to the directory containing the generated grobid training files
  --out OUT             path to the directory where to add the selected training files

For example:

> python3 select_error_cases.py --grobid-tei result/ --jats /media/lopez/data1/biblio/bioRxiv/biorxiv-10k-validation-2000 --grobid-training training --out training_selected

The script will select error cases that would be worth adding as training data, and will copy all the pre-labeld training files
for these error cases under the repository given by the --output parameter. 

'''

import os
import lxml
import argparse
from lxml import etree as ET
import shutil

# Grobid TEI xpath
grobid_title = "//tei:titleStmt/tei:title/text()"
grobid_authors = "//tei:sourceDesc/tei:biblStruct/tei:analytic/tei:author/tei:persName/tei:surname/text()"
grobid_first_author = "//tei:sourceDesc/tei:biblStruct/tei:analytic/tei:author[1]/tei:persName/tei:surname/text()"
grobid_affiliation = "//tei:sourceDesc/tei:biblStruct/tei:analytic/tei:author/tei:affiliation/tei:orgName/text()"
grobid_abstract = "//tei:profileDesc/tei:abstract//text()"
grobid_citations_base = "//tei:back/tei:div/tei:listBibl/tei:biblStruct"
grobid_availability = "//tei:back/tei:div[@type=\"availability\"]"

# JATS xpath
jats_title = "/article/front/article-meta/title-group/article-title//text()"
jats_authors = "/article/front/article-meta/contrib-group/contrib[@contrib-type=\"author\"]/name/surname/text()"
jats_first_author = "/article/front/article-meta/contrib-group/contrib[@contrib-type=\"author\"][1]/name/surname/text()"
jats_affiliation = "/article/front/article-meta/contrib-group/aff//text()"
jats_abstract = "/article/front/article-meta/abstract//text()"
jats_citations_base = "//ref-list/ref"

# this is an example of availability section titles
interesting_availability_patterns = [ 
    "Availability of data and materials", 
    "Availability of data and material",
    "AVAILABILITY OF DATA AND MATERIALS",
    "Availability of supporting data", 
    "Data access",
    "DATA ACCESS", 
    "Data accessibility"
    "DATA ACCESSIBILITY",
    "Code availability", 
    "Code Accessibility",
    "Software availability",
    "Data availability",
    "Data preprocessing and availability",
    "Software and Data Availability",
    "Availability and implementation",
    "Availability of code and data",
    "Implementation and availability",
    "Data availability statement",
    "Data and code availability",
    "Accessibility of biological resources",
    "Availability of Data materials",
    "DATA AND SOFTWARE AVAILABILITY",
    "Data and software availability",
    "Computer code and data availability",
    "Software availability and documentation",
    "Data Accession Information",
    "Availability of data and code",
    "Data and Materials Availability",
    "Data and Reagent availability",
    "Data access in NCBI",
    "Data availability and distribution",
    "DATABASE AVAILABILITY",
    "Code and data availability",
    "Code dependencies and availability",
    "AVAILABILITY"
]

def evaluate_error_cases(grobid_tei, jats_files, grobid_training=None, output=None):
    interesting_error_cases = []
    ns = {"tei": "http://www.tei-c.org/ns/1.0"}
    for root, directories, filenames in os.walk(grobid_tei):
        for filename in filenames: 
            if filename.endswith(".tei.xml"):
                tei_file = os.path.join(root, filename)
                jats_file = os.path.join(jats_files, filename.replace(".tei.xml", ""), filename.replace(".tei.xml", ".xml"))

                # apply some xpath on the files
                parser = ET.XMLParser(remove_comments=True)

                try:
                    tei_xml = ET.parse(tei_file, parser=parser)
                except:
                    #print("XML parsing error with", tei_file)
                    continue

                try:
                    jats_xml = ET.parse(jats_file, parser=parser)
                except:
                    #print("XML parsing error with", jats_file)
                    continue

                # titles
                tei_title = tei_xml.xpath(grobid_title, namespaces=ns)
                if tei_title != None and len(tei_title)>0:
                    tei_title = tei_title[0]
                else:
                    tei_title = None 
                nlm_title = jats_xml.xpath(jats_title)
                if nlm_title != None and len(nlm_title)>0:
                    nlm_title = nlm_title[0]
                else:
                    nlm_title = None

                #print("tei_title:", tei_title)
                #print("nlm_title:", nlm_title)
                
                # authors
                tei_authors = tei_xml.xpath(grobid_authors, namespaces=ns)
                nlm_authors = jats_xml.xpath(jats_authors)

                #print("tei_authors:", tei_authors)
                #print("nlm_authors:", nlm_authors)

                # first author
                tei_first_author = tei_xml.xpath(grobid_first_author, namespaces=ns)
                nlm_first_author = jats_xml.xpath(jats_first_author)

                #print("tei_first_author:", tei_first_author)
                #print("nlm_first_author:", nlm_first_author)

                # affiliation
                tei_affiliation = tei_xml.xpath(grobid_affiliation, namespaces=ns)
                nlm_affiliation = jats_xml.xpath(jats_affiliation)

                #print("tei_affiliation:", tei_affiliation)
                #print("nlm_affiliation:", nlm_affiliation)

                # abstract
                tei_abstract = tei_xml.xpath(grobid_abstract, namespaces=ns)
                nlm_abstract = jats_xml.xpath(jats_abstract)

                #print("tei_abstract:", tei_abstract)
                #print("nlm_abstract:", nlm_abstract)

                # citations
                tei_citations_base = tei_xml.xpath(grobid_citations_base, namespaces=ns)
                nlm_citations_base = jats_xml.xpath(jats_citations_base)

                # availability section (Grobid only)
                tei_availability = tei_xml.xpath(grobid_availability, namespaces=ns)

                # check availability statement patterns
                all_text_tei = "".join(tei_xml.xpath(".//text()"))
                all_text_nlm = " ".join(jats_xml.xpath(".//text()"))

                # check conditions
                main_error = False

                if tei_title == None or len(tei_title.strip()) == 0:
                    main_error = True
                elif (tei_citations_base == None or len(tei_citations_base)<2) and (nlm_citations_base != None and len(nlm_citations_base)>0):
                    main_error = True
                elif (tei_authors == None or len(tei_authors)==0) and (nlm_authors != None and len(nlm_authors)>0):
                    main_error = True
                elif (tei_abstract == None or len(tei_abstract)==0) and (nlm_abstract != None and len(nlm_abstract)>0):
                    main_error = True

                avail_stat_match = False

                # consider only file without Grobid found availability section, but with an availability pattern
                if (tei_availability == None or len(tei_availability) ==0):
                    for pattern in interesting_availability_patterns:
                        if all_text_nlm.find(pattern) != -1:
                            avail_stat_match = True
                            break

                if main_error and avail_stat_match:
                    print("selected case:", jats_file)

                    # if available, copy the selected training files into the out directory 
                    if grobid_training != None and output != None:
                        # training file
                        file_base = filename.replace(".tei.xml", "")
                        
                        segmentation_file = os.path.join(grobid_training, file_base + ".training.segmentation.tei.xml")
                        segmentation_raw = os.path.join(grobid_training, file_base + ".training.segmentation")
                        shutil.copy2(segmentation_file, output)
                        shutil.copy2(segmentation_raw, output)
                        
                        header_file = os.path.join(grobid_training, file_base + ".training.header.tei.xml")
                        header_raw = os.path.join(grobid_training, file_base + ".training.header")
                        if os.path.isfile(header_file):
                            shutil.copy2(header_file, output)
                            shutil.copy2(header_raw, output)

                        fulltext_file = os.path.join(grobid_training, file_base + ".training.fulltext.tei.xml")
                        fulltext_raw = os.path.join(grobid_training, file_base + ".training.fulltext")
                        shutil.copy2(fulltext_file, output)
                        shutil.copy2(fulltext_raw, output)

                        affiliation_file = os.path.join(grobid_training, file_base + ".training.affiliation.tei.xml")
                        if os.path.isfile(affiliation_file):
                            shutil.copy2(affiliation_file, output)
                        
                        authors_file = os.path.join(grobid_training, file_base + ".training.header.authors.tei.xml")
                        if os.path.isfile(authors_file):
                            shutil.copy2(authors_file, output)
                        
                        references_authors_file = os.path.join(grobid_training, file_base + ".training.references.authors.tei.xml")
                        if os.path.isfile(references_authors_file):
                            shutil.copy2(references_authors_file, output)

                        references_file = os.path.join(grobid_training, file_base + ".training.references.tei.xml")
                        if os.path.isfile(references_file):
                            shutil.copy2(references_file, output)
                            referenceSegmenter_file = os.path.join(grobid_training, file_base + ".training.referenceSegmenter.tei.xml")
                            referenceSegmenter_raw = os.path.join(grobid_training, file_base + ".training.referenceSegmenter")
                            if os.path.isfile(referenceSegmenter_file):
                                shutil.copy2(referenceSegmenter_file, output)
                                shutil.copy2(referenceSegmenter_raw, output)

                        table_file = os.path.join(grobid_training, file_base + ".training.table.tei.xml")
                        table_raw = os.path.join(grobid_training, file_base + ".training.table")
                        if os.path.isfile(table_file):
                            shutil.copy2(table_file, output)
                            shutil.copy2(table_raw, output)

                        figure_file = os.path.join(grobid_training, file_base + ".training.figure.tei.xml")
                        figure_raw = os.path.join(grobid_training, file_base + ".training.figure")
                        if os.path.isfile(figure_file):
                            shutil.copy2(figure_file, output)
                            shutil.copy2(figure_raw, output)

                        # and we copy the PDF for reference/corrections
                        pdf_file = os.path.join(jats_files, file_base, file_base + ".pdf")
                        if os.path.isfile(pdf_file):
                            shutil.copy2(pdf_file, output)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description = "Select ineresting error cases from biorxiv for Grobid training data")
    parser.add_argument("--grobid-tei", type=str, required=True,
        help="path to the generated Grobid TEI result directory")
    parser.add_argument("--jats", type=str, required=True,
        help="path to the bioRxiv JATS directory")
    parser.add_argument("--grobid-training", type=str, required=False,
        help="path to the directory containing the generated grobid training files")
    parser.add_argument("--out", type=str, required=False,
        help="path to the directory where to add the selected training files")

    args = parser.parse_args()
    grobid_tei = args.grobid_tei
    jats_files = args.jats
    grobid_training = args.grobid_training
    output = args.out

    # check path and call methods
    if grobid_tei is None or not os.path.isdir(grobid_tei):
        print("error: the path to the grobid TEI result directory is not valid: ", grobid_tei)
        exit(0)

    if jats_files is None or not os.path.isdir(jats_files):
        print("error: the path to the JATS files is not valid: ", jats_files)
        exit(0)

    if grobid_training is not None and not os.path.isdir(grobid_training):
        print("warning: the path to the Grobid training directory is not valid: ", grobid_training)
        print("selected training files will not be copied")
        grobid_training = None

    if output is not None and not os.path.isdir(output):
        print("warning: the path to the output directory is not valid: ", output)
        print("selected training files will not be copied")
        output = None

    evaluate_error_cases(grobid_tei, jats_files, grobid_training, output)