myenv/lib/python3.10/site-packages/Bio/Entrez/Parser.py

# Copyright 2008-2014 by Michiel de Hoon.  All rights reserved.
# Revisions copyright 2008-2015 by Peter Cock. All rights reserved.
#
# This file is part of the Biopython distribution and governed by your
# choice of the "Biopython License Agreement" or the "BSD 3-Clause License".
# Please see the LICENSE file that should have been included as part of this
# package.

"""Parser for XML results returned by NCBI's Entrez Utilities.

This parser is used by the read() function in Bio.Entrez, and is not
intended be used directly.

The question is how to represent an XML file as Python objects. Some
XML files returned by NCBI look like lists, others look like dictionaries,
and others look like a mix of lists and dictionaries.

My approach is to classify each possible element in the XML as a plain
string, an integer, a list, a dictionary, or a structure. The latter is a
dictionary where the same key can occur multiple times; in Python, it is
represented as a dictionary where that key occurs once, pointing to a list
of values found in the XML file.

The parser then goes through the XML and creates the appropriate Python
object for each element. The different levels encountered in the XML are
preserved on the Python side. So a subelement of a subelement of an element
is a value in a dictionary that is stored in a list which is a value in
some other dictionary (or a value in a list which itself belongs to a list
which is a value in a dictionary, and so on). Attributes encountered in
the XML are stored as a dictionary in a member .attributes of each element,
and the tag name is saved in a member .tag.

To decide which kind of Python object corresponds to each element in the
XML, the parser analyzes the DTD referred at the top of (almost) every
XML file returned by the Entrez Utilities. This is preferred over a hand-
written solution, since the number of DTDs is rather large and their
contents may change over time. About half the code in this parser deals
with parsing the DTD, and the other half with the XML itself.
"""
import os
import warnings
from collections import Counter
from xml.parsers import expat
from io import BytesIO
import xml.etree.ElementTree as ET
from xml.sax.saxutils import escape

from urllib.request import urlopen, urlparse


# The following four classes are used to add a member .attributes to integers,
# strings, lists, and dictionaries, respectively.


class NoneElement:
    """NCBI Entrez XML element mapped to None."""

    def __init__(self, tag, attributes, key):
        """Create a NoneElement."""
        self.tag = tag
        self.key = key
        self.attributes = attributes

    def __eq__(self, other):
        """Define equality with other None objects."""
        if other is None:
            return True
        elif other.__eq__(None):
            return True
        else:
            return False

    def __ne__(self, other):
        """Define non-equality."""
        if other is None:
            return False
        elif other.__eq__(None):
            return False
        else:
            return True

    def __repr__(self):
        """Return a string representation of the object."""
        try:
            attributes = self.attributes
        except AttributeError:
            return "NoneElement"
        return "NoneElement(attributes=%r)" % attributes


class IntegerElement(int):
    """NCBI Entrez XML element mapped to an integer."""

    def __new__(cls, value, *args, **kwargs):
        """Create an IntegerElement."""
        return int.__new__(cls, value)

    def __init__(self, value, tag, attributes, key):
        """Initialize an IntegerElement."""
        self.tag = tag
        self.attributes = attributes
        self.key = key

    def __repr__(self):
        """Return a string representation of the object."""
        text = int.__repr__(self)
        try:
            attributes = self.attributes
        except AttributeError:
            return text
        return f"IntegerElement({text}, attributes={attributes!r})"


class StringElement(str):
    """NCBI Entrez XML element mapped to a string."""

    def __new__(cls, value, *args, **kwargs):
        """Create a StringElement."""
        return str.__new__(cls, value)

    def __init__(self, value, tag, attributes, key):
        """Initialize a StringElement."""
        self.tag = tag
        self.attributes = attributes
        self.key = key

    def __repr__(self):
        """Return a string representation of the object."""
        text = str.__repr__(self)
        attributes = self.attributes
        if not attributes:
            return text
        return f"StringElement({text}, attributes={attributes!r})"


class ListElement(list):
    """NCBI Entrez XML element mapped to a list."""

    def __init__(self, tag, attributes, allowed_tags, key=None):
        """Create a ListElement."""
        self.tag = tag
        if key is None:
            self.key = tag
        else:
            self.key = key
        self.attributes = attributes
        self.allowed_tags = allowed_tags

    def __repr__(self):
        """Return a string representation of the object."""
        text = list.__repr__(self)
        attributes = self.attributes
        if not attributes:
            return text
        return f"ListElement({text}, attributes={attributes!r})"

    def store(self, value):
        """Append an element to the list, checking tags."""
        key = value.key
        if self.allowed_tags is not None and key not in self.allowed_tags:
            raise ValueError("Unexpected item '%s' in list" % key)
        del value.key
        self.append(value)


class DictionaryElement(dict):
    """NCBI Entrez XML element mapped to a dictionaray."""

    def __init__(self, tag, attrs, allowed_tags, repeated_tags=None, key=None):
        """Create a DictionaryElement."""
        self.tag = tag
        if key is None:
            self.key = tag
        else:
            self.key = key
        self.attributes = attrs
        self.allowed_tags = allowed_tags
        self.repeated_tags = repeated_tags
        if repeated_tags:
            for key in repeated_tags:
                self[key] = []

    def __repr__(self):
        """Return a string representation of the object."""
        text = dict.__repr__(self)
        attributes = self.attributes
        if not attributes:
            return text
        return f"DictElement({text}, attributes={attributes!r})"

    def store(self, value):
        """Add an entry to the dictionary, checking tags."""
        key = value.key
        tag = value.tag
        if self.allowed_tags is not None and tag not in self.allowed_tags:
            raise ValueError("Unexpected item '%s' in dictionary" % key)
        del value.key
        if self.repeated_tags and key in self.repeated_tags:
            self[key].append(value)
        else:
            self[key] = value


class OrderedListElement(list):
    """NCBI Entrez XML element mapped to a list of lists.

    OrderedListElement is used to describe a list of repeating elements such as
    A, B, C, A, B, C, A, B, C ... where each set of A, B, C forms a group. This
    is then stored as [[A, B, C], [A, B, C], [A, B, C], ...]
    """

    def __init__(self, tag, attributes, allowed_tags, first_tag, key=None):
        """Create an OrderedListElement."""
        self.tag = tag
        if key is None:
            self.key = tag
        else:
            self.key = key
        self.attributes = attributes
        self.allowed_tags = allowed_tags
        self.first_tag = first_tag

    def __repr__(self):
        """Return a string representation of the object."""
        text = list.__repr__(self)
        attributes = self.attributes
        if not attributes:
            return text
        return f"OrderedListElement({text}, attributes={attributes!r})"

    def store(self, value):
        """Append an element to the list, checking tags."""
        key = value.key
        if self.allowed_tags is not None and key not in self.allowed_tags:
            raise ValueError("Unexpected item '%s' in list" % key)
        if key == self.first_tag:
            self.append([])
        self[-1].append(value)


class ErrorElement(str):
    """NCBI Entrez XML element containing an error message."""

    def __new__(cls, value, *args, **kwargs):
        """Create an ErrorElement."""
        return str.__new__(cls, value)

    def __init__(self, value, tag):
        """Initialize an ErrorElement."""
        self.tag = tag
        self.key = tag

    def __repr__(self):
        """Return the error message as a string."""
        text = str.__repr__(self)
        return f"ErrorElement({text})"


class NotXMLError(ValueError):
    """Failed to parse file as XML."""

    def __init__(self, message):
        """Initialize the class."""
        self.msg = message

    def __str__(self):
        """Return a string summary of the exception."""
        return (
            "Failed to parse the XML data (%s). Please make sure that the input data "
            "are in XML format." % self.msg
        )


class CorruptedXMLError(ValueError):
    """Corrupted XML."""

    def __init__(self, message):
        """Initialize the class."""
        self.msg = message

    def __str__(self):
        """Return a string summary of the exception."""
        return (
            "Failed to parse the XML data (%s). Please make sure that the input data "
            "are not corrupted." % self.msg
        )


class ValidationError(ValueError):
    """XML tag found which was not defined in the DTD.

    Validating parsers raise this error if the parser finds a tag in the XML
    that is not defined in the DTD. Non-validating parsers do not raise this
    error. The Bio.Entrez.read and Bio.Entrez.parse functions use validating
    parsers by default (see those functions for more information).
    """

    def __init__(self, name):
        """Initialize the class."""
        self.name = name

    def __str__(self):
        """Return a string summary of the exception."""
        return (
            "Failed to find tag '%s' in the DTD. To skip all tags that "
            "are not represented in the DTD, please call Bio.Entrez.read "
            "or Bio.Entrez.parse with validate=False." % self.name
        )


class DataHandlerMeta(type):
    """A metaclass is needed until Python supports @classproperty."""

    def __init__(cls, *args, **kwargs):
        """Initialize the class."""
        try:
            cls.directory = None  # use default directory for local cache
        except PermissionError:
            cls._directory = None  # no local cache

    @property
    def directory(cls):
        """Directory for caching XSD and DTD files."""
        return cls._directory

    @directory.setter
    def directory(cls, value):
        """Set a custom directory for the local DTD/XSD directories."""
        if value is None:
            import platform

            if platform.system() == "Windows":
                value = os.path.join(os.getenv("APPDATA"), "biopython")
            else:  # Unix/Linux/Mac
                home = os.path.expanduser("~")
                value = os.path.join(home, ".config", "biopython")
        # Create DTD local directory
        cls.local_dtd_dir = os.path.join(value, "Bio", "Entrez", "DTDs")
        os.makedirs(cls.local_dtd_dir, exist_ok=True)
        # Create XSD local directory
        cls.local_xsd_dir = os.path.join(value, "Bio", "Entrez", "XSDs")
        os.makedirs(cls.local_xsd_dir, exist_ok=True)
        # Save the directory name after creating the DTD and XSD local
        # directories to ensure there was no PermissionError.
        cls._directory = value


class DataHandler(metaclass=DataHandlerMeta):
    """Data handler for parsing NCBI XML from Entrez."""

    from Bio import Entrez

    global_dtd_dir = os.path.join(Entrez.__path__[0], "DTDs")
    global_xsd_dir = os.path.join(Entrez.__path__[0], "XSDs")
    local_dtd_dir = None
    local_xsd_dir = None

    del Entrez

    def __init__(self, validate, escape, ignore_errors):
        """Create a DataHandler object."""
        self.dtd_urls = []
        self.element = None
        self.level = 0
        self.data = []
        self.attributes = None
        self.allowed_tags = None
        self.constructors = {}
        self.strings = {}
        self.items = set()
        self.errors = set()
        self.validating = validate
        self.ignore_errors = ignore_errors
        self.parser = expat.ParserCreate(namespace_separator=" ")
        self.parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS)
        self.parser.XmlDeclHandler = self.xmlDeclHandler
        self.schema_namespace = None
        self.namespace_level = Counter()
        self.namespace_prefix = {}
        if escape:
            self.characterDataHandler = self.characterDataHandlerEscape
        else:
            self.characterDataHandler = self.characterDataHandlerRaw

    def read(self, handle):
        """Set up the parser and let it parse the XML results."""
        # Expat's parser.ParseFile function only accepts binary data;
        # see also the comment below for Entrez.parse.
        if handle.read(0) != b"":
            raise TypeError("file should be opened in binary mode")
        try:
            self.parser.ParseFile(handle)
        except expat.ExpatError as e:
            if self.parser.StartElementHandler:
                # We saw the initial <!xml declaration, so we can be sure that
                # we are parsing XML data. Most likely, the XML file is
                # corrupted.
                raise CorruptedXMLError(e) from None
            else:
                # We have not seen the initial <!xml declaration, so probably
                # the input data is not in XML format.
                raise NotXMLError(e) from None
        try:
            record = self.record
        except AttributeError:
            if self.parser.StartElementHandler:
                # We saw the initial <!xml declaration, and expat didn't notice
                # any errors, so self.record should be defined. If not, this is
                # a bug.
                raise RuntimeError(
                    "Failed to parse the XML file correctly, possibly due to a bug "
                    "in Bio.Entrez. Please contact the Biopython developers via "
                    "the mailing list or GitHub for assistance."
                ) from None
            else:
                # We did not see the initial <!xml declaration, so probably
                # the input data is not in XML format.
                raise NotXMLError("XML declaration not found") from None
        else:
            del record.key
            return record

    def parse(self, handle):
        """Parse the XML in the given file handle."""
        # The handle should have been opened in binary mode; data read from
        # the handle are then bytes. Expat will pick up the encoding from the
        # XML declaration (or assume UTF-8 if it is missing), and use this
        # encoding to convert the binary data to a string before giving it to
        # characterDataHandler.
        # While parser.ParseFile only accepts binary data, parser.Parse accepts
        # both binary data and strings. However, a file in text mode may have
        # been opened with an encoding different from the encoding specified in
        # the XML declaration at the top of the file. If so, the data in the
        # file will have been decoded with an incorrect encoding. To avoid
        # this, and to be consistent with parser.ParseFile (which is used in
        # the Entrez.read function above), we require the handle to be in
        # binary mode here as well.
        if handle.read(0) != b"":
            raise TypeError("file should be opened in binary mode")
        BLOCK = 1024
        while True:
            # Read in another block of data from the file.
            data = handle.read(BLOCK)
            try:
                self.parser.Parse(data, False)
            except expat.ExpatError as e:
                if self.parser.StartElementHandler:
                    # We saw the initial <!xml declaration, so we can be sure
                    # that we are parsing XML data. Most likely, the XML file
                    # is corrupted.
                    raise CorruptedXMLError(e) from None
                else:
                    # We have not seen the initial <!xml declaration, so
                    # probably the input data is not in XML format.
                    raise NotXMLError(e) from None
            try:
                records = self.record
            except AttributeError:
                if self.parser.StartElementHandler:
                    # We saw the initial <!xml declaration, and expat
                    # didn't notice any errors, so self.record should be
                    # defined. If not, this is a bug.

                    raise RuntimeError(
                        "Failed to parse the XML file correctly, possibly due to a "
                        "bug in Bio.Entrez. Please contact the Biopython "
                        "developers via the mailing list or GitHub for assistance."
                    ) from None
                else:
                    # We did not see the initial <!xml declaration, so
                    # probably the input data is not in XML format.
                    raise NotXMLError("XML declaration not found") from None

            if not isinstance(records, list):
                raise ValueError(
                    "The XML file does not represent a list. Please use Entrez.read "
                    "instead of Entrez.parse"
                )

            if not data:
                break

            while len(records) >= 2:
                # Then the first record is finished, while the second record
                # is still a work in progress.
                record = records.pop(0)
                yield record

        # We have reached the end of the XML file
        self.parser = None
        if self.element is not None:
            # No more XML data, but there is still some unfinished business
            raise CorruptedXMLError("Premature end of data")

        # Send out the remaining records
        yield from records

    def xmlDeclHandler(self, version, encoding, standalone):
        """Set XML handlers when an XML declaration is found."""
        self.parser.CharacterDataHandler = self.characterDataHandler
        self.parser.ExternalEntityRefHandler = self.externalEntityRefHandler
        self.parser.StartNamespaceDeclHandler = self.startNamespaceDeclHandler
        self.parser.EndNamespaceDeclHandler = self.endNamespaceDeclHandler
        self.parser.StartElementHandler = self.handleMissingDocumentDefinition

    def handleMissingDocumentDefinition(self, tag, attrs):
        """Raise an Exception if neither a DTD nor an XML Schema is found."""
        raise ValueError(
            "As the XML data contained neither a Document Type Definition (DTD) nor an XML Schema, Bio.Entrez is unable to parse these data. We recommend using a generic XML parser from the Python standard library instead, for example ElementTree."
        )

    def startNamespaceDeclHandler(self, prefix, uri):
        """Handle start of an XML namespace declaration."""
        if prefix == "xsi":
            # This is an xml schema
            self.schema_namespace = uri
            self.parser.StartElementHandler = self.schemaHandler
        else:
            # Note that the DTD for MathML specifies a default attribute
            # that declares the namespace for each MathML element. This means
            # that MathML element in the XML has an invisible MathML namespace
            # declaration that triggers a call to startNamespaceDeclHandler
            # and endNamespaceDeclHandler. Therefore we need to count how often
            # startNamespaceDeclHandler and endNamespaceDeclHandler were called
            # to find out their first and last invocation for each namespace.
            if prefix == "mml":
                assert uri == "http://www.w3.org/1998/Math/MathML"
            elif prefix == "xlink":
                assert uri == "http://www.w3.org/1999/xlink"
            elif prefix == "ali":
                assert uri.rstrip("/") == "http://www.niso.org/schemas/ali/1.0"
            else:
                raise ValueError(f"Unknown prefix '{prefix}' with uri '{uri}'")
            self.namespace_level[prefix] += 1
            self.namespace_prefix[uri] = prefix

    def endNamespaceDeclHandler(self, prefix):
        """Handle end of an XML namespace declaration."""
        if prefix != "xsi":
            self.namespace_level[prefix] -= 1
            if self.namespace_level[prefix] == 0:
                for key, value in self.namespace_prefix.items():
                    if value == prefix:
                        break
                else:
                    raise RuntimeError("Failed to find namespace prefix")
                del self.namespace_prefix[key]

    def schemaHandler(self, name, attrs):
        """Process the XML schema (before processing the element)."""
        key = "%s noNamespaceSchemaLocation" % self.schema_namespace
        schema = attrs[key]
        handle = self.open_xsd_file(os.path.basename(schema))
        # if there is no local xsd file grab the url and parse the file
        if not handle:
            handle = urlopen(schema)
            text = handle.read()
            self.save_xsd_file(os.path.basename(schema), text)
            handle.close()
            self.parse_xsd(ET.fromstring(text))
        else:
            self.parse_xsd(ET.fromstring(handle.read()))
            handle.close()
        # continue handling the element
        self.startElementHandler(name, attrs)
        # reset the element handler
        self.parser.StartElementHandler = self.startElementHandler

    def startElementHandler(self, tag, attrs):
        """Handle start of an XML element."""
        prefix = None
        if self.namespace_prefix:
            try:
                uri, name = tag.split()
            except ValueError:
                pass
            else:
                prefix = self.namespace_prefix[uri]
                tag = f"{prefix}:{name}"
        if tag in self.items:
            assert tag == "Item"
            name = attrs["Name"]
            itemtype = attrs["Type"]
            del attrs["Type"]
            if itemtype == "Structure":
                del attrs["Name"]
                element = DictionaryElement(
                    name, attrs, allowed_tags=None, repeated_tags=None
                )
                parent = self.element
                element.parent = parent
                # For consistency with lists below, store the element here
                if parent is None:
                    self.record = element
                else:
                    parent.store(element)
                self.element = element
                self.parser.EndElementHandler = self.endElementHandler
                self.parser.CharacterDataHandler = self.skipCharacterDataHandler
            elif name in ("ArticleIds", "History"):
                del attrs["Name"]
                allowed_tags = None  # allowed tags are unknown
                repeated_tags = frozenset(["pubmed", "medline"])
                element = DictionaryElement(
                    tag,
                    attrs,
                    allowed_tags=allowed_tags,
                    repeated_tags=repeated_tags,
                    key=name,
                )
                parent = self.element
                element.parent = parent
                # For consistency with lists below, store the element here
                if parent is None:
                    self.record = element
                else:
                    parent.store(element)
                self.element = element
                self.parser.EndElementHandler = self.endElementHandler
                self.parser.CharacterDataHandler = self.skipCharacterDataHandler
            elif itemtype == "List":
                del attrs["Name"]
                allowed_tags = None  # allowed tags are unknown
                element = ListElement(tag, attrs, allowed_tags, name)
                parent = self.element
                element.parent = parent
                if self.element is None:
                    # Set self.record here to let Entrez.parse iterate over it
                    self.record = element
                else:
                    parent.store(element)
                self.element = element
                self.parser.EndElementHandler = self.endElementHandler
                self.parser.CharacterDataHandler = self.skipCharacterDataHandler
            elif itemtype == "Integer":
                self.parser.EndElementHandler = self.endIntegerElementHandler
                self.parser.CharacterDataHandler = self.characterDataHandler
                self.attributes = attrs
            elif itemtype in ("String", "Unknown", "Date", "Enumerator"):
                assert self.attributes is None
                self.attributes = attrs
                self.parser.StartElementHandler = self.startRawElementHandler
                self.parser.EndElementHandler = self.endStringElementHandler
                self.parser.CharacterDataHandler = self.characterDataHandler
            else:
                raise ValueError("Unknown item type %s" % name)
        elif tag in self.errors:
            self.parser.EndElementHandler = self.endErrorElementHandler
            self.parser.CharacterDataHandler = self.characterDataHandler
        elif tag in self.strings:
            self.parser.StartElementHandler = self.startRawElementHandler
            self.parser.EndElementHandler = self.endStringElementHandler
            self.parser.CharacterDataHandler = self.characterDataHandler
            assert self.allowed_tags is None
            self.allowed_tags = self.strings[tag]
            assert self.attributes is None
            self.attributes = attrs
        elif tag in self.constructors:
            cls, allowed_tags = self.constructors[tag]
            element = cls(tag, attrs, *allowed_tags)
            parent = self.element
            element.parent = parent
            if parent is None:
                # Set self.record here to let Entrez.parse iterate over it
                self.record = element
            else:
                parent.store(element)
            self.element = element
            self.parser.EndElementHandler = self.endElementHandler
            self.parser.CharacterDataHandler = self.skipCharacterDataHandler
        else:
            # Element not found in DTD
            if tag == "processing-meta":
                terms = []
                dtd_version = "1.3"
                if attrs["tagset-family"] == "jats":
                    terms.append("JATS")
                if attrs["base-tagset"] == "archiving":
                    term = "archivearticle" + dtd_version.replace(".", "-")
                    terms.append(term)
                if attrs.get("mathml-version") == "3.0":
                    terms.append("mathml3")
                basename = "-".join(terms)
                url = f"https://{attrs['tagset-family']}.nlm.nih.gov/{attrs['base-tagset']}/{dtd_version}/{basename}.dtd"
                self.xmlDeclHandler(None, None, None)
                self.externalEntityRefHandler(None, None, url, None)
                # remainder will be ignored and will not be stored in the record
            elif self.validating:
                raise ValidationError(tag)
            # this will not be stored in the record
            self.parser.StartElementHandler = self.startSkipElementHandler
            self.parser.EndElementHandler = self.endSkipElementHandler
            self.parser.CharacterDataHandler = self.skipCharacterDataHandler
            self.level = 1

    def startRawElementHandler(self, name, attrs):
        """Handle start of an XML raw element."""
        # check if the name is in a namespace
        prefix = None
        if self.namespace_prefix:
            try:
                uri, name = name.split()
            except ValueError:
                pass
            else:
                prefix = self.namespace_prefix[uri]
                if self.namespace_level[prefix] == 1:
                    attrs = {"xmlns": uri}
        if prefix:
            key = f"{prefix}:{name}"
        else:
            key = name
        # self.allowed_tags is ignored for now. Anyway we know what to do
        # with this tag.
        tag = "<%s" % name
        for key, value in attrs.items():
            tag += f' {key}="{value}"'
        tag += ">"
        self.data.append(tag)
        self.parser.EndElementHandler = self.endRawElementHandler
        self.level += 1

    def startSkipElementHandler(self, name, attrs):
        """Handle start of an XML skip element."""
        self.level += 1

    def endStringElementHandler(self, tag):
        """Handle end of an XML string element."""
        element = self.element
        if element is not None:
            self.parser.StartElementHandler = self.startElementHandler
            self.parser.EndElementHandler = self.endElementHandler
            self.parser.CharacterDataHandler = self.skipCharacterDataHandler
        data = "".join(self.data)
        self.data = []
        attributes = self.attributes
        self.attributes = None
        if self.namespace_prefix:
            try:
                uri, name = tag.split()
            except ValueError:
                pass
            else:
                prefix = self.namespace_prefix[uri]
                tag = f"{prefix}:{name}"
        if tag in self.items:
            assert tag == "Item"
            key = attributes["Name"]
            del attributes["Name"]
        else:
            key = tag
        value = StringElement(data, tag, attributes, key)
        if element is None:
            self.record = element
        else:
            element.store(value)
        self.allowed_tags = None

    def endRawElementHandler(self, name):
        """Handle end of an XML raw element."""
        self.level -= 1
        if self.level == 0:
            self.parser.EndElementHandler = self.endStringElementHandler
        if self.namespace_prefix:
            try:
                uri, name = name.split()
            except ValueError:
                pass
        tag = "</%s>" % name
        self.data.append(tag)

    def endSkipElementHandler(self, name):
        """Handle end of an XML skip element."""
        self.level -= 1
        if self.level == 0:
            self.parser.StartElementHandler = self.startElementHandler
            self.parser.EndElementHandler = self.endElementHandler

    def endErrorElementHandler(self, tag):
        """Handle end of an XML error element."""
        element = self.element
        if element is not None:
            self.parser.StartElementHandler = self.startElementHandler
            self.parser.EndElementHandler = self.endElementHandler
            self.parser.CharacterDataHandler = self.skipCharacterDataHandler
        data = "".join(self.data)
        if data == "":
            return
        if self.ignore_errors is False:
            raise RuntimeError(data)
        self.data = []
        value = ErrorElement(data, tag)
        if element is None:
            self.record = element
        else:
            element.store(value)

    def endElementHandler(self, name):
        """Handle end of an XML element."""
        element = self.element
        self.element = element.parent
        del element.parent

    def endIntegerElementHandler(self, tag):
        """Handle end of an XML integer element."""
        attributes = self.attributes
        self.attributes = None
        assert tag == "Item"
        key = attributes["Name"]
        del attributes["Name"]
        if self.data:
            value = int("".join(self.data))
            self.data = []
            value = IntegerElement(value, tag, attributes, key)
        else:
            value = NoneElement(tag, attributes, key)
        element = self.element
        if element is None:
            self.record = value
        else:
            self.parser.EndElementHandler = self.endElementHandler
            self.parser.CharacterDataHandler = self.skipCharacterDataHandler
            if value is None:
                return
            element.store(value)

    def characterDataHandlerRaw(self, content):
        """Handle character data as-is (raw)."""
        self.data.append(content)

    def characterDataHandlerEscape(self, content):
        """Handle character data by encoding it."""
        content = escape(content)
        self.data.append(content)

    def skipCharacterDataHandler(self, content):
        """Handle character data by skipping it."""

    def parse_xsd(self, root):
        """Parse an XSD file."""
        prefix = "{http://www.w3.org/2001/XMLSchema}"
        for element in root:
            isSimpleContent = False
            attribute_keys = []
            keys = []
            multiple = []
            assert element.tag == prefix + "element"
            name = element.attrib["name"]
            assert len(element) == 1
            complexType = element[0]
            assert complexType.tag == prefix + "complexType"
            for component in complexType:
                tag = component.tag
                if tag == prefix + "attribute":
                    # we could distinguish by type; keeping string for now
                    attribute_keys.append(component.attrib["name"])
                elif tag == prefix + "sequence":
                    maxOccurs = component.attrib.get("maxOccurs", "1")
                    for key in component:
                        assert key.tag == prefix + "element"
                        ref = key.attrib["ref"]
                        keys.append(ref)
                        if maxOccurs != "1" or key.attrib.get("maxOccurs", "1") != "1":
                            multiple.append(ref)
                elif tag == prefix + "simpleContent":
                    assert len(component) == 1
                    extension = component[0]
                    assert extension.tag == prefix + "extension"
                    assert extension.attrib["base"] == "xs:string"
                    for attribute in extension:
                        assert attribute.tag == prefix + "attribute"
                        # we could distinguish by type; keeping string for now
                        attribute_keys.append(attribute.attrib["name"])
                    isSimpleContent = True
            allowed_tags = frozenset(keys)
            if len(keys) == 1 and keys == multiple:
                assert not isSimpleContent
                args = (allowed_tags,)
                self.constructors[name] = (ListElement, args)
            elif len(keys) >= 1:
                assert not isSimpleContent
                repeated_tags = frozenset(multiple)
                args = (allowed_tags, repeated_tags)
                self.constructors[name] = (DictionaryElement, args)
            else:
                self.strings[name] = allowed_tags

    def elementDecl(self, name, model):
        """Call a call-back function for each element declaration in a DTD.

        This is used for each element declaration in a DTD like::

            <!ELEMENT       name          (...)>

        The purpose of this function is to determine whether this element
        should be regarded as a string, integer, list, dictionary, structure,
        or error.
        """
        if name.upper() == "ERROR":
            self.errors.add(name)
            return
        if name == "Item" and model == (
            expat.model.XML_CTYPE_MIXED,
            expat.model.XML_CQUANT_REP,
            None,
            ((expat.model.XML_CTYPE_NAME, expat.model.XML_CQUANT_NONE, "Item", ()),),
        ):
            # Special case. As far as I can tell, this only occurs in the
            # eSummary DTD.
            self.items.add(name)
            return
        # First, remove ignorable parentheses around declarations
        while (
            model[0] in (expat.model.XML_CTYPE_SEQ, expat.model.XML_CTYPE_CHOICE)
            and model[1] in (expat.model.XML_CQUANT_NONE, expat.model.XML_CQUANT_OPT)
            and len(model[3]) == 1
        ):
            model = model[3][0]
        # PCDATA declarations correspond to strings
        if model[0] in (expat.model.XML_CTYPE_MIXED, expat.model.XML_CTYPE_EMPTY):
            if model[1] == expat.model.XML_CQUANT_REP:
                children = model[3]
                allowed_tags = frozenset(child[2] for child in children)
            else:
                allowed_tags = frozenset()
            self.strings[name] = allowed_tags
            return
        # Children can be anything; use a dictionary-type element
        if model == (expat.model.XML_CTYPE_ANY, expat.model.XML_CQUANT_NONE, None, ()):
            allowed_tags = None
            repeated_tags = None
            args = (allowed_tags, repeated_tags)
            self.constructors[name] = (DictionaryElement, args)
            return
        # List-type elements
        if model[0] in (
            expat.model.XML_CTYPE_CHOICE,
            expat.model.XML_CTYPE_SEQ,
        ) and model[1] in (expat.model.XML_CQUANT_PLUS, expat.model.XML_CQUANT_REP):
            children = model[3]
            allowed_tags = frozenset(child[2] for child in children)
            if model[0] == expat.model.XML_CTYPE_SEQ:
                if len(children) > 1:
                    assert model[1] == expat.model.XML_CQUANT_PLUS
                    first_child = children[0]
                    assert first_child[1] == expat.model.XML_CQUANT_NONE
                    first_tag = first_child[2]
                    args = allowed_tags, first_tag
                    self.constructors[name] = (OrderedListElement, args)
                    return
                assert len(children) == 1
            self.constructors[name] = (ListElement, (allowed_tags,))
            return
        # This is the tricky case. Check which keys can occur multiple
        # times. If only one key is possible, and it can occur multiple
        # times, then this is a list. If more than one key is possible,
        # but none of them can occur multiple times, then this is a
        # dictionary. Otherwise, this is a structure.
        # In 'single' and 'multiple', we keep track which keys can occur
        # only once, and which can occur multiple times.
        single = []
        multiple = []
        errors = []
        # The 'count' function is called recursively to make sure all the
        # children in this model are counted.

        def count(model):
            quantifier, key, children = model[1:]
            if key is None:
                if quantifier in (
                    expat.model.XML_CQUANT_PLUS,
                    expat.model.XML_CQUANT_REP,
                ):
                    for child in children:
                        multiple.append(child[2])
                else:
                    for child in children:
                        count(child)
            elif key.upper() == "ERROR":
                errors.append(key)
            else:
                if quantifier in (
                    expat.model.XML_CQUANT_NONE,
                    expat.model.XML_CQUANT_OPT,
                ):
                    single.append(key)
                elif quantifier in (
                    expat.model.XML_CQUANT_PLUS,
                    expat.model.XML_CQUANT_REP,
                ):
                    multiple.append(key)

        count(model)
        if len(single) == 0 and len(multiple) == 1:
            allowed_tags = frozenset(multiple + errors)
            self.constructors[name] = (ListElement, (allowed_tags,))
        else:
            allowed_tags = frozenset(single + multiple + errors)
            repeated_tags = frozenset(multiple)
            args = (allowed_tags, repeated_tags)
            self.constructors[name] = (DictionaryElement, args)

    def open_dtd_file(self, filename):
        """Open specified DTD file."""
        if DataHandler.local_dtd_dir is not None:
            path = os.path.join(DataHandler.local_dtd_dir, filename)
            try:
                handle = open(path, "rb")
            except FileNotFoundError:
                pass
            else:
                return handle
        path = os.path.join(DataHandler.global_dtd_dir, filename)
        try:
            handle = open(path, "rb")
        except FileNotFoundError:
            pass
        else:
            return handle
        return None

    def open_xsd_file(self, filename):
        """Open specified XSD file."""
        if DataHandler.local_xsd_dir is not None:
            path = os.path.join(DataHandler.local_xsd_dir, filename)
            try:
                handle = open(path, "rb")
            except FileNotFoundError:
                pass
            else:
                return handle
        path = os.path.join(DataHandler.global_xsd_dir, filename)
        try:
            handle = open(path, "rb")
        except FileNotFoundError:
            pass
        else:
            return handle
        return None

    def save_dtd_file(self, filename, text):
        """Save DTD file to cache."""
        if DataHandler.local_dtd_dir is None:
            return
        path = os.path.join(DataHandler.local_dtd_dir, filename)
        try:
            handle = open(path, "wb")
        except OSError:
            warnings.warn(f"Failed to save {filename} at {path}")
        else:
            handle.write(text)
            handle.close()

    def save_xsd_file(self, filename, text):
        """Save XSD file to cache."""
        if DataHandler.local_xsd_dir is None:
            return
        path = os.path.join(DataHandler.local_xsd_dir, filename)
        try:
            handle = open(path, "wb")
        except OSError:
            warnings.warn(f"Failed to save {filename} at {path}")
        else:
            handle.write(text)
            handle.close()

    def externalEntityRefHandler(self, context, base, systemId, publicId):
        """Handle external entity reference in order to cache DTD locally.

        The purpose of this function is to load the DTD locally, instead
        of downloading it from the URL specified in the XML. Using the local
        DTD results in much faster parsing. If the DTD is not found locally,
        we try to download it. If new DTDs become available from NCBI,
        putting them in Bio/Entrez/DTDs will allow the parser to see them.
        """
        urlinfo = urlparse(systemId)
        if urlinfo.scheme in ["http", "https", "ftp"]:
            # Then this is an absolute path to the DTD.
            url = systemId
        elif urlinfo.scheme == "":
            # Then this is a relative path to the DTD.
            # Look at the parent URL to find the full path.
            try:
                source = self.dtd_urls[-1]
            except IndexError:
                # Assume the default URL for DTDs if the top parent
                # does not contain an absolute path
                source = "http://www.ncbi.nlm.nih.gov/dtd/"
            else:
                source = os.path.dirname(source)
            # urls always have a forward slash, don't use os.path.join
            url = source.rstrip("/") + "/" + systemId
        else:
            raise ValueError("Unexpected URL scheme %r" % urlinfo.scheme)
        self.dtd_urls.append(url)
        # First, try to load the local version of the DTD file
        location, filename = os.path.split(systemId)
        handle = self.open_dtd_file(filename)
        if not handle:
            # DTD is not available as a local file. Try accessing it through
            # the internet instead.
            try:
                handle = urlopen(url)
            except OSError:
                raise RuntimeError(f"Failed to access {filename} at {url}") from None
            text = handle.read()
            handle.close()
            self.save_dtd_file(filename, text)
            handle = BytesIO(text)

        parser = self.parser.ExternalEntityParserCreate(context)
        parser.ElementDeclHandler = self.elementDecl
        parser.ParseFile(handle)
        handle.close()
        self.dtd_urls.pop()
        self.parser.StartElementHandler = self.startElementHandler
        return 1