| """Build the English name lookup used by the Genome Atlas labels. |
| |
| Two sources are merged, curated names winning over NCBI: |
| |
| * NCBI Taxonomy ``names.dmp`` supplies ``genbank common name`` and ``common name`` |
| for roughly 9,000 taxa, almost all of them species, genera and familiar classes. |
| * A curated table below covers the large unranked clades that NCBI leaves |
| unnamed. These are exactly the groups the atlas shows first — Opisthokonta, |
| Eumetazoa, Ecdysozoa and their kin — so without them the top of the tree reads |
| as untranslated Latin. |
| |
| Curated glosses are short, lower-case noun phrases describing the living members |
| of the clade. They are plain-language summaries, not formal synonyms; the |
| scientific name stays the primary label everywhere in the interface. |
| """ |
| import argparse |
| from collections import Counter |
| from datetime import datetime, timezone |
| import json |
| from pathlib import Path |
| import sqlite3 |
| import tarfile |
|
|
| ROOT = Path(__file__).resolve().parent |
| NAME_CLASSES = ("genbank common name", "common name") |
|
|
| |
| |
| |
| CURATED = { |
| 2698737: ("Sar", "marine algae and protists"), |
| 554915: ("Amoebozoa", "amoebas"), |
| 2611352: ("Discoba", "flagellate protists"), |
| 33682: ("Euglenozoa", "euglenids and trypanosomes"), |
| 2704647: ("Metakinetoplastina", "trypanosomes and kin"), |
| 2696291: ("Ochrophyta", "brown algae and diatoms"), |
| 431838: ("Intramacronucleata", "ciliates"), |
| 422676: ("Aconoidasida", "malaria parasites and kin"), |
| 1280412: ("Conoidasida", "gut parasites"), |
| 2692248: ("core chlorophytes", "green algae"), |
| 3166: ("Chlorophyceae", "green algae"), |
| 75966: ("Trebouxiophyceae", "green algae"), |
| 1035538: ("Mamiellophyceae", "tiny marine green algae"), |
| 13792: ("Mamiellales", "tiny marine green algae"), |
| 127916: ("Ichthyosporea", "parasites of fish and kin"), |
| 2687318: ("Filasterea", "single-celled animal kin"), |
| 2686024: ("Rotosphaerida", "amoeboid protists"), |
| 6042: ("Demospongiae", "common sponges"), |
| 1913637: ("Mucoromycota", "bread moulds and kin"), |
| 451507: ("Mucoromycotina", "bread moulds"), |
| 3704128: ("Kickxellomycota", "thread-like fungi"), |
| 112252: ("Fungi incertae sedis", "fungi of uncertain position"), |
| 33154: ("Opisthokonta", "animals and fungi"), |
| 6072: ("Eumetazoa", "animals with true tissues"), |
| 33213: ("Bilateria", "two-sided animals"), |
| 33317: ("Protostomia", "mouth-first animals"), |
| 1206794: ("Ecdysozoa", "moulting animals"), |
| 2697495: ("Spiralia", "spiral-cleaving animals"), |
| 1206795: ("Lophotrochozoa", "molluscs, worms and kin"), |
| 88770: ("Panarthropoda", "arthropods and kin"), |
| 3701028: ("Altocrustacea", "crustaceans and insects"), |
| 3701030: ("Allotriocarida", "insects and some crustaceans"), |
| 85512: ("Dicondylia", "most insects"), |
| 33340: ("Neoptera", "wing-folding insects"), |
| 33342: ("Paraneoptera", "bugs, lice and thrips"), |
| 3701061: ("Eumetabola", "advanced insects"), |
| 33392: ("Endopterygota", "insects with metamorphosis"), |
| 3701062: ("Aparaglossata", "most metamorphic insects"), |
| 3701063: ("Panorpida", "flies, moths and kin"), |
| 3701064: ("Neuropteroidea", "beetles and lacewings"), |
| 41084: ("Polyphaga", "most beetles"), |
| 41088: ("Cucujiformia", "plant-eating beetles"), |
| 7434: ("Aculeata", "bees, wasps and ants"), |
| 85604: ("Amphiesmenoptera", "moths and caddisflies"), |
| 41191: ("Glossata", "moths and butterflies"), |
| 41196: ("Neolepidoptera", "moths and butterflies"), |
| 41197: ("Heteroneura", "moths and butterflies"), |
| 37567: ("Ditrysia", "most moths and butterflies"), |
| 104430: ("Apoditrysia", "moths and butterflies"), |
| 104431: ("Obtectomera", "macro-moths and butterflies"), |
| 37570: ("Noctuoidea", "owlet moths"), |
| 40092: ("Hesperioidea", "skipper butterflies"), |
| 37568: ("Tortricoidea", "leafroller moths"), |
| 7139: ("Tortricidae", "leafroller moths"), |
| 7203: ("Brachycera", "short-horned flies"), |
| 480118: ("Eremoneura", "advanced flies"), |
| 480117: ("Cyclorrhapha", "advanced flies"), |
| 43733: ("Muscomorpha", "house flies and kin"), |
| 43738: ("Schizophora", "advanced flies"), |
| 43741: ("Acalyptratae", "small flies"), |
| 43746: ("Ephydroidea", "fruit flies and kin"), |
| 43845: ("Drosophilinae", "fruit flies"), |
| 46877: ("Drosophilini", "fruit flies"), |
| 119089: ("Chromadorea", "roundworms"), |
| 6236: ("Rhabditida", "roundworms"), |
| 89593: ("Craniata", "animals with a skull"), |
| 117570: ("Teleostomi", "jawed vertebrates with bone"), |
| 8287: ("Sarcopterygii", "lobe-finned fish and kin"), |
| 1338369: ("Dipnotetrapodomorpha", "lungfish and tetrapods"), |
| 186623: ("Actinopteri", "ray-finned fish"), |
| 41665: ("Neopterygii", "modern ray-finned fish"), |
| 1489341: ("Osteoglossocephalai", "bony-tongued fish and kin"), |
| 186625: ("Clupeocephala", "most teleost fish"), |
| 1489388: ("Euteleosteomorpha", "advanced teleost fish"), |
| 123365: ("Neoteleostei", "advanced bony fish"), |
| 123366: ("Eurypterygia", "advanced bony fish"), |
| 123367: ("Ctenosquamata", "advanced bony fish"), |
| 123368: ("Acanthomorphata", "spiny-rayed fish"), |
| 123369: ("Euacanthomorphacea", "spiny-rayed fish"), |
| 1489872: ("Percomorphaceae", "perch-like fish"), |
| 1489922: ("Eupercaria", "perch-like fish"), |
| 1489908: ("Ovalentaria", "cichlids and relatives"), |
| 1489910: ("Cichlomorphae", "cichlids and kin"), |
| 318546: ("Pseudocrenilabrinae", "African cichlids"), |
| 186634: ("Otomorpha", "carps, catfish and herrings"), |
| 32519: ("Ostariophysi", "carps and catfish"), |
| 186626: ("Otophysi", "carps and catfish"), |
| 186627: ("Cypriniphysae", "carps and loaches"), |
| 1329799: ("Archelosauria", "turtles, birds, crocodiles"), |
| 8492: ("Archosauria", "birds and crocodiles"), |
| 436489: ("Saurischia", "dinosaurs, birds today"), |
| 436491: ("Theropoda", "dinosaurs, birds today"), |
| 436492: ("Coelurosauria", "dinosaurs, birds today"), |
| 8825: ("Neognathae", "most modern birds"), |
| 3078114: ("Neoaves", "most modern birds"), |
| 3073809: ("Australaves", "parrots and songbirds"), |
| 1329950: ("Unidentata", "lizards and snakes"), |
| 1329912: ("Episquamata", "lizards and snakes"), |
| 32525: ("Theria", "live-bearing mammals"), |
| 1437010: ("Boreoeutheria", "most placental mammals"), |
| 314146: ("Euarchontoglires", "primates and rodents"), |
| 314145: ("Laurasiatheria", "carnivores, bats, ungulates"), |
| 376913: ("Haplorrhini", "monkeys, apes and tarsiers"), |
| 314293: ("Simiiformes", "monkeys and apes"), |
| 9526: ("Catarrhini", "Old World monkeys and apes"), |
| 207598: ("Homininae", "great apes and humans"), |
| 9845: ("Ruminantia", "ruminants"), |
| 35500: ("Pecora", "deer, cattle and kin"), |
| 9895: ("Bovidae", "cattle, sheep and antelope"), |
| 451864: ("Dikarya", "higher fungi"), |
| 716545: ("saccharomyceta", "yeasts and sac fungi"), |
| 716546: ("leotiomyceta", "filamentous sac fungi"), |
| 715989: ("sordariomyceta", "flask fungi and kin"), |
| 1520881: ("OSLEUM clade", "filamentous sac fungi"), |
| 147550: ("Sordariomycetes", "flask fungi"), |
| 222544: ("Sordariomycetidae", "flask fungi"), |
| 5139: ("Sordariales", "flask fungi"), |
| 222543: ("Hypocreomycetidae", "mould-forming sac fungi"), |
| 5125: ("Hypocreales", "moulds and insect fungi"), |
| 110618: ("Nectriaceae", "Fusarium and relatives"), |
| 474943: ("Cordycipitaceae", "insect-killing fungi"), |
| 1028384: ("Glomerellales", "anthracnose fungi"), |
| 681950: ("Glomerellaceae", "anthracnose fungi"), |
| 5455: ("Colletotrichum", "anthracnose fungi"), |
| 639021: ("Magnaporthales", "rice blast fungi"), |
| 2528436: ("Pyriculariaceae", "rice blast fungi"), |
| 48558: ("Pyricularia", "rice blast fungi"), |
| 715962: ("dothideomyceta", "sooty moulds and kin"), |
| 147541: ("Dothideomycetes", "sooty moulds and leaf fungi"), |
| 451867: ("Dothideomycetidae", "sooty moulds"), |
| 451868: ("Pleosporomycetidae", "leaf-spot fungi"), |
| 92860: ("Pleosporales", "leaf-spot fungi"), |
| 715340: ("Pleosporineae", "leaf-spot fungi"), |
| 28556: ("Pleosporaceae", "leaf-spot fungi"), |
| 451869: ("Botryosphaeriales", "canker fungi"), |
| 45131: ("Botryosphaeriaceae", "canker fungi"), |
| 35724: ("Macrophomina", "charcoal rot fungi"), |
| 147545: ("Eurotiomycetes", "moulds and relatives"), |
| 451871: ("Eurotiomycetidae", "moulds and relatives"), |
| 1131492: ("Aspergillaceae", "Aspergillus and Penicillium"), |
| 33183: ("Onygenales", "skin and hair fungi"), |
| 147548: ("Leotiomycetes", "cup fungi and moulds"), |
| 4891: ("Saccharomycetes", "budding yeasts"), |
| 4892: ("Saccharomycetales", "budding yeasts"), |
| 4893: ("Saccharomycetaceae", "baker's yeast family"), |
| 4930: ("Saccharomyces", "baker's yeasts"), |
| 3239874: ("Pichiomycetes", "budding yeasts"), |
| 3243775: ("Pichiales", "budding yeasts"), |
| 1156497: ("Pichiaceae", "budding yeasts"), |
| 3239873: ("Dipodascomycetes", "budding yeasts"), |
| 3243772: ("Dipodascales", "budding yeasts"), |
| 766764: ("Debaryomycetaceae", "yeasts"), |
| 27319: ("Metschnikowiaceae", "yeasts"), |
| 5302: ("Agaricomycotina", "mushroom-forming fungi"), |
| 155619: ("Agaricomycetes", "mushrooms and bracket fungi"), |
| 452333: ("Agaricomycetidae", "mushroom-forming fungi"), |
| 68889: ("Boletales", "boletes"), |
| 155616: ("Tremellomycetes", "jelly fungi"), |
| 4762: ("Oomycota", "water moulds"), |
| 3418804: ("Peronosporomycetes", "water moulds"), |
| 35493: ("Streptophyta", "land plants and their algae"), |
| 131221: ("Streptophytina", "land plants and stoneworts"), |
| 78536: ("Euphyllophyta", "ferns and seed plants"), |
| 1437183: ("Mesangiospermae", "core flowering plants"), |
| 1437197: ("Petrosaviidae", "most monocot plants"), |
| 4734: ("commelinids", "grasses, palms and kin"), |
| 38820: ("Poales", "grasses and sedges"), |
| 147368: ("Pooideae", "temperate grasses"), |
| 359160: ("BOP clade", "bamboo, rice, wheat grasses"), |
| 1648038: ("Triticodae", "wheat and barley grasses"), |
| 91827: ("Gunneridae", "core eudicot plants"), |
| 1437201: ("Pentapetalae", "core eudicot plants"), |
| 71275: ("rosids", "rose-related plants"), |
| 91835: ("fabids", "legume-related plants"), |
| 91836: ("malvids", "mallow-related plants"), |
| 71274: ("asterids", "daisy-related plants"), |
| 91888: ("lamiids", "mint-related plants"), |
| 72025: ("Fabales", "legumes and kin"), |
| 3814: ("Papilionoideae", "pea-flowered legumes"), |
| 2231393: ("50 kb inversion clade", "legumes"), |
| 3699: ("Brassicales", "cabbages and mustards"), |
| 980083: ("Camelineae", "thale cress and kin"), |
| 3701: ("Arabidopsis", "thale cresses"), |
| 3744: ("Rosales", "roses, figs and kin"), |
| 3524: ("Caryophyllales", "cacti, beets and kin"), |
| 3646: ("Malpighiales", "willows and spurges"), |
| 4069: ("Solanales", "nightshades and kin"), |
| 4143: ("Lamiales", "mints and kin"), |
| } |
|
|
|
|
| def dump_rows(archive, name): |
| with archive.extractfile(name) as stream: |
| for line in stream: |
| yield [part.strip() for part in line.decode("utf-8").split("|")] |
|
|
|
|
| def build(taxdump, database, output): |
| with sqlite3.connect(f"{Path(database).resolve().as_uri()}?mode=ro", uri=True) as conn: |
| known = dict(conn.execute("SELECT taxid, name FROM taxa")) |
| wrong = {t: (want, known.get(t)) for t, (want, _) in CURATED.items() if known.get(t) != want} |
| if wrong: |
| raise ValueError(f"Curated taxids do not match the snapshot: {wrong}") |
|
|
| names, ranked = {}, {} |
| with tarfile.open(taxdump, "r:gz") as archive: |
| for row in dump_rows(archive, "names.dmp"): |
| taxid = int(row[0]) |
| if taxid not in known or row[3] not in NAME_CLASSES: |
| continue |
| rank = NAME_CLASSES.index(row[3]) |
| if taxid not in ranked or rank < ranked[taxid]: |
| names[taxid], ranked[taxid] = row[1], rank |
| sources = Counter(ncbi=len(names)) |
| for taxid, (_, english) in CURATED.items(): |
| sources["curated_override" if taxid in names else "curated"] += 1 |
| names[taxid] = english |
| |
| |
| redundant = [t for t, english in names.items() if english.casefold() == known[t].casefold()] |
| for taxid in redundant: |
| del names[taxid] |
| payload = {"created_at": datetime.now(timezone.utc).isoformat(), |
| "sources": {"ncbi_taxonomy": "https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz", |
| "name_classes": list(NAME_CLASSES), |
| "curated": "refresh_common_names.py, plain-language glosses for unnamed clades"}, |
| "counts": dict(sources, redundant_dropped=len(redundant), total=len(names)), |
| "names": {str(t): names[t] for t in sorted(names)}} |
| Path(output).write_text(json.dumps(payload, indent=1, sort_keys=False)) |
| print(json.dumps(payload["counts"], indent=2)) |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument("--taxdump", type=Path, default=ROOT / ".cache/coverage/taxdump.tar.gz") |
| parser.add_argument("--database", type=Path, default=ROOT / "data/taxonomy.sqlite") |
| parser.add_argument("--output", type=Path, default=ROOT / "data/common_names.json") |
| args = parser.parse_args() |
| build(args.taxdump, args.database, args.output) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
