File size: 4,143 Bytes
718badd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bdf35c9
718badd
 
 
 
 
 
 
 
 
 
 
 
 
42969b4
bdf35c9
 
 
 
42969b4
bdf35c9
 
 
 
42969b4
 
bdf35c9
 
 
 
 
 
718badd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
"""Provision a complete official Swiss-Prot snapshot for local BLAST+.

The two release-label reads guard against a release changing during download.
All source and index bytes are fingerprinted; requests cannot provide URLs.
"""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import gzip
import hashlib
import json
from pathlib import Path
import re
import shutil
import subprocess
import requests

BASE = "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/"


def digest(path):
    value = hashlib.sha256()
    with path.open("rb") as handle:
        for block in iter(lambda: handle.read(1024 * 1024), b""):
            value.update(block)
    return value.hexdigest()


def download(name, target):
    if name == "reldate.txt":
        response = requests.get(
            "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/reldate.txt",
            stream=True, timeout=120, allow_redirects=False,
        )
    elif name == "uniprot_sprot.fasta.gz":
        response = requests.get(
            "https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/uniprot_sprot.fasta.gz",
            stream=True, timeout=120, allow_redirects=False,
        )
    else:
        raise ValueError("Only the two fixed official Swiss-Prot resources may be downloaded")
    with response as source:
        if source.status_code != 200:
            raise RuntimeError(f"Official Swiss-Prot download returned HTTP {source.status_code}")
        with target.open("wb") as out:
            for block in source.iter_content(chunk_size=1024 * 1024):
                out.write(block)
    return target


def provision(root):
    root = Path(root).resolve()
    root.mkdir(parents=True, exist_ok=True)
    (root / "manifest.json").unlink(missing_ok=True)
    before = download("reldate.txt", root / "reldate.before.txt").read_text()
    archive = download("uniprot_sprot.fasta.gz", root / "uniprot_sprot.fasta.gz")
    after = download("reldate.txt", root / "reldate.txt").read_text()
    if before != after:
        raise RuntimeError("UniProt release changed during reference download")
    match = re.search(r"(20\d\d_\d\d)", after)
    if not match:
        raise RuntimeError("Could not establish the UniProt reference release")
    fasta = root / "swissprot.fasta"
    with gzip.open(archive, "rb") as source, fasta.open("wb") as out:
        shutil.copyfileobj(source, out)
    with fasta.open("rb") as handle:
        count = sum(line.startswith(b">sp|") for line in handle)
    if count < 100000:
        raise RuntimeError("Downloaded source is not a complete Swiss-Prot snapshot")
    process = subprocess.run(["makeblastdb", "-in", str(fasta), "-dbtype", "prot",
        "-parse_seqids", "-out", str(root / "swissprot"), "-title", "UniProtKB/Swiss-Prot " + match.group(1)],
        check=True, capture_output=True, text=True, timeout=600)
    index_files = {path.name: digest(path) for path in sorted(root.glob("swissprot.*"))
                   if path.name != "swissprot.fasta" and path.is_file()}
    manifest = {"database": "swissprot", "release": match.group(1),
        "scope": "complete UniProtKB/Swiss-Prot canonical sequence snapshot",
        "source_url": BASE + "uniprot_sprot.fasta.gz",
        "release_url": BASE + "reldate.txt", "release_text": after,
        "retrieved_at": datetime.now(timezone.utc).isoformat(), "sequence_count": count,
        "archive_sha256": digest(archive), "fasta_sha256": digest(fasta),
        "index_files": index_files,
        "indexer": subprocess.run(["makeblastdb", "-version"], capture_output=True,
                                 text=True, check=True, timeout=15).stdout.strip()}
    (root / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
    archive.unlink()
    fasta.unlink()
    (root / "reldate.before.txt").unlink()
    print(json.dumps(manifest, sort_keys=True))
    return manifest


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("root")
    provision(parser.parse_args().root)