diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..6ab0c2641e8977432e37362c9a340a529a691845 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,35 +1,89 @@ *.7z filter=lfs diff=lfs merge=lfs -text *.arrow filter=lfs diff=lfs merge=lfs -text *.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text *.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text *.ftz filter=lfs diff=lfs merge=lfs -text *.gz filter=lfs diff=lfs merge=lfs -text *.h5 filter=lfs diff=lfs merge=lfs -text *.joblib filter=lfs diff=lfs merge=lfs -text *.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text *.model filter=lfs diff=lfs merge=lfs -text *.msgpack filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.npz filter=lfs diff=lfs merge=lfs -text *.onnx filter=lfs diff=lfs merge=lfs -text *.ot filter=lfs diff=lfs merge=lfs -text *.parquet filter=lfs diff=lfs merge=lfs -text *.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text + *.pth filter=lfs diff=lfs merge=lfs -text *.rar filter=lfs diff=lfs merge=lfs -text -*.safetensors filter=lfs diff=lfs merge=lfs -text saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.tar.* filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text *.tflite filter=lfs diff=lfs merge=lfs -text *.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text *.xz filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +*.wav filter=lfs diff=lfs merge=lfs -text +*.mp3 filter=lfs diff=lfs merge=lfs -text +*.flac filter=lfs diff=lfs merge=lfs -text +*.ogg filter=lfs diff=lfs merge=lfs -text +*.m4a filter=lfs diff=lfs merge=lfs -text +*.aac filter=lfs diff=lfs merge=lfs -text +*.opus filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.mov filter=lfs diff=lfs merge=lfs -text +*.avi filter=lfs diff=lfs merge=lfs -text +*.mkv filter=lfs diff=lfs merge=lfs -text +*.webm filter=lfs diff=lfs merge=lfs -text +*.obj filter=lfs diff=lfs merge=lfs -text +*.glb filter=lfs diff=lfs merge=lfs -text +*.gltf filter=lfs diff=lfs merge=lfs -text +*.ply filter=lfs diff=lfs merge=lfs -text +*.stl filter=lfs diff=lfs merge=lfs -text +*.fbx filter=lfs diff=lfs merge=lfs -text + +data/inverse_folding/5YH2.cif filter=lfs diff=lfs merge=lfs -text +data/inverse_folding/5YH2.pdb filter=lfs diff=lfs merge=lfs -text +data/variant_prediction/BLAT_ECOLX_Ranganathan2015.csv filter=lfs diff=lfs merge=lfs -text + + + + +weight/esm-main/examples/data/P62593.fasta filter=lfs diff=lfs merge=lfs -text +weight/esm-main/examples/inverse_folding/data/4uv3.cif filter=lfs diff=lfs merge=lfs -text +weight/esm-main/examples/inverse_folding/data/4uv3.pdb filter=lfs diff=lfs merge=lfs -text +weight/esm-main/examples/inverse_folding/data/5YH2.cif filter=lfs diff=lfs merge=lfs -text +weight/esm-main/examples/inverse_folding/data/5YH2.pdb filter=lfs diff=lfs merge=lfs -text +weight/esm-main/examples/inverse_folding/notebook.ipynb filter=lfs diff=lfs merge=lfs -text +weight/esm-main/examples/inverse_folding/notebook_multichain.ipynb filter=lfs diff=lfs merge=lfs -text + +weight/esm-main/examples/variant-prediction/data/BLAT_ECOLX_1_b0.5.a3m filter=lfs diff=lfs merge=lfs -text +weight/esm-main/examples/variant-prediction/data/BLAT_ECOLX_Ranganathan2015.csv filter=lfs diff=lfs merge=lfs -text +weight/esm-main/examples/variant-prediction/data/BLAT_ECOLX_Ranganathan2015_labeled.csv filter=lfs diff=lfs merge=lfs -text +weight/esm-main/examples/variant-prediction/data/BLAT_ECOLX_Ranganathan2015_labeled_masked.csv filter=lfs diff=lfs merge=lfs -text + + + +*.pt filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..30a9c5faf43545ae34031a75ec04b84bd595b118 --- /dev/null +++ b/README.md @@ -0,0 +1,283 @@ +--- +license: mit +tasks: + - protein-structure-prediction +frameworks: + - pytorch +language: + - en + - zh +tags: + - OneScience + - Life Sciences + - Protein Language Model + - Protein Structure Prediction + - Variant Effect Prediction + - ESM +huggingface: + model: OneScience-Sugon/ESM + data_path: data/ +--- + +

+ + ESM + +

+ +# Model Introduction + +ESM (Evolutionary Scale Modeling) is a family of protein language models released by Meta AI / FAIR. It can be used for protein representation extraction, structure prediction, variant effect scoring, and fixed-backbone sequence design. + +Paper: Evolutionary-scale prediction of atomic-level protein structure with a language model +https://www.science.org/doi/10.1126/science.ade2574 + +# Model Description + +This model package provides PyTorch inference support for ESM-1, ESM-2, MSA Transformer, ESMFold, ESM-1v, and ESM-IF1, together with adaptations for running on DCUs. Sample data is distributed with the Hugging Face model repository `OneScience-Sugon/ESM`. + +# Use Cases + +| Scenario | Description | +| :---: | :--- | +| Protein representation extraction | Takes a FASTA file as input and outputs per-token, mean-pooled, BOS, or contact representations | +| Protein structure prediction | Takes one or more amino acid sequences as input and outputs corresponding PDB structure files | +| Variant effect scoring | Takes a wild-type sequence and a DMS mutation table as input and outputs mutation effect scores | +| Fixed-backbone sequence design | Takes a PDB / CIF structure and chain ID as input and samples candidate sequences that satisfy the backbone constraints | +| Structure-conditioned sequence scoring | Takes a structure and candidate sequences as input and computes their conditional log-likelihoods | +| Hugging Face / OneCode execution | After downloading the model project, quickly verifies that the scripts run correctly in a life-sciences runtime environment | + + +# Usage Guide + +## 1. OneCode Usage + +Try one-click AI4S development in the OneCode online environment: + +[Try one-click AI4S development](https://web-2069360198568017922-iaaj.ksai.scnet.cn:58043/home) + +## 2. Manual Installation and Usage + +**Hardware Requirements** + +- GPU or DCU is recommended. +- A CPU can be used for import checks and lightweight configuration tests; full training and inference will be slow. +- DCU users must install DTK in advance. DTK 25.04.2 or later is recommended, or a OneScience-recommended version matching the current cluster. + + + + + +**Environment Check** + +- NVIDIA GPU: + +```bash +nvidia-smi +``` + +- Hygon DCU: + +```bash +hy-smi +``` + +### Download the Model Package + +```bash +hf download --model OneScience-Sugon/ESM --local-dir ./ESM +cd ESM +``` + +This model package includes a small set of sample data that can be used directly to validate the default workflow. + +### Install the Runtime Environment + +**DCU Environment** + +```bash +# Activate DTK and CONDA first +conda create -n onescience311 python=3.11 -y +conda activate onescience311 +# uv installation supported +pip install onescience[bio-dcu] -i http://mirrors.onescience.ai:3141/pypi/simple/ --trusted-host mirrors.onescience.ai +``` + +```bash +# If required libraries cannot be found, activate the CUDA compatibility environment as follows: +source ${ROCM_PATH}/cuda/env.sh +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:$LD_LIBRARY_PATH" +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib/python3.11/site-packages/fastpt/torch/lib:$LD_LIBRARY_PATH" +``` + +After installation, return to the model package directory: + +```bash +cd ./ESM +``` + +### Training and Inference Data Overview + +The FASTA, PDB / CIF, and DMS files used in the ESM examples are distributed with the [Hugging Face model repository OneScience-Sugon/ESM](https://huggingface.co/OneScience-Sugon/ESM). After downloading the complete model package, the files are available under `data/`. This model package does not include a training entry point; the data is intended for example inference and workflow validation. You can also download only the data directory: + +```bash +hf download --model OneScience-Sugon/ESM ESM/data --local-dir ./data +``` +### Model Weights + +The repository includes multiple ESM model checkpoints under `weight/`; select the appropriate checkpoint for inference. + +### Preparing Weights + +Place the required ESM weights in the following directory: + +```text +weight/ + checkpoints/ + esm2_t6_8M_UR50D.pt + esmfold_3B_v1.pt + esm1v_t33_650M_UR90S_1.pt + esm_if1_gvp4_t16_142M_UR50.pt + ... +``` + +When using a shared runtime environment, you can specify the weights location through an environment variable: + +```bash +export ESM_WEIGHT_DIR=/path/to/esm/weight +``` + +The default example uses: + +- `weight/checkpoints/esm2_t6_8M_UR50D.pt` + +The ESMFold, ESM-1v, and ESM-IF1 examples require their respective weights to be available. + +### Default Example + +```bash +bash scripts/infer.sh +``` + +The default example reads `data/fasta/few_proteins.fasta`, extracts protein representations using `esm2_t6_8M_UR50D.pt`, and saves the results to `outputs/embeddings/`. + +### Sequence Representation Extraction + +```bash +python scripts/extract.py \ + weight/checkpoints/esm2_t6_8M_UR50D.pt \ + data/fasta/few_proteins.fasta \ + outputs/embeddings \ + --include mean per_tok \ + --repr_layers 6 +``` + +### ESMFold Structure Prediction + +```bash +python scripts/fold.py \ + -i data/fasta/few_proteins.fasta \ + -o outputs/pdb \ + --model-dir weight \ + --cpu-only +``` + +The output directory will contain one or more `.pdb` files. For production GPU / DCU inference, remove `--cpu-only` and configure `--chunk-size` or `--max-tokens-per-batch` according to the available accelerator memory. + +You can also explicitly enable ESMFold via the default script: + +```bash +RUN_ESMFOLD=1 bash scripts/infer.sh +``` + +### Inverse Folding — Sequence Sampling + +```bash +python scripts/inverse_folding/sample_sequences.py \ + data/inverse_folding/5YH2.pdb \ + --chain A \ + --outpath outputs/sampled_seqs.fasta \ + --num-samples 1 \ + --nogpu +``` + +### Inverse Folding — Sequence Scoring + +```bash +python scripts/inverse_folding/score_log_likelihoods.py \ + data/inverse_folding/5YH2.pdb \ + data/inverse_folding/5YH2_mutated_seqs.fasta \ + --chain A \ + --outpath outputs/sequence_scores.csv \ + --nogpu +``` + +### Variant Effect Prediction + +Variant effect prediction requires a wild-type sequence that is consistent with the mutation annotations in the DMS table: + +```bash +python scripts/variant_prediction/predict.py \ + --model-location esm1v_t33_650M_UR90S_1 \ + --sequence "${ESM_VARIANT_SEQUENCE}" \ + --dms-input data/variant_prediction/BLAT_ECOLX_Ranganathan2015.csv \ + --mutation-col mutant \ + --dms-output outputs/variant_prediction.csv \ + --offset-idx 24 \ + --scoring-strategy wt-marginals +``` + +# Data Format + +Sample data is stored under `data/` by default: + +```text +data/ + fasta/ + few_proteins.fasta + some_proteins.fasta + inverse_folding/ + 5YH2.pdb + 5YH2.cif + 5YH2_mutated_seqs.fasta + example.json + variant_prediction/ + BLAT_ECOLX_Ranganathan2015.csv + rho_pp.csv + aggregated_rho.csv + aggregated_rho_round3.csv +``` + +In this structure: + +- FASTA files are used for sequence representation extraction and structure prediction. +- PDB / CIF files are used for inverse folding sampling and structure-conditioned sequence scoring. +- Variant effect prediction CSV files must include a mutation column. The default column name is `mutant`, and mutations use notation such as `A123B`. +- For custom DMS data, the wild-type amino acid at each mutated position in the sequence provided via `--sequence` must match the corresponding mutation annotation. + +# Verification + +Static import check: + +```bash +python scripts/check_import_boundaries.py +``` + +Syntax check: + +```bash +python -B -c "import ast, pathlib; [ast.parse(p.read_text(encoding='utf-8'), filename=str(p)) for root in ['model', 'scripts', 'tests'] for p in pathlib.Path(root).rglob('*.py')]" +``` + +# OneScience Official Information + +| Platform | OneScience Main Repository | Skills Repository | +| --- | --- | --- | +| Gitee | https://gitee.com/onescience-ai/onescience | https://gitee.com/onescience-ai/oneskills | +| GitHub | https://github.com/onescience-ai/OneScience | https://github.com/onescience-ai/oneskills | + +# Citation & License + +- This repository is adapted from the open-source ESM model to support DCUs. +- The ESM source code is licensed under the MIT License; see `LICENSE`. For the usage terms governing model weights and data, refer to the documentation provided by the respective publishers. +- For scientific use, please cite the corresponding original ESM paper for each submodel used. For ESM-2 / ESMFold, cite: [Evolutionary-scale prediction of atomic-level protein structure with a language model](https://www.science.org/doi/10.1126/science.ade2574). diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000000000000000000000000000000000000..f3d7612cc06c4fda48a089c80f73cdfd48ba6716 --- /dev/null +++ b/configuration.json @@ -0,0 +1,2 @@ +{"framework":"Pytorch","task":"other"} + diff --git a/model/README.md b/model/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d33f3339fe18860356f36f5e505c75314de6760b --- /dev/null +++ b/model/README.md @@ -0,0 +1,10 @@ +# Local Model Source + +This directory contains OneScience `models` sources that ESM imports directly. + +- `esm/`: local copy of `onescience.models.esm` +- `openfold/`: local copy of `onescience.models.openfold`, required by ESMFold +- `protenix/layer_norm/`: optional fused layer norm dependency used by OpenFold when `LAYERNORM_TYPE=fast_layernorm` + +Other OneScience namespaces such as `onescience.datapipes`, `onescience.modules`, and `onescience.utils` are intentionally imported from the installed OneScience environment. + diff --git a/model/__init__.py b/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7dc8f21e7864f50b9b9ef9c765f5358a61c6d252 --- /dev/null +++ b/model/__init__.py @@ -0,0 +1,2 @@ +"""Local model packages copied from OneScience for the standalone ESM project.""" + diff --git a/model/__pycache__/__init__.cpython-311.pyc b/model/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..26b2c1be950cf7f590b6fdc9a724af280053c112 Binary files /dev/null and b/model/__pycache__/__init__.cpython-311.pyc differ diff --git a/model/esm/__init__.py b/model/esm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f40d172517dea9b8233246dd079aa1709674fb10 --- /dev/null +++ b/model/esm/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from onescience.datapipes import esm as data # noqa +from onescience.datapipes.esm import Alphabet, BatchConverter, FastaBatchedDataset # noqa + +from .version import version as __version__ # noqa +from .esm1 import ProteinBertModel # noqa +from .esm2 import ESM2 # noqa +from .msa_transformer import MSATransformer #noqa +from . import pretrained # noqa diff --git a/model/esm/__pycache__/__init__.cpython-311.pyc b/model/esm/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3826c86a6152dac0cb1a47523af646c95f470f51 Binary files /dev/null and b/model/esm/__pycache__/__init__.cpython-311.pyc differ diff --git a/model/esm/__pycache__/esm1.cpython-311.pyc b/model/esm/__pycache__/esm1.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a57097126ca3e40d8819b05bd38ca6b6cd8d3c8a Binary files /dev/null and b/model/esm/__pycache__/esm1.cpython-311.pyc differ diff --git a/model/esm/__pycache__/esm2.cpython-311.pyc b/model/esm/__pycache__/esm2.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba62f422b80e3645d4e7a484ac7eba70557427c9 Binary files /dev/null and b/model/esm/__pycache__/esm2.cpython-311.pyc differ diff --git a/model/esm/__pycache__/msa_transformer.cpython-311.pyc b/model/esm/__pycache__/msa_transformer.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d18f0d9c51be7bfe6ecb156527f446b5560f9c3c Binary files /dev/null and b/model/esm/__pycache__/msa_transformer.cpython-311.pyc differ diff --git a/model/esm/__pycache__/pretrained.cpython-311.pyc b/model/esm/__pycache__/pretrained.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9578daf98ebc5d300fa7fc1e9836eb72e98df0b2 Binary files /dev/null and b/model/esm/__pycache__/pretrained.cpython-311.pyc differ diff --git a/model/esm/__pycache__/version.cpython-311.pyc b/model/esm/__pycache__/version.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9b9426a2d487e97e2bb3b68a498d240c6f9cdf2 Binary files /dev/null and b/model/esm/__pycache__/version.cpython-311.pyc differ diff --git a/model/esm/esm1.py b/model/esm/esm1.py new file mode 100644 index 0000000000000000000000000000000000000000..35fbdf824682322c0e886e73d214dad79554837d --- /dev/null +++ b/model/esm/esm1.py @@ -0,0 +1,200 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from onescience.modules.esm import ( + TransformerLayer, + LearnedPositionalEmbedding, + SinusoidalPositionalEmbedding, + RobertaLMHead, + ESM1bLayerNorm, + ContactPredictionHead, +) + + +class ProteinBertModel(nn.Module): + @classmethod + def add_args(cls, parser): + parser.add_argument( + "--num_layers", default=36, type=int, metavar="N", help="number of layers" + ) + parser.add_argument( + "--embed_dim", default=1280, type=int, metavar="N", help="embedding dimension" + ) + parser.add_argument( + "--logit_bias", action="store_true", help="whether to apply bias to logits" + ) + parser.add_argument( + "--ffn_embed_dim", + default=5120, + type=int, + metavar="N", + help="embedding dimension for FFN", + ) + parser.add_argument( + "--attention_heads", + default=20, + type=int, + metavar="N", + help="number of attention heads", + ) + + def __init__(self, args, alphabet): + super().__init__() + self.args = args + self.alphabet_size = len(alphabet) + self.padding_idx = alphabet.padding_idx + self.mask_idx = alphabet.mask_idx + self.cls_idx = alphabet.cls_idx + self.eos_idx = alphabet.eos_idx + self.prepend_bos = alphabet.prepend_bos + self.append_eos = alphabet.append_eos + self.emb_layer_norm_before = getattr(self.args, "emb_layer_norm_before", False) + if self.args.arch == "roberta_large": + self.model_version = "ESM-1b" + self._init_submodules_esm1b() + else: + self.model_version = "ESM-1" + self._init_submodules_esm1() + + def _init_submodules_common(self): + self.embed_tokens = nn.Embedding( + self.alphabet_size, self.args.embed_dim, padding_idx=self.padding_idx + ) + self.layers = nn.ModuleList( + [ + TransformerLayer( + self.args.embed_dim, + self.args.ffn_embed_dim, + self.args.attention_heads, + add_bias_kv=(self.model_version != "ESM-1b"), + use_esm1b_layer_norm=(self.model_version == "ESM-1b"), + ) + for _ in range(self.args.layers) + ] + ) + + self.contact_head = ContactPredictionHead( + self.args.layers * self.args.attention_heads, + self.prepend_bos, + self.append_eos, + eos_idx=self.eos_idx, + ) + + def _init_submodules_esm1b(self): + self._init_submodules_common() + self.embed_scale = 1 + self.embed_positions = LearnedPositionalEmbedding( + self.args.max_positions, self.args.embed_dim, self.padding_idx + ) + self.emb_layer_norm_before = ( + ESM1bLayerNorm(self.args.embed_dim) if self.emb_layer_norm_before else None + ) + self.emb_layer_norm_after = ESM1bLayerNorm(self.args.embed_dim) + self.lm_head = RobertaLMHead( + embed_dim=self.args.embed_dim, + output_dim=self.alphabet_size, + weight=self.embed_tokens.weight, + ) + + def _init_submodules_esm1(self): + self._init_submodules_common() + self.embed_scale = math.sqrt(self.args.embed_dim) + self.embed_positions = SinusoidalPositionalEmbedding(self.args.embed_dim, self.padding_idx) + self.embed_out = nn.Parameter(torch.zeros((self.alphabet_size, self.args.embed_dim))) + self.embed_out_bias = None + if self.args.final_bias: + self.embed_out_bias = nn.Parameter(torch.zeros(self.alphabet_size)) + + def forward(self, tokens, repr_layers=[], need_head_weights=False, return_contacts=False): + if return_contacts: + need_head_weights = True + + assert tokens.ndim == 2 + padding_mask = tokens.eq(self.padding_idx) # B, T + + x = self.embed_scale * self.embed_tokens(tokens) + + if getattr(self.args, "token_dropout", False): + x.masked_fill_((tokens == self.mask_idx).unsqueeze(-1), 0.0) + # x: B x T x C + mask_ratio_train = 0.15 * 0.8 + src_lengths = (~padding_mask).sum(-1) + mask_ratio_observed = (tokens == self.mask_idx).sum(-1).float() / src_lengths + x = x * (1 - mask_ratio_train) / (1 - mask_ratio_observed)[:, None, None] + + x = x + self.embed_positions(tokens) + + if self.model_version == "ESM-1b": + if self.emb_layer_norm_before: + x = self.emb_layer_norm_before(x) + if padding_mask is not None: + x = x * (1 - padding_mask.unsqueeze(-1).type_as(x)) + + repr_layers = set(repr_layers) + hidden_representations = {} + if 0 in repr_layers: + hidden_representations[0] = x + + if need_head_weights: + attn_weights = [] + + # (B, T, E) => (T, B, E) + x = x.transpose(0, 1) + + if not padding_mask.any(): + padding_mask = None + + for layer_idx, layer in enumerate(self.layers): + x, attn = layer( + x, self_attn_padding_mask=padding_mask, need_head_weights=need_head_weights + ) + if (layer_idx + 1) in repr_layers: + hidden_representations[layer_idx + 1] = x.transpose(0, 1) + if need_head_weights: + # (H, B, T, T) => (B, H, T, T) + attn_weights.append(attn.transpose(1, 0)) + + if self.model_version == "ESM-1b": + x = self.emb_layer_norm_after(x) + x = x.transpose(0, 1) # (T, B, E) => (B, T, E) + + # last hidden representation should have layer norm applied + if (layer_idx + 1) in repr_layers: + hidden_representations[layer_idx + 1] = x + x = self.lm_head(x) + else: + x = F.linear(x, self.embed_out, bias=self.embed_out_bias) + x = x.transpose(0, 1) # (T, B, E) => (B, T, E) + + result = {"logits": x, "representations": hidden_representations} + if need_head_weights: + # attentions: B x L x H x T x T + attentions = torch.stack(attn_weights, 1) + if self.model_version == "ESM-1": + # ESM-1 models have an additional null-token for attention, which we remove + attentions = attentions[..., :-1] + if padding_mask is not None: + attention_mask = 1 - padding_mask.type_as(attentions) + attention_mask = attention_mask.unsqueeze(1) * attention_mask.unsqueeze(2) + attentions = attentions * attention_mask[:, None, None, :, :] + result["attentions"] = attentions + if return_contacts: + contacts = self.contact_head(tokens, attentions) + result["contacts"] = contacts + + return result + + def predict_contacts(self, tokens): + return self(tokens, return_contacts=True)["contacts"] + + @property + def num_layers(self): + return self.args.layers diff --git a/model/esm/esm2.py b/model/esm/esm2.py new file mode 100644 index 0000000000000000000000000000000000000000..fcb5814085e2e9e35bc6410219f42620b19219f0 --- /dev/null +++ b/model/esm/esm2.py @@ -0,0 +1,152 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Union +import torch +import torch.nn as nn + +from onescience.datapipes.esm import Alphabet +from onescience.modules.esm import ( + ContactPredictionHead, + ESM1bLayerNorm, + RobertaLMHead, + TransformerLayer, +) + + +class ESM2(nn.Module): + def __init__( + self, + num_layers: int = 33, + embed_dim: int = 1280, + attention_heads: int = 20, + alphabet: Union[Alphabet, str] = "ESM-1b", + token_dropout: bool = True, + ): + super().__init__() + self.num_layers = num_layers + self.embed_dim = embed_dim + self.attention_heads = attention_heads + if not isinstance(alphabet, Alphabet): + alphabet = Alphabet.from_architecture(alphabet) + self.alphabet = alphabet + self.alphabet_size = len(alphabet) + self.padding_idx = alphabet.padding_idx + self.mask_idx = alphabet.mask_idx + self.cls_idx = alphabet.cls_idx + self.eos_idx = alphabet.eos_idx + self.prepend_bos = alphabet.prepend_bos + self.append_eos = alphabet.append_eos + self.token_dropout = token_dropout + + self._init_submodules() + + def _init_submodules(self): + self.embed_scale = 1 + self.embed_tokens = nn.Embedding( + self.alphabet_size, + self.embed_dim, + padding_idx=self.padding_idx, + ) + + self.layers = nn.ModuleList( + [ + TransformerLayer( + self.embed_dim, + 4 * self.embed_dim, + self.attention_heads, + add_bias_kv=False, + use_esm1b_layer_norm=True, + use_rotary_embeddings=True, + ) + for _ in range(self.num_layers) + ] + ) + + self.contact_head = ContactPredictionHead( + self.num_layers * self.attention_heads, + self.prepend_bos, + self.append_eos, + eos_idx=self.eos_idx, + ) + self.emb_layer_norm_after = ESM1bLayerNorm(self.embed_dim) + + self.lm_head = RobertaLMHead( + embed_dim=self.embed_dim, + output_dim=self.alphabet_size, + weight=self.embed_tokens.weight, + ) + + def forward(self, tokens, repr_layers=[], need_head_weights=False, return_contacts=False): + if return_contacts: + need_head_weights = True + + assert tokens.ndim == 2 + padding_mask = tokens.eq(self.padding_idx) # B, T + + x = self.embed_scale * self.embed_tokens(tokens) + + if self.token_dropout: + x.masked_fill_((tokens == self.mask_idx).unsqueeze(-1), 0.0) + # x: B x T x C + mask_ratio_train = 0.15 * 0.8 + src_lengths = (~padding_mask).sum(-1) + mask_ratio_observed = (tokens == self.mask_idx).sum(-1).to(x.dtype) / src_lengths + x = x * (1 - mask_ratio_train) / (1 - mask_ratio_observed)[:, None, None] + + if padding_mask is not None: + x = x * (1 - padding_mask.unsqueeze(-1).type_as(x)) + + repr_layers = set(repr_layers) + hidden_representations = {} + if 0 in repr_layers: + hidden_representations[0] = x + + if need_head_weights: + attn_weights = [] + + # (B, T, E) => (T, B, E) + x = x.transpose(0, 1) + + if not padding_mask.any(): + padding_mask = None + + for layer_idx, layer in enumerate(self.layers): + x, attn = layer( + x, + self_attn_padding_mask=padding_mask, + need_head_weights=need_head_weights, + ) + if (layer_idx + 1) in repr_layers: + hidden_representations[layer_idx + 1] = x.transpose(0, 1) + if need_head_weights: + # (H, B, T, T) => (B, H, T, T) + attn_weights.append(attn.transpose(1, 0)) + + x = self.emb_layer_norm_after(x) + x = x.transpose(0, 1) # (T, B, E) => (B, T, E) + + # last hidden representation should have layer norm applied + if (layer_idx + 1) in repr_layers: + hidden_representations[layer_idx + 1] = x + x = self.lm_head(x) + + result = {"logits": x, "representations": hidden_representations} + if need_head_weights: + # attentions: B x L x H x T x T + attentions = torch.stack(attn_weights, 1) + if padding_mask is not None: + attention_mask = 1 - padding_mask.type_as(attentions) + attention_mask = attention_mask.unsqueeze(1) * attention_mask.unsqueeze(2) + attentions = attentions * attention_mask[:, None, None, :, :] + result["attentions"] = attentions + if return_contacts: + contacts = self.contact_head(tokens, attentions) + result["contacts"] = contacts + + return result + + def predict_contacts(self, tokens): + return self(tokens, return_contacts=True)["contacts"] diff --git a/model/esm/esmfold/__init__.py b/model/esm/esmfold/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/model/esm/esmfold/v1/categorical_mixture.py b/model/esm/esmfold/v1/categorical_mixture.py new file mode 100644 index 0000000000000000000000000000000000000000..aedff861c8568d9b6389ab2ab57aa50040543f25 --- /dev/null +++ b/model/esm/esmfold/v1/categorical_mixture.py @@ -0,0 +1,43 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import torch + + +class CategoricalMixture: + def __init__(self, param, bins=50, start=0, end=1): + # All tensors are of shape ..., bins. + self.logits = param + bins = torch.linspace( + start, end, bins + 1, device=self.logits.device, dtype=self.logits.dtype + ) + self.v_bins = (bins[:-1] + bins[1:]) / 2 + + def log_prob(self, true): + # Shapes are: + # self.probs: ... x bins + # true : ... + true_index = ( + ( + true.unsqueeze(-1) + - self.v_bins[ + [ + None, + ] + * true.ndim + ] + ) + .abs() + .argmin(-1) + ) + nll = self.logits.log_softmax(-1) + return torch.take_along_dim(nll, true_index.unsqueeze(-1), dim=-1).squeeze(-1) + + def mean(self): + return (self.logits.softmax(-1) @ self.v_bins.unsqueeze(1)).squeeze(-1) + + +def categorical_lddt(logits, bins=50): + # Logits are ..., 37, bins. + return CategoricalMixture(logits, bins=bins).mean() diff --git a/model/esm/esmfold/v1/misc.py b/model/esm/esmfold/v1/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..364853710fafd87c0370b14e6a7679725d2401d4 --- /dev/null +++ b/model/esm/esmfold/v1/misc.py @@ -0,0 +1,309 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +import typing as T + +import numpy as np +import torch +import torch.nn.functional as F +from einops import rearrange, repeat +from torch import nn +from onescience.utils.openfold.np import residue_constants +from onescience.utils.openfold.np.protein import Protein as OFProtein +from onescience.utils.openfold.np.protein import to_pdb +from onescience.utils.openfold.feats import atom14_to_atom37 + + +def encode_sequence( + seq: str, + residue_index_offset: T.Optional[int] = 512, + chain_linker: T.Optional[str] = "G" * 25, +) -> T.Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + if chain_linker is None: + chain_linker = "" + if residue_index_offset is None: + residue_index_offset = 0 + + chains = seq.split(":") + seq = chain_linker.join(chains) + + unk_idx = residue_constants.restype_order_with_x["X"] + encoded = torch.tensor( + [residue_constants.restype_order_with_x.get(aa, unk_idx) for aa in seq] + ) + residx = torch.arange(len(encoded)) + + if residue_index_offset > 0: + start = 0 + for i, chain in enumerate(chains): + residx[start : start + len(chain) + len(chain_linker)] += ( + i * residue_index_offset + ) + start += len(chain) + len(chain_linker) + + linker_mask = torch.ones_like(encoded, dtype=torch.float32) + chain_index = [] + offset = 0 + for i, chain in enumerate(chains): + if i > 0: + chain_index.extend([i - 1] * len(chain_linker)) + chain_index.extend([i] * len(chain)) + offset += len(chain) + linker_mask[offset : offset + len(chain_linker)] = 0 + offset += len(chain_linker) + + chain_index = torch.tensor(chain_index, dtype=torch.int64) + + return encoded, residx, linker_mask, chain_index + + +def batch_encode_sequences( + sequences: T.Sequence[str], + residue_index_offset: T.Optional[int] = 512, + chain_linker: T.Optional[str] = "G" * 25, +) -> T.Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + + aatype_list = [] + residx_list = [] + linker_mask_list = [] + chain_index_list = [] + for seq in sequences: + aatype_seq, residx_seq, linker_mask_seq, chain_index_seq = encode_sequence( + seq, + residue_index_offset=residue_index_offset, + chain_linker=chain_linker, + ) + aatype_list.append(aatype_seq) + residx_list.append(residx_seq) + linker_mask_list.append(linker_mask_seq) + chain_index_list.append(chain_index_seq) + + aatype = collate_dense_tensors(aatype_list) + mask = collate_dense_tensors( + [aatype.new_ones(len(aatype_seq)) for aatype_seq in aatype_list] + ) + residx = collate_dense_tensors(residx_list) + linker_mask = collate_dense_tensors(linker_mask_list) + chain_index_list = collate_dense_tensors(chain_index_list, -1) + + return aatype, mask, residx, linker_mask, chain_index_list + + +def output_to_pdb(output: T.Dict) -> T.List[str]: + """Returns the pbd (file) string from the model given the model output.""" + # atom14_to_atom37 must be called first, as it fails on latest numpy if the + # input is a numpy array. It will work if the input is a torch tensor. + final_atom_positions = atom14_to_atom37(output["positions"][-1], output) + output = {k: v.to("cpu").numpy() for k, v in output.items()} + final_atom_positions = final_atom_positions.cpu().numpy() + final_atom_mask = output["atom37_atom_exists"] + pdbs = [] + for i in range(output["aatype"].shape[0]): + aa = output["aatype"][i] + pred_pos = final_atom_positions[i] + mask = final_atom_mask[i] + resid = output["residue_index"][i] + 1 + pred = OFProtein( + aatype=aa, + atom_positions=pred_pos, + atom_mask=mask, + residue_index=resid, + b_factors=output["plddt"][i], + chain_index=output["chain_index"][i] if "chain_index" in output else None, + ) + pdbs.append(to_pdb(pred)) + return pdbs + + +def collate_dense_tensors( + samples: T.List[torch.Tensor], pad_v: float = 0 +) -> torch.Tensor: + """ + Takes a list of tensors with the following dimensions: + [(d_11, ..., d_1K), + (d_21, ..., d_2K), + ..., + (d_N1, ..., d_NK)] + and stack + pads them into a single tensor of: + (N, max_i=1,N { d_i1 }, ..., max_i=1,N {diK}) + """ + if len(samples) == 0: + return torch.Tensor() + if len(set(x.dim() for x in samples)) != 1: + raise RuntimeError( + f"Samples has varying dimensions: {[x.dim() for x in samples]}" + ) + (device,) = tuple(set(x.device for x in samples)) # assumes all on same device + max_shape = [max(lst) for lst in zip(*[x.shape for x in samples])] + result = torch.empty( + len(samples), *max_shape, dtype=samples[0].dtype, device=device + ) + result.fill_(pad_v) + for i in range(len(samples)): + result_i = result[i] + t = samples[i] + result_i[tuple(slice(0, k) for k in t.shape)] = t + return result + + +class Attention(nn.Module): + def __init__(self, embed_dim, num_heads, head_width, gated=False): + super().__init__() + assert embed_dim == num_heads * head_width + + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_width = head_width + + self.proj = nn.Linear(embed_dim, embed_dim * 3, bias=False) + self.o_proj = nn.Linear(embed_dim, embed_dim, bias=True) + self.gated = gated + if gated: + self.g_proj = nn.Linear(embed_dim, embed_dim) + torch.nn.init.zeros_(self.g_proj.weight) + torch.nn.init.ones_(self.g_proj.bias) + + self.rescale_factor = self.head_width**-0.5 + + torch.nn.init.zeros_(self.o_proj.bias) + + def forward(self, x, mask=None, bias=None, indices=None): + """ + Basic self attention with optional mask and external pairwise bias. + To handle sequences of different lengths, use mask. + + Inputs: + x: batch of input sequneces (.. x L x C) + mask: batch of boolean masks where 1=valid, 0=padding position (.. x L_k). optional. + bias: batch of scalar pairwise attention biases (.. x Lq x Lk x num_heads). optional. + + Outputs: + sequence projection (B x L x embed_dim), attention maps (B x L x L x num_heads) + """ + + t = rearrange(self.proj(x), "... l (h c) -> ... h l c", h=self.num_heads) + q, k, v = t.chunk(3, dim=-1) + + q = self.rescale_factor * q + a = torch.einsum("...qc,...kc->...qk", q, k) + + # Add external attention bias. + if bias is not None: + a = a + rearrange(bias, "... lq lk h -> ... h lq lk") + + # Do not attend to padding tokens. + if mask is not None: + mask = repeat( + mask, "... lk -> ... h lq lk", h=self.num_heads, lq=q.shape[-2] + ) + a = a.masked_fill(mask == False, -np.inf) + + a = F.softmax(a, dim=-1) + + y = torch.einsum("...hqk,...hkc->...qhc", a, v) + y = rearrange(y, "... h c -> ... (h c)", h=self.num_heads) + + if self.gated: + y = self.g_proj(x).sigmoid() * y + y = self.o_proj(y) + + return y, rearrange(a, "... lq lk h -> ... h lq lk") + + +class Dropout(nn.Module): + """ + Implementation of dropout with the ability to share the dropout mask + along a particular dimension. + """ + + def __init__(self, r: float, batch_dim: T.Union[int, T.List[int]]): + super(Dropout, self).__init__() + + self.r = r + if type(batch_dim) == int: + batch_dim = [batch_dim] + self.batch_dim = batch_dim + self.dropout = nn.Dropout(self.r) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + shape = list(x.shape) + if self.batch_dim is not None: + for bd in self.batch_dim: + shape[bd] = 1 + return x * self.dropout(x.new_ones(shape)) + + +class SequenceToPair(nn.Module): + def __init__(self, sequence_state_dim, inner_dim, pairwise_state_dim): + super().__init__() + + self.layernorm = nn.LayerNorm(sequence_state_dim) + self.proj = nn.Linear(sequence_state_dim, inner_dim * 2, bias=True) + self.o_proj = nn.Linear(2 * inner_dim, pairwise_state_dim, bias=True) + + torch.nn.init.zeros_(self.proj.bias) + torch.nn.init.zeros_(self.o_proj.bias) + + def forward(self, sequence_state): + """ + Inputs: + sequence_state: B x L x sequence_state_dim + + Output: + pairwise_state: B x L x L x pairwise_state_dim + + Intermediate state: + B x L x L x 2*inner_dim + """ + + assert len(sequence_state.shape) == 3 + + s = self.layernorm(sequence_state) + s = self.proj(s) + q, k = s.chunk(2, dim=-1) + + prod = q[:, None, :, :] * k[:, :, None, :] + diff = q[:, None, :, :] - k[:, :, None, :] + + x = torch.cat([prod, diff], dim=-1) + x = self.o_proj(x) + + return x + + +class PairToSequence(nn.Module): + def __init__(self, pairwise_state_dim, num_heads): + super().__init__() + + self.layernorm = nn.LayerNorm(pairwise_state_dim) + self.linear = nn.Linear(pairwise_state_dim, num_heads, bias=False) + + def forward(self, pairwise_state): + """ + Inputs: + pairwise_state: B x L x L x pairwise_state_dim + + Output: + pairwise_bias: B x L x L x num_heads + """ + assert len(pairwise_state.shape) == 4 + z = self.layernorm(pairwise_state) + pairwise_bias = self.linear(z) + return pairwise_bias + + +class ResidueMLP(nn.Module): + def __init__(self, embed_dim, inner_dim, norm=nn.LayerNorm, dropout=0): + super().__init__() + + self.mlp = nn.Sequential( + norm(embed_dim), + nn.Linear(embed_dim, inner_dim), + nn.ReLU(), + nn.Linear(inner_dim, embed_dim), + nn.Dropout(dropout), + ) + + def forward(self, x): + return x + self.mlp(x) diff --git a/model/esm/esmfold/v1/pretrained.py b/model/esm/esmfold/v1/pretrained.py new file mode 100644 index 0000000000000000000000000000000000000000..855ae9fae4a4d38791dfa0ecc7eb9a166144cfc4 --- /dev/null +++ b/model/esm/esmfold/v1/pretrained.py @@ -0,0 +1,197 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from pathlib import Path + +import torch + +from model.esm.esmfold.v1.esmfold import ESMFold + +def _upgrade_state_dict(model_state): + """Map legacy ESMFold checkpoint keys to the current OpenFold modules.""" + point_projection_prefixes = ( + "trunk.structure_module.ipa.linear_q_points", + "trunk.structure_module.ipa.linear_kv_points", + "trunk.structure_module.ipa.linear_k_points", + "trunk.structure_module.ipa.linear_v_points", + ) + for prefix in point_projection_prefixes: + for suffix in ("weight", "bias"): + old_key = f"{prefix}.{suffix}" + new_key = f"{prefix}.linear.{suffix}" + if old_key in model_state and new_key not in model_state: + model_state[new_key] = model_state.pop(old_key) + return model_state + +def _load_model(model_name): + if model_name.endswith(".pt"): # local, treat as filepath + model_path = Path(model_name) + model_data = torch.load(str(model_path), map_location="cpu") + else: # load from hub + url = f"https://dl.fbaipublicfiles.com/fair-esm/models/{model_name}.pt" + model_data = torch.hub.load_state_dict_from_url(url, progress=False, map_location="cpu") + + cfg = model_data["cfg"]["model"] + model_state = _upgrade_state_dict(model_data["model"]) + # model_state = model_data["model"] + model = ESMFold(esmfold_config=cfg) + + expected_keys = set(model.state_dict().keys()) + found_keys = set(model_state.keys()) + + missing_essential_keys = [] + for missing_key in expected_keys - found_keys: + if not missing_key.startswith("esm."): + missing_essential_keys.append(missing_key) + + if missing_essential_keys: + raise RuntimeError(f"Keys '{', '.join(missing_essential_keys)}' are missing.") + + model.load_state_dict(model_state, strict=False) + + return model + + +def esmfold_v0(): + """ + ESMFold v0 model with 3B ESM-2, 48 folding blocks. + This version was used for the paper (Lin et al, 2022). It was trained + on all PDB chains until 2020-05, to ensure temporal holdout with CASP14 + and the CAMEO validation and test set reported there. + """ + return _load_model("esmfold_3B_v0") + + +def esmfold_v1(): + """ + ESMFold v1 model using 3B ESM-2, 48 folding blocks. + ESMFold provides fast high accuracy atomic level structure prediction + directly from the individual sequence of a protein. ESMFold uses the ESM2 + protein language model to extract meaningful representations from the + protein sequence. + """ + return _load_model("esmfold_3B_v1") + + +def esmfold_structure_module_only_8M(): + """ + ESMFold baseline model using 8M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 500K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + return _load_model("esmfold_structure_module_only_8M") + + +def esmfold_structure_module_only_8M_270K(): + """ + ESMFold baseline model using 8M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + return _load_model("esmfold_structure_module_only_8M_270K") + + +def esmfold_structure_module_only_35M(): + """ + ESMFold baseline model using 35M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 500K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + return _load_model("esmfold_structure_module_only_35M") + + +def esmfold_structure_module_only_35M_270K(): + """ + ESMFold baseline model using 35M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + return _load_model("esmfold_structure_module_only_35M_270K") + + +def esmfold_structure_module_only_150M(): + """ + ESMFold baseline model using 150M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 500K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + return _load_model("esmfold_structure_module_only_150M") + + +def esmfold_structure_module_only_150M_270K(): + """ + ESMFold baseline model using 150M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + return _load_model("esmfold_structure_module_only_150M_270K") + + +def esmfold_structure_module_only_650M(): + """ + ESMFold baseline model using 650M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 500K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + return _load_model("esmfold_structure_module_only_650M") + + +def esmfold_structure_module_only_650M_270K(): + """ + ESMFold baseline model using 650M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + return _load_model("esmfold_structure_module_only_650M_270K") + + +def esmfold_structure_module_only_3B(): + """ + ESMFold baseline model using 3B ESM-2, 0 folding blocks. + ESM-2 here is trained out to 500K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + return _load_model("esmfold_structure_module_only_3B") + + +def esmfold_structure_module_only_3B_270K(): + """ + ESMFold baseline model using 3B ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + return _load_model("esmfold_structure_module_only_3B_270K") + + +def esmfold_structure_module_only_15B(): + """ + ESMFold baseline model using 15B ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + The 15B parameter ESM-2 was not trained out to 500K updates + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + return _load_model("esmfold_structure_module_only_15B") diff --git a/model/esm/inverse_folding/__init__.py b/model/esm/inverse_folding/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2906fc5a84b8c3cf64af778f8c78938d57d2f7da --- /dev/null +++ b/model/esm/inverse_folding/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from . import gvp_transformer +from . import util +from . import multichain_util diff --git a/model/esm/inverse_folding/features.py b/model/esm/inverse_folding/features.py new file mode 100644 index 0000000000000000000000000000000000000000..f67f3fe8d122b30162fcb4c136f65f2886150431 --- /dev/null +++ b/model/esm/inverse_folding/features.py @@ -0,0 +1,352 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +# +# Portions of this file were adapted from the open source code for the following +# two papers: +# +# Ingraham, J., Garg, V., Barzilay, R., & Jaakkola, T. (2019). Generative +# models for graph-based protein design. Advances in Neural Information +# Processing Systems, 32. +# +# Jing, B., Eismann, S., Suriana, P., Townshend, R. J. L., & Dror, R. (2020). +# Learning from Protein Structure with Geometric Vector Perceptrons. In +# International Conference on Learning Representations. +# +# MIT License +# +# Copyright (c) 2020 Bowen Jing, Stephan Eismann, Patricia Suriana, Raphael Townshend, Ron Dror +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# ================================================================ +# The below license applies to the portions of the code (parts of +# src/datasets.py and src/models.py) adapted from Ingraham, et al. +# ================================================================ +# +# MIT License +# +# Copyright (c) 2019 John Ingraham, Vikas Garg, Regina Barzilay, Tommi Jaakkola +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import math +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .gvp_utils import flatten_graph +from .gvp_modules import GVP, LayerNorm +from .util import normalize, norm, nan_to_num, rbf + + +class GVPInputFeaturizer(nn.Module): + + @staticmethod + def get_node_features(coords, coord_mask, with_coord_mask=True): + # scalar features + node_scalar_features = GVPInputFeaturizer._dihedrals(coords) + if with_coord_mask: + node_scalar_features = torch.cat([ + node_scalar_features, + coord_mask.float().unsqueeze(-1) + ], dim=-1) + # vector features + X_ca = coords[:, :, 1] + orientations = GVPInputFeaturizer._orientations(X_ca) + sidechains = GVPInputFeaturizer._sidechains(coords) + node_vector_features = torch.cat([orientations, sidechains.unsqueeze(-2)], dim=-2) + return node_scalar_features, node_vector_features + + @staticmethod + def _orientations(X): + forward = normalize(X[:, 1:] - X[:, :-1]) + backward = normalize(X[:, :-1] - X[:, 1:]) + forward = F.pad(forward, [0, 0, 0, 1]) + backward = F.pad(backward, [0, 0, 1, 0]) + return torch.cat([forward.unsqueeze(-2), backward.unsqueeze(-2)], -2) + + @staticmethod + def _sidechains(X): + n, origin, c = X[:, :, 0], X[:, :, 1], X[:, :, 2] + c, n = normalize(c - origin), normalize(n - origin) + bisector = normalize(c + n) + perp = normalize(torch.cross(c, n, dim=-1)) + vec = -bisector * math.sqrt(1 / 3) - perp * math.sqrt(2 / 3) + return vec + + @staticmethod + def _dihedrals(X, eps=1e-7): + X = torch.flatten(X[:, :, :3], 1, 2) + bsz = X.shape[0] + dX = X[:, 1:] - X[:, :-1] + U = normalize(dX, dim=-1) + u_2 = U[:, :-2] + u_1 = U[:, 1:-1] + u_0 = U[:, 2:] + + # Backbone normals + n_2 = normalize(torch.cross(u_2, u_1, dim=-1), dim=-1) + n_1 = normalize(torch.cross(u_1, u_0, dim=-1), dim=-1) + + # Angle between normals + cosD = torch.sum(n_2 * n_1, -1) + cosD = torch.clamp(cosD, -1 + eps, 1 - eps) + D = torch.sign(torch.sum(u_2 * n_1, -1)) * torch.acos(cosD) + + # This scheme will remove phi[0], psi[-1], omega[-1] + D = F.pad(D, [1, 2]) + D = torch.reshape(D, [bsz, -1, 3]) + # Lift angle representations to the circle + D_features = torch.cat([torch.cos(D), torch.sin(D)], -1) + return D_features + + @staticmethod + def _positional_embeddings(edge_index, + num_embeddings=None, + num_positional_embeddings=16, + period_range=[2, 1000]): + # From https://github.com/jingraham/neurips19-graph-protein-design + num_embeddings = num_embeddings or num_positional_embeddings + d = edge_index[0] - edge_index[1] + + frequency = torch.exp( + torch.arange(0, num_embeddings, 2, dtype=torch.float32, + device=edge_index.device) + * -(np.log(10000.0) / num_embeddings) + ) + angles = d.unsqueeze(-1) * frequency + E = torch.cat((torch.cos(angles), torch.sin(angles)), -1) + return E + + @staticmethod + def _dist(X, coord_mask, padding_mask, top_k_neighbors, eps=1e-8): + """ Pairwise euclidean distances """ + bsz, maxlen = X.size(0), X.size(1) + coord_mask_2D = torch.unsqueeze(coord_mask,1) * torch.unsqueeze(coord_mask,2) + residue_mask = ~padding_mask + residue_mask_2D = torch.unsqueeze(residue_mask,1) * torch.unsqueeze(residue_mask,2) + dX = torch.unsqueeze(X,1) - torch.unsqueeze(X,2) + D = coord_mask_2D * norm(dX, dim=-1) + + # sorting preference: first those with coords, then among the residues that + # exist but are masked use distance in sequence as tie breaker, and then the + # residues that came from padding are last + seqpos = torch.arange(maxlen, device=X.device) + Dseq = torch.abs(seqpos.unsqueeze(1) - seqpos.unsqueeze(0)).repeat(bsz, 1, 1) + D_adjust = nan_to_num(D) + (~coord_mask_2D) * (1e8 + Dseq*1e6) + ( + ~residue_mask_2D) * (1e10) + + if top_k_neighbors == -1: + D_neighbors = D_adjust + E_idx = seqpos.repeat( + *D_neighbors.shape[:-1], 1) + else: + # Identify k nearest neighbors (including self) + k = min(top_k_neighbors, X.size(1)) + D_neighbors, E_idx = torch.topk(D_adjust, k, dim=-1, largest=False) + + coord_mask_neighbors = (D_neighbors < 5e7) + residue_mask_neighbors = (D_neighbors < 5e9) + return D_neighbors, E_idx, coord_mask_neighbors, residue_mask_neighbors + + +class Normalize(nn.Module): + def __init__(self, features, epsilon=1e-6): + super(Normalize, self).__init__() + self.gain = nn.Parameter(torch.ones(features)) + self.bias = nn.Parameter(torch.zeros(features)) + self.epsilon = epsilon + + def forward(self, x, dim=-1): + mu = x.mean(dim, keepdim=True) + sigma = torch.sqrt(x.var(dim, keepdim=True) + self.epsilon) + gain = self.gain + bias = self.bias + # Reshape + if dim != -1: + shape = [1] * len(mu.size()) + shape[dim] = self.gain.size()[0] + gain = gain.view(shape) + bias = bias.view(shape) + return gain * (x - mu) / (sigma + self.epsilon) + bias + + +class DihedralFeatures(nn.Module): + def __init__(self, node_embed_dim): + """ Embed dihedral angle features. """ + super(DihedralFeatures, self).__init__() + # 3 dihedral angles; sin and cos of each angle + node_in = 6 + # Normalization and embedding + self.node_embedding = nn.Linear(node_in, node_embed_dim, bias=True) + self.norm_nodes = Normalize(node_embed_dim) + + def forward(self, X): + """ Featurize coordinates as an attributed graph """ + V = self._dihedrals(X) + V = self.node_embedding(V) + V = self.norm_nodes(V) + return V + + @staticmethod + def _dihedrals(X, eps=1e-7, return_angles=False): + # First 3 coordinates are N, CA, C + X = X[:,:,:3,:].reshape(X.shape[0], 3*X.shape[1], 3) + + # Shifted slices of unit vectors + dX = X[:,1:,:] - X[:,:-1,:] + U = F.normalize(dX, dim=-1) + u_2 = U[:,:-2,:] + u_1 = U[:,1:-1,:] + u_0 = U[:,2:,:] + # Backbone normals + n_2 = F.normalize(torch.cross(u_2, u_1, dim=-1), dim=-1) + n_1 = F.normalize(torch.cross(u_1, u_0, dim=-1), dim=-1) + + # Angle between normals + cosD = (n_2 * n_1).sum(-1) + cosD = torch.clamp(cosD, -1+eps, 1-eps) + D = torch.sign((u_2 * n_1).sum(-1)) * torch.acos(cosD) + + # This scheme will remove phi[0], psi[-1], omega[-1] + D = F.pad(D, (1,2), 'constant', 0) + D = D.view((D.size(0), int(D.size(1)/3), 3)) + phi, psi, omega = torch.unbind(D,-1) + + if return_angles: + return phi, psi, omega + + # Lift angle representations to the circle + D_features = torch.cat((torch.cos(D), torch.sin(D)), 2) + return D_features + + +class GVPGraphEmbedding(GVPInputFeaturizer): + + def __init__(self, args): + super().__init__() + self.top_k_neighbors = args.top_k_neighbors + self.num_positional_embeddings = 16 + self.remove_edges_without_coords = True + node_input_dim = (7, 3) + edge_input_dim = (34, 1) + node_hidden_dim = (args.node_hidden_dim_scalar, + args.node_hidden_dim_vector) + edge_hidden_dim = (args.edge_hidden_dim_scalar, + args.edge_hidden_dim_vector) + self.embed_node = nn.Sequential( + GVP(node_input_dim, node_hidden_dim, activations=(None, None)), + LayerNorm(node_hidden_dim, eps=1e-4) + ) + self.embed_edge = nn.Sequential( + GVP(edge_input_dim, edge_hidden_dim, activations=(None, None)), + LayerNorm(edge_hidden_dim, eps=1e-4) + ) + self.embed_confidence = nn.Linear(16, args.node_hidden_dim_scalar) + + def forward(self, coords, coord_mask, padding_mask, confidence): + with torch.no_grad(): + node_features = self.get_node_features(coords, coord_mask) + edge_features, edge_index = self.get_edge_features( + coords, coord_mask, padding_mask) + node_embeddings_scalar, node_embeddings_vector = self.embed_node(node_features) + edge_embeddings = self.embed_edge(edge_features) + + rbf_rep = rbf(confidence, 0., 1.) + node_embeddings = ( + node_embeddings_scalar + self.embed_confidence(rbf_rep), + node_embeddings_vector + ) + + node_embeddings, edge_embeddings, edge_index = flatten_graph( + node_embeddings, edge_embeddings, edge_index) + return node_embeddings, edge_embeddings, edge_index + + def get_edge_features(self, coords, coord_mask, padding_mask): + X_ca = coords[:, :, 1] + # Get distances to the top k neighbors + E_dist, E_idx, E_coord_mask, E_residue_mask = GVPInputFeaturizer._dist( + X_ca, coord_mask, padding_mask, self.top_k_neighbors) + # Flatten the graph to be batch size 1 for torch_geometric package + dest = E_idx + B, L, k = E_idx.shape[:3] + src = torch.arange(L, device=E_idx.device).view([1, L, 1]).expand(B, L, k) + # After flattening, [2, B, E] + edge_index = torch.stack([src, dest], dim=0).flatten(2, 3) + # After flattening, [B, E] + E_dist = E_dist.flatten(1, 2) + E_coord_mask = E_coord_mask.flatten(1, 2).unsqueeze(-1) + E_residue_mask = E_residue_mask.flatten(1, 2) + # Calculate relative positional embeddings and distance RBF + pos_embeddings = GVPInputFeaturizer._positional_embeddings( + edge_index, + num_positional_embeddings=self.num_positional_embeddings, + ) + D_rbf = rbf(E_dist, 0., 20.) + # Calculate relative orientation + X_src = X_ca.unsqueeze(2).expand(-1, -1, k, -1).flatten(1, 2) + X_dest = torch.gather( + X_ca, + 1, + edge_index[1, :, :].unsqueeze(-1).expand([B, L*k, 3]) + ) + coord_mask_src = coord_mask.unsqueeze(2).expand(-1, -1, k).flatten(1, 2) + coord_mask_dest = torch.gather( + coord_mask, + 1, + edge_index[1, :, :].expand([B, L*k]) + ) + E_vectors = X_src - X_dest + # For the ones without coordinates, substitute in the average vector + E_vector_mean = torch.sum(E_vectors * E_coord_mask, dim=1, + keepdims=True) / torch.sum(E_coord_mask, dim=1, keepdims=True) + E_vectors = E_vectors * E_coord_mask + E_vector_mean * ~(E_coord_mask) + # Normalize and remove nans + edge_s = torch.cat([D_rbf, pos_embeddings], dim=-1) + edge_v = normalize(E_vectors).unsqueeze(-2) + edge_s, edge_v = map(nan_to_num, (edge_s, edge_v)) + # Also add indications of whether the coordinates are present + edge_s = torch.cat([ + edge_s, + (~coord_mask_src).float().unsqueeze(-1), + (~coord_mask_dest).float().unsqueeze(-1), + ], dim=-1) + edge_index[:, ~E_residue_mask] = -1 + if self.remove_edges_without_coords: + edge_index[:, ~E_coord_mask.squeeze(-1)] = -1 + return (edge_s, edge_v), edge_index.transpose(0, 1) diff --git a/model/esm/inverse_folding/gvp_encoder.py b/model/esm/inverse_folding/gvp_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..a565201090624b71c272c9e23b58feb7f1cca30d --- /dev/null +++ b/model/esm/inverse_folding/gvp_encoder.py @@ -0,0 +1,56 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from argparse import Namespace + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .features import GVPGraphEmbedding +from .gvp_modules import GVPConvLayer, LayerNorm +from .gvp_utils import unflatten_graph + + + +class GVPEncoder(nn.Module): + + def __init__(self, args): + super().__init__() + self.args = args + self.embed_graph = GVPGraphEmbedding(args) + + node_hidden_dim = (args.node_hidden_dim_scalar, + args.node_hidden_dim_vector) + edge_hidden_dim = (args.edge_hidden_dim_scalar, + args.edge_hidden_dim_vector) + + conv_activations = (F.relu, torch.sigmoid) + self.encoder_layers = nn.ModuleList( + GVPConvLayer( + node_hidden_dim, + edge_hidden_dim, + drop_rate=args.dropout, + vector_gate=True, + attention_heads=0, + n_message=3, + conv_activations=conv_activations, + n_edge_gvps=0, + eps=1e-4, + layernorm=True, + ) + for i in range(args.num_encoder_layers) + ) + + def forward(self, coords, coord_mask, padding_mask, confidence): + node_embeddings, edge_embeddings, edge_index = self.embed_graph( + coords, coord_mask, padding_mask, confidence) + + for i, layer in enumerate(self.encoder_layers): + node_embeddings, edge_embeddings = layer(node_embeddings, + edge_index, edge_embeddings) + + node_embeddings = unflatten_graph(node_embeddings, coords.shape[0]) + return node_embeddings diff --git a/model/esm/inverse_folding/gvp_transformer_encoder.py b/model/esm/inverse_folding/gvp_transformer_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..7918af2ef45ab83c5866c2c9ecea90899f76dd32 --- /dev/null +++ b/model/esm/inverse_folding/gvp_transformer_encoder.py @@ -0,0 +1,184 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Contents of this file were adapted from the open source fairseq repository. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import argparse +import math +from typing import Dict, List, Optional + +import torch +import torch.nn as nn +from torch import Tensor + +from onescience.modules.esm import SinusoidalPositionalEmbedding +from .features import GVPInputFeaturizer, DihedralFeatures +from .gvp_encoder import GVPEncoder +from .transformer_layer import TransformerEncoderLayer +from .util import nan_to_num, get_rotation_frames, rotate, rbf + + +class GVPTransformerEncoder(nn.Module): + """ + Transformer encoder consisting of *args.encoder.layers* layers. Each layer + is a :class:`TransformerEncoderLayer`. + + Args: + args (argparse.Namespace): parsed command-line arguments + dictionary (~fairseq.data.Dictionary): encoding dictionary + embed_tokens (torch.nn.Embedding): input embedding + """ + + def __init__(self, args, dictionary, embed_tokens): + super().__init__() + self.args = args + self.dictionary = dictionary + + self.dropout_module = nn.Dropout(args.dropout) + + embed_dim = embed_tokens.embedding_dim + self.padding_idx = embed_tokens.padding_idx + + self.embed_tokens = embed_tokens + self.embed_scale = math.sqrt(embed_dim) + self.embed_positions = SinusoidalPositionalEmbedding( + embed_dim, + self.padding_idx, + ) + self.embed_gvp_input_features = nn.Linear(15, embed_dim) + self.embed_confidence = nn.Linear(16, embed_dim) + self.embed_dihedrals = DihedralFeatures(embed_dim) + + gvp_args = argparse.Namespace() + for k, v in vars(args).items(): + if k.startswith("gvp_"): + setattr(gvp_args, k[4:], v) + self.gvp_encoder = GVPEncoder(gvp_args) + gvp_out_dim = gvp_args.node_hidden_dim_scalar + (3 * + gvp_args.node_hidden_dim_vector) + self.embed_gvp_output = nn.Linear(gvp_out_dim, embed_dim) + + self.layers = nn.ModuleList([]) + self.layers.extend( + [self.build_encoder_layer(args) for i in range(args.encoder_layers)] + ) + self.num_layers = len(self.layers) + self.layer_norm = nn.LayerNorm(embed_dim) + + def build_encoder_layer(self, args): + return TransformerEncoderLayer(args) + + def forward_embedding(self, coords, padding_mask, confidence): + """ + Args: + coords: N, CA, C backbone coordinates in shape length x 3 (atoms) x 3 + padding_mask: boolean Tensor (true for padding) of shape length + confidence: confidence scores between 0 and 1 of shape length + """ + components = dict() + coord_mask = torch.all(torch.all(torch.isfinite(coords), dim=-1), dim=-1) + coords = nan_to_num(coords) + mask_tokens = ( + padding_mask * self.dictionary.padding_idx + + ~padding_mask * self.dictionary.get_idx("") + ) + components["tokens"] = self.embed_tokens(mask_tokens) * self.embed_scale + components["diherals"] = self.embed_dihedrals(coords) + + # GVP encoder + gvp_out_scalars, gvp_out_vectors = self.gvp_encoder(coords, + coord_mask, padding_mask, confidence) + R = get_rotation_frames(coords) + # Rotate to local rotation frame for rotation-invariance + gvp_out_features = torch.cat([ + gvp_out_scalars, + rotate(gvp_out_vectors, R.transpose(-2, -1)).flatten(-2, -1), + ], dim=-1) + components["gvp_out"] = self.embed_gvp_output(gvp_out_features) + + components["confidence"] = self.embed_confidence( + rbf(confidence, 0., 1.)) + + # In addition to GVP encoder outputs, also directly embed GVP input node + # features to the Transformer + scalar_features, vector_features = GVPInputFeaturizer.get_node_features( + coords, coord_mask, with_coord_mask=False) + features = torch.cat([ + scalar_features, + rotate(vector_features, R.transpose(-2, -1)).flatten(-2, -1), + ], dim=-1) + components["gvp_input_features"] = self.embed_gvp_input_features(features) + + embed = sum(components.values()) + # for k, v in components.items(): + # print(k, torch.mean(v, dim=(0,1)), torch.std(v, dim=(0,1))) + + x = embed + x = x + self.embed_positions(mask_tokens) + x = self.dropout_module(x) + return x, components + + def forward( + self, + coords, + encoder_padding_mask, + confidence, + return_all_hiddens: bool = False, + ): + """ + Args: + coords (Tensor): backbone coordinates + shape batch_size x num_residues x num_atoms (3 for N, CA, C) x 3 + encoder_padding_mask (ByteTensor): the positions of + padding elements of shape `(batch_size x num_residues)` + confidence (Tensor): the confidence score of shape (batch_size x + num_residues). The value is between 0. and 1. for each residue + coordinate, or -1. if no coordinate is given + return_all_hiddens (bool, optional): also return all of the + intermediate hidden states (default: False). + + Returns: + dict: + - **encoder_out** (Tensor): the last encoder layer's output of + shape `(num_residues, batch_size, embed_dim)` + - **encoder_padding_mask** (ByteTensor): the positions of + padding elements of shape `(batch_size, num_residues)` + - **encoder_embedding** (Tensor): the (scaled) embedding lookup + of shape `(batch_size, num_residues, embed_dim)` + - **encoder_states** (List[Tensor]): all intermediate + hidden states of shape `(num_residues, batch_size, embed_dim)`. + Only populated if *return_all_hiddens* is True. + """ + x, encoder_embedding = self.forward_embedding(coords, + encoder_padding_mask, confidence) + # account for padding while computing the representation + x = x * (1 - encoder_padding_mask.unsqueeze(-1).type_as(x)) + + # B x T x C -> T x B x C + x = x.transpose(0, 1) + + encoder_states = [] + + if return_all_hiddens: + encoder_states.append(x) + + # encoder layers + for layer in self.layers: + x = layer( + x, encoder_padding_mask=encoder_padding_mask + ) + if return_all_hiddens: + assert encoder_states is not None + encoder_states.append(x) + + if self.layer_norm is not None: + x = self.layer_norm(x) + + return { + "encoder_out": [x], # T x B x C + "encoder_padding_mask": [encoder_padding_mask], # B x T + "encoder_embedding": [encoder_embedding], # dictionary + "encoder_states": encoder_states, # List[T x B x C] + } diff --git a/model/esm/inverse_folding/gvp_utils.py b/model/esm/inverse_folding/gvp_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fc0617fff0913f0f8b0ecace8e1debd334c00299 --- /dev/null +++ b/model/esm/inverse_folding/gvp_utils.py @@ -0,0 +1,68 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import torch + + +def flatten_graph(node_embeddings, edge_embeddings, edge_index): + """ + Flattens the graph into a batch size one (with disconnected subgraphs for + each example) to be compatible with pytorch-geometric package. + Args: + node_embeddings: node embeddings in tuple form (scalar, vector) + - scalar: shape batch size x nodes x node_embed_dim + - vector: shape batch size x nodes x node_embed_dim x 3 + edge_embeddings: edge embeddings of in tuple form (scalar, vector) + - scalar: shape batch size x edges x edge_embed_dim + - vector: shape batch size x edges x edge_embed_dim x 3 + edge_index: shape batch_size x 2 (source node and target node) x edges + Returns: + node_embeddings: node embeddings in tuple form (scalar, vector) + - scalar: shape batch total_nodes x node_embed_dim + - vector: shape batch total_nodes x node_embed_dim x 3 + edge_embeddings: edge embeddings of in tuple form (scalar, vector) + - scalar: shape batch total_edges x edge_embed_dim + - vector: shape batch total_edges x edge_embed_dim x 3 + edge_index: shape 2 x total_edges + """ + x_s, x_v = node_embeddings + e_s, e_v = edge_embeddings + batch_size, N = x_s.shape[0], x_s.shape[1] + node_embeddings = (torch.flatten(x_s, 0, 1), torch.flatten(x_v, 0, 1)) + edge_embeddings = (torch.flatten(e_s, 0, 1), torch.flatten(e_v, 0, 1)) + + edge_mask = torch.any(edge_index != -1, dim=1) + # Re-number the nodes by adding batch_idx * N to each batch + edge_index = edge_index + (torch.arange(batch_size, device=edge_index.device) * + N).unsqueeze(-1).unsqueeze(-1) + edge_index = edge_index.permute(1, 0, 2).flatten(1, 2) + edge_mask = edge_mask.flatten() + edge_index = edge_index[:, edge_mask] + edge_embeddings = ( + edge_embeddings[0][edge_mask, :], + edge_embeddings[1][edge_mask, :] + ) + return node_embeddings, edge_embeddings, edge_index + + +def unflatten_graph(node_embeddings, batch_size): + """ + Unflattens node embeddings. + Args: + node_embeddings: node embeddings in tuple form (scalar, vector) + - scalar: shape batch total_nodes x node_embed_dim + - vector: shape batch total_nodes x node_embed_dim x 3 + batch_size: int + Returns: + node_embeddings: node embeddings in tuple form (scalar, vector) + - scalar: shape batch size x nodes x node_embed_dim + - vector: shape batch size x nodes x node_embed_dim x 3 + """ + x_s, x_v = node_embeddings + x_s = x_s.reshape(batch_size, -1, x_s.shape[1]) + x_v = x_v.reshape(batch_size, -1, x_v.shape[1], x_v.shape[2]) + return (x_s, x_v) + + diff --git a/model/esm/inverse_folding/multichain_util.py b/model/esm/inverse_folding/multichain_util.py new file mode 100644 index 0000000000000000000000000000000000000000..ed0070abf94a5f9fae3b2565e6d27a496202d5f8 --- /dev/null +++ b/model/esm/inverse_folding/multichain_util.py @@ -0,0 +1,152 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import biotite.structure +import numpy as np +import torch +from typing import Sequence, Tuple, List + +from model.esm.inverse_folding.util import ( + load_structure, + extract_coords_from_structure, + load_coords, + get_sequence_loss, + get_encoder_output, +) + + +def extract_coords_from_complex(structure: biotite.structure.AtomArray): + """ + Args: + structure: biotite AtomArray + Returns: + Tuple (coords_list, seq_list) + - coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C + coordinates representing the backbone of each chain + - seqs: Dictionary mapping chain ids to native sequences of each chain + """ + coords = {} + seqs = {} + all_chains = biotite.structure.get_chains(structure) + for chain_id in all_chains: + chain = structure[structure.chain_id == chain_id] + coords[chain_id], seqs[chain_id] = extract_coords_from_structure(chain) + return coords, seqs + + +def load_complex_coords(fpath, chains): + """ + Args: + fpath: filepath to either pdb or cif file + chains: the chain ids (the order matters for autoregressive model) + Returns: + Tuple (coords_list, seq_list) + - coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C + coordinates representing the backbone of each chain + - seqs: Dictionary mapping chain ids to native sequences of each chain + """ + structure = load_structure(fpath, chains) + return extract_coords_from_complex(structure) + + +def _concatenate_coords(coords, target_chain_id, padding_length=10): + """ + Args: + coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C + coordinates representing the backbone of each chain + target_chain_id: The chain id to sample sequences for + padding_length: Length of padding between concatenated chains + Returns: + Tuple (coords, seq) + - coords is an L x 3 x 3 array for N, CA, C coordinates, a + concatenation of the chains with padding in between + - seq is the extracted sequence, with padding tokens inserted + between the concatenated chains + """ + pad_coords = np.full((padding_length, 3, 3), np.nan, dtype=np.float32) + # For best performance, put the target chain first in concatenation. + coords_list = [coords[target_chain_id]] + for chain_id in coords: + if chain_id == target_chain_id: + continue + coords_list.append(pad_coords) + coords_list.append(coords[chain_id]) + coords_concatenated = np.concatenate(coords_list, axis=0) + return coords_concatenated + + +def sample_sequence_in_complex(model, coords, target_chain_id, temperature=1., + padding_length=10): + """ + Samples sequence for one chain in a complex. + Args: + model: An instance of the GVPTransformer model + coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C + coordinates representing the backbone of each chain + target_chain_id: The chain id to sample sequences for + padding_length: padding length in between chains + Returns: + Sampled sequence for the target chain + """ + target_chain_len = coords[target_chain_id].shape[0] + all_coords = _concatenate_coords(coords, target_chain_id) + device = next(model.parameters()).device + + # Supply padding tokens for other chains to avoid unused sampling for speed + padding_pattern = [''] * all_coords.shape[0] + for i in range(target_chain_len): + padding_pattern[i] = '' + sampled = model.sample(all_coords, partial_seq=padding_pattern, + temperature=temperature, device=device) + sampled = sampled[:target_chain_len] + return sampled + + +def score_sequence_in_complex(model, alphabet, coords, target_chain_id, + target_seq, padding_length=10): + """ + Scores sequence for one chain in a complex. + Args: + model: An instance of the GVPTransformer model + alphabet: Alphabet for the model + coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C + coordinates representing the backbone of each chain + target_chain_id: The chain id to sample sequences for + target_seq: Target sequence for the target chain for scoring. + padding_length: padding length in between chains + Returns: + Tuple (ll_fullseq, ll_withcoord) + - ll_fullseq: Average log-likelihood over the full target chain + - ll_withcoord: Average log-likelihood in target chain excluding those + residues without coordinates + """ + all_coords = _concatenate_coords(coords, target_chain_id) + + loss, target_padding_mask = get_sequence_loss(model, alphabet, all_coords, + target_seq) + ll_fullseq = -np.sum(loss * ~target_padding_mask) / np.sum( + ~target_padding_mask) + + # Also calculate average when excluding masked portions + coord_mask = np.all(np.isfinite(coords[target_chain_id]), axis=(-1, -2)) + ll_withcoord = -np.sum(loss * coord_mask) / np.sum(coord_mask) + return ll_fullseq, ll_withcoord + + +def get_encoder_output_for_complex(model, alphabet, coords, target_chain_id): + """ + Args: + model: An instance of the GVPTransformer model + alphabet: Alphabet for the model + coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C + coordinates representing the backbone of each chain + target_chain_id: The chain id to sample sequences for + Returns: + Dictionary mapping chain id to encoder output for each chain + """ + all_coords = _concatenate_coords(coords, target_chain_id) + all_rep = get_encoder_output(model, alphabet, all_coords) + target_chain_len = coords[target_chain_id].shape[0] + return all_rep[:target_chain_len] diff --git a/model/esm/inverse_folding/transformer_decoder.py b/model/esm/inverse_folding/transformer_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..93d4c4294617ebae10b0399b091ae4b2404a6e2e --- /dev/null +++ b/model/esm/inverse_folding/transformer_decoder.py @@ -0,0 +1,228 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Contents of this file were adapted from the open source fairseq repository. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import math +from typing import Any, Dict, List, Optional + +import torch +import torch.nn as nn +from torch import Tensor + +from onescience.modules.esm import SinusoidalPositionalEmbedding +from .transformer_layer import TransformerDecoderLayer + + +def fill_with_neg_inf(t): + """FP16-compatible function that fills a tensor with -inf.""" + return t.float().fill_(float("-inf")).type_as(t) + + +class TransformerDecoder(nn.Module): + """ + Transformer decoder consisting of *args.decoder.layers* layers. Each layer + is a :class:`TransformerDecoderLayer`. + + Args: + args (argparse.Namespace): parsed command-line arguments + dictionary (~fairseq.data.Dictionary): decoding dictionary + embed_tokens (torch.nn.Embedding): output embedding + no_encoder_attn (bool, optional): whether to attend to encoder outputs + (default: False). + """ + + def __init__( + self, + args, + dictionary, + embed_tokens, + ): + super().__init__() + self.args = args + self.dictionary = dictionary + self._future_mask = torch.empty(0) + + self.dropout_module = nn.Dropout(args.dropout) + + input_embed_dim = embed_tokens.embedding_dim + embed_dim = args.decoder_embed_dim + self.embed_dim = embed_dim + + self.padding_idx = embed_tokens.padding_idx + + self.embed_tokens = embed_tokens + self.embed_scale = math.sqrt(embed_dim) + + self.project_in_dim = ( + nn.Linear(input_embed_dim, embed_dim, bias=False) + if embed_dim != input_embed_dim + else None + ) + self.embed_positions = SinusoidalPositionalEmbedding( + embed_dim, + self.padding_idx, + ) + + self.layers = nn.ModuleList([]) + self.layers.extend( + [ + self.build_decoder_layer(args) + for _ in range(args.decoder_layers) + ] + ) + self.num_layers = len(self.layers) + self.layer_norm = nn.LayerNorm(embed_dim) + + self.build_output_projection(args, dictionary) + + def build_output_projection(self, args, dictionary): + self.output_projection = nn.Linear( + args.decoder_embed_dim, len(dictionary), bias=False + ) + nn.init.normal_( + self.output_projection.weight, mean=0, std=args.decoder_embed_dim ** -0.5 + ) + + def build_decoder_layer(self, args): + return TransformerDecoderLayer(args) + + def forward( + self, + prev_output_tokens, + encoder_out: Optional[Dict[str, List[Tensor]]] = None, + incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None, + features_only: bool = False, + return_all_hiddens: bool = False, + ): + """ + Args: + prev_output_tokens (LongTensor): previous decoder outputs of shape + `(batch, tgt_len)`, for teacher forcing + encoder_out (optional): output from the encoder, used for + encoder-side attention, should be of size T x B x C + incremental_state (dict): dictionary used for storing state during + :ref:`Incremental decoding` + features_only (bool, optional): only return features without + applying output layer (default: False). + + Returns: + tuple: + - the decoder's output of shape `(batch, tgt_len, vocab)` + - a dictionary with any model-specific outputs + """ + + x, extra = self.extract_features( + prev_output_tokens, + encoder_out=encoder_out, + incremental_state=incremental_state, + ) + + if not features_only: + x = self.output_layer(x) + x = x.transpose(1, 2) # B x T x C -> B x C x T + return x, extra + + def extract_features( + self, + prev_output_tokens, + encoder_out: Optional[Dict[str, List[Tensor]]], + incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None, + ): + """ + Similar to *forward* but only return features. + + Includes several features from "Jointly Learning to Align and + Translate with Transformer Models" (Garg et al., EMNLP 2019). + + Returns: + tuple: + - the decoder's features of shape `(batch, tgt_len, embed_dim)` + - a dictionary with any model-specific outputs + """ + bs, slen = prev_output_tokens.size() + + enc: Optional[Tensor] = None + padding_mask: Optional[Tensor] = None + if encoder_out is not None and len(encoder_out["encoder_out"]) > 0: + enc = encoder_out["encoder_out"][0] + assert ( + enc.size()[1] == bs + ), f"Expected enc.shape == (t, {bs}, c) got {enc.shape}" + if encoder_out is not None and len(encoder_out["encoder_padding_mask"]) > 0: + padding_mask = encoder_out["encoder_padding_mask"][0] + + # embed positions + positions = self.embed_positions( + prev_output_tokens + ) + + if incremental_state is not None: + prev_output_tokens = prev_output_tokens[:, -1:] + positions = positions[:, -1:] + + # embed tokens and positions + x = self.embed_scale * self.embed_tokens(prev_output_tokens) + + if self.project_in_dim is not None: + x = self.project_in_dim(x) + + x += positions + + x = self.dropout_module(x) + + # B x T x C -> T x B x C + x = x.transpose(0, 1) + + self_attn_padding_mask: Optional[Tensor] = None + if prev_output_tokens.eq(self.padding_idx).any(): + self_attn_padding_mask = prev_output_tokens.eq(self.padding_idx) + + # decoder layers + attn: Optional[Tensor] = None + inner_states: List[Optional[Tensor]] = [x] + for idx, layer in enumerate(self.layers): + if incremental_state is None: + self_attn_mask = self.buffered_future_mask(x) + else: + self_attn_mask = None + + x, layer_attn, _ = layer( + x, + enc, + padding_mask, + incremental_state, + self_attn_mask=self_attn_mask, + self_attn_padding_mask=self_attn_padding_mask, + need_attn=False, + need_head_weights=False, + ) + inner_states.append(x) + + if self.layer_norm is not None: + x = self.layer_norm(x) + + # T x B x C -> B x C x T + x = x.transpose(0, 1) + + return x, {"inner_states": inner_states} + + def output_layer(self, features): + """Project features to the vocabulary size.""" + return self.output_projection(features) + + def buffered_future_mask(self, tensor): + dim = tensor.size(0) + # self._future_mask.device != tensor.device is not working in TorchScript. This is a workaround. + if ( + self._future_mask.size(0) == 0 + or (not self._future_mask.device == tensor.device) + or self._future_mask.size(0) < dim + ): + self._future_mask = torch.triu( + fill_with_neg_inf(torch.zeros([dim, dim])), 1 + ) + self._future_mask = self._future_mask.to(tensor) + return self._future_mask[:dim, :dim] diff --git a/model/esm/inverse_folding/transformer_layer.py b/model/esm/inverse_folding/transformer_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..90f1c9bef9aadde0b0fd760504e0267398e38521 --- /dev/null +++ b/model/esm/inverse_folding/transformer_layer.py @@ -0,0 +1,304 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Contents of this file were adapted from the open source fairseq repository. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Dict, List, Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +from onescience.modules.attention import MultiheadAttention +from torch import Tensor + + +class TransformerEncoderLayer(nn.Module): + """Encoder layer block. + `layernorm -> dropout -> add residual` + + Args: + args (argparse.Namespace): parsed command-line arguments + """ + + def __init__(self, args): + super().__init__() + self.args = args + self.embed_dim = args.encoder_embed_dim + self.self_attn = self.build_self_attention(self.embed_dim, args) + self.self_attn_layer_norm = torch.nn.LayerNorm(self.embed_dim) + self.dropout_module = nn.Dropout(args.dropout) + self.activation_fn = F.relu + self.fc1 = self.build_fc1( + self.embed_dim, + args.encoder_ffn_embed_dim, + ) + self.fc2 = self.build_fc2( + args.encoder_ffn_embed_dim, + self.embed_dim, + ) + + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + + def build_fc1(self, input_dim, output_dim): + return nn.Linear(input_dim, output_dim) + + def build_fc2(self, input_dim, output_dim): + return nn.Linear(input_dim, output_dim) + + def build_self_attention(self, embed_dim, args): + return MultiheadAttention( + embed_dim, + args.encoder_attention_heads, + dropout=args.attention_dropout, + self_attention=True, + ) + + def residual_connection(self, x, residual): + return residual + x + + def forward( + self, + x, + encoder_padding_mask: Optional[Tensor], + attn_mask: Optional[Tensor] = None, + ): + """ + Args: + x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)` + encoder_padding_mask (ByteTensor): binary ByteTensor of shape + `(batch, seq_len)` where padding elements are indicated by ``1``. + attn_mask (ByteTensor): binary tensor of shape `(tgt_len, src_len)`, + where `tgt_len` is the length of output and `src_len` is the + length of input, though here both are equal to `seq_len`. + `attn_mask[tgt_i, src_j] = 1` means that when calculating the + embedding for `tgt_i`, we exclude (mask out) `src_j`. This is + useful for strided self-attention. + + Returns: + encoded output of shape `(seq_len, batch, embed_dim)` + """ + # anything in original attn_mask = 1, becomes -1e8 + # anything in original attn_mask = 0, becomes 0 + # Note that we cannot use -inf here, because at some edge cases, + # the attention weight (before softmax) for some padded element in query + # will become -inf, which results in NaN in model parameters + if attn_mask is not None: + attn_mask = attn_mask.masked_fill( + attn_mask.to(torch.bool), -1e8 if x.dtype == torch.float32 else -1e4 + ) + + residual = x + x = self.self_attn_layer_norm(x) + x, _ = self.self_attn( + query=x, + key=x, + value=x, + key_padding_mask=encoder_padding_mask, + need_weights=False, + attn_mask=attn_mask, + ) + x = self.dropout_module(x) + x = self.residual_connection(x, residual) + + residual = x + x = self.final_layer_norm(x) + x = self.activation_fn(self.fc1(x)) + x = self.fc2(x) + x = self.dropout_module(x) + x = self.residual_connection(x, residual) + return x + + +class TransformerDecoderLayer(nn.Module): + """Decoder layer block. + `layernorm -> dropout -> add residual` + + Args: + args (argparse.Namespace): parsed command-line arguments + no_encoder_attn (bool, optional): whether to attend to encoder outputs + (default: False). + """ + + def __init__( + self, args, no_encoder_attn=False, add_bias_kv=False, add_zero_attn=False + ): + super().__init__() + self.embed_dim = args.decoder_embed_dim + self.dropout_module = nn.Dropout(args.dropout) + + self.self_attn = self.build_self_attention( + self.embed_dim, + args, + add_bias_kv=add_bias_kv, + add_zero_attn=add_zero_attn, + ) + self.nh = self.self_attn.num_heads + self.head_dim = self.self_attn.head_dim + + self.activation_fn = F.relu + + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + + if no_encoder_attn: + self.encoder_attn = None + self.encoder_attn_layer_norm = None + else: + self.encoder_attn = self.build_encoder_attention(self.embed_dim, args) + self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim) + + self.ffn_layernorm = ( + LayerNorm(args.decoder_ffn_embed_dim) + if getattr(args, "scale_fc", False) + else None + ) + self.w_resid = ( + nn.Parameter( + torch.ones( + self.embed_dim, + ), + requires_grad=True, + ) + if getattr(args, "scale_resids", False) + else None + ) + + self.fc1 = self.build_fc1( + self.embed_dim, + args.decoder_ffn_embed_dim, + ) + self.fc2 = self.build_fc2( + args.decoder_ffn_embed_dim, + self.embed_dim, + ) + + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + self.need_attn = True + + def build_fc1(self, input_dim, output_dim): + return nn.Linear(input_dim, output_dim) + + def build_fc2(self, input_dim, output_dim): + return nn.Linear(input_dim, output_dim) + + def build_self_attention( + self, embed_dim, args, add_bias_kv=False, add_zero_attn=False + ): + return MultiheadAttention( + embed_dim, + args.decoder_attention_heads, + dropout=args.attention_dropout, + add_bias_kv=add_bias_kv, + add_zero_attn=add_zero_attn, + self_attention=True, + ) + + def build_encoder_attention(self, embed_dim, args): + return MultiheadAttention( + embed_dim, + args.decoder_attention_heads, + kdim=args.encoder_embed_dim, + vdim=args.encoder_embed_dim, + dropout=args.attention_dropout, + encoder_decoder_attention=True, + ) + + def residual_connection(self, x, residual): + return residual + x + + def forward( + self, + x, + encoder_out: Optional[torch.Tensor] = None, + encoder_padding_mask: Optional[torch.Tensor] = None, + incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None, + prev_self_attn_state: Optional[List[torch.Tensor]] = None, + prev_attn_state: Optional[List[torch.Tensor]] = None, + self_attn_mask: Optional[torch.Tensor] = None, + self_attn_padding_mask: Optional[torch.Tensor] = None, + need_attn: bool = False, + need_head_weights: bool = False, + ): + """ + Args: + x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)` + encoder_padding_mask (ByteTensor, optional): binary + ByteTensor of shape `(batch, src_len)` where padding + elements are indicated by ``1``. + need_attn (bool, optional): return attention weights + need_head_weights (bool, optional): return attention weights + for each head (default: return average over heads). + + Returns: + encoded output of shape `(seq_len, batch, embed_dim)` + """ + if need_head_weights: + need_attn = True + + residual = x + x = self.self_attn_layer_norm(x) + if prev_self_attn_state is not None: + prev_key, prev_value = prev_self_attn_state[:2] + saved_state: Dict[str, Optional[Tensor]] = { + "prev_key": prev_key, + "prev_value": prev_value, + } + if len(prev_self_attn_state) >= 3: + saved_state["prev_key_padding_mask"] = prev_self_attn_state[2] + assert incremental_state is not None + self.self_attn._set_input_buffer(incremental_state, saved_state) + _self_attn_input_buffer = self.self_attn._get_input_buffer(incremental_state) + y = x + + x, attn = self.self_attn( + query=x, + key=y, + value=y, + key_padding_mask=self_attn_padding_mask, + incremental_state=incremental_state, + need_weights=False, + attn_mask=self_attn_mask, + ) + x = self.dropout_module(x) + x = self.residual_connection(x, residual) + + if self.encoder_attn is not None and encoder_out is not None: + residual = x + x = self.encoder_attn_layer_norm(x) + if prev_attn_state is not None: + prev_key, prev_value = prev_attn_state[:2] + saved_state: Dict[str, Optional[Tensor]] = { + "prev_key": prev_key, + "prev_value": prev_value, + } + if len(prev_attn_state) >= 3: + saved_state["prev_key_padding_mask"] = prev_attn_state[2] + assert incremental_state is not None + self.encoder_attn._set_input_buffer(incremental_state, saved_state) + + x, attn = self.encoder_attn( + query=x, + key=encoder_out, + value=encoder_out, + key_padding_mask=encoder_padding_mask, + incremental_state=incremental_state, + static_kv=True, + need_weights=need_attn or (not self.training and self.need_attn), + need_head_weights=need_head_weights, + ) + x = self.dropout_module(x) + x = self.residual_connection(x, residual) + + residual = x + x = self.final_layer_norm(x) + + x = self.activation_fn(self.fc1(x)) + if self.ffn_layernorm is not None: + x = self.ffn_layernorm(x) + x = self.fc2(x) + x = self.dropout_module(x) + if self.w_resid is not None: + residual = torch.mul(self.w_resid, residual) + x = self.residual_connection(x, residual) + return x, attn, None diff --git a/model/esm/inverse_folding/util.py b/model/esm/inverse_folding/util.py new file mode 100644 index 0000000000000000000000000000000000000000..08c187ba1c930e773c78c56bb8c86341f76b3a36 --- /dev/null +++ b/model/esm/inverse_folding/util.py @@ -0,0 +1,323 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import json +import math + +import biotite.structure +from biotite.structure.io import pdbx, pdb +from biotite.structure.residues import get_residues +from biotite.structure import filter_backbone +from biotite.structure import get_chains +from biotite.sequence import ProteinSequence +import numpy as np +from scipy.spatial import transform +from scipy.stats import special_ortho_group +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.data as data +from typing import Sequence, Tuple, List + +from onescience.datapipes.esm import BatchConverter + + +def load_structure(fpath, chain=None): + """ + Args: + fpath: filepath to either pdb or cif file + chain: the chain id or list of chain ids to load + Returns: + biotite.structure.AtomArray + """ + if fpath.endswith('cif'): + with open(fpath) as fin: + pdbxf = pdbx.PDBxFile.read(fin) + structure = pdbx.get_structure(pdbxf, model=1) + elif fpath.endswith('pdb'): + with open(fpath) as fin: + pdbf = pdb.PDBFile.read(fin) + structure = pdb.get_structure(pdbf, model=1) + bbmask = filter_backbone(structure) + structure = structure[bbmask] + all_chains = get_chains(structure) + if len(all_chains) == 0: + raise ValueError('No chains found in the input file.') + if chain is None: + chain_ids = all_chains + elif isinstance(chain, list): + chain_ids = chain + else: + chain_ids = [chain] + for chain in chain_ids: + if chain not in all_chains: + raise ValueError(f'Chain {chain} not found in input file') + chain_filter = [a.chain_id in chain_ids for a in structure] + structure = structure[chain_filter] + return structure + + +def extract_coords_from_structure(structure: biotite.structure.AtomArray): + """ + Args: + structure: An instance of biotite AtomArray + Returns: + Tuple (coords, seq) + - coords is an L x 3 x 3 array for N, CA, C coordinates + - seq is the extracted sequence + """ + coords = get_atom_coords_residuewise(["N", "CA", "C"], structure) + residue_identities = get_residues(structure)[1] + seq = ''.join([ProteinSequence.convert_letter_3to1(r) for r in residue_identities]) + return coords, seq + + +def load_coords(fpath, chain): + """ + Args: + fpath: filepath to either pdb or cif file + chain: the chain id + Returns: + Tuple (coords, seq) + - coords is an L x 3 x 3 array for N, CA, C coordinates + - seq is the extracted sequence + """ + structure = load_structure(fpath, chain) + return extract_coords_from_structure(structure) + + +def get_atom_coords_residuewise(atoms: List[str], struct: biotite.structure.AtomArray): + """ + Example for atoms argument: ["N", "CA", "C"] + """ + def filterfn(s, axis=None): + filters = np.stack([s.atom_name == name for name in atoms], axis=1) + sum = filters.sum(0) + if not np.all(sum <= np.ones(filters.shape[1])): + raise RuntimeError("structure has multiple atoms with same name") + index = filters.argmax(0) + coords = s[index].coord + coords[sum == 0] = float("nan") + return coords + + return biotite.structure.apply_residue_wise(struct, struct, filterfn) + + +def get_sequence_loss(model, alphabet, coords, seq): + device = next(model.parameters()).device + batch_converter = CoordBatchConverter(alphabet) + batch = [(coords, None, seq)] + coords, confidence, strs, tokens, padding_mask = batch_converter( + batch, device=device) + + prev_output_tokens = tokens[:, :-1].to(device) + target = tokens[:, 1:] + target_padding_mask = (target == alphabet.padding_idx) + logits, _ = model.forward(coords, padding_mask, confidence, prev_output_tokens) + loss = F.cross_entropy(logits, target, reduction='none') + loss = loss[0].cpu().detach().numpy() + target_padding_mask = target_padding_mask[0].cpu().numpy() + return loss, target_padding_mask + + +def score_sequence(model, alphabet, coords, seq): + loss, target_padding_mask = get_sequence_loss(model, alphabet, coords, seq) + ll_fullseq = -np.sum(loss * ~target_padding_mask) / np.sum(~target_padding_mask) + # Also calculate average when excluding masked portions + coord_mask = np.all(np.isfinite(coords), axis=(-1, -2)) + ll_withcoord = -np.sum(loss * coord_mask) / np.sum(coord_mask) + return ll_fullseq, ll_withcoord + + +def get_encoder_output(model, alphabet, coords): + device = next(model.parameters()).device + batch_converter = CoordBatchConverter(alphabet) + batch = [(coords, None, None)] + coords, confidence, strs, tokens, padding_mask = batch_converter( + batch, device=device) + encoder_out = model.encoder.forward(coords, padding_mask, confidence, + return_all_hiddens=False) + # remove beginning and end (bos and eos tokens) + return encoder_out['encoder_out'][0][1:-1, 0] + + +def rotate(v, R): + """ + Rotates a vector by a rotation matrix. + + Args: + v: 3D vector, tensor of shape (length x batch_size x channels x 3) + R: rotation matrix, tensor of shape (length x batch_size x 3 x 3) + + Returns: + Rotated version of v by rotation matrix R. + """ + R = R.unsqueeze(-3) + v = v.unsqueeze(-1) + return torch.sum(v * R, dim=-2) + + +def get_rotation_frames(coords): + """ + Returns a local rotation frame defined by N, CA, C positions. + + Args: + coords: coordinates, tensor of shape (batch_size x length x 3 x 3) + where the third dimension is in order of N, CA, C + + Returns: + Local relative rotation frames in shape (batch_size x length x 3 x 3) + """ + v1 = coords[:, :, 2] - coords[:, :, 1] + v2 = coords[:, :, 0] - coords[:, :, 1] + e1 = normalize(v1, dim=-1) + u2 = v2 - e1 * torch.sum(e1 * v2, dim=-1, keepdim=True) + e2 = normalize(u2, dim=-1) + e3 = torch.cross(e1, e2, dim=-1) + R = torch.stack([e1, e2, e3], dim=-2) + return R + + +def nan_to_num(ts, val=0.0): + """ + Replaces nans in tensor with a fixed value. + """ + val = torch.tensor(val, dtype=ts.dtype, device=ts.device) + return torch.where(~torch.isfinite(ts), val, ts) + + +def rbf(values, v_min, v_max, n_bins=16): + """ + Returns RBF encodings in a new dimension at the end. + """ + rbf_centers = torch.linspace(v_min, v_max, n_bins, device=values.device) + rbf_centers = rbf_centers.view([1] * len(values.shape) + [-1]) + rbf_std = (v_max - v_min) / n_bins + v_expand = torch.unsqueeze(values, -1) + z = (values.unsqueeze(-1) - rbf_centers) / rbf_std + return torch.exp(-z ** 2) + + +def norm(tensor, dim, eps=1e-8, keepdim=False): + """ + Returns L2 norm along a dimension. + """ + return torch.sqrt( + torch.sum(torch.square(tensor), dim=dim, keepdim=keepdim) + eps) + + +def normalize(tensor, dim=-1): + """ + Normalizes a tensor along a dimension after removing nans. + """ + return nan_to_num( + torch.div(tensor, norm(tensor, dim=dim, keepdim=True)) + ) + + +class CoordBatchConverter(BatchConverter): + def __call__(self, raw_batch: Sequence[Tuple[Sequence, str]], device=None): + """ + Args: + raw_batch: List of tuples (coords, confidence, seq) + In each tuple, + coords: list of floats, shape L x 3 x 3 + confidence: list of floats, shape L; or scalar float; or None + seq: string of length L + Returns: + coords: Tensor of shape batch_size x L x 3 x 3 + confidence: Tensor of shape batch_size x L + strs: list of strings + tokens: LongTensor of shape batch_size x L + padding_mask: ByteTensor of shape batch_size x L + """ + self.alphabet.cls_idx = self.alphabet.get_idx("") + batch = [] + for coords, confidence, seq in raw_batch: + if confidence is None: + confidence = 1. + if isinstance(confidence, float) or isinstance(confidence, int): + confidence = [float(confidence)] * len(coords) + if seq is None: + seq = 'X' * len(coords) + batch.append(((coords, confidence), seq)) + + coords_and_confidence, strs, tokens = super().__call__(batch) + + # pad beginning and end of each protein due to legacy reasons + coords = [ + F.pad(torch.tensor(cd), (0, 0, 0, 0, 1, 1), value=np.inf) + for cd, _ in coords_and_confidence + ] + confidence = [ + F.pad(torch.tensor(cf), (1, 1), value=-1.) + for _, cf in coords_and_confidence + ] + coords = self.collate_dense_tensors(coords, pad_v=np.nan) + confidence = self.collate_dense_tensors(confidence, pad_v=-1.) + if device is not None: + coords = coords.to(device) + confidence = confidence.to(device) + tokens = tokens.to(device) + padding_mask = torch.isnan(coords[:,:,0,0]) + coord_mask = torch.isfinite(coords.sum(-2).sum(-1)) + confidence = confidence * coord_mask + (-1.) * padding_mask + return coords, confidence, strs, tokens, padding_mask + + def from_lists(self, coords_list, confidence_list=None, seq_list=None, device=None): + """ + Args: + coords_list: list of length batch_size, each item is a list of + floats in shape L x 3 x 3 to describe a backbone + confidence_list: one of + - None, default to highest confidence + - list of length batch_size, each item is a scalar + - list of length batch_size, each item is a list of floats of + length L to describe the confidence scores for the backbone + with values between 0. and 1. + seq_list: either None or a list of strings + Returns: + coords: Tensor of shape batch_size x L x 3 x 3 + confidence: Tensor of shape batch_size x L + strs: list of strings + tokens: LongTensor of shape batch_size x L + padding_mask: ByteTensor of shape batch_size x L + """ + batch_size = len(coords_list) + if confidence_list is None: + confidence_list = [None] * batch_size + if seq_list is None: + seq_list = [None] * batch_size + raw_batch = zip(coords_list, confidence_list, seq_list) + return self.__call__(raw_batch, device) + + @staticmethod + def collate_dense_tensors(samples, pad_v): + """ + Takes a list of tensors with the following dimensions: + [(d_11, ..., d_1K), + (d_21, ..., d_2K), + ..., + (d_N1, ..., d_NK)] + and stack + pads them into a single tensor of: + (N, max_i=1,N { d_i1 }, ..., max_i=1,N {diK}) + """ + if len(samples) == 0: + return torch.Tensor() + if len(set(x.dim() for x in samples)) != 1: + raise RuntimeError( + f"Samples has varying dimensions: {[x.dim() for x in samples]}" + ) + (device,) = tuple(set(x.device for x in samples)) # assumes all on same device + max_shape = [max(lst) for lst in zip(*[x.shape for x in samples])] + result = torch.empty( + len(samples), *max_shape, dtype=samples[0].dtype, device=device + ) + result.fill_(pad_v) + for i in range(len(samples)): + result_i = result[i] + t = samples[i] + result_i[tuple(slice(0, k) for k in t.shape)] = t + return result diff --git a/model/esm/msa_transformer.py b/model/esm/msa_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..5f164e956940dce024d992b762051f345226cbde --- /dev/null +++ b/model/esm/msa_transformer.py @@ -0,0 +1,238 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import torch +import torch.nn as nn + +from onescience.modules.esm import ( + AxialTransformerLayer, + LearnedPositionalEmbedding, + RobertaLMHead, + ESM1bLayerNorm, + ContactPredictionHead, +) + +from onescience.modules.attention import RowSelfAttention, ColumnSelfAttention + + + +class MSATransformer(nn.Module): + @classmethod + def add_args(cls, parser): + # fmt: off + parser.add_argument( + "--num_layers", + default=12, + type=int, + metavar="N", + help="number of layers" + ) + parser.add_argument( + "--embed_dim", + default=768, + type=int, + metavar="N", + help="embedding dimension" + ) + parser.add_argument( + "--logit_bias", + action="store_true", + help="whether to apply bias to logits" + ) + parser.add_argument( + "--ffn_embed_dim", + default=3072, + type=int, + metavar="N", + help="embedding dimension for FFN", + ) + parser.add_argument( + "--attention_heads", + default=12, + type=int, + metavar="N", + help="number of attention heads", + ) + parser.add_argument( + "--dropout", + default=0.1, + type=float, + help="Dropout to apply." + ) + parser.add_argument( + "--attention_dropout", + default=0.1, + type=float, + help="Dropout to apply." + ) + parser.add_argument( + "--activation_dropout", + default=0.1, + type=float, + help="Dropout to apply." + ) + parser.add_argument( + "--max_tokens_per_msa", + default=2 ** 14, + type=int, + help=( + "Used during inference to batch attention computations in a single " + "forward pass. This allows increased input sizes with less memory." + ), + ) + # fmt: on + + def __init__(self, args, alphabet): + super().__init__() + self.args = args + self.alphabet_size = len(alphabet) + self.padding_idx = alphabet.padding_idx + self.mask_idx = alphabet.mask_idx + self.cls_idx = alphabet.cls_idx + self.eos_idx = alphabet.eos_idx + self.prepend_bos = alphabet.prepend_bos + self.append_eos = alphabet.append_eos + + self.embed_tokens = nn.Embedding( + self.alphabet_size, self.args.embed_dim, padding_idx=self.padding_idx + ) + + if getattr(self.args, "embed_positions_msa", False): + emb_dim = getattr(self.args, "embed_positions_msa_dim", self.args.embed_dim) + self.msa_position_embedding = nn.Parameter( + 0.01 * torch.randn(1, 1024, 1, emb_dim), + requires_grad=True, + ) + else: + self.register_parameter("msa_position_embedding", None) + + self.dropout_module = nn.Dropout(self.args.dropout) + self.layers = nn.ModuleList( + [ + AxialTransformerLayer( + self.args.embed_dim, + self.args.ffn_embed_dim, + self.args.attention_heads, + self.args.dropout, + self.args.attention_dropout, + self.args.activation_dropout, + getattr(self.args, "max_tokens_per_msa", self.args.max_tokens), + ) + for _ in range(self.args.layers) + ] + ) + + self.contact_head = ContactPredictionHead( + self.args.layers * self.args.attention_heads, + self.prepend_bos, + self.append_eos, + eos_idx=self.eos_idx, + ) + self.embed_positions = LearnedPositionalEmbedding( + self.args.max_positions, + self.args.embed_dim, + self.padding_idx, + ) + self.emb_layer_norm_before = ESM1bLayerNorm(self.args.embed_dim) + self.emb_layer_norm_after = ESM1bLayerNorm(self.args.embed_dim) + self.lm_head = RobertaLMHead( + embed_dim=self.args.embed_dim, + output_dim=self.alphabet_size, + weight=self.embed_tokens.weight, + ) + + def forward(self, tokens, repr_layers=[], need_head_weights=False, return_contacts=False): + if return_contacts: + need_head_weights = True + + assert tokens.ndim == 3 + batch_size, num_alignments, seqlen = tokens.size() + padding_mask = tokens.eq(self.padding_idx) # B, R, C + if not padding_mask.any(): + padding_mask = None + + x = self.embed_tokens(tokens) + x += self.embed_positions(tokens.view(batch_size * num_alignments, seqlen)).view(x.size()) + if self.msa_position_embedding is not None: + if x.size(1) > 1024: + raise RuntimeError( + "Using model with MSA position embedding trained on maximum MSA " + f"depth of 1024, but received {x.size(1)} alignments." + ) + x += self.msa_position_embedding[:, :num_alignments] + + x = self.emb_layer_norm_before(x) + + x = self.dropout_module(x) + + if padding_mask is not None: + x = x * (1 - padding_mask.unsqueeze(-1).type_as(x)) + + repr_layers = set(repr_layers) + hidden_representations = {} + if 0 in repr_layers: + hidden_representations[0] = x + + if need_head_weights: + row_attn_weights = [] + col_attn_weights = [] + + # B x R x C x D -> R x C x B x D + x = x.permute(1, 2, 0, 3) + + for layer_idx, layer in enumerate(self.layers): + x = layer( + x, + self_attn_padding_mask=padding_mask, + need_head_weights=need_head_weights, + ) + if need_head_weights: + x, col_attn, row_attn = x + # H x C x B x R x R -> B x H x C x R x R + col_attn_weights.append(col_attn.permute(2, 0, 1, 3, 4)) + # H x B x C x C -> B x H x C x C + row_attn_weights.append(row_attn.permute(1, 0, 2, 3)) + if (layer_idx + 1) in repr_layers: + hidden_representations[layer_idx + 1] = x.permute(2, 0, 1, 3) + + x = self.emb_layer_norm_after(x) + x = x.permute(2, 0, 1, 3) # R x C x B x D -> B x R x C x D + + # last hidden representation should have layer norm applied + if (layer_idx + 1) in repr_layers: + hidden_representations[layer_idx + 1] = x + x = self.lm_head(x) + + result = {"logits": x, "representations": hidden_representations} + if need_head_weights: + # col_attentions: B x L x H x C x R x R + col_attentions = torch.stack(col_attn_weights, 1) + # row_attentions: B x L x H x C x C + row_attentions = torch.stack(row_attn_weights, 1) + result["col_attentions"] = col_attentions + result["row_attentions"] = row_attentions + if return_contacts: + contacts = self.contact_head(tokens, row_attentions) + result["contacts"] = contacts + + return result + + def predict_contacts(self, tokens): + return self(tokens, return_contacts=True)["contacts"] + + @property + def num_layers(self): + return self.args.layers + + def max_tokens_per_msa_(self, value: int) -> None: + """The MSA Transformer automatically batches attention computations when + gradients are disabled to allow you to pass in larger MSAs at test time than + you can fit in GPU memory. By default this occurs when more than 2^14 tokens + are passed in the input MSA. You can set this value to infinity to disable + this behavior. + """ + for module in self.modules(): + if isinstance(module, (RowSelfAttention, ColumnSelfAttention)): + module.max_tokens_per_msa = value diff --git a/model/esm/pretrained.py b/model/esm/pretrained.py new file mode 100644 index 0000000000000000000000000000000000000000..7b0c08f4e1b52f7a332049f32a83301c57a58fe3 --- /dev/null +++ b/model/esm/pretrained.py @@ -0,0 +1,553 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import re +import urllib +import warnings +from argparse import Namespace +from pathlib import Path + +import torch + +import model.esm as esm +from onescience.datapipes.esm import Alphabet +from model.esm.esm2 import ESM2 + + +def _has_regression_weights(model_name): + """Return whether we expect / require regression weights; + Right now that is all models except ESM-1v, ESM-IF, and partially trained ESM2 models""" + return not ("esm1v" in model_name or "esm_if" in model_name or "270K" in model_name or "500K" in model_name) + + +def load_model_and_alphabet(model_name): + if model_name.endswith(".pt"): # treat as filepath + return load_model_and_alphabet_local(model_name) + else: + return load_model_and_alphabet_hub(model_name) + + +def load_hub_workaround(url): + try: + data = torch.hub.load_state_dict_from_url(url, progress=False, map_location="cpu") + except RuntimeError: + # Pytorch version issue - see https://github.com/pytorch/pytorch/issues/43106 + fn = Path(url).name + data = torch.load( + f"{torch.hub.get_dir()}/checkpoints/{fn}", + map_location="cpu", + ) + except urllib.error.HTTPError as e: + raise Exception(f"Could not load {url}, check if you specified a correct model name?") + return data + + +def load_regression_hub(model_name): + url = f"https://dl.fbaipublicfiles.com/fair-esm/regression/{model_name}-contact-regression.pt" + regression_data = load_hub_workaround(url) + return regression_data + + +def _download_model_and_regression_data(model_name): + url = f"https://dl.fbaipublicfiles.com/fair-esm/models/{model_name}.pt" + model_data = load_hub_workaround(url) + if _has_regression_weights(model_name): + regression_data = load_regression_hub(model_name) + else: + regression_data = None + return model_data, regression_data + + +def load_model_and_alphabet_hub(model_name): + model_data, regression_data = _download_model_and_regression_data(model_name) + return load_model_and_alphabet_core(model_name, model_data, regression_data) + + +def load_model_and_alphabet_local(model_location): + """Load from local path. The regression weights need to be co-located""" + model_location = Path(model_location) + model_data = torch.load(str(model_location), map_location="cpu") + model_name = model_location.stem + if _has_regression_weights(model_name): + regression_location = str(model_location.with_suffix("")) + "-contact-regression.pt" + regression_data = torch.load(regression_location, map_location="cpu") + else: + regression_data = None + return load_model_and_alphabet_core(model_name, model_data, regression_data) + + +def has_emb_layer_norm_before(model_state): + """Determine whether layer norm needs to be applied before the encoder""" + return any(k.startswith("emb_layer_norm_before") for k, param in model_state.items()) + + +def _load_model_and_alphabet_core_v1(model_data): + import model.esm as esm # since esm.inverse_folding is imported below, you actually have to re-import esm here + + alphabet = esm.Alphabet.from_architecture(model_data["args"].arch) + + if model_data["args"].arch == "roberta_large": + # upgrade state dict + pra = lambda s: "".join(s.split("encoder_")[1:] if "encoder" in s else s) + prs1 = lambda s: "".join(s.split("encoder.")[1:] if "encoder" in s else s) + prs2 = lambda s: "".join( + s.split("sentence_encoder.")[1:] if "sentence_encoder" in s else s + ) + model_args = {pra(arg[0]): arg[1] for arg in vars(model_data["args"]).items()} + model_state = {prs1(prs2(arg[0])): arg[1] for arg in model_data["model"].items()} + model_state["embed_tokens.weight"][alphabet.mask_idx].zero_() # For token drop + model_args["emb_layer_norm_before"] = has_emb_layer_norm_before(model_state) + model_type = esm.ProteinBertModel + + elif model_data["args"].arch == "protein_bert_base": + + # upgrade state dict + pra = lambda s: "".join(s.split("decoder_")[1:] if "decoder" in s else s) + prs = lambda s: "".join(s.split("decoder.")[1:] if "decoder" in s else s) + model_args = {pra(arg[0]): arg[1] for arg in vars(model_data["args"]).items()} + model_state = {prs(arg[0]): arg[1] for arg in model_data["model"].items()} + model_type = esm.ProteinBertModel + elif model_data["args"].arch == "msa_transformer": + + # upgrade state dict + pra = lambda s: "".join(s.split("encoder_")[1:] if "encoder" in s else s) + prs1 = lambda s: "".join(s.split("encoder.")[1:] if "encoder" in s else s) + prs2 = lambda s: "".join( + s.split("sentence_encoder.")[1:] if "sentence_encoder" in s else s + ) + prs3 = lambda s: s.replace("row", "column") if "row" in s else s.replace("column", "row") + model_args = {pra(arg[0]): arg[1] for arg in vars(model_data["args"]).items()} + model_state = {prs1(prs2(prs3(arg[0]))): arg[1] for arg in model_data["model"].items()} + if model_args.get("embed_positions_msa", False): + emb_dim = model_state["msa_position_embedding"].size(-1) + model_args["embed_positions_msa_dim"] = emb_dim # initial release, bug: emb_dim==1 + + model_type = esm.MSATransformer + + elif "invariant_gvp" in model_data["args"].arch: + import model.esm.inverse_folding + + model_type = esm.inverse_folding.gvp_transformer.GVPTransformerModel + model_args = vars(model_data["args"]) # convert Namespace -> dict + + def update_name(s): + # Map the module names in checkpoints trained with internal code to + # the updated module names in open source code + s = s.replace("W_v", "embed_graph.embed_node") + s = s.replace("W_e", "embed_graph.embed_edge") + s = s.replace("embed_scores.0", "embed_confidence") + s = s.replace("embed_score.", "embed_graph.embed_confidence.") + s = s.replace("seq_logits_projection.", "") + s = s.replace("embed_ingraham_features", "embed_dihedrals") + s = s.replace("embed_gvp_in_local_frame.0", "embed_gvp_output") + s = s.replace("embed_features_in_local_frame.0", "embed_gvp_input_features") + return s + + model_state = { + update_name(sname): svalue + for sname, svalue in model_data["model"].items() + if "version" not in sname + } + + else: + raise ValueError("Unknown architecture selected") + + model = model_type( + Namespace(**model_args), + alphabet, + ) + + return model, alphabet, model_state + + +def _load_model_and_alphabet_core_v2(model_data): + def upgrade_state_dict(state_dict): + """Removes prefixes 'model.encoder.sentence_encoder.' and 'model.encoder.'.""" + prefixes = ["encoder.sentence_encoder.", "encoder."] + pattern = re.compile("^" + "|".join(prefixes)) + state_dict = {pattern.sub("", name): param for name, param in state_dict.items()} + return state_dict + + cfg = model_data["cfg"]["model"] + state_dict = model_data["model"] + state_dict = upgrade_state_dict(state_dict) + alphabet = Alphabet.from_architecture("ESM-1b") + model = ESM2( + num_layers=cfg.encoder_layers, + embed_dim=cfg.encoder_embed_dim, + attention_heads=cfg.encoder_attention_heads, + alphabet=alphabet, + token_dropout=cfg.token_dropout, + ) + return model, alphabet, state_dict + + +def load_model_and_alphabet_core(model_name, model_data, regression_data=None): + if regression_data is not None: + model_data["model"].update(regression_data["model"]) + + if model_name.startswith("esm2"): + model, alphabet, model_state = _load_model_and_alphabet_core_v2(model_data) + else: + model, alphabet, model_state = _load_model_and_alphabet_core_v1(model_data) + + expected_keys = set(model.state_dict().keys()) + found_keys = set(model_state.keys()) + + if regression_data is None: + expected_missing = {"contact_head.regression.weight", "contact_head.regression.bias"} + error_msgs = [] + missing = (expected_keys - found_keys) - expected_missing + if missing: + error_msgs.append(f"Missing key(s) in state_dict: {missing}.") + unexpected = found_keys - expected_keys + if unexpected: + error_msgs.append(f"Unexpected key(s) in state_dict: {unexpected}.") + + if error_msgs: + raise RuntimeError( + "Error(s) in loading state_dict for {}:\n\t{}".format( + model.__class__.__name__, "\n\t".join(error_msgs) + ) + ) + if expected_missing - found_keys: + warnings.warn( + "Regression weights not found, predicting contacts will not produce correct results." + ) + + model.load_state_dict(model_state, strict=regression_data is not None) + + return model, alphabet + + +def esm1_t34_670M_UR50S(): + """34 layer transformer model with 670M params, trained on Uniref50 Sparse. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1_t34_670M_UR50S") + + +def esm1_t34_670M_UR50D(): + """34 layer transformer model with 670M params, trained on Uniref50 Dense. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1_t34_670M_UR50D") + + +def esm1_t34_670M_UR100(): + """34 layer transformer model with 670M params, trained on Uniref100. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1_t34_670M_UR100") + + +def esm1_t12_85M_UR50S(): + """12 layer transformer model with 85M params, trained on Uniref50 Sparse. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1_t12_85M_UR50S") + + +def esm1_t6_43M_UR50S(): + """6 layer transformer model with 43M params, trained on Uniref50 Sparse. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1_t6_43M_UR50S") + + +def esm1b_t33_650M_UR50S(): + """33 layer transformer model with 650M params, trained on Uniref50 Sparse. + This is our best performing model, which will be described in a future publication. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1b_t33_650M_UR50S") + + +def esm_msa1_t12_100M_UR50S(): + warnings.warn( + "This model had a minor bug in the positional embeddings, " + "please use ESM-MSA-1b: esm.pretrained.esm_msa1b_t12_100M_UR50S()", + ) + return load_model_and_alphabet_hub("esm_msa1_t12_100M_UR50S") + + +def esm_msa1b_t12_100M_UR50S(): + return load_model_and_alphabet_hub("esm_msa1b_t12_100M_UR50S") + + +def esm1v_t33_650M_UR90S(): + """33 layer transformer model with 650M params, trained on Uniref90. + This is model 1 of a 5 model ensemble. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_1") + + +def esm1v_t33_650M_UR90S_1(): + """33 layer transformer model with 650M params, trained on Uniref90. + This is model 1 of a 5 model ensemble. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_1") + + +def esm1v_t33_650M_UR90S_2(): + """33 layer transformer model with 650M params, trained on Uniref90. + This is model 2 of a 5 model ensemble. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_2") + + +def esm1v_t33_650M_UR90S_3(): + """33 layer transformer model with 650M params, trained on Uniref90. + This is model 3 of a 5 model ensemble. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_3") + + +def esm1v_t33_650M_UR90S_4(): + """33 layer transformer model with 650M params, trained on Uniref90. + This is model 4 of a 5 model ensemble. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_4") + + +def esm1v_t33_650M_UR90S_5(): + """33 layer transformer model with 650M params, trained on Uniref90. + This is model 5 of a 5 model ensemble. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_5") + + +def esm_if1_gvp4_t16_142M_UR50(): + """Inverse folding model with 142M params, with 4 GVP-GNN layers, 8 + Transformer encoder layers, and 8 Transformer decoder layers, trained on + CATH structures and 12 million alphafold2 predicted structures from UniRef50 + sequences. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm_if1_gvp4_t16_142M_UR50") + + +def esm2_t6_8M_UR50D(): + """6 layer ESM-2 model with 8M params, trained on UniRef50. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm2_t6_8M_UR50D") + + +def esm2_t12_35M_UR50D(): + """12 layer ESM-2 model with 35M params, trained on UniRef50. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm2_t12_35M_UR50D") + + +def esm2_t30_150M_UR50D(): + """30 layer ESM-2 model with 150M params, trained on UniRef50. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm2_t30_150M_UR50D") + + +def esm2_t33_650M_UR50D(): + """33 layer ESM-2 model with 650M params, trained on UniRef50. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm2_t33_650M_UR50D") + + +def esm2_t36_3B_UR50D(): + """36 layer ESM-2 model with 3B params, trained on UniRef50. + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm2_t36_3B_UR50D") + + +def esm2_t48_15B_UR50D(): + """48 layer ESM-2 model with 15B params, trained on UniRef50. + If you have OOM while loading this model, please refer to README + on how to employ FSDP and ZeRO CPU offloading + + Returns a tuple of (Model, Alphabet). + """ + return load_model_and_alphabet_hub("esm2_t48_15B_UR50D") + + +def esmfold_v0(): + """ + ESMFold v0 model with 3B ESM-2, 48 folding blocks. + This version was used for the paper (Lin et al, 2022). It was trained + on all PDB chains until 2020-05, to ensure temporal holdout with CASP14 + and the CAMEO validation and test set reported there. + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_v0() + + +def esmfold_v1(): + """ + ESMFold v1 model using 3B ESM-2, 48 folding blocks. + ESMFold provides fast high accuracy atomic level structure prediction + directly from the individual sequence of a protein. ESMFold uses the ESM2 + protein language model to extract meaningful representations from the + protein sequence. + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_v1() + +def esmfold_structure_module_only_8M(): + """ + ESMFold baseline model using 8M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 500K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_structure_module_only_8M() + + +def esmfold_structure_module_only_8M_270K(): + """ + ESMFold baseline model using 8M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_structure_module_only_8M_270K() + + +def esmfold_structure_module_only_35M(): + """ + ESMFold baseline model using 35M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 500K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_structure_module_only_35M() + + +def esmfold_structure_module_only_35M_270K(): + """ + ESMFold baseline model using 35M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_structure_module_only_35M_270K() + + +def esmfold_structure_module_only_150M(): + """ + ESMFold baseline model using 150M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 500K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_structure_module_only_150M() + + +def esmfold_structure_module_only_150M_270K(): + """ + ESMFold baseline model using 150M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_structure_module_only_150M_270K() + + +def esmfold_structure_module_only_650M(): + """ + ESMFold baseline model using 650M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 500K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_structure_module_only_650M() + + +def esmfold_structure_module_only_650M_270K(): + """ + ESMFold baseline model using 650M ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_structure_module_only_650M_270K() + + +def esmfold_structure_module_only_3B(): + """ + ESMFold baseline model using 3B ESM-2, 0 folding blocks. + ESM-2 here is trained out to 500K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_structure_module_only_3B() + + +def esmfold_structure_module_only_3B_270K(): + """ + ESMFold baseline model using 3B ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_structure_module_only_3B_270K() + + +def esmfold_structure_module_only_15B(): + """ + ESMFold baseline model using 15B ESM-2, 0 folding blocks. + ESM-2 here is trained out to 270K updates. + The 15B parameter ESM-2 was not trained out to 500K updates + This is a model designed to test the capabilities of the language model + when ablated for number of parameters in the language model. + See table S1 in (Lin et al, 2022). + """ + import model.esm.esmfold.v1.pretrained + return esm.esmfold.v1.pretrained.esmfold_structure_module_only_15B() diff --git a/model/esm/version.py b/model/esm/version.py new file mode 100644 index 0000000000000000000000000000000000000000..271f36efa775e1a23929f69a3d5c4883291eabe7 --- /dev/null +++ b/model/esm/version.py @@ -0,0 +1,6 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +version = "2.0.1" diff --git a/model/openfold/__init__.py b/model/openfold/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/model/openfold/dropout.py b/model/openfold/dropout.py new file mode 100644 index 0000000000000000000000000000000000000000..651b9775ef44fba20dec75c60703f00beac66e0c --- /dev/null +++ b/model/openfold/dropout.py @@ -0,0 +1,78 @@ +# Copyright 2021 AlQuraishi Laboratory +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import torch +import torch.nn as nn +from functools import partialmethod +from typing import Union, List + + +class Dropout(nn.Module): + """ + Implementation of dropout with the ability to share the dropout mask + along a particular dimension. + + If not in training mode, this module computes the identity function. + """ + + def __init__(self, r: float, batch_dim: Union[int, List[int]]): + """ + Args: + r: + Dropout rate + batch_dim: + Dimension(s) along which the dropout mask is shared + """ + super(Dropout, self).__init__() + + self.r = r + if type(batch_dim) == int: + batch_dim = [batch_dim] + self.batch_dim = batch_dim + self.dropout = nn.Dropout(self.r) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Args: + x: + Tensor to which dropout is applied. Can have any shape + compatible with self.batch_dim + """ + shape = list(x.shape) + if self.batch_dim is not None: + for bd in self.batch_dim: + shape[bd] = 1 + mask = x.new_ones(shape) + mask = self.dropout(mask) + x *= mask + return x + + +class DropoutRowwise(Dropout): + """ + Convenience class for rowwise dropout as described in subsection + 1.11.6. + """ + + __init__ = partialmethod(Dropout.__init__, batch_dim=-3) + + +class DropoutColumnwise(Dropout): + """ + Convenience class for columnwise dropout as described in subsection + 1.11.6. + """ + + __init__ = partialmethod(Dropout.__init__, batch_dim=-2) diff --git a/model/openfold/embedders.py b/model/openfold/embedders.py new file mode 100644 index 0000000000000000000000000000000000000000..ad77e319f3a8e13cc99eabd1cec3dd0e6bc46550 --- /dev/null +++ b/model/openfold/embedders.py @@ -0,0 +1,984 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import partial + +import torch +import torch.nn as nn +from typing import Tuple, Optional + +from onescience.utils.openfold import all_atom_multimer +from onescience.utils.openfold.feats import ( + pseudo_beta_fn, + dgram_from_positions, + build_template_angle_feat, + build_template_pair_feat, +) +from model.openfold.primitives import Linear, LayerNorm +from model.openfold.template import ( + TemplatePairStack, + TemplatePointwiseAttention, +) +from onescience.utils.openfold import geometry +from onescience.utils.openfold.tensor_utils import add, one_hot, tensor_tree_map, dict_multimap + + +class InputEmbedder(nn.Module): + """ + Embeds a subset of the input features. + + Implements Algorithms 3 (InputEmbedder) and 4 (relpos). + """ + + def __init__( + self, + tf_dim: int, + msa_dim: int, + c_z: int, + c_m: int, + relpos_k: int, + **kwargs, + ): + """ + Args: + tf_dim: + Final dimension of the target features + msa_dim: + Final dimension of the MSA features + c_z: + Pair embedding dimension + c_m: + MSA embedding dimension + relpos_k: + Window size used in relative positional encoding + """ + super(InputEmbedder, self).__init__() + + self.tf_dim = tf_dim + self.msa_dim = msa_dim + + self.c_z = c_z + self.c_m = c_m + + self.linear_tf_z_i = Linear(tf_dim, c_z) + self.linear_tf_z_j = Linear(tf_dim, c_z) + self.linear_tf_m = Linear(tf_dim, c_m) + self.linear_msa_m = Linear(msa_dim, c_m) + + # RPE stuff + self.relpos_k = relpos_k + self.no_bins = 2 * relpos_k + 1 + self.linear_relpos = Linear(self.no_bins, c_z) + + def relpos(self, ri: torch.Tensor): + """ + Computes relative positional encodings + + Implements Algorithm 4. + + Args: + ri: + "residue_index" features of shape [*, N] + """ + d = ri[..., None] - ri[..., None, :] + boundaries = torch.arange( + start=-self.relpos_k, end=self.relpos_k + 1, device=d.device + ) + reshaped_bins = boundaries.view(((1,) * len(d.shape)) + (len(boundaries),)) + d = d[..., None] - reshaped_bins + d = torch.abs(d) + d = torch.argmin(d, dim=-1) + d = nn.functional.one_hot(d, num_classes=len(boundaries)).float() + d = d.to(ri.dtype) + return self.linear_relpos(d) + + def forward( + self, + tf: torch.Tensor, + ri: torch.Tensor, + msa: torch.Tensor, + inplace_safe: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + batch: Dict containing + "target_feat": + Features of shape [*, N_res, tf_dim] + "residue_index": + Features of shape [*, N_res] + "msa_feat": + Features of shape [*, N_clust, N_res, msa_dim] + Returns: + msa_emb: + [*, N_clust, N_res, C_m] MSA embedding + pair_emb: + [*, N_res, N_res, C_z] pair embedding + + """ + # [*, N_res, c_z] + tf_emb_i = self.linear_tf_z_i(tf) + tf_emb_j = self.linear_tf_z_j(tf) + + # [*, N_res, N_res, c_z] + pair_emb = self.relpos(ri.type(tf_emb_i.dtype)) + pair_emb = add(pair_emb, + tf_emb_i[..., None, :], + inplace=inplace_safe + ) + pair_emb = add(pair_emb, + tf_emb_j[..., None, :, :], + inplace=inplace_safe + ) + + # [*, N_clust, N_res, c_m] + n_clust = msa.shape[-3] + tf_m = ( + self.linear_tf_m(tf) + .unsqueeze(-3) + .expand(((-1,) * len(tf.shape[:-2]) + (n_clust, -1, -1))) + ) + msa_emb = self.linear_msa_m(msa) + tf_m + + return msa_emb, pair_emb + + +class InputEmbedderMultimer(nn.Module): + """ + Embeds a subset of the input features. + + Implements Algorithms 3 (InputEmbedder) and 4 (relpos). + """ + + def __init__( + self, + tf_dim: int, + msa_dim: int, + c_z: int, + c_m: int, + max_relative_idx: int, + use_chain_relative: bool, + max_relative_chain: int, + **kwargs, + ): + """ + Args: + tf_dim: + Final dimension of the target features + msa_dim: + Final dimension of the MSA features + c_z: + Pair embedding dimension + c_m: + MSA embedding dimension + relpos_k: + Window size used in relative positional encoding + """ + super(InputEmbedderMultimer, self).__init__() + + self.tf_dim = tf_dim + self.msa_dim = msa_dim + + self.c_z = c_z + self.c_m = c_m + + self.linear_tf_z_i = Linear(tf_dim, c_z) + self.linear_tf_z_j = Linear(tf_dim, c_z) + self.linear_tf_m = Linear(tf_dim, c_m) + self.linear_msa_m = Linear(msa_dim, c_m) + + # RPE stuff + self.max_relative_idx = max_relative_idx + self.use_chain_relative = use_chain_relative + self.max_relative_chain = max_relative_chain + if(self.use_chain_relative): + self.no_bins = ( + 2 * max_relative_idx + 2 + + 1 + + 2 * max_relative_chain + 2 + ) + else: + self.no_bins = 2 * max_relative_idx + 1 + self.linear_relpos = Linear(self.no_bins, c_z) + + def relpos(self, batch): + pos = batch["residue_index"] + asym_id = batch["asym_id"] + asym_id_same = (asym_id[..., None] == asym_id[..., None, :]) + offset = pos[..., None] - pos[..., None, :] + + clipped_offset = torch.clamp( + offset + self.max_relative_idx, 0, 2 * self.max_relative_idx + ) + + rel_feats = [] + if(self.use_chain_relative): + final_offset = torch.where( + asym_id_same, + clipped_offset, + (2 * self.max_relative_idx + 1) * + torch.ones_like(clipped_offset) + ) + boundaries = torch.arange( + start=0, end=2 * self.max_relative_idx + 2, device=final_offset.device + ) + rel_pos = one_hot( + final_offset, + boundaries, + ) + + rel_feats.append(rel_pos) + + entity_id = batch["entity_id"] + entity_id_same = (entity_id[..., None] == entity_id[..., None, :]) + rel_feats.append(entity_id_same[..., None].to(dtype=rel_pos.dtype)) + + sym_id = batch["sym_id"] + rel_sym_id = sym_id[..., None] - sym_id[..., None, :] + + max_rel_chain = self.max_relative_chain + clipped_rel_chain = torch.clamp( + rel_sym_id + max_rel_chain, + 0, + 2 * max_rel_chain, + ) + + final_rel_chain = torch.where( + entity_id_same, + clipped_rel_chain, + (2 * max_rel_chain + 1) * + torch.ones_like(clipped_rel_chain) + ) + + boundaries = torch.arange( + start=0, end=2 * max_rel_chain + 2, device=final_rel_chain.device + ) + rel_chain = one_hot( + final_rel_chain, + boundaries, + ) + + rel_feats.append(rel_chain) + else: + boundaries = torch.arange( + start=0, end=2 * self.max_relative_idx + 1, device=clipped_offset.device + ) + rel_pos = one_hot( + clipped_offset, boundaries, + ) + rel_feats.append(rel_pos) + + rel_feat = torch.cat(rel_feats, dim=-1).to( + self.linear_relpos.weight.dtype + ) + + return self.linear_relpos(rel_feat) + + def forward(self, batch) -> Tuple[torch.Tensor, torch.Tensor]: + tf = batch["target_feat"] + msa = batch["msa_feat"] + + # [*, N_res, c_z] + tf_emb_i = self.linear_tf_z_i(tf) + tf_emb_j = self.linear_tf_z_j(tf) + + # [*, N_res, N_res, c_z] + pair_emb = tf_emb_i[..., None, :] + tf_emb_j[..., None, :, :] + pair_emb = pair_emb + self.relpos(batch) + + # [*, N_clust, N_res, c_m] + n_clust = msa.shape[-3] + tf_m = ( + self.linear_tf_m(tf) + .unsqueeze(-3) + .expand(((-1,) * len(tf.shape[:-2]) + (n_clust, -1, -1))) + ) + msa_emb = self.linear_msa_m(msa) + tf_m + + return msa_emb, pair_emb + + +class PreembeddingEmbedder(nn.Module): + """ + Embeds the sequence pre-embedding passed to the model and the target_feat features. + """ + + def __init__( + self, + tf_dim: int, + preembedding_dim: int, + c_z: int, + c_m: int, + relpos_k: int, + **kwargs, + ): + """ + Args: + tf_dim: + End channel dimension of the incoming target features + preembedding_dim: + End channel dimension of the incoming embeddings + c_z: + Pair embedding dimension + c_m: + Single-Seq embedding dimension + relpos_k: + Window size used in relative position encoding + """ + super(PreembeddingEmbedder, self).__init__() + + self.tf_dim = tf_dim + self.preembedding_dim = preembedding_dim + + self.c_z = c_z + self.c_m = c_m + + self.linear_tf_m = Linear(tf_dim, c_m) + self.linear_preemb_m = Linear(self.preembedding_dim, c_m) + self.linear_preemb_z_i = Linear(self.preembedding_dim, c_z) + self.linear_preemb_z_j = Linear(self.preembedding_dim, c_z) + + # Relative Positional Encoding + self.relpos_k = relpos_k + self.no_bins = 2 * relpos_k + 1 + self.linear_relpos = Linear(self.no_bins, c_z) + + def relpos(self, ri: torch.Tensor): + """ + Computes relative positional encodings + Args: + ri: + "residue_index" feature of shape [*, N] + Returns: + Relative positional encoding of protein using the + residue_index feature + """ + d = ri[..., None] - ri[..., None, :] + boundaries = torch.arange( + start=-self.relpos_k, end=self.relpos_k + 1, device=d.device + ) + reshaped_bins = boundaries.view(((1,) * len(d.shape)) + (len(boundaries),)) + d = d[..., None] - reshaped_bins + d = torch.abs(d) + d = torch.argmin(d, dim=-1) + d = nn.functional.one_hot(d, num_classes=len(boundaries)).float() + d = d.to(ri.dtype) + return self.linear_relpos(d) + + def forward( + self, + tf: torch.Tensor, + ri: torch.Tensor, + preemb: torch.Tensor, + inplace_safe: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor]: + + tf_m = ( + self.linear_tf_m(tf) + .unsqueeze(-3) + ) + preemb_emb = self.linear_preemb_m(preemb[..., None, :, :]) + tf_m + preemb_emb_i = self.linear_preemb_z_i(preemb) + preemb_emb_j = self.linear_preemb_z_j(preemb) + + pair_emb = self.relpos(ri.type(preemb_emb_i.dtype)) + pair_emb = add(pair_emb, + preemb_emb_i[..., None, :], + inplace=inplace_safe) + pair_emb = add(pair_emb, + preemb_emb_j[..., None, :, :], + inplace=inplace_safe) + + return preemb_emb, pair_emb + + +class RecyclingEmbedder(nn.Module): + """ + Embeds the output of an iteration of the model for recycling. + + Implements Algorithm 32. + """ + def __init__( + self, + c_m: int, + c_z: int, + min_bin: float, + max_bin: float, + no_bins: int, + inf: float = 1e8, + **kwargs, + ): + """ + Args: + c_m: + MSA channel dimension + c_z: + Pair embedding channel dimension + min_bin: + Smallest distogram bin (Angstroms) + max_bin: + Largest distogram bin (Angstroms) + no_bins: + Number of distogram bins + """ + super(RecyclingEmbedder, self).__init__() + + self.c_m = c_m + self.c_z = c_z + self.min_bin = min_bin + self.max_bin = max_bin + self.no_bins = no_bins + self.inf = inf + + self.linear = Linear(self.no_bins, self.c_z) + self.layer_norm_m = LayerNorm(self.c_m) + self.layer_norm_z = LayerNorm(self.c_z) + + def forward( + self, + m: torch.Tensor, + z: torch.Tensor, + x: torch.Tensor, + inplace_safe: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + m: + First row of the MSA embedding. [*, N_res, C_m] + z: + [*, N_res, N_res, C_z] pair embedding + x: + [*, N_res, 3] predicted C_beta coordinates + Returns: + m: + [*, N_res, C_m] MSA embedding update + z: + [*, N_res, N_res, C_z] pair embedding update + """ + # [*, N, C_m] + m_update = self.layer_norm_m(m) + if(inplace_safe): + m.copy_(m_update) + m_update = m + + # [*, N, N, C_z] + z_update = self.layer_norm_z(z) + if(inplace_safe): + z.copy_(z_update) + z_update = z + + # This squared method might become problematic in FP16 mode. + bins = torch.linspace( + self.min_bin, + self.max_bin, + self.no_bins, + dtype=x.dtype, + device=x.device, + requires_grad=False, + ) + squared_bins = bins ** 2 + upper = torch.cat( + [squared_bins[1:], squared_bins.new_tensor([self.inf])], dim=-1 + ) + d = torch.sum( + (x[..., None, :] - x[..., None, :, :]) ** 2, dim=-1, keepdims=True + ) + + # [*, N, N, no_bins] + d = ((d > squared_bins) * (d < upper)).type(x.dtype) + + # [*, N, N, C_z] + d = self.linear(d) + z_update = add(z_update, d, inplace_safe) + + return m_update, z_update + + +class TemplateSingleEmbedder(nn.Module): + """ + Embeds the "template_angle_feat" feature. + + Implements Algorithm 2, line 7. + """ + + def __init__( + self, + c_in: int, + c_out: int, + **kwargs, + ): + """ + Args: + c_in: + Final dimension of "template_angle_feat" + c_out: + Output channel dimension + """ + super(TemplateSingleEmbedder, self).__init__() + + self.c_out = c_out + self.c_in = c_in + + self.linear_1 = Linear(self.c_in, self.c_out, init="relu") + self.relu = nn.ReLU() + self.linear_2 = Linear(self.c_out, self.c_out, init="relu") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Args: + x: [*, N_templ, N_res, c_in] "template_angle_feat" features + Returns: + x: [*, N_templ, N_res, C_out] embedding + """ + x = self.linear_1(x) + x = self.relu(x) + x = self.linear_2(x) + + return x + + +class TemplatePairEmbedder(nn.Module): + """ + Embeds "template_pair_feat" features. + + Implements Algorithm 2, line 9. + """ + + def __init__( + self, + c_in: int, + c_out: int, + **kwargs, + ): + """ + Args: + c_in: + + c_out: + Output channel dimension + """ + super(TemplatePairEmbedder, self).__init__() + + self.c_in = c_in + self.c_out = c_out + + # Despite there being no relu nearby, the source uses that initializer + self.linear = Linear(self.c_in, self.c_out, init="relu") + + def forward( + self, + x: torch.Tensor, + ) -> torch.Tensor: + """ + Args: + x: + [*, C_in] input tensor + Returns: + [*, C_out] output tensor + """ + x = self.linear(x) + + return x + + +class ExtraMSAEmbedder(nn.Module): + """ + Embeds unclustered MSA sequences. + + Implements Algorithm 2, line 15 + """ + def __init__( + self, + c_in: int, + c_out: int, + **kwargs, + ): + """ + Args: + c_in: + Input channel dimension + c_out: + Output channel dimension + """ + super(ExtraMSAEmbedder, self).__init__() + + self.c_in = c_in + self.c_out = c_out + + self.linear = Linear(self.c_in, self.c_out) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Args: + x: + [*, N_extra_seq, N_res, C_in] "extra_msa_feat" features + Returns: + [*, N_extra_seq, N_res, C_out] embedding + """ + x = self.linear(x) + + return x + + +class TemplateEmbedder(nn.Module): + def __init__(self, config): + super(TemplateEmbedder, self).__init__() + + self.config = config + self.template_single_embedder = TemplateSingleEmbedder( + **config["template_single_embedder"], + ) + self.template_pair_embedder = TemplatePairEmbedder( + **config["template_pair_embedder"], + ) + self.template_pair_stack = TemplatePairStack( + **config["template_pair_stack"], + ) + self.template_pointwise_att = TemplatePointwiseAttention( + **config["template_pointwise_attention"], + ) + + def forward( + self, + batch, + z, + pair_mask, + templ_dim, + chunk_size, + _mask_trans=True, + use_deepspeed_evo_attention=False, + use_lma=False, + inplace_safe=False + ): + # Embed the templates one at a time (with a poor man's vmap) + pair_embeds = [] + n = z.shape[-2] + n_templ = batch["template_aatype"].shape[templ_dim] + + if (inplace_safe): + # We'll preallocate the full pair tensor now to avoid manifesting + # a second copy during the stack later on + t_pair = z.new_zeros( + z.shape[:-3] + + (n_templ, n, n, self.config.template_pair_embedder.c_out) + ) + + for i in range(n_templ): + idx = batch["template_aatype"].new_tensor(i) + single_template_feats = tensor_tree_map( + lambda t: torch.index_select(t, templ_dim, idx).squeeze(templ_dim), + batch, + ) + + # [*, N, N, C_t] + t = build_template_pair_feat( + single_template_feats, + use_unit_vector=self.config.use_unit_vector, + inf=self.config.inf, + eps=self.config.eps, + **self.config.distogram, + ).to(z.dtype) + t = self.template_pair_embedder(t) + + if (inplace_safe): + t_pair[..., i, :, :, :] = t + else: + pair_embeds.append(t) + + del t + + if (not inplace_safe): + t_pair = torch.stack(pair_embeds, dim=templ_dim) + + del pair_embeds + + # [*, S_t, N, N, C_z] + t = self.template_pair_stack( + t_pair, + pair_mask.unsqueeze(-3).to(dtype=z.dtype), + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe, + _mask_trans=_mask_trans, + ) + del t_pair + + # [*, N, N, C_z] + t = self.template_pointwise_att( + t, + z, + template_mask=batch["template_mask"].to(dtype=z.dtype), + use_lma=use_lma, + ) + + t_mask = torch.sum(batch["template_mask"], dim=-1) > 0 + # Append singletons + t_mask = t_mask.reshape( + *t_mask.shape, *([1] * (len(t.shape) - len(t_mask.shape))) + ) + + if (inplace_safe): + t *= t_mask + else: + t = t * t_mask + + ret = {} + + ret.update({"template_pair_embedding": t}) + + del t + + if self.config.embed_angles: + template_angle_feat = build_template_angle_feat( + batch + ) + + # [*, S_t, N, C_m] + a = self.template_single_embedder(template_angle_feat) + + ret["template_single_embedding"] = a + + return ret + + +class TemplatePairEmbedderMultimer(nn.Module): + def __init__(self, + c_in: int, + c_out: int, + c_dgram: int, + c_aatype: int, + ): + super(TemplatePairEmbedderMultimer, self).__init__() + + self.dgram_linear = Linear(c_dgram, c_out, init='relu') + self.aatype_linear_1 = Linear(c_aatype, c_out, init='relu') + self.aatype_linear_2 = Linear(c_aatype, c_out, init='relu') + self.query_embedding_layer_norm = LayerNorm(c_in) + self.query_embedding_linear = Linear(c_in, c_out, init='relu') + + self.pseudo_beta_mask_linear = Linear(1, c_out, init='relu') + self.x_linear = Linear(1, c_out, init='relu') + self.y_linear = Linear(1, c_out, init='relu') + self.z_linear = Linear(1, c_out, init='relu') + self.backbone_mask_linear = Linear(1, c_out, init='relu') + + def forward(self, + template_dgram: torch.Tensor, + aatype_one_hot: torch.Tensor, + query_embedding: torch.Tensor, + pseudo_beta_mask: torch.Tensor, + backbone_mask: torch.Tensor, + multichain_mask_2d: torch.Tensor, + unit_vector: geometry.Vec3Array, + ) -> torch.Tensor: + act = 0. + + pseudo_beta_mask_2d = ( + pseudo_beta_mask[..., None] * pseudo_beta_mask[..., None, :] + ) + pseudo_beta_mask_2d *= multichain_mask_2d + template_dgram *= pseudo_beta_mask_2d[..., None] + act += self.dgram_linear(template_dgram) + act += self.pseudo_beta_mask_linear(pseudo_beta_mask_2d[..., None]) + + aatype_one_hot = aatype_one_hot.to(template_dgram.dtype) + act += self.aatype_linear_1(aatype_one_hot[..., None, :, :]) + act += self.aatype_linear_2(aatype_one_hot[..., None, :]) + + backbone_mask_2d = ( + backbone_mask[..., None] * backbone_mask[..., None, :] + ) + backbone_mask_2d *= multichain_mask_2d + x, y, z = [(coord * backbone_mask_2d).to(dtype=query_embedding.dtype) for coord in unit_vector] + act += self.x_linear(x[..., None]) + act += self.y_linear(y[..., None]) + act += self.z_linear(z[..., None]) + + act += self.backbone_mask_linear(backbone_mask_2d[..., None].to(dtype=query_embedding.dtype)) + + query_embedding = self.query_embedding_layer_norm(query_embedding) + act += self.query_embedding_linear(query_embedding) + + return act + + +class TemplateSingleEmbedderMultimer(nn.Module): + def __init__(self, + c_in: int, + c_out: int, + ): + super(TemplateSingleEmbedderMultimer, self).__init__() + self.template_single_embedder = Linear(c_in, c_out) + self.template_projector = Linear(c_out, c_out) + + def forward(self, + batch, + atom_pos, + aatype_one_hot, + ): + out = {} + + dtype = batch["template_all_atom_positions"].dtype + + template_chi_angles, template_chi_mask = ( + all_atom_multimer.compute_chi_angles( + atom_pos, + batch["template_all_atom_mask"], + batch["template_aatype"], + ) + ) + + template_features = torch.cat( + [ + aatype_one_hot, + torch.sin(template_chi_angles) * template_chi_mask, + torch.cos(template_chi_angles) * template_chi_mask, + template_chi_mask, + ], + dim=-1, + ).to(dtype=dtype) + + template_mask = template_chi_mask[..., 0].to(dtype=dtype) + + template_activations = self.template_single_embedder( + template_features + ) + template_activations = torch.nn.functional.relu( + template_activations + ) + template_activations = self.template_projector( + template_activations, + ) + + out["template_single_embedding"] = ( + template_activations + ) + out["template_mask"] = template_mask + + return out + + +class TemplateEmbedderMultimer(nn.Module): + def __init__(self, config): + super(TemplateEmbedderMultimer, self).__init__() + + self.config = config + self.template_pair_embedder = TemplatePairEmbedderMultimer( + **config["template_pair_embedder"], + ) + self.template_single_embedder = TemplateSingleEmbedderMultimer( + **config["template_single_embedder"], + ) + self.template_pair_stack = TemplatePairStack( + **config["template_pair_stack"], + ) + + self.linear_t = Linear(config.c_t, config.c_z) + + def forward(self, + batch, + z, + padding_mask_2d, + templ_dim, + chunk_size, + multichain_mask_2d, + _mask_trans=True, + use_deepspeed_evo_attention=False, + use_lma=False, + inplace_safe=False + ): + template_embeds = [] + n_templ = batch["template_aatype"].shape[templ_dim] + for i in range(n_templ): + idx = batch["template_aatype"].new_tensor(i) + single_template_feats = tensor_tree_map( + lambda t: torch.index_select(t, templ_dim, idx), + batch, + ) + + single_template_embeds = {} + act = 0. + + template_positions, pseudo_beta_mask = pseudo_beta_fn( + single_template_feats["template_aatype"], + single_template_feats["template_all_atom_positions"], + single_template_feats["template_all_atom_mask"]) + + template_dgram = dgram_from_positions( + template_positions, + inf=self.config.inf, + **self.config.distogram, + ) + + aatype_one_hot = torch.nn.functional.one_hot( + single_template_feats["template_aatype"], 22, + ) + + raw_atom_pos = single_template_feats["template_all_atom_positions"] + + # Vec3Arrays are required to be float32 + atom_pos = geometry.Vec3Array.from_array(raw_atom_pos.to(dtype=torch.float32)) + + rigid, backbone_mask = all_atom_multimer.make_backbone_affine( + atom_pos, + single_template_feats["template_all_atom_mask"], + single_template_feats["template_aatype"], + ) + points = rigid.translation + rigid_vec = rigid[..., None].inverse().apply_to_point(points) + unit_vector = rigid_vec.normalized() + + pair_act = self.template_pair_embedder( + template_dgram, + aatype_one_hot, + z, + pseudo_beta_mask, + backbone_mask, + multichain_mask_2d, + unit_vector, + ) + + single_template_embeds["template_pair_embedding"] = pair_act + single_template_embeds.update( + self.template_single_embedder( + single_template_feats, + atom_pos, + aatype_one_hot, + ) + ) + template_embeds.append(single_template_embeds) + + template_embeds = dict_multimap( + partial(torch.cat, dim=templ_dim), + template_embeds, + ) + + # [*, S_t, N, N, C_z] + t = self.template_pair_stack( + template_embeds["template_pair_embedding"], + padding_mask_2d.unsqueeze(-3).to(dtype=z.dtype), + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe, + _mask_trans=_mask_trans, + ) + # [*, N, N, C_z] + t = torch.sum(t, dim=-4) / n_templ + t = torch.nn.functional.relu(t) + t = self.linear_t(t) + template_embeds["template_pair_embedding"] = t + + return template_embeds diff --git a/model/openfold/evoformer.py b/model/openfold/evoformer.py new file mode 100644 index 0000000000000000000000000000000000000000..8d5a3124b7610f66ba68ee74bc0f3068b9429b0b --- /dev/null +++ b/model/openfold/evoformer.py @@ -0,0 +1,1219 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +import sys +import torch +import torch.nn as nn +from typing import Tuple, Sequence, Optional +from functools import partial +from abc import ABC, abstractmethod + +from model.openfold.primitives import Linear, LayerNorm +from model.openfold.dropout import DropoutRowwise, DropoutColumnwise +from model.openfold.msa import ( + MSARowAttentionWithPairBias, + MSAColumnAttention, + MSAColumnGlobalAttention, +) +from model.openfold.outer_product_mean import OuterProductMean +from model.openfold.pair_transition import PairTransition +from model.openfold.triangular_attention import ( + TriangleAttention, + TriangleAttentionStartingNode, + TriangleAttentionEndingNode, +) +from model.openfold.triangular_multiplicative_update import ( + TriangleMultiplicationOutgoing, + TriangleMultiplicationIncoming, + FusedTriangleMultiplicationIncoming, + FusedTriangleMultiplicationOutgoing +) +from onescience.utils.openfold.checkpointing import checkpoint_blocks, get_checkpoint_fn +from onescience.utils.openfold.chunk_utils import chunk_layer, ChunkSizeTuner +from onescience.utils.openfold.tensor_utils import add + + +class MSATransition(nn.Module): + """ + Feed-forward network applied to MSA activations after attention. + + Implements Algorithm 9 + """ + def __init__(self, c_m, n): + """ + Args: + c_m: + MSA channel dimension + n: + Factor multiplied to c_m to obtain the hidden channel + dimension + """ + super(MSATransition, self).__init__() + + self.c_m = c_m + self.n = n + + self.layer_norm = LayerNorm(self.c_m) + self.linear_1 = Linear(self.c_m, self.n * self.c_m, init="relu") + self.relu = nn.ReLU() + self.linear_2 = Linear(self.n * self.c_m, self.c_m, init="final") + + def _transition(self, m, mask): + m = self.layer_norm(m) + m = self.linear_1(m) + m = self.relu(m) + m = self.linear_2(m) * mask + return m + + @torch.jit.ignore + def _chunk(self, + m: torch.Tensor, + mask: torch.Tensor, + chunk_size: int, + ) -> torch.Tensor: + return chunk_layer( + self._transition, + {"m": m, "mask": mask}, + chunk_size=chunk_size, + no_batch_dims=len(m.shape[:-2]), + ) + + def forward( + self, + m: torch.Tensor, + mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + ) -> torch.Tensor: + """ + Args: + m: + [*, N_seq, N_res, C_m] MSA activation + mask: + [*, N_seq, N_res, C_m] MSA mask + Returns: + m: + [*, N_seq, N_res, C_m] MSA activation update + """ + # DISCREPANCY: DeepMind forgets to apply the MSA mask here. + if mask is None: + mask = m.new_ones(m.shape[:-1]) + + mask = mask.unsqueeze(-1) + + if chunk_size is not None: + m = self._chunk(m, mask, chunk_size) + else: + m = self._transition(m, mask) + + return m + + +class PairStack(nn.Module): + def __init__( + self, + c_z: int, + c_hidden_mul: int, + c_hidden_pair_att: int, + no_heads_pair: int, + transition_n: int, + pair_dropout: float, + fuse_projection_weights: bool, + inf: float, + eps: float + ): + super(PairStack, self).__init__() + + if fuse_projection_weights: + self.tri_mul_out = FusedTriangleMultiplicationOutgoing( + c_z, + c_hidden_mul, + ) + self.tri_mul_in = FusedTriangleMultiplicationIncoming( + c_z, + c_hidden_mul, + ) + else: + self.tri_mul_out = TriangleMultiplicationOutgoing( + c_z, + c_hidden_mul, + ) + self.tri_mul_in = TriangleMultiplicationIncoming( + c_z, + c_hidden_mul, + ) + + self.tri_att_start = TriangleAttention( + c_z, + c_hidden_pair_att, + no_heads_pair, + inf=inf, + ) + self.tri_att_end = TriangleAttention( + c_z, + c_hidden_pair_att, + no_heads_pair, + inf=inf, + ) + + self.pair_transition = PairTransition( + c_z, + transition_n, + ) + + self.ps_dropout_row_layer = DropoutRowwise(pair_dropout) + + def forward(self, + z: torch.Tensor, + pair_mask: torch.Tensor, + chunk_size: Optional[int] = None, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + inplace_safe: bool = False, + _mask_trans: bool = True, + _attn_chunk_size: Optional[int] = None + ) -> torch.Tensor: + # DeepMind doesn't mask these transitions in the source, so _mask_trans + # should be disabled to better approximate the exact activations of + # the original. + pair_trans_mask = pair_mask if _mask_trans else None + + if (_attn_chunk_size is None): + _attn_chunk_size = chunk_size + + tmu_update = self.tri_mul_out( + z, + mask=pair_mask, + inplace_safe=inplace_safe, + _add_with_inplace=True, + ) + if (not inplace_safe): + z = z + self.ps_dropout_row_layer(tmu_update) + else: + z = tmu_update + + del tmu_update + + tmu_update = self.tri_mul_in( + z, + mask=pair_mask, + inplace_safe=inplace_safe, + _add_with_inplace=True, + ) + if (not inplace_safe): + z = z + self.ps_dropout_row_layer(tmu_update) + else: + z = tmu_update + + del tmu_update + + z = add(z, + self.ps_dropout_row_layer( + self.tri_att_start( + z, + mask=pair_mask, + chunk_size=_attn_chunk_size, + use_memory_efficient_kernel=False, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe, + ) + ), + inplace=inplace_safe, + ) + + z = z.transpose(-2, -3) + if (inplace_safe): + z = z.contiguous() + + z = add(z, + self.ps_dropout_row_layer( + self.tri_att_end( + z, + mask=pair_mask.transpose(-1, -2), + chunk_size=_attn_chunk_size, + use_memory_efficient_kernel=False, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe, + ) + ), + inplace=inplace_safe, + ) + + z = z.transpose(-2, -3) + if (inplace_safe): + z = z.contiguous() + + z = add(z, + self.pair_transition( + z, mask=pair_trans_mask, chunk_size=chunk_size, + ), + inplace=inplace_safe, + ) + + return z + + +class MSABlock(nn.Module, ABC): + @abstractmethod + def __init__(self, + c_m: int, + c_z: int, + c_hidden_msa_att: int, + c_hidden_opm: int, + c_hidden_mul: int, + c_hidden_pair_att: int, + no_heads_msa: int, + no_heads_pair: int, + transition_n: int, + msa_dropout: float, + pair_dropout: float, + opm_first: bool, + fuse_projection_weights: bool, + inf: float, + eps: float, + ): + super(MSABlock, self).__init__() + + self.opm_first = opm_first + + self.msa_att_row = MSARowAttentionWithPairBias( + c_m=c_m, + c_z=c_z, + c_hidden=c_hidden_msa_att, + no_heads=no_heads_msa, + inf=inf, + ) + + self.msa_dropout_layer = DropoutRowwise(msa_dropout) + + self.msa_transition = MSATransition( + c_m=c_m, + n=transition_n, + ) + + self.outer_product_mean = OuterProductMean( + c_m, + c_z, + c_hidden_opm, + ) + + self.pair_stack = PairStack( + c_z=c_z, + c_hidden_mul=c_hidden_mul, + c_hidden_pair_att=c_hidden_pair_att, + no_heads_pair=no_heads_pair, + transition_n=transition_n, + pair_dropout=pair_dropout, + fuse_projection_weights=fuse_projection_weights, + inf=inf, + eps=eps + ) + + def _compute_opm(self, + input_tensors: Sequence[torch.Tensor], + msa_mask: torch.Tensor, + chunk_size: Optional[int] = None, + inplace_safe: bool = False, + _offload_inference: bool = False + ) -> Tuple[torch.Tensor, torch.Tensor]: + + m, z = input_tensors + + if (_offload_inference and inplace_safe): + # m: GPU, z: CPU + del m, z + assert (sys.getrefcount(input_tensors[1]) == 2) + input_tensors[1] = input_tensors[1].cpu() + m, z = input_tensors + + opm = self.outer_product_mean( + m, mask=msa_mask, chunk_size=chunk_size, inplace_safe=inplace_safe + ) + + if (_offload_inference and inplace_safe): + # m: GPU, z: GPU + del m, z + assert (sys.getrefcount(input_tensors[0]) == 2) + input_tensors[1] = input_tensors[1].to(opm.device) + m, z = input_tensors + + z = add(z, opm, inplace=inplace_safe) + del opm + + return m, z + + @abstractmethod + def forward(self, + m: Optional[torch.Tensor], + z: Optional[torch.Tensor], + msa_mask: torch.Tensor, + pair_mask: torch.Tensor, + chunk_size: Optional[int] = None, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + use_flash: bool = False, + inplace_safe: bool = False, + _mask_trans: bool = True, + _attn_chunk_size: Optional[int] = None, + _offload_inference: bool = False, + _offloadable_inputs: Optional[Sequence[torch.Tensor]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + pass + + +class EvoformerBlock(MSABlock): + def __init__(self, + c_m: int, + c_z: int, + c_hidden_msa_att: int, + c_hidden_opm: int, + c_hidden_mul: int, + c_hidden_pair_att: int, + no_heads_msa: int, + no_heads_pair: int, + transition_n: int, + msa_dropout: float, + pair_dropout: float, + no_column_attention: bool, + opm_first: bool, + fuse_projection_weights: bool, + inf: float, + eps: float, + ): + super(EvoformerBlock, self).__init__(c_m=c_m, + c_z=c_z, + c_hidden_msa_att=c_hidden_msa_att, + c_hidden_opm=c_hidden_opm, + c_hidden_mul=c_hidden_mul, + c_hidden_pair_att=c_hidden_pair_att, + no_heads_msa=no_heads_msa, + no_heads_pair=no_heads_pair, + transition_n=transition_n, + msa_dropout=msa_dropout, + pair_dropout=pair_dropout, + opm_first=opm_first, + fuse_projection_weights=fuse_projection_weights, + inf=inf, + eps=eps) + + # Specifically, seqemb mode does not use column attention + self.no_column_attention = no_column_attention + + if not self.no_column_attention: + self.msa_att_col = MSAColumnAttention( + c_m, + c_hidden_msa_att, + no_heads_msa, + inf=inf, + ) + + def forward(self, + m: Optional[torch.Tensor], + z: Optional[torch.Tensor], + msa_mask: torch.Tensor, + pair_mask: torch.Tensor, + chunk_size: Optional[int] = None, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + use_flash: bool = False, + inplace_safe: bool = False, + _mask_trans: bool = True, + _attn_chunk_size: Optional[int] = None, + _offload_inference: bool = False, + _offloadable_inputs: Optional[Sequence[torch.Tensor]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + + msa_trans_mask = msa_mask if _mask_trans else None + + if(_attn_chunk_size is None): + _attn_chunk_size = chunk_size + + if(_offload_inference and inplace_safe): + input_tensors = _offloadable_inputs + del _offloadable_inputs + else: + input_tensors = [m, z] + + m, z = input_tensors + + if self.opm_first: + del m, z + + m, z = self._compute_opm(input_tensors=input_tensors, + msa_mask=msa_mask, + chunk_size=chunk_size, + inplace_safe=inplace_safe, + _offload_inference=_offload_inference) + + m = add(m, + self.msa_dropout_layer( + self.msa_att_row( + m, + z=z, + mask=msa_mask, + chunk_size=_attn_chunk_size, + use_memory_efficient_kernel=False, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + ) + ), + inplace=inplace_safe, + ) + + if (_offload_inference and inplace_safe): + # m: GPU, z: CPU + del m, z + assert (sys.getrefcount(input_tensors[1]) == 2) + input_tensors[1] = input_tensors[1].cpu() + torch.cuda.empty_cache() + m, z = input_tensors + + # Specifically, column attention is not used in seqemb mode. + if not self.no_column_attention: + m = add(m, + self.msa_att_col( + m, + mask=msa_mask, + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + use_flash=use_flash, + ), + inplace=inplace_safe, + ) + + m = add( + m, + self.msa_transition( + m, mask=msa_trans_mask, chunk_size=chunk_size, + ), + inplace=inplace_safe, + ) + + if not self.opm_first: + if (not inplace_safe): + input_tensors = [m, z] + + del m, z + + m, z = self._compute_opm(input_tensors=input_tensors, + msa_mask=msa_mask, + chunk_size=chunk_size, + inplace_safe=inplace_safe, + _offload_inference=_offload_inference) + + if (_offload_inference and inplace_safe): + # m: CPU, z: GPU + del m, z + assert (sys.getrefcount(input_tensors[0]) == 2) + device = input_tensors[0].device + input_tensors[0] = input_tensors[0].cpu() + input_tensors[1] = input_tensors[1].to(device) + m, z = input_tensors + + if (not inplace_safe): + input_tensors = [m, z] + + del m, z + + z = self.pair_stack( + z=input_tensors[1], + pair_mask=pair_mask, + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe, + _mask_trans=_mask_trans, + _attn_chunk_size=_attn_chunk_size + ) + + if (_offload_inference and inplace_safe): + # m: GPU, z: GPU + device = z.device + assert (sys.getrefcount(input_tensors[0]) == 2) + input_tensors[0] = input_tensors[0].to(device) + m, _ = input_tensors + else: + m = input_tensors[0] + + return m, z + + +class ExtraMSABlock(MSABlock): + """ + Almost identical to the standard EvoformerBlock, except in that the + ExtraMSABlock uses GlobalAttention for MSA column attention and + requires more fine-grained control over checkpointing. Separated from + its twin to preserve the TorchScript-ability of the latter. + """ + def __init__(self, + c_m: int, + c_z: int, + c_hidden_msa_att: int, + c_hidden_opm: int, + c_hidden_mul: int, + c_hidden_pair_att: int, + no_heads_msa: int, + no_heads_pair: int, + transition_n: int, + msa_dropout: float, + pair_dropout: float, + opm_first: bool, + fuse_projection_weights: bool, + inf: float, + eps: float, + ckpt: bool, + ): + super(ExtraMSABlock, self).__init__(c_m=c_m, + c_z=c_z, + c_hidden_msa_att=c_hidden_msa_att, + c_hidden_opm=c_hidden_opm, + c_hidden_mul=c_hidden_mul, + c_hidden_pair_att=c_hidden_pair_att, + no_heads_msa=no_heads_msa, + no_heads_pair=no_heads_pair, + transition_n=transition_n, + msa_dropout=msa_dropout, + pair_dropout=pair_dropout, + opm_first=opm_first, + fuse_projection_weights=fuse_projection_weights, + inf=inf, + eps=eps) + + self.ckpt = ckpt + + self.msa_att_col = MSAColumnGlobalAttention( + c_in=c_m, + c_hidden=c_hidden_msa_att, + no_heads=no_heads_msa, + inf=inf, + eps=eps, + ) + + def forward(self, + m: Optional[torch.Tensor], + z: Optional[torch.Tensor], + msa_mask: torch.Tensor, + pair_mask: torch.Tensor, + chunk_size: Optional[int] = None, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + inplace_safe: bool = False, + _mask_trans: bool = True, + _attn_chunk_size: Optional[int] = None, + _offload_inference: bool = False, + _offloadable_inputs: Optional[Sequence[torch.Tensor]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + if(_attn_chunk_size is None): + _attn_chunk_size = chunk_size + + if(_offload_inference and inplace_safe): + input_tensors = _offloadable_inputs + del _offloadable_inputs + else: + input_tensors = [m, z] + + m, z = input_tensors + + if self.opm_first: + del m, z + + m, z = self._compute_opm(input_tensors=input_tensors, + msa_mask=msa_mask, + chunk_size=chunk_size, + inplace_safe=inplace_safe, + _offload_inference=_offload_inference) + + m = add(m, + self.msa_dropout_layer( + self.msa_att_row( + m.clone() if torch.is_grad_enabled() else m, + z=z.clone() if torch.is_grad_enabled() else z, + mask=msa_mask, + chunk_size=_attn_chunk_size, + use_lma=use_lma, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_memory_efficient_kernel=not (use_lma or use_deepspeed_evo_attention), + _checkpoint_chunks= + self.ckpt if torch.is_grad_enabled() else False, + ) + ), + inplace=inplace_safe, + ) + + if (not inplace_safe): + input_tensors = [m, z] + + del m, z + + def fn(input_tensors): + m, z = input_tensors + + if (_offload_inference and inplace_safe): + # m: GPU, z: CPU + del m, z + assert (sys.getrefcount(input_tensors[1]) == 2) + input_tensors[1] = input_tensors[1].cpu() + torch.cuda.empty_cache() + m, z = input_tensors + + m = add(m, + self.msa_att_col( + m, + mask=msa_mask, + chunk_size=chunk_size, + use_lma=use_lma, + ), + inplace=inplace_safe, + ) + + m = add( + m, + self.msa_transition( + m, mask=msa_mask, chunk_size=chunk_size, + ), + inplace=inplace_safe, + ) + + if not self.opm_first: + if (not inplace_safe): + input_tensors = [m, z] + + del m, z + + m, z = self._compute_opm(input_tensors=input_tensors, + msa_mask=msa_mask, + chunk_size=chunk_size, + inplace_safe=inplace_safe, + _offload_inference=_offload_inference) + + if (_offload_inference and inplace_safe): + # m: CPU, z: GPU + del m, z + assert (sys.getrefcount(input_tensors[0]) == 2) + device = input_tensors[0].device + input_tensors[0] = input_tensors[0].cpu() + input_tensors[1] = input_tensors[1].to(device) + m, z = input_tensors + + if (not inplace_safe): + input_tensors = [m, z] + + del m, z + + z = self.pair_stack( + input_tensors[1], + pair_mask=pair_mask, + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe, + _mask_trans=_mask_trans, + _attn_chunk_size=_attn_chunk_size + ) + + m = input_tensors[0] + if (_offload_inference and inplace_safe): + # m: GPU, z: GPU + device = z.device + del m + assert (sys.getrefcount(input_tensors[0]) == 2) + input_tensors[0] = input_tensors[0].to(device) + m, _ = input_tensors + + return m, z + + if (torch.is_grad_enabled() and self.ckpt): + checkpoint_fn = get_checkpoint_fn() + m, z = checkpoint_fn(fn, input_tensors) + else: + m, z = fn(input_tensors) + + return m, z + + +class EvoformerStack(nn.Module): + """ + Main Evoformer trunk. + + Implements Algorithm 6. + """ + + def __init__( + self, + c_m: int, + c_z: int, + c_hidden_msa_att: int, + c_hidden_opm: int, + c_hidden_mul: int, + c_hidden_pair_att: int, + c_s: int, + no_heads_msa: int, + no_heads_pair: int, + no_blocks: int, + transition_n: int, + msa_dropout: float, + pair_dropout: float, + no_column_attention: bool, + opm_first: bool, + fuse_projection_weights: bool, + blocks_per_ckpt: int, + inf: float, + eps: float, + clear_cache_between_blocks: bool = False, + tune_chunk_size: bool = False, + **kwargs, + ): + """ + Args: + c_m: + MSA channel dimension + c_z: + Pair channel dimension + c_hidden_msa_att: + Hidden dimension in MSA attention + c_hidden_opm: + Hidden dimension in outer product mean module + c_hidden_mul: + Hidden dimension in multiplicative updates + c_hidden_pair_att: + Hidden dimension in triangular attention + c_s: + Channel dimension of the output "single" embedding + no_heads_msa: + Number of heads used for MSA attention + no_heads_pair: + Number of heads used for pair attention + no_blocks: + Number of Evoformer blocks in the stack + transition_n: + Factor by which to multiply c_m to obtain the MSATransition + hidden dimension + msa_dropout: + Dropout rate for MSA activations + pair_dropout: + Dropout used for pair activations + no_column_attention: + When True, doesn't use column attention. Required for running + sequence embedding mode + opm_first: + When True, Outer Product Mean is performed at the beginning of + the Evoformer block instead of after the MSA Stack. + Used in Multimer pipeline. + fuse_projection_weights: + When True, uses FusedTriangleMultiplicativeUpdate variant in + the Pair Stack. Used in Multimer pipeline. + blocks_per_ckpt: + Number of Evoformer blocks in each activation checkpoint + clear_cache_between_blocks: + Whether to clear CUDA's GPU memory cache between blocks of the + stack. Slows down each block but can reduce fragmentation + tune_chunk_size: + Whether to dynamically tune the module's chunk size + """ + super(EvoformerStack, self).__init__() + + self.blocks_per_ckpt = blocks_per_ckpt + self.clear_cache_between_blocks = clear_cache_between_blocks + + self.blocks = nn.ModuleList() + + for _ in range(no_blocks): + block = EvoformerBlock( + c_m=c_m, + c_z=c_z, + c_hidden_msa_att=c_hidden_msa_att, + c_hidden_opm=c_hidden_opm, + c_hidden_mul=c_hidden_mul, + c_hidden_pair_att=c_hidden_pair_att, + no_heads_msa=no_heads_msa, + no_heads_pair=no_heads_pair, + transition_n=transition_n, + msa_dropout=msa_dropout, + pair_dropout=pair_dropout, + no_column_attention=no_column_attention, + opm_first=opm_first, + fuse_projection_weights=fuse_projection_weights, + inf=inf, + eps=eps, + ) + self.blocks.append(block) + + self.linear = Linear(c_m, c_s) + + self.tune_chunk_size = tune_chunk_size + self.chunk_size_tuner = None + if(tune_chunk_size): + self.chunk_size_tuner = ChunkSizeTuner() + + def _prep_blocks(self, + m: torch.Tensor, + z: torch.Tensor, + chunk_size: int, + use_deepspeed_evo_attention: bool, + use_lma: bool, + use_flash: bool, + msa_mask: Optional[torch.Tensor], + pair_mask: Optional[torch.Tensor], + inplace_safe: bool, + _mask_trans: bool, + ): + blocks = [ + partial( + b, + msa_mask=msa_mask, + pair_mask=pair_mask, + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + use_flash=use_flash, + inplace_safe=inplace_safe, + _mask_trans=_mask_trans, + ) + for b in self.blocks + ] + + if(self.clear_cache_between_blocks): + def block_with_cache_clear(block, *args, **kwargs): + torch.cuda.empty_cache() + return block(*args, **kwargs) + + blocks = [partial(block_with_cache_clear, b) for b in blocks] + + if(chunk_size is not None and self.chunk_size_tuner is not None): + assert(not self.training) + tuned_chunk_size = self.chunk_size_tuner.tune_chunk_size( + representative_fn=blocks[0], + # We don't want to write in-place during chunk tuning runs + args=(m.clone(), z.clone(),), + min_chunk_size=chunk_size, + ) + blocks = [ + partial(b, + chunk_size=tuned_chunk_size, + # A temporary measure to address torch's occasional + # inability to allocate large tensors + _attn_chunk_size=max(chunk_size, tuned_chunk_size // 4), + ) for b in blocks + ] + + return blocks + + def _forward_offload(self, + input_tensors: Sequence[torch.Tensor], + msa_mask: torch.Tensor, + pair_mask: torch.Tensor, + chunk_size: int, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + use_flash: bool = False, + _mask_trans: bool = True, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + assert(not (self.training or torch.is_grad_enabled())) + blocks = self._prep_blocks( + # We are very careful not to create references to these tensors in + # this function + m=input_tensors[0], + z=input_tensors[1], + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + use_flash=use_flash, + msa_mask=msa_mask, + pair_mask=pair_mask, + inplace_safe=True, + _mask_trans=_mask_trans, + ) + + for b in blocks: + m, z = b( + None, + None, + _offload_inference=True, + _offloadable_inputs=input_tensors, + ) + input_tensors[0] = m + input_tensors[1] = z + del m, z + + m, z = input_tensors + + s = self.linear(m[..., 0, :, :]) + + return m, z, s + + def forward(self, + m: torch.Tensor, + z: torch.Tensor, + msa_mask: torch.Tensor, + pair_mask: torch.Tensor, + chunk_size: int, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + use_flash: bool = False, + inplace_safe: bool = False, + _mask_trans: bool = True, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Args: + m: + [*, N_seq, N_res, C_m] MSA embedding + z: + [*, N_res, N_res, C_z] pair embedding + msa_mask: + [*, N_seq, N_res] MSA mask + pair_mask: + [*, N_res, N_res] pair mask + chunk_size: + Inference-time subbatch size. Acts as a minimum if + self.tune_chunk_size is True + use_deepspeed_evo_attention: + Whether to use DeepSpeed memory efficient kernel. + Mutually exclusive with use_lma and use_flash. + use_lma: + Whether to use low-memory attention during inference. + Mutually exclusive with use_flash and use_deepspeed_evo_attention. + use_flash: + Whether to use FlashAttention where possible. Mutually + exclusive with use_lma and use_deepspeed_evo_attention. + Returns: + m: + [*, N_seq, N_res, C_m] MSA embedding + z: + [*, N_res, N_res, C_z] pair embedding + s: + [*, N_res, C_s] single embedding (or None if extra MSA stack) + """ + blocks = self._prep_blocks( + m=m, + z=z, + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + use_flash=use_flash, + msa_mask=msa_mask, + pair_mask=pair_mask, + inplace_safe=inplace_safe, + _mask_trans=_mask_trans, + ) + + blocks_per_ckpt = self.blocks_per_ckpt + if(not torch.is_grad_enabled()): + blocks_per_ckpt = None + + m, z = checkpoint_blocks( + blocks, + args=(m, z), + blocks_per_ckpt=blocks_per_ckpt, + ) + + s = self.linear(m[..., 0, :, :]) + + return m, z, s + + +class ExtraMSAStack(nn.Module): + """ + Implements Algorithm 18. + """ + def __init__(self, + c_m: int, + c_z: int, + c_hidden_msa_att: int, + c_hidden_opm: int, + c_hidden_mul: int, + c_hidden_pair_att: int, + no_heads_msa: int, + no_heads_pair: int, + no_blocks: int, + transition_n: int, + msa_dropout: float, + pair_dropout: float, + opm_first: bool, + fuse_projection_weights: bool, + inf: float, + eps: float, + ckpt: bool, + clear_cache_between_blocks: bool = False, + tune_chunk_size: bool = False, + **kwargs, + ): + super(ExtraMSAStack, self).__init__() + + self.ckpt = ckpt + self.clear_cache_between_blocks = clear_cache_between_blocks + self.blocks = nn.ModuleList() + for _ in range(no_blocks): + block = ExtraMSABlock( + c_m=c_m, + c_z=c_z, + c_hidden_msa_att=c_hidden_msa_att, + c_hidden_opm=c_hidden_opm, + c_hidden_mul=c_hidden_mul, + c_hidden_pair_att=c_hidden_pair_att, + no_heads_msa=no_heads_msa, + no_heads_pair=no_heads_pair, + transition_n=transition_n, + msa_dropout=msa_dropout, + pair_dropout=pair_dropout, + opm_first=opm_first, + fuse_projection_weights=fuse_projection_weights, + inf=inf, + eps=eps, + ckpt=False, + ) + self.blocks.append(block) + + self.tune_chunk_size = tune_chunk_size + self.chunk_size_tuner = None + if(tune_chunk_size): + self.chunk_size_tuner = ChunkSizeTuner() + + def _prep_blocks(self, + m: torch.Tensor, + z: torch.Tensor, + chunk_size: int, + use_deepspeed_evo_attention: bool, + use_lma: bool, + msa_mask: Optional[torch.Tensor], + pair_mask: Optional[torch.Tensor], + inplace_safe: bool, + _mask_trans: bool, + ): + blocks = [ + partial( + b, + msa_mask=msa_mask, + pair_mask=pair_mask, + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe, + _mask_trans=_mask_trans, + ) for b in self.blocks + ] + + def clear_cache(b, *args, **kwargs): + torch.cuda.empty_cache() + return b(*args, **kwargs) + + if(self.clear_cache_between_blocks): + blocks = [partial(clear_cache, b) for b in blocks] + + if(chunk_size is not None and self.chunk_size_tuner is not None): + tuned_chunk_size = self.chunk_size_tuner.tune_chunk_size( + representative_fn=blocks[0], + # Tensors cloned to avoid getting written to in-place + # A corollary is that chunk size tuning should be disabled for + # large N, when z gets really big + args=(m.clone(), z.clone(),), + min_chunk_size=chunk_size, + ) + blocks = [ + partial(b, + chunk_size=tuned_chunk_size, + # A temporary measure to address torch's occasional + # inability to allocate large tensors + _attn_chunk_size=max(chunk_size, tuned_chunk_size // 4), + ) for b in blocks + ] + + return blocks + + def _forward_offload(self, + input_tensors: Sequence[torch.Tensor], + chunk_size: int, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + msa_mask: Optional[torch.Tensor] = None, + pair_mask: Optional[torch.Tensor] = None, + _mask_trans: bool = True, + ) -> torch.Tensor: + assert(not (self.training or torch.is_grad_enabled())) + blocks = self._prep_blocks( + # We are very careful not to create references to these tensors in + # this function + m=input_tensors[0], + z=input_tensors[1], + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + msa_mask=msa_mask, + pair_mask=pair_mask, + inplace_safe=True, + _mask_trans=_mask_trans, + ) + + for b in blocks: + m, z = b( + None, + None, + _offload_inference=True, + _offloadable_inputs=input_tensors, + ) + input_tensors[0] = m + input_tensors[1] = z + del m, z + + return input_tensors[1] + + def forward(self, + m: torch.Tensor, + z: torch.Tensor, + msa_mask: Optional[torch.Tensor], + pair_mask: Optional[torch.Tensor], + chunk_size: int, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + inplace_safe: bool = False, + _mask_trans: bool = True, + ) -> torch.Tensor: + """ + Args: + m: + [*, N_extra, N_res, C_m] extra MSA embedding + z: + [*, N_res, N_res, C_z] pair embedding + chunk_size: Inference-time subbatch size for Evoformer modules + use_deepspeed_evo_attention: Whether to use DeepSpeed memory-efficient kernel + use_lma: Whether to use low-memory attention during inference + msa_mask: + Optional [*, N_extra, N_res] MSA mask + pair_mask: + Optional [*, N_res, N_res] pair mask + Returns: + [*, N_res, N_res, C_z] pair update + """ + checkpoint_fn = get_checkpoint_fn() + blocks = self._prep_blocks( + m=m, + z=z, + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + msa_mask=msa_mask, + pair_mask=pair_mask, + inplace_safe=inplace_safe, + _mask_trans=_mask_trans, + ) + + for b in blocks: + if(self.ckpt and torch.is_grad_enabled()): + m, z = checkpoint_fn(b, m, z) + else: + m, z = b(m, z) + + return z diff --git a/model/openfold/heads.py b/model/openfold/heads.py new file mode 100644 index 0000000000000000000000000000000000000000..38ba203ca994c4e919d58b63467a754ec6be08b3 --- /dev/null +++ b/model/openfold/heads.py @@ -0,0 +1,267 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn + +from model.openfold.primitives import Linear, LayerNorm +from onescience.utils.openfold.loss import ( + compute_plddt, + compute_tm, + compute_predicted_aligned_error, +) +from onescience.utils.openfold.precision_utils import is_fp16_enabled + + +class AuxiliaryHeads(nn.Module): + def __init__(self, config): + super(AuxiliaryHeads, self).__init__() + + self.plddt = PerResidueLDDTCaPredictor( + **config["lddt"], + ) + + self.distogram = DistogramHead( + **config["distogram"], + ) + + self.masked_msa = MaskedMSAHead( + **config["masked_msa"], + ) + + self.experimentally_resolved = ExperimentallyResolvedHead( + **config["experimentally_resolved"], + ) + + if config.tm.enabled: + self.tm = TMScoreHead( + **config.tm, + ) + + self.config = config + + def forward(self, outputs): + aux_out = {} + lddt_logits = self.plddt(outputs["sm"]["single"]) + aux_out["lddt_logits"] = lddt_logits + + # Required for relaxation later on + aux_out["plddt"] = compute_plddt(lddt_logits) + + distogram_logits = self.distogram(outputs["pair"]) + aux_out["distogram_logits"] = distogram_logits + + masked_msa_logits = self.masked_msa(outputs["msa"]) + aux_out["masked_msa_logits"] = masked_msa_logits + + experimentally_resolved_logits = self.experimentally_resolved( + outputs["single"] + ) + aux_out[ + "experimentally_resolved_logits" + ] = experimentally_resolved_logits + + if self.config.tm.enabled: + tm_logits = self.tm(outputs["pair"]) + aux_out["tm_logits"] = tm_logits + aux_out["ptm_score"] = compute_tm( + tm_logits, **self.config.tm + ) + asym_id = outputs.get("asym_id") + if asym_id is not None: + aux_out["iptm_score"] = compute_tm( + tm_logits, asym_id=asym_id, interface=True, **self.config.tm + ) + aux_out["weighted_ptm_score"] = (self.config.tm["iptm_weight"] * aux_out["iptm_score"] + + self.config.tm["ptm_weight"] * aux_out["ptm_score"]) + + aux_out.update( + compute_predicted_aligned_error( + tm_logits, + **self.config.tm, + ) + ) + + return aux_out + + +class PerResidueLDDTCaPredictor(nn.Module): + def __init__(self, no_bins, c_in, c_hidden): + super(PerResidueLDDTCaPredictor, self).__init__() + + self.no_bins = no_bins + self.c_in = c_in + self.c_hidden = c_hidden + + self.layer_norm = LayerNorm(self.c_in) + + self.linear_1 = Linear(self.c_in, self.c_hidden, init="relu") + self.linear_2 = Linear(self.c_hidden, self.c_hidden, init="relu") + self.linear_3 = Linear(self.c_hidden, self.no_bins, init="final") + + self.relu = nn.ReLU() + + def forward(self, s): + s = self.layer_norm(s) + s = self.linear_1(s) + s = self.relu(s) + s = self.linear_2(s) + s = self.relu(s) + s = self.linear_3(s) + + return s + + +class DistogramHead(nn.Module): + """ + Computes a distogram probability distribution. + + For use in computation of distogram loss, subsection 1.9.8 + """ + + def __init__(self, c_z, no_bins, **kwargs): + """ + Args: + c_z: + Input channel dimension + no_bins: + Number of distogram bins + """ + super(DistogramHead, self).__init__() + + self.c_z = c_z + self.no_bins = no_bins + + self.linear = Linear(self.c_z, self.no_bins, init="final") + + def _forward(self, z): # [*, N, N, C_z] + """ + Args: + z: + [*, N_res, N_res, C_z] pair embedding + Returns: + [*, N, N, no_bins] distogram probability distribution + """ + # [*, N, N, no_bins] + logits = self.linear(z) + logits = logits + logits.transpose(-2, -3) + return logits + + def forward(self, z): + if(is_fp16_enabled()): + with torch.cuda.amp.autocast(enabled=False): + return self._forward(z.float()) + else: + return self._forward(z) + + +class TMScoreHead(nn.Module): + """ + For use in computation of TM-score, subsection 1.9.7 + """ + + def __init__(self, c_z, no_bins, **kwargs): + """ + Args: + c_z: + Input channel dimension + no_bins: + Number of bins + """ + super(TMScoreHead, self).__init__() + + self.c_z = c_z + self.no_bins = no_bins + + self.linear = Linear(self.c_z, self.no_bins, init="final") + + def forward(self, z): + """ + Args: + z: + [*, N_res, N_res, C_z] pairwise embedding + Returns: + [*, N_res, N_res, no_bins] prediction + """ + # [*, N, N, no_bins] + logits = self.linear(z) + return logits + + +class MaskedMSAHead(nn.Module): + """ + For use in computation of masked MSA loss, subsection 1.9.9 + """ + + def __init__(self, c_m, c_out, **kwargs): + """ + Args: + c_m: + MSA channel dimension + c_out: + Output channel dimension + """ + super(MaskedMSAHead, self).__init__() + + self.c_m = c_m + self.c_out = c_out + + self.linear = Linear(self.c_m, self.c_out, init="final") + + def forward(self, m): + """ + Args: + m: + [*, N_seq, N_res, C_m] MSA embedding + Returns: + [*, N_seq, N_res, C_out] reconstruction + """ + # [*, N_seq, N_res, C_out] + logits = self.linear(m) + return logits + + +class ExperimentallyResolvedHead(nn.Module): + """ + For use in computation of "experimentally resolved" loss, subsection + 1.9.10 + """ + + def __init__(self, c_s, c_out, **kwargs): + """ + Args: + c_s: + Input channel dimension + c_out: + Number of distogram bins + """ + super(ExperimentallyResolvedHead, self).__init__() + + self.c_s = c_s + self.c_out = c_out + + self.linear = Linear(self.c_s, self.c_out, init="final") + + def forward(self, s): + """ + Args: + s: + [*, N_res, C_s] single embedding + Returns: + [*, N, C_out] logits + """ + # [*, N, C_out] + logits = self.linear(s) + return logits diff --git a/model/openfold/model.py b/model/openfold/model.py new file mode 100644 index 0000000000000000000000000000000000000000..5523e21b3633cde0032953f719541f6431d39796 --- /dev/null +++ b/model/openfold/model.py @@ -0,0 +1,591 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from functools import partial +import weakref + +import torch +import torch.nn as nn + +from onescience.datapipes.openfold import data_transforms_multimer +from onescience.utils.openfold.feats import ( + pseudo_beta_fn, + build_extra_msa_feat, + dgram_from_positions, + atom14_to_atom37, +) +from onescience.utils.openfold.tensor_utils import masked_mean +from model.openfold.embedders import ( + InputEmbedder, + InputEmbedderMultimer, + RecyclingEmbedder, + TemplateEmbedder, + TemplateEmbedderMultimer, + ExtraMSAEmbedder, + PreembeddingEmbedder, +) +from model.openfold.evoformer import EvoformerStack, ExtraMSAStack +from model.openfold.heads import AuxiliaryHeads +from model.openfold.structure_module import StructureModule +from model.openfold.template import ( + TemplatePairStack, + TemplatePointwiseAttention, + embed_templates_average, + embed_templates_offload, +) +import onescience.utils.openfold.np.residue_constants as residue_constants +from onescience.utils.openfold.feats import ( + pseudo_beta_fn, + build_extra_msa_feat, + build_template_angle_feat, + build_template_pair_feat, + atom14_to_atom37, +) +from onescience.utils.openfold.loss import ( + compute_plddt, +) +from onescience.utils.openfold.tensor_utils import ( + add, + dict_multimap, + tensor_tree_map, +) + + +class AlphaFold(nn.Module): + """ + Alphafold 2. + + Implements Algorithm 2 (but with training). + """ + + def __init__(self, config): + """ + Args: + config: + A dict-like config object (like the one in config.py) + """ + super(AlphaFold, self).__init__() + + self.globals = config.globals + self.config = config.model + self.template_config = self.config.template + self.extra_msa_config = self.config.extra_msa + self.seqemb_mode = config.globals.seqemb_mode_enabled + + # Main trunk + structure module + if self.globals.is_multimer: + self.input_embedder = InputEmbedderMultimer( + **self.config["input_embedder"] + ) + elif self.seqemb_mode: + # If using seqemb mode, embed the sequence embeddings passed + # to the model ("preembeddings") instead of embedding the sequence + self.input_embedder = PreembeddingEmbedder( + **self.config["preembedding_embedder"], + ) + else: + self.input_embedder = InputEmbedder( + **self.config["input_embedder"], + ) + + self.recycling_embedder = RecyclingEmbedder( + **self.config["recycling_embedder"], + ) + + if self.template_config.enabled: + if self.globals.is_multimer: + self.template_embedder = TemplateEmbedderMultimer( + self.template_config, + ) + else: + self.template_embedder = TemplateEmbedder( + self.template_config, + ) + + if self.extra_msa_config.enabled: + self.extra_msa_embedder = ExtraMSAEmbedder( + **self.extra_msa_config["extra_msa_embedder"], + ) + self.extra_msa_stack = ExtraMSAStack( + **self.extra_msa_config["extra_msa_stack"], + ) + + self.evoformer = EvoformerStack( + **self.config["evoformer_stack"], + ) + + self.structure_module = StructureModule( + is_multimer=self.globals.is_multimer, + **self.config["structure_module"], + ) + self.aux_heads = AuxiliaryHeads( + self.config["heads"], + ) + + def embed_templates(self, batch, feats, z, pair_mask, templ_dim, inplace_safe): + if self.globals.is_multimer: + asym_id = feats["asym_id"] + multichain_mask_2d = ( + asym_id[..., None] == asym_id[..., None, :] + ) + template_embeds = self.template_embedder( + batch, + z, + pair_mask.to(dtype=z.dtype), + templ_dim, + chunk_size=self.globals.chunk_size, + multichain_mask_2d=multichain_mask_2d, + use_deepspeed_evo_attention=self.globals.use_deepspeed_evo_attention, + use_lma=self.globals.use_lma, + inplace_safe=inplace_safe, + _mask_trans=self.config._mask_trans + ) + feats["template_torsion_angles_mask"] = ( + template_embeds["template_mask"] + ) + else: + if self.template_config.offload_templates: + return embed_templates_offload(self, + batch, z, pair_mask, templ_dim, inplace_safe=inplace_safe, + ) + elif self.template_config.average_templates: + return embed_templates_average(self, + batch, z, pair_mask, templ_dim, inplace_safe=inplace_safe, + ) + + template_embeds = self.template_embedder( + batch, + z, + pair_mask.to(dtype=z.dtype), + templ_dim, + chunk_size=self.globals.chunk_size, + use_deepspeed_evo_attention=self.globals.use_deepspeed_evo_attention, + use_lma=self.globals.use_lma, + inplace_safe=inplace_safe, + _mask_trans=self.config._mask_trans + ) + + return template_embeds + + def tolerance_reached(self, prev_pos, next_pos, mask, eps=1e-8) -> bool: + """ + Early stopping criteria based on criteria used in + AF2Complex: https://www.nature.com/articles/s41467-022-29394-2 + Args: + prev_pos: Previous atom positions in atom37/14 representation + next_pos: Current atom positions in atom37/14 representation + mask: 1-D sequence mask + eps: Epsilon used in square root calculation + Returns: + Whether to stop recycling early based on the desired tolerance. + """ + + def distances(points): + """Compute all pairwise distances for a set of points.""" + d = points[..., None, :] - points[..., None, :, :] + return torch.sqrt(torch.sum(d ** 2, dim=-1)) + + if self.config.recycle_early_stop_tolerance < 0: + return False + + ca_idx = residue_constants.atom_order['CA'] + sq_diff = (distances(prev_pos[..., ca_idx, :]) - distances(next_pos[..., ca_idx, :])) ** 2 + mask = mask[..., None] * mask[..., None, :] + sq_diff = masked_mean(mask=mask, value=sq_diff, dim=list(range(len(mask.shape)))) + diff = torch.sqrt(sq_diff + eps).item() + return diff <= self.config.recycle_early_stop_tolerance + + def iteration(self, feats, prevs, _recycle=True): + # Primary output dictionary + outputs = {} + + # This needs to be done manually for DeepSpeed's sake + dtype = next(self.parameters()).dtype + for k in feats: + if feats[k].dtype == torch.float32: + feats[k] = feats[k].to(dtype=dtype) + + # Grab some data about the input + batch_dims = feats["target_feat"].shape[:-2] + no_batch_dims = len(batch_dims) + n = feats["target_feat"].shape[-2] + n_seq = feats["msa_feat"].shape[-3] + device = feats["target_feat"].device + + # Controls whether the model uses in-place operations throughout + # The dual condition accounts for activation checkpoints + inplace_safe = not (self.training or torch.is_grad_enabled()) + + # Prep some features + seq_mask = feats["seq_mask"] + pair_mask = seq_mask[..., None] * seq_mask[..., None, :] + msa_mask = feats["msa_mask"] + + if self.globals.is_multimer: + # Initialize the MSA and pair representations + # m: [*, S_c, N, C_m] + # z: [*, N, N, C_z] + m, z = self.input_embedder(feats) + elif self.seqemb_mode: + # Initialize the SingleSeq and pair representations + # m: [*, 1, N, C_m] + # z: [*, N, N, C_z] + m, z = self.input_embedder( + feats["target_feat"], + feats["residue_index"], + feats["seq_embedding"] + ) + else: + # Initialize the MSA and pair representations + # m: [*, S_c, N, C_m] + # z: [*, N, N, C_z] + m, z = self.input_embedder( + feats["target_feat"], + feats["residue_index"], + feats["msa_feat"], + inplace_safe=inplace_safe, + ) + + # Unpack the recycling embeddings. Removing them from the list allows + # them to be freed further down in this function, saving memory + m_1_prev, z_prev, x_prev = reversed([prevs.pop() for _ in range(3)]) + + # Initialize the recycling embeddings, if needs be + if None in [m_1_prev, z_prev, x_prev]: + # [*, N, C_m] + m_1_prev = m.new_zeros( + (*batch_dims, n, self.config.input_embedder.c_m), + requires_grad=False, + ) + + # [*, N, N, C_z] + z_prev = z.new_zeros( + (*batch_dims, n, n, self.config.input_embedder.c_z), + requires_grad=False, + ) + + # [*, N, 3] + x_prev = z.new_zeros( + (*batch_dims, n, residue_constants.atom_type_num, 3), + requires_grad=False, + ) + + pseudo_beta_x_prev = pseudo_beta_fn( + feats["aatype"], x_prev, None + ).to(dtype=z.dtype) + + # The recycling embedder is memory-intensive, so we offload first + if self.globals.offload_inference and inplace_safe: + m = m.cpu() + z = z.cpu() + + # m_1_prev_emb: [*, N, C_m] + # z_prev_emb: [*, N, N, C_z] + m_1_prev_emb, z_prev_emb = self.recycling_embedder( + m_1_prev, + z_prev, + pseudo_beta_x_prev, + inplace_safe=inplace_safe, + ) + + del pseudo_beta_x_prev + + if self.globals.offload_inference and inplace_safe: + m = m.to(m_1_prev_emb.device) + z = z.to(z_prev.device) + + # [*, S_c, N, C_m] + m[..., 0, :, :] += m_1_prev_emb + + # [*, N, N, C_z] + z = add(z, z_prev_emb, inplace=inplace_safe) + + # Deletions like these become significant for inference with large N, + # where they free unused tensors and remove references to others such + # that they can be offloaded later + del m_1_prev, z_prev, m_1_prev_emb, z_prev_emb + + # Embed the templates + merge with MSA/pair embeddings + if self.config.template.enabled: + template_feats = { + k: v for k, v in feats.items() if k.startswith("template_") + } + + template_embeds = self.embed_templates( + template_feats, + feats, + z, + pair_mask.to(dtype=z.dtype), + no_batch_dims, + inplace_safe=inplace_safe, + ) + + # [*, N, N, C_z] + z = add(z, + template_embeds.pop("template_pair_embedding"), + inplace_safe, + ) + + if ( + "template_single_embedding" in template_embeds + ): + # [*, S = S_c + S_t, N, C_m] + m = torch.cat( + [m, template_embeds["template_single_embedding"]], + dim=-3 + ) + + # [*, S, N] + if not self.globals.is_multimer: + torsion_angles_mask = feats["template_torsion_angles_mask"] + msa_mask = torch.cat( + [feats["msa_mask"], torsion_angles_mask[..., 2]], + dim=-2 + ) + else: + msa_mask = torch.cat( + [feats["msa_mask"], template_embeds["template_mask"]], + dim=-2, + ) + + # Embed extra MSA features + merge with pairwise embeddings + if self.config.extra_msa.enabled: + if self.globals.is_multimer: + extra_msa_fn = data_transforms_multimer.build_extra_msa_feat + else: + extra_msa_fn = build_extra_msa_feat + + # [*, S_e, N, C_e] + extra_msa_feat = extra_msa_fn(feats).to(dtype=z.dtype) + a = self.extra_msa_embedder(extra_msa_feat) + + if self.globals.offload_inference: + # To allow the extra MSA stack (and later the evoformer) to + # offload its inputs, we remove all references to them here + input_tensors = [a, z] + del a, z + + # [*, N, N, C_z] + z = self.extra_msa_stack._forward_offload( + input_tensors, + msa_mask=feats["extra_msa_mask"].to(dtype=m.dtype), + chunk_size=self.globals.chunk_size, + use_deepspeed_evo_attention=self.globals.use_deepspeed_evo_attention, + use_lma=self.globals.use_lma, + pair_mask=pair_mask.to(dtype=m.dtype), + _mask_trans=self.config._mask_trans, + ) + + del input_tensors + else: + # [*, N, N, C_z] + z = self.extra_msa_stack( + a, z, + msa_mask=feats["extra_msa_mask"].to(dtype=m.dtype), + chunk_size=self.globals.chunk_size, + use_deepspeed_evo_attention=self.globals.use_deepspeed_evo_attention, + use_lma=self.globals.use_lma, + pair_mask=pair_mask.to(dtype=m.dtype), + inplace_safe=inplace_safe, + _mask_trans=self.config._mask_trans, + ) + + # Run MSA + pair embeddings through the trunk of the network + # m: [*, S, N, C_m] + # z: [*, N, N, C_z] + # s: [*, N, C_s] + if self.globals.offload_inference: + input_tensors = [m, z] + del m, z + m, z, s = self.evoformer._forward_offload( + input_tensors, + msa_mask=msa_mask.to(dtype=input_tensors[0].dtype), + pair_mask=pair_mask.to(dtype=input_tensors[1].dtype), + chunk_size=self.globals.chunk_size, + use_deepspeed_evo_attention=self.globals.use_deepspeed_evo_attention, + use_lma=self.globals.use_lma, + _mask_trans=self.config._mask_trans, + ) + + del input_tensors + else: + m, z, s = self.evoformer( + m, + z, + msa_mask=msa_mask.to(dtype=m.dtype), + pair_mask=pair_mask.to(dtype=z.dtype), + chunk_size=self.globals.chunk_size, + use_deepspeed_evo_attention=self.globals.use_deepspeed_evo_attention, + use_lma=self.globals.use_lma, + use_flash=self.globals.use_flash, + inplace_safe=inplace_safe, + _mask_trans=self.config._mask_trans, + ) + + outputs["msa"] = m[..., :n_seq, :, :] + outputs["pair"] = z + outputs["single"] = s + + del z + + # Predict 3D structure + outputs["sm"] = self.structure_module( + outputs, + feats["aatype"], + mask=feats["seq_mask"].to(dtype=s.dtype), + inplace_safe=inplace_safe, + _offload_inference=self.globals.offload_inference, + ) + outputs["final_atom_positions"] = atom14_to_atom37( + outputs["sm"]["positions"][-1], feats + ) + outputs["final_atom_mask"] = feats["atom37_atom_exists"] + outputs["final_affine_tensor"] = outputs["sm"]["frames"][-1] + + # Save embeddings for use during the next recycling iteration + + # [*, N, C_m] + m_1_prev = m[..., 0, :, :] + + # [*, N, N, C_z] + z_prev = outputs["pair"] + + early_stop = False + if self.globals.is_multimer: + early_stop = self.tolerance_reached(x_prev, outputs["final_atom_positions"], seq_mask) + + del x_prev + + # [*, N, 3] + x_prev = outputs["final_atom_positions"] + + return outputs, m_1_prev, z_prev, x_prev, early_stop + + def _disable_activation_checkpointing(self): + self.template_embedder.template_pair_stack.blocks_per_ckpt = None + self.evoformer.blocks_per_ckpt = None + + for b in self.extra_msa_stack.blocks: + b.ckpt = False + + def _enable_activation_checkpointing(self): + self.template_embedder.template_pair_stack.blocks_per_ckpt = ( + self.config.template.template_pair_stack.blocks_per_ckpt + ) + self.evoformer.blocks_per_ckpt = ( + self.config.evoformer_stack.blocks_per_ckpt + ) + + for b in self.extra_msa_stack.blocks: + b.ckpt = self.config.extra_msa.extra_msa_stack.ckpt + + def forward(self, batch): + """ + Args: + batch: + Dictionary of arguments outlined in Algorithm 2. Keys must + include the official names of the features in the + supplement subsection 1.2.9. + + The final dimension of each input must have length equal to + the number of recycling iterations. + + Features (without the recycling dimension): + + "aatype" ([*, N_res]): + Contrary to the supplement, this tensor of residue + indices is not one-hot. + "target_feat" ([*, N_res, C_tf]) + One-hot encoding of the target sequence. C_tf is + config.model.input_embedder.tf_dim. + "residue_index" ([*, N_res]) + Tensor whose final dimension consists of + consecutive indices from 0 to N_res. + "msa_feat" ([*, N_seq, N_res, C_msa]) + MSA features, constructed as in the supplement. + C_msa is config.model.input_embedder.msa_dim. + "seq_mask" ([*, N_res]) + 1-D sequence mask + "msa_mask" ([*, N_seq, N_res]) + MSA mask + "pair_mask" ([*, N_res, N_res]) + 2-D pair mask + "extra_msa_mask" ([*, N_extra, N_res]) + Extra MSA mask + "template_mask" ([*, N_templ]) + Template mask (on the level of templates, not + residues) + "template_aatype" ([*, N_templ, N_res]) + Tensor of template residue indices (indices greater + than 19 are clamped to 20 (Unknown)) + "template_all_atom_positions" + ([*, N_templ, N_res, 37, 3]) + Template atom coordinates in atom37 format + "template_all_atom_mask" ([*, N_templ, N_res, 37]) + Template atom coordinate mask + "template_pseudo_beta" ([*, N_templ, N_res, 3]) + Positions of template carbon "pseudo-beta" atoms + (i.e. C_beta for all residues but glycine, for + for which C_alpha is used instead) + "template_pseudo_beta_mask" ([*, N_templ, N_res]) + Pseudo-beta mask + """ + # Initialize recycling embeddings + m_1_prev, z_prev, x_prev = None, None, None + prevs = [m_1_prev, z_prev, x_prev] + + is_grad_enabled = torch.is_grad_enabled() + + # Main recycling loop + num_iters = batch["aatype"].shape[-1] + early_stop = False + num_recycles = 0 + for cycle_no in range(num_iters): + # Select the features for the current recycling cycle + fetch_cur_batch = lambda t: t[..., cycle_no] + feats = tensor_tree_map(fetch_cur_batch, batch) + + # Enable grad iff we're training and it's the final recycling layer + is_final_iter = cycle_no == (num_iters - 1) or early_stop + with torch.set_grad_enabled(is_grad_enabled and is_final_iter): + if is_final_iter: + # Sidestep AMP bug (PyTorch issue #65766) + if torch.is_autocast_enabled(): + torch.clear_autocast_cache() + + # Run the next iteration of the model + outputs, m_1_prev, z_prev, x_prev, early_stop = self.iteration( + feats, + prevs, + _recycle=(num_iters > 1) + ) + + num_recycles += 1 + + if not is_final_iter: + del outputs + prevs = [m_1_prev, z_prev, x_prev] + del m_1_prev, z_prev, x_prev + else: + break + + outputs["num_recycles"] = torch.tensor(num_recycles, device=feats["aatype"].device) + + if "asym_id" in batch: + outputs["asym_id"] = feats["asym_id"] + + # Run auxiliary heads + outputs.update(self.aux_heads(outputs)) + + return outputs diff --git a/model/openfold/msa.py b/model/openfold/msa.py new file mode 100644 index 0000000000000000000000000000000000000000..3d3772c927d3102f2724543e7082b0d904866bfe --- /dev/null +++ b/model/openfold/msa.py @@ -0,0 +1,476 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from functools import partial +import math +import torch +import torch.nn as nn +from typing import Optional, List, Tuple + +from model.openfold.primitives import ( + Linear, + LayerNorm, + Attention, + GlobalAttention, + _attention_chunked_trainable, +) +from onescience.utils.openfold.checkpointing import get_checkpoint_fn +from onescience.utils.openfold.chunk_utils import chunk_layer +from onescience.utils.openfold.tensor_utils import ( + permute_final_dims, + flatten_final_dims, +) + + +class MSAAttention(nn.Module): + def __init__( + self, + c_in, + c_hidden, + no_heads, + pair_bias=False, + c_z=None, + inf=1e9, + ): + """ + Args: + c_in: + Input channel dimension + c_hidden: + Per-head hidden channel dimension + no_heads: + Number of attention heads + pair_bias: + Whether to use pair embedding bias + c_z: + Pair embedding channel dimension. Ignored unless pair_bias + is true + inf: + A large number to be used in computing the attention mask + """ + super(MSAAttention, self).__init__() + + self.c_in = c_in + self.c_hidden = c_hidden + self.no_heads = no_heads + self.pair_bias = pair_bias + self.c_z = c_z + self.inf = inf + + self.layer_norm_m = LayerNorm(self.c_in) + + self.layer_norm_z = None + self.linear_z = None + if self.pair_bias: + self.layer_norm_z = LayerNorm(self.c_z) + self.linear_z = Linear( + self.c_z, self.no_heads, bias=False, init="normal" + ) + + self.mha = Attention( + self.c_in, + self.c_in, + self.c_in, + self.c_hidden, + self.no_heads, + ) + + @torch.jit.ignore + def _chunk(self, + m: torch.Tensor, + biases: Optional[List[torch.Tensor]], + chunk_size: int, + use_memory_efficient_kernel: bool, + use_deepspeed_evo_attention: bool, + use_lma: bool, + use_flash: bool, + flash_mask: Optional[torch.Tensor], + ) -> torch.Tensor: + def fn(m, biases, flash_mask): + m = self.layer_norm_m(m) + return self.mha( + q_x=m, + kv_x=m, + biases=biases, + use_memory_efficient_kernel=use_memory_efficient_kernel, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + use_flash=use_flash, + flash_mask=flash_mask, + ) + + inputs = {"m": m} + if(biases is not None): + inputs["biases"] = biases + else: + fn = partial(fn, biases=None) + if(use_flash and flash_mask is not None): + inputs["flash_mask"] = flash_mask + else: + fn = partial(fn, flash_mask=None) + + return chunk_layer( + fn, + inputs, + chunk_size=chunk_size, + no_batch_dims=len(m.shape[:-2]) + ) + + def _prep_inputs(self, + m: torch.Tensor, + z: Optional[torch.Tensor], + mask: Optional[torch.Tensor], + inplace_safe: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + n_seq, n_res = m.shape[-3:-1] + if mask is None: + # [*, N_seq, N_res] + mask = m.new_ones( + m.shape[:-3] + (n_seq, n_res), + ) + + # [*, N_seq, 1, 1, N_res] + mask_bias = (self.inf * (mask - 1))[..., :, None, None, :] + + if (self.pair_bias and + z is not None and # For the + self.layer_norm_z is not None and # benefit of + self.linear_z is not None # TorchScript + ): + chunks = [] + + for i in range(0, z.shape[-3], 256): + z_chunk = z[..., i: i + 256, :, :] + + # [*, N_res, N_res, C_z] + z_chunk = self.layer_norm_z(z_chunk) + + # [*, N_res, N_res, no_heads] + z_chunk = self.linear_z(z_chunk) + + chunks.append(z_chunk) + + z = torch.cat(chunks, dim=-3) + + # [*, 1, no_heads, N_res, N_res] + z = permute_final_dims(z, (2, 0, 1)).unsqueeze(-4) + + return m, mask_bias, z + + @torch.jit.ignore + def _chunked_msa_attn(self, + m: torch.Tensor, + z: Optional[torch.Tensor], + mask: Optional[torch.Tensor], + chunk_logits: int, + checkpoint: bool, + inplace_safe: bool = False + ) -> torch.Tensor: + """ + MSA attention with training-time chunking of the softmax computation. + Saves memory in the extra MSA stack. Probably obviated by our fused + attention kernel, which is now used by default. + """ + MSA_DIM = -4 + + def _get_qkv(m, z): + m, mask_bias, z = self._prep_inputs( + m, z, mask, inplace_safe=inplace_safe + ) + m = self.layer_norm_m(m) + q, k, v = self.mha._prep_qkv(m, m) + return m, q, k, v, mask_bias, z + + checkpoint_fn = get_checkpoint_fn() + + if(torch.is_grad_enabled() and checkpoint): + m, q, k, v, mask_bias, z = checkpoint_fn(_get_qkv, m, z) + else: + m, q, k, v, mask_bias, z = _get_qkv(m, z) + + o = _attention_chunked_trainable( + query=q, + key=k, + value=v, + biases=[mask_bias, z], + chunk_size=chunk_logits, + chunk_dim=MSA_DIM, + checkpoint=checkpoint, + ) + + if(torch.is_grad_enabled() and checkpoint): + # Storing an additional m here is far from ideal + m = checkpoint_fn(self.mha._wrap_up, o, m) + else: + m = self.mha._wrap_up(o, m) + + return m + + def forward(self, + m: torch.Tensor, + z: Optional[torch.Tensor] = None, + mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + use_memory_efficient_kernel: bool = False, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + use_flash: bool = False, + inplace_safe: bool = False, + _chunk_logits: Optional[int] = None, + _checkpoint_chunks: Optional[bool] = None, + ) -> torch.Tensor: + """ + Args: + m: + [*, N_seq, N_res, C_m] MSA embedding + z: + [*, N_res, N_res, C_z] pair embedding. Required only if + pair_bias is True + mask: + [*, N_seq, N_res] MSA mask + chunk_size: + Size of chunks into which the inputs are split along their + batch dimensions. A low value decreases memory overhead at the + cost of slower execution. Chunking is not performed by default. + + """ + if(_chunk_logits is not None): + return self._chunked_msa_attn( + m=m, z=z, mask=mask, + chunk_logits=_chunk_logits, + checkpoint=_checkpoint_chunks, + inplace_safe=inplace_safe, + ) + + if(use_flash): + assert z is None + biases = None + else: + m, mask_bias, z = self._prep_inputs( + m, z, mask, inplace_safe=inplace_safe + ) + + biases = [mask_bias] + if(z is not None): + biases.append(z) + + if chunk_size is not None: + m = self._chunk( + m, + biases, + chunk_size, + use_memory_efficient_kernel=use_memory_efficient_kernel, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + use_flash=use_flash, + flash_mask=mask, + ) + else: + m = self.layer_norm_m(m) + m = self.mha( + q_x=m, + kv_x=m, + biases=biases, + use_memory_efficient_kernel=use_memory_efficient_kernel, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + use_flash=use_flash, + flash_mask=mask, + ) + + return m + + +class MSARowAttentionWithPairBias(MSAAttention): + """ + Implements Algorithm 7. + """ + + def __init__(self, c_m, c_z, c_hidden, no_heads, inf=1e9): + """ + Args: + c_m: + Input channel dimension + c_z: + Pair embedding channel dimension + c_hidden: + Per-head hidden channel dimension + no_heads: + Number of attention heads + inf: + Large number used to construct attention masks + """ + super(MSARowAttentionWithPairBias, self).__init__( + c_m, + c_hidden, + no_heads, + pair_bias=True, + c_z=c_z, + inf=inf, + ) + + +class MSAColumnAttention(nn.Module): + """ + Implements Algorithm 8. + + By rights, this should also be a subclass of MSAAttention. Alas, + most inheritance isn't supported by TorchScript. + """ + + def __init__(self, c_m, c_hidden, no_heads, inf=1e9): + """ + Args: + c_m: + MSA channel dimension + c_hidden: + Per-head hidden channel dimension + no_heads: + Number of attention heads + inf: + Large number used to construct attention masks + """ + super(MSAColumnAttention, self).__init__() + + self.c_m = c_m + self.c_hidden = c_hidden + self.no_heads = no_heads + self.inf = inf + + self._msa_att = MSAAttention( + c_in=c_m, + c_hidden=c_hidden, + no_heads=no_heads, + pair_bias=False, + c_z=None, + inf=inf, + ) + + def forward(self, + m: torch.Tensor, + mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + use_flash: bool = False, + ) -> torch.Tensor: + """ + Args: + m: + [*, N_seq, N_res, C_m] MSA embedding + mask: + [*, N_seq, N_res] MSA mask + chunk_size: + Size of chunks into which the inputs are split along their + batch dimensions. A low value decreases memory overhead at the + cost of slower execution. Chunking is not performed by default. + """ + # [*, N_res, N_seq, C_in] + m = m.transpose(-2, -3) + if mask is not None: + mask = mask.transpose(-1, -2) + + m = self._msa_att( + m, + mask=mask, + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + use_flash=use_flash, + ) + + # [*, N_seq, N_res, C_in] + m = m.transpose(-2, -3) + if mask is not None: + mask = mask.transpose(-1, -2) + + return m + + +class MSAColumnGlobalAttention(nn.Module): + def __init__( + self, c_in, c_hidden, no_heads, inf=1e9, eps=1e-10, + ): + super(MSAColumnGlobalAttention, self).__init__() + + self.c_in = c_in + self.c_hidden = c_hidden + self.no_heads = no_heads + self.inf = inf + self.eps = eps + + self.layer_norm_m = nn.LayerNorm(c_in) + + self.global_attention = GlobalAttention( + c_in=c_in, + c_hidden=c_hidden, + no_heads=no_heads, + inf=inf, + eps=eps, + ) + + @torch.jit.ignore + def _chunk(self, + m: torch.Tensor, + mask: torch.Tensor, + chunk_size: int, + use_lma: bool = False, + ) -> torch.Tensor: + mha_input = { + "m": m, + "mask": mask, + } + + def fn(m, mask): + m = self.layer_norm_m(m) + return self.global_attention(m, mask, use_lma=use_lma) + + return chunk_layer( + fn, + mha_input, + chunk_size=chunk_size, + no_batch_dims=len(m.shape[:-2]), + ) + + def forward( + self, + m: torch.Tensor, + mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + use_lma: bool = False, + ) -> torch.Tensor: + n_seq, n_res, c_in = m.shape[-3:] + + if mask is None: + # [*, N_seq, N_res] + mask = torch.ones( + m.shape[:-1], + dtype=m.dtype, + device=m.device, + ).detach() + + # [*, N_res, N_seq, C_in] + m = m.transpose(-2, -3) + mask = mask.transpose(-1, -2) + + if chunk_size is not None: + m = self._chunk(m, mask, chunk_size, use_lma=use_lma) + else: + m = self.layer_norm_m(m) + m = self.global_attention(m=m, mask=mask, use_lma=use_lma) + + # [*, N_seq, N_res, C_in] + m = m.transpose(-2, -3) + + return m diff --git a/model/openfold/outer_product_mean.py b/model/openfold/outer_product_mean.py new file mode 100644 index 0000000000000000000000000000000000000000..2d7a41496ca9e54e0474c92c461c56400ff645e5 --- /dev/null +++ b/model/openfold/outer_product_mean.py @@ -0,0 +1,159 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import partial +from typing import Optional + +import torch +import torch.nn as nn + +from model.openfold.primitives import Linear +from onescience.utils.openfold.chunk_utils import chunk_layer +from onescience.utils.openfold.precision_utils import is_fp16_enabled + + +class OuterProductMean(nn.Module): + """ + Implements Algorithm 10. + """ + + def __init__(self, c_m, c_z, c_hidden, eps=1e-3, bias: bool=True): + """ + Args: + c_m: + MSA embedding channel dimension + c_z: + Pair embedding channel dimension + c_hidden: + Hidden channel dimension + """ + super(OuterProductMean, self).__init__() + + self.c_m = c_m + self.c_z = c_z + self.c_hidden = c_hidden + self.eps = eps + + self.layer_norm = nn.LayerNorm(c_m) + self.linear_1 = Linear(c_m, c_hidden, bias=bias) + self.linear_2 = Linear(c_m, c_hidden, bias=bias) + self.linear_out = Linear(c_hidden ** 2, c_z, init="final") + + def _opm(self, a, b): + # [*, N_res, N_res, C, C] + outer = torch.einsum("...bac,...dae->...bdce", a, b) + + # [*, N_res, N_res, C * C] + outer = outer.reshape(outer.shape[:-2] + (-1,)) + + # [*, N_res, N_res, C_z] + outer = self.linear_out(outer) + + return outer + + @torch.jit.ignore + def _chunk(self, + a: torch.Tensor, + b: torch.Tensor, + chunk_size: int + ) -> torch.Tensor: + # Since the "batch dim" in this case is not a true batch dimension + # (in that the shape of the output depends on it), we need to + # iterate over it ourselves + a_reshape = a.reshape((-1,) + a.shape[-3:]) + b_reshape = b.reshape((-1,) + b.shape[-3:]) + out = [] + for a_prime, b_prime in zip(a_reshape, b_reshape): + outer = chunk_layer( + partial(self._opm, b=b_prime), + {"a": a_prime}, + chunk_size=chunk_size, + no_batch_dims=1, + ) + out.append(outer) + + # For some cursed reason making this distinction saves memory + if(len(out) == 1): + outer = out[0].unsqueeze(0) + else: + outer = torch.stack(out, dim=0) + + outer = outer.reshape(a.shape[:-3] + outer.shape[1:]) + + return outer + + def _forward(self, + m: torch.Tensor, + mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + inplace_safe: bool = False, + ) -> torch.Tensor: + """ + Args: + m: + [*, N_seq, N_res, C_m] MSA embedding + mask: + [*, N_seq, N_res] MSA mask + Returns: + [*, N_res, N_res, C_z] pair embedding update + """ + if mask is None: + mask = m.new_ones(m.shape[:-1]) + + # [*, N_seq, N_res, C_m] + ln = self.layer_norm(m) + + # [*, N_seq, N_res, C] + mask = mask.unsqueeze(-1) + a = self.linear_1(ln) + a = a * mask + + b = self.linear_2(ln) + b = b * mask + + del ln + + a = a.transpose(-2, -3) + b = b.transpose(-2, -3) + + if chunk_size is not None: + outer = self._chunk(a, b, chunk_size) + else: + outer = self._opm(a, b) + + # [*, N_res, N_res, 1] + norm = torch.einsum("...abc,...adc->...bdc", mask, mask) + norm = norm + self.eps + + # [*, N_res, N_res, C_z] + if(inplace_safe): + outer /= norm + else: + outer = outer / norm + + return outer + + def forward(self, + m: torch.Tensor, + mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + inplace_safe: bool = False, + ) -> torch.Tensor: + if(is_fp16_enabled()): + with torch.cuda.amp.autocast(enabled=False): + return self._forward(m.float(), mask, chunk_size, inplace_safe) + else: + return self._forward(m, mask, chunk_size, inplace_safe) + diff --git a/model/openfold/pair_transition.py b/model/openfold/pair_transition.py new file mode 100644 index 0000000000000000000000000000000000000000..5967959b956fb65713d9cafc57bf2ff577744b8c --- /dev/null +++ b/model/openfold/pair_transition.py @@ -0,0 +1,99 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional + +import torch +import torch.nn as nn + +from model.openfold.primitives import Linear, LayerNorm +from onescience.utils.openfold.chunk_utils import chunk_layer + + +class PairTransition(nn.Module): + """ + Implements Algorithm 15. + """ + + def __init__(self, c_z, n): + """ + Args: + c_z: + Pair transition channel dimension + n: + Factor by which c_z is multiplied to obtain hidden channel + dimension + """ + super(PairTransition, self).__init__() + + self.c_z = c_z + self.n = n + + self.layer_norm = LayerNorm(self.c_z) + self.linear_1 = Linear(self.c_z, self.n * self.c_z, init="relu") + self.relu = nn.ReLU() + self.linear_2 = Linear(self.n * self.c_z, c_z, init="final") + + def _transition(self, z, mask): + # [*, N_res, N_res, C_z] + z = self.layer_norm(z) + + # [*, N_res, N_res, C_hidden] + z = self.linear_1(z) + z = self.relu(z) + + # [*, N_res, N_res, C_z] + z = self.linear_2(z) + z = z * mask + + return z + + @torch.jit.ignore + def _chunk(self, + z: torch.Tensor, + mask: torch.Tensor, + chunk_size: int, + ) -> torch.Tensor: + return chunk_layer( + self._transition, + {"z": z, "mask": mask}, + chunk_size=chunk_size, + no_batch_dims=len(z.shape[:-2]), + ) + + def forward(self, + z: torch.Tensor, + mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + ) -> torch.Tensor: + """ + Args: + z: + [*, N_res, N_res, C_z] pair embedding + Returns: + [*, N_res, N_res, C_z] pair embedding update + """ + # DISCREPANCY: DeepMind forgets to apply the mask in this module. + if mask is None: + mask = z.new_ones(z.shape[:-1]) + + # [*, N_res, N_res, 1] + mask = mask.unsqueeze(-1) + + if chunk_size is not None: + z = self._chunk(z, mask, chunk_size) + else: + z = self._transition(z=z, mask=mask) + + return z diff --git a/model/openfold/primitives.py b/model/openfold/primitives.py new file mode 100644 index 0000000000000000000000000000000000000000..2e0ea520f2a68c76fb21ad4be64e2efa61225c57 --- /dev/null +++ b/model/openfold/primitives.py @@ -0,0 +1,902 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import importlib +import math +from typing import Optional, Callable, List, Tuple +import numpy as np + +deepspeed_is_installed = importlib.util.find_spec("deepspeed") is not None +ds4s_is_installed = deepspeed_is_installed and importlib.util.find_spec("deepspeed.ops.deepspeed4science") is not None +if deepspeed_is_installed: + import deepspeed + +if ds4s_is_installed: + from deepspeed.ops.deepspeed4science import DS4Sci_EvoformerAttention + +fa_is_installed = importlib.util.find_spec("flash_attn") is not None +if fa_is_installed: + from flash_attn.bert_padding import unpad_input + from flash_attn.flash_attn_interface import flash_attn_varlen_kvpacked_func + +import torch +import torch.nn as nn +from scipy.stats import truncnorm + +from onescience.utils.openfold.checkpointing import get_checkpoint_fn +from onescience.utils.openfold.kernel.attention_core import attention_core +from onescience.utils.openfold.precision_utils import is_fp16_enabled +from onescience.utils.openfold.tensor_utils import ( + permute_final_dims, + flatten_final_dims, +) + + +DEFAULT_LMA_Q_CHUNK_SIZE = 1024 +DEFAULT_LMA_KV_CHUNK_SIZE = 4096 + + +def _prod(nums): + out = 1 + for n in nums: + out = out * n + return out + + +def _calculate_fan(linear_weight_shape, fan="fan_in"): + fan_out, fan_in = linear_weight_shape + + if fan == "fan_in": + f = fan_in + elif fan == "fan_out": + f = fan_out + elif fan == "fan_avg": + f = (fan_in + fan_out) / 2 + else: + raise ValueError("Invalid fan option") + + return f + + +def trunc_normal_init_(weights, scale=1.0, fan="fan_in"): + shape = weights.shape + f = _calculate_fan(shape, fan) + scale = scale / max(1, f) + a = -2 + b = 2 + std = math.sqrt(scale) / truncnorm.std(a=a, b=b, loc=0, scale=1) + size = _prod(shape) + samples = truncnorm.rvs(a=a, b=b, loc=0, scale=std, size=size) + samples = np.reshape(samples, shape) + with torch.no_grad(): + weights.copy_(torch.tensor(samples, device=weights.device)) + + +def lecun_normal_init_(weights): + trunc_normal_init_(weights, scale=1.0) + + +def he_normal_init_(weights): + trunc_normal_init_(weights, scale=2.0) + + +def glorot_uniform_init_(weights): + nn.init.xavier_uniform_(weights, gain=1) + + +def final_init_(weights): + with torch.no_grad(): + weights.fill_(0.0) + + +def gating_init_(weights): + with torch.no_grad(): + weights.fill_(0.0) + + +def normal_init_(weights): + torch.nn.init.kaiming_normal_(weights, nonlinearity="linear") + + +def ipa_point_weights_init_(weights): + with torch.no_grad(): + softplus_inverse_1 = 0.541324854612918 + weights.fill_(softplus_inverse_1) + + +class Linear(nn.Linear): + """ + A Linear layer with built-in nonstandard initializations. Called just + like torch.nn.Linear. + + Implements the initializers in 1.11.4, plus some additional ones found + in the code. + """ + + def __init__( + self, + in_dim: int, + out_dim: int, + bias: bool = True, + init: str = "default", + init_fn: Optional[Callable[[torch.Tensor, torch.Tensor], None]] = None, + precision=None + ): + """ + Args: + in_dim: + The final dimension of inputs to the layer + out_dim: + The final dimension of layer outputs + bias: + Whether to learn an additive bias. True by default + init: + The initializer to use. Choose from: + + "default": LeCun fan-in truncated normal initialization + "relu": He initialization w/ truncated normal distribution + "glorot": Fan-average Glorot uniform initialization + "gating": Weights=0, Bias=1 + "normal": Normal initialization with std=1/sqrt(fan_in) + "final": Weights=0, Bias=0 + + Overridden by init_fn if the latter is not None. + init_fn: + A custom initializer taking weight and bias as inputs. + Overrides init if not None. + """ + super(Linear, self).__init__(in_dim, out_dim, bias=bias) + + if bias: + with torch.no_grad(): + self.bias.fill_(0) + + with torch.no_grad(): + if init_fn is not None: + init_fn(self.weight, self.bias) + else: + if init == "default": + lecun_normal_init_(self.weight) + elif init == "relu": + he_normal_init_(self.weight) + elif init == "glorot": + glorot_uniform_init_(self.weight) + elif init == "gating": + gating_init_(self.weight) + if bias: + self.bias.fill_(1.0) + elif init == "normal": + normal_init_(self.weight) + elif init == "final": + final_init_(self.weight) + else: + raise ValueError("Invalid init string.") + + self.precision = precision + + def forward(self, input: torch.Tensor) -> torch.Tensor: + d = input.dtype + deepspeed_is_initialized = ( + deepspeed_is_installed and + deepspeed.comm.comm.is_initialized() + ) + if self.precision is not None: + with torch.cuda.amp.autocast(enabled=False): + bias = self.bias.to(dtype=self.precision) if self.bias is not None else None + return nn.functional.linear(input.to(dtype=self.precision), + self.weight.to(dtype=self.precision), + bias).to(dtype=d) + + if d is torch.bfloat16 and not deepspeed_is_initialized: + with torch.cuda.amp.autocast(enabled=False): + bias = self.bias.to(dtype=d) if self.bias is not None else None + return nn.functional.linear(input, self.weight.to(dtype=d), bias) + + return nn.functional.linear(input, self.weight, self.bias) + + +class LayerNorm(nn.Module): + def __init__(self, c_in, eps=1e-5): + super(LayerNorm, self).__init__() + + self.c_in = (c_in,) + self.eps = eps + + self.weight = nn.Parameter(torch.ones(c_in)) + self.bias = nn.Parameter(torch.zeros(c_in)) + + def forward(self, x): + d = x.dtype + deepspeed_is_initialized = ( + deepspeed_is_installed and + deepspeed.comm.comm.is_initialized() + ) + if d is torch.bfloat16 and not deepspeed_is_initialized: + with torch.cuda.amp.autocast(enabled=False): + out = nn.functional.layer_norm( + x, + self.c_in, + self.weight.to(dtype=d), + self.bias.to(dtype=d), + self.eps + ) + else: + out = nn.functional.layer_norm( + x, + self.c_in, + self.weight, + self.bias, + self.eps, + ) + + return out +import os +fastln_is_installed = os.getenv("LAYERNORM_TYPE", None) == "fast_layernorm" +if fastln_is_installed: + # LayerNorm is a time bottomneck, so we use a custom implementation. + from model.protenix.layer_norm.layer_norm import FusedLayerNorm + +class OpenFoldLayerNorm(nn.Module): + def __init__( + self, + c_in, + create_scale: bool = True, + create_offset: bool = True, + eps=1e-5, + ): + super(OpenFoldLayerNorm, self).__init__() + + self.c_in = (c_in,) + self.create_scale = create_scale + self.create_offset = create_offset + self.eps = eps + + if self.create_scale: + self.weight = nn.Parameter(torch.ones(c_in)) + else: + self.weight = None + if self.create_offset: + self.bias = nn.Parameter(torch.zeros(c_in)) + else: + self.bias = None + + def forward(self, x): + d = x.dtype + deepspeed_is_initialized = ( + deepspeed_is_installed and deepspeed.comm.comm.is_initialized() + ) + if d is torch.bfloat16 and not deepspeed_is_initialized: + with torch.cuda.amp.autocast(enabled=False): + out = nn.functional.layer_norm( + x, + self.c_in, + self.weight.to(dtype=d) if self.weight is not None else None, + self.bias.to(dtype=d) if self.bias is not None else None, + self.eps, + ) + else: + out = nn.functional.layer_norm( + x, + self.c_in, + self.weight, + self.bias, + self.eps, + ) + return out + + +# Keep the function name for code simplicity +def ProtenixLayerNorm( + c_in, + create_scale: bool = True, + create_offset: bool = True, + eps: float = 1e-5, +): + # if specify "fast_layernorm" and fastln_is_installed, use the FusedLayerNorm, + # Otherwise, OpenFoldLayerNorm is used! + if fastln_is_installed: + # print("use fast layernorm") + return FusedLayerNorm( + c_in, create_scale=create_scale, create_offset=create_offset, eps=eps + ) + # print("use openfold layernorm") + return OpenFoldLayerNorm(c_in, create_scale, create_offset, eps) + + +@torch.jit.ignore +def softmax_no_cast(t: torch.Tensor, dim: int = -1) -> torch.Tensor: + """ + Softmax, but without automatic casting to fp32 when the input is of + type bfloat16 + """ + d = t.dtype + deepspeed_is_initialized = ( + deepspeed_is_installed and + deepspeed.comm.comm.is_initialized() + ) + if d is torch.bfloat16 and not deepspeed_is_initialized: + with torch.cuda.amp.autocast(enabled=False): + s = torch.nn.functional.softmax(t, dim=dim) + else: + s = torch.nn.functional.softmax(t, dim=dim) + + return s + + +#@torch.jit.script +def _attention(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, biases: List[torch.Tensor]) -> torch.Tensor: + # [*, H, C_hidden, K] + key = permute_final_dims(key, (1, 0)) + + # [*, H, Q, K] + a = torch.matmul(query, key) + + for b in biases: + a += b + + a = softmax_no_cast(a, -1) + + # [*, H, Q, C_hidden] + a = torch.matmul(a, value) + + return a + + +@torch.jit.ignore +def _attention_chunked_trainable( + query, key, value, biases, chunk_size, chunk_dim, checkpoint, +): + if checkpoint and len(biases) > 2: + raise ValueError( + "Checkpointed version permits only permits two bias terms" + ) + + def _checkpointable_attention(q, k, v, b1, b2): + bs = [b for b in [b1, b2] if b is not None] + a = _attention(q, k, v, bs) + return a + + o_chunks = [] + checkpoint_fn = get_checkpoint_fn() + count = query.shape[chunk_dim] + for start in range(0, count, chunk_size): + end = start + chunk_size + idx = [slice(None)] * len(query.shape) + idx[chunk_dim] = slice(start, end) + idx_tup = tuple(idx) + q_chunk = query[idx_tup] + k_chunk = key[idx_tup] + v_chunk = value[idx_tup] + + def _slice_bias(b): + idx[chunk_dim] = ( + slice(start, end) if b.shape[chunk_dim] != 1 else slice(None) + ) + return b[tuple(idx)] + + if checkpoint: + bias_1_chunk, bias_2_chunk = [ + _slice_bias(b) if b is not None else None + for b in (biases + [None, None])[:2] + ] + + o_chunk = checkpoint_fn(_checkpointable_attention, + q_chunk, k_chunk, v_chunk, bias_1_chunk, bias_2_chunk + ) + else: + bias_chunks = [ + _slice_bias(b) for b in biases + ] + + o_chunk = _attention(q_chunk, k_chunk, v_chunk, bias_chunks) + + o_chunk = o_chunk.transpose(-2, -3) + o_chunks.append(o_chunk) + + o = torch.cat(o_chunks, dim=chunk_dim) + return o + + +class Attention(nn.Module): + """ + Standard multi-head attention using AlphaFold's default layer + initialization. Allows multiple bias vectors. + """ + def __init__( + self, + c_q: int, + c_k: int, + c_v: int, + c_hidden: int, + no_heads: int, + gating: bool = True, + bias: bool = True + ): + """ + Args: + c_q: + Input dimension of query data + c_k: + Input dimension of key data + c_v: + Input dimension of value data + c_hidden: + Per-head hidden dimension + no_heads: + Number of attention heads + gating: + Whether the output should be gated using query data + """ + super(Attention, self).__init__() + + self.c_q = c_q + self.c_k = c_k + self.c_v = c_v + self.c_hidden = c_hidden + self.no_heads = no_heads + self.gating = gating + + # DISCREPANCY: c_hidden is not the per-head channel dimension, as + # stated in the supplement, but the overall channel dimension. + + self.linear_q = Linear( + self.c_q, self.c_hidden * self.no_heads, bias=False, init="glorot" + ) + self.linear_k = Linear( + self.c_k, self.c_hidden * self.no_heads, bias=False, init="glorot" + ) + self.linear_v = Linear( + self.c_v, self.c_hidden * self.no_heads, bias=False, init="glorot" + ) + self.linear_o = Linear( + self.c_hidden * self.no_heads, self.c_q, bias=bias, init="final" + ) + + self.linear_g = None + if self.gating: + self.linear_g = Linear( + self.c_q, self.c_hidden * self.no_heads, bias=bias, init="gating" + ) + + self.sigmoid = nn.Sigmoid() + + def _prep_qkv(self, + q_x: torch.Tensor, + kv_x: torch.Tensor, + apply_scale: bool = True + ) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor + ]: + # [*, Q/K/V, H * C_hidden] + q = self.linear_q(q_x) + k = self.linear_k(kv_x) + v = self.linear_v(kv_x) + + # [*, Q/K, H, C_hidden] + q = q.view(q.shape[:-1] + (self.no_heads, -1)) + k = k.view(k.shape[:-1] + (self.no_heads, -1)) + v = v.view(v.shape[:-1] + (self.no_heads, -1)) + + # [*, H, Q/K, C_hidden] + q = q.transpose(-2, -3) + k = k.transpose(-2, -3) + v = v.transpose(-2, -3) + + if apply_scale: + q /= math.sqrt(self.c_hidden) + + return q, k, v + + def _wrap_up(self, + o: torch.Tensor, + q_x: torch.Tensor + ) -> torch.Tensor: + if self.linear_g is not None: + g = self.sigmoid(self.linear_g(q_x)) + + # [*, Q, H, C_hidden] + g = g.view(g.shape[:-1] + (self.no_heads, -1)) + o = o * g + + # [*, Q, H * C_hidden] + o = flatten_final_dims(o, 2) + + # [*, Q, C_q] + o = self.linear_o(o) + + return o + + def forward( + self, + q_x: torch.Tensor, + kv_x: torch.Tensor, + biases: Optional[List[torch.Tensor]] = None, + use_memory_efficient_kernel: bool = False, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + lma_q_chunk_size: int = DEFAULT_LMA_Q_CHUNK_SIZE, + lma_kv_chunk_size: int = DEFAULT_LMA_KV_CHUNK_SIZE, + use_flash: bool = False, + flash_mask: Optional[torch.Tensor] = None + ) -> torch.Tensor: + """ + Args: + q_x: + [*, Q, C_q] query data + kv_x: + [*, K, C_k] key data + biases: + List of biases that broadcast to [*, H, Q, K] + use_memory_efficient_kernel: + Whether to use a custom memory-efficient attention kernel. + This should be the default choice for most. If none of the + "use_<...>" flags are True, a stock PyTorch implementation + is used instead + use_deepspeed_evo_attention: + Whether to use DeepSpeed memory-efficient attention kernel. + If none of the "use_<...>" flags are True, a stock PyTorch + implementation is used instead + use_lma: + Whether to use low-memory attention (Staats & Rabe 2021). If + none of the "use_<...>" flags are True, a stock PyTorch + implementation is used instead + lma_q_chunk_size: + Query chunk size (for LMA) + lma_kv_chunk_size: + Key/Value chunk size (for LMA) + Returns + [*, Q, C_q] attention update + """ + if use_lma and (lma_q_chunk_size is None or lma_kv_chunk_size is None): + raise ValueError( + "If use_lma is specified, lma_q_chunk_size and " + "lma_kv_chunk_size must be provided" + ) + + if use_flash and biases is not None: + raise ValueError( + "use_flash is incompatible with the bias option. For masking, " + "use flash_mask instead" + ) + + attn_options = [use_memory_efficient_kernel, use_deepspeed_evo_attention, use_lma, use_flash] + if sum(attn_options) > 1: + raise ValueError( + "Choose at most one alternative attention algorithm" + ) + + if biases is None: + biases = [] + + # DeepSpeed attention kernel applies scaling internally + q, k, v = self._prep_qkv(q_x, kv_x, + apply_scale=not use_deepspeed_evo_attention) + + if is_fp16_enabled(): + use_memory_efficient_kernel = False + + if use_memory_efficient_kernel: + if len(biases) > 2: + raise ValueError( + "If use_memory_efficient_kernel is True, you may only " + "provide up to two bias terms" + ) + o = attention_core(q, k, v, *((biases + [None] * 2)[:2])) + o = o.transpose(-2, -3) + elif use_deepspeed_evo_attention: + if len(biases) > 2: + raise ValueError( + "If use_deepspeed_evo_attention is True, you may only " + "provide up to two bias terms" + ) + o = _deepspeed_evo_attn(q, k, v, biases) + elif use_lma: + biases = [ + b.expand(b.shape[:-2] + (q_x.shape[-2],) + (kv_x.shape[-2],)) + for b in biases + ] + o = _lma(q, k, v, biases, lma_q_chunk_size, lma_kv_chunk_size) + o = o.transpose(-2, -3) + elif use_flash: + o = _flash_attn(q, k, v, flash_mask) + else: + o = _attention(q, k, v, biases) + o = o.transpose(-2, -3) + + o = self._wrap_up(o, q_x) + + return o + + +class GlobalAttention(nn.Module): + def __init__(self, c_in, c_hidden, no_heads, inf, eps): + super(GlobalAttention, self).__init__() + + self.c_in = c_in + self.c_hidden = c_hidden + self.no_heads = no_heads + self.inf = inf + self.eps = eps + + self.linear_q = Linear( + c_in, c_hidden * no_heads, bias=False, init="glorot" + ) + + self.linear_k = Linear( + c_in, c_hidden, bias=False, init="glorot", + ) + self.linear_v = Linear( + c_in, c_hidden, bias=False, init="glorot", + ) + self.linear_g = Linear(c_in, c_hidden * no_heads, init="gating") + self.linear_o = Linear(c_hidden * no_heads, c_in, init="final") + + self.sigmoid = nn.Sigmoid() + + def forward(self, + m: torch.Tensor, + mask: torch.Tensor, + use_lma: bool = False, + ) -> torch.Tensor: + # [*, N_res, C_in] + q = torch.sum(m * mask.unsqueeze(-1), dim=-2) / ( + torch.sum(mask, dim=-1)[..., None] + self.eps + ) + + # [*, N_res, H * C_hidden] + q = self.linear_q(q) + q *= (self.c_hidden ** (-0.5)) + + # [*, N_res, H, C_hidden] + q = q.view(q.shape[:-1] + (self.no_heads, -1)) + + # [*, N_res, N_seq, C_hidden] + k = self.linear_k(m) + v = self.linear_v(m) + + bias = (self.inf * (mask - 1))[..., :, None, :] + if not use_lma: + # [*, N_res, H, N_seq] + a = torch.matmul( + q, + k.transpose(-1, -2), # [*, N_res, C_hidden, N_seq] + ) + a += bias + a = softmax_no_cast(a) + + # [*, N_res, H, C_hidden] + o = torch.matmul( + a, + v, + ) + else: + o = _lma( + q, + k, + v, + [bias], + DEFAULT_LMA_Q_CHUNK_SIZE, + DEFAULT_LMA_KV_CHUNK_SIZE + ) + + # [*, N_res, N_seq, C_hidden] + g = self.sigmoid(self.linear_g(m)) + + # [*, N_res, N_seq, H, C_hidden] + g = g.view(g.shape[:-1] + (self.no_heads, -1)) + + # [*, N_res, N_seq, H, C_hidden] + o = o.unsqueeze(-3) * g + + # [*, N_res, N_seq, H * C_hidden] + o = o.reshape(o.shape[:-2] + (-1,)) + + # [*, N_res, N_seq, C_in] + m = self.linear_o(o) + + return m + + +@torch.jit.ignore +def _deepspeed_evo_attn( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + biases: List[torch.Tensor], +): + """"" + Compute attention using the DeepSpeed DS4Sci_EvoformerAttention kernel. + + Args: + q: + [*, H, Q, C_hidden] query data + k: + [*, H, K, C_hidden] key data + v: + [*, H, V, C_hidden] value data + biases: + List of biases that broadcast to [*, H, Q, K] + """ + + if not ds4s_is_installed: + raise ValueError( + "_deepspeed_evo_attn requires that DeepSpeed be installed " + "and that the deepspeed.ops.deepspeed4science package exists" + ) + + def reshape_dims(x): + no_batch_dims = len(x.shape[:-3]) + if no_batch_dims < 2: + return x.reshape(*((1,) * (2 - no_batch_dims) + x.shape)) + if no_batch_dims > 2: + return x.reshape(*((x.shape[0], -1) + x.shape[-3:])) + return x + + # [*, Q/K, H, C_hidden] + q = q.transpose(-2, -3) + k = k.transpose(-2, -3) + v = v.transpose(-2, -3) + + # Reshape tensors to match expected input shape [B, N, Q/K, H, C_hidden] + # for DS4Sci_EvoformerAttention() by adding or flattening batch dims as needed. + orig_shape = q.shape + if len(orig_shape[:-3]) != 2: + q = reshape_dims(q) + k = reshape_dims(k) + v = reshape_dims(v) + biases = [reshape_dims(b) for b in biases] + + # DeepSpeed attn. kernel requires inputs to be type bf16 or fp16 + # Cast to bf16 so kernel can be used during inference + orig_dtype = q.dtype + if orig_dtype not in [torch.bfloat16, torch.float16]: + o = DS4Sci_EvoformerAttention(q.to(dtype=torch.bfloat16), + k.to(dtype=torch.bfloat16), + v.to(dtype=torch.bfloat16), + [b.to(dtype=torch.bfloat16) for b in biases]) + + o = o.to(dtype=orig_dtype) + else: + o = DS4Sci_EvoformerAttention(q, k, v, biases) + + o = o.reshape(orig_shape) + return o + + +def _lma( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + biases: List[torch.Tensor], + q_chunk_size: int, + kv_chunk_size: int, +): + no_q, no_kv = q.shape[-2], k.shape[-2] + + # [*, H, Q, C_hidden] + o = q.new_zeros(q.shape) + for q_s in range(0, no_q, q_chunk_size): + q_chunk = q[..., q_s: q_s + q_chunk_size, :] + large_bias_chunks = [ + b[..., q_s: q_s + q_chunk_size, :] for b in biases + ] + + maxes = [] + weights = [] + values = [] + for kv_s in range(0, no_kv, kv_chunk_size): + k_chunk = k[..., kv_s: kv_s + kv_chunk_size, :] + v_chunk = v[..., kv_s: kv_s + kv_chunk_size, :] + small_bias_chunks = [ + b[..., kv_s: kv_s + kv_chunk_size] for b in large_bias_chunks + ] + + a = torch.einsum( + "...hqd,...hkd->...hqk", q_chunk, k_chunk, + ) + + for b in small_bias_chunks: + a += b + + max_a = torch.max(a, dim=-1, keepdim=True)[0] + exp_a = torch.exp(a - max_a) + exp_v = torch.einsum("...hvf,...hqv->...hqf", v_chunk, exp_a) + + maxes.append(max_a.detach().squeeze(-1)) + weights.append(torch.sum(exp_a, dim=-1)) + values.append(exp_v) + + chunk_max = torch.stack(maxes, dim=-3) + chunk_weights = torch.stack(weights, dim=-3) + chunk_values = torch.stack(values, dim=-4) + + global_max = torch.max(chunk_max, dim=-3, keepdim=True)[0] + max_diffs = torch.exp(chunk_max - global_max) + chunk_values = chunk_values * max_diffs.unsqueeze(-1) + chunk_weights = chunk_weights * max_diffs + + all_values = torch.sum(chunk_values, dim=-4) + all_weights = torch.sum(chunk_weights.unsqueeze(-1), dim=-4) + + q_chunk_out = all_values / all_weights + + o[..., q_s: q_s + q_chunk_size, :] = q_chunk_out + + return o + + +@torch.jit.ignore +def _flash_attn(q, k, v, kv_mask): + if not fa_is_installed: + raise ValueError( + "_flash_attn requires that FlashAttention be installed" + ) + + batch_dims = q.shape[:-3] + no_heads, n, c = q.shape[-3:] + dtype = q.dtype + + q = q.half() + k = k.half() + v = v.half() + kv_mask = kv_mask.half() + + # [*, B, N, H, C] + q = q.transpose(-2, -3) + k = k.transpose(-2, -3) + v = v.transpose(-2, -3) + + # [B_flat, N, H, C] + q = q.reshape(-1, *q.shape[-3:]) + k = k.reshape(-1, *k.shape[-3:]) + v = v.reshape(-1, *v.shape[-3:]) + + # Flattened batch size + batch_size = q.shape[0] + + # [B_flat * N, H, C] + q = q.reshape(-1, *q.shape[-2:]) + + q_max_s = n + q_cu_seqlens = torch.arange( + 0, (batch_size + 1) * n, step=n, dtype=torch.int32, device=q.device + ) + + # [B_flat, N, 2, H, C] + kv = torch.stack([k, v], dim=-3) + kv_shape = kv.shape + + # [B_flat, N, 2 * H * C] + kv = kv.reshape(*kv.shape[:-3], -1) + + kv_unpad, _, kv_cu_seqlens, kv_max_s, _ = unpad_input(kv, kv_mask) + kv_unpad = kv_unpad.reshape(-1, *kv_shape[-3:]) + + out = flash_attn_varlen_kvpacked_func( + q, + kv_unpad, + q_cu_seqlens, + kv_cu_seqlens, + q_max_s, + kv_max_s, + dropout_p=0., + softmax_scale=1., # q has been scaled already + ) + + # [*, B, N, H, C] + out = out.reshape(*batch_dims, n, no_heads, c) + + out = out.to(dtype=dtype) + + return out diff --git a/model/openfold/structure_module.py b/model/openfold/structure_module.py new file mode 100644 index 0000000000000000000000000000000000000000..8ef8252a3c4c29e16d6b06068f2734ff1456f93f --- /dev/null +++ b/model/openfold/structure_module.py @@ -0,0 +1,1252 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from functools import reduce +import importlib +import math +import sys +from operator import mul + +import torch +import torch.nn as nn +from typing import Optional, Tuple, Sequence, Union + +from model.openfold.primitives import Linear, LayerNorm, ipa_point_weights_init_ +from onescience.utils.openfold.np.residue_constants import ( + restype_rigid_group_default_frame, + restype_atom14_to_rigid_group, + restype_atom14_mask, + restype_atom14_rigid_group_positions, +) +from onescience.utils.openfold.geometry.quat_rigid import QuatRigid +from onescience.utils.openfold.geometry.rigid_matrix_vector import Rigid3Array +from onescience.utils.openfold.geometry.vector import Vec3Array, square_euclidean_distance +from onescience.utils.openfold.feats import ( + frames_and_literature_positions_to_atom14_pos, + torsion_angles_to_frames, +) +from onescience.utils.openfold.precision_utils import is_fp16_enabled +from onescience.utils.openfold.rigid_utils import Rotation, Rigid +from onescience.utils.openfold.tensor_utils import ( + dict_multimap, + permute_final_dims, + flatten_final_dims, +) + +attn_core_inplace_cuda = importlib.import_module("attn_core_inplace_cuda") + + +class AngleResnetBlock(nn.Module): + def __init__(self, c_hidden): + """ + Args: + c_hidden: + Hidden channel dimension + """ + super(AngleResnetBlock, self).__init__() + + self.c_hidden = c_hidden + + self.linear_1 = Linear(self.c_hidden, self.c_hidden, init="relu") + self.linear_2 = Linear(self.c_hidden, self.c_hidden, init="final") + + self.relu = nn.ReLU() + + def forward(self, a: torch.Tensor) -> torch.Tensor: + + s_initial = a + + a = self.relu(a) + a = self.linear_1(a) + a = self.relu(a) + a = self.linear_2(a) + + return a + s_initial + + +class AngleResnet(nn.Module): + """ + Implements Algorithm 20, lines 11-14 + """ + + def __init__(self, c_in, c_hidden, no_blocks, no_angles, epsilon): + """ + Args: + c_in: + Input channel dimension + c_hidden: + Hidden channel dimension + no_blocks: + Number of resnet blocks + no_angles: + Number of torsion angles to generate + epsilon: + Small constant for normalization + """ + super(AngleResnet, self).__init__() + + self.c_in = c_in + self.c_hidden = c_hidden + self.no_blocks = no_blocks + self.no_angles = no_angles + self.eps = epsilon + + self.linear_in = Linear(self.c_in, self.c_hidden) + self.linear_initial = Linear(self.c_in, self.c_hidden) + + self.layers = nn.ModuleList() + for _ in range(self.no_blocks): + layer = AngleResnetBlock(c_hidden=self.c_hidden) + self.layers.append(layer) + + self.linear_out = Linear(self.c_hidden, self.no_angles * 2) + + self.relu = nn.ReLU() + + def forward( + self, s: torch.Tensor, s_initial: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + s: + [*, C_hidden] single embedding + s_initial: + [*, C_hidden] single embedding as of the start of the + StructureModule + Returns: + [*, no_angles, 2] predicted angles + """ + # NOTE: The ReLU's applied to the inputs are absent from the supplement + # pseudocode but present in the source. For maximal compatibility with + # the pretrained weights, I'm going with the source. + + # [*, C_hidden] + s_initial = self.relu(s_initial) + s_initial = self.linear_initial(s_initial) + s = self.relu(s) + s = self.linear_in(s) + s = s + s_initial + + for l in self.layers: + s = l(s) + + s = self.relu(s) + + # [*, no_angles * 2] + s = self.linear_out(s) + + # [*, no_angles, 2] + s = s.view(s.shape[:-1] + (-1, 2)) + + unnormalized_s = s + norm_denom = torch.sqrt( + torch.clamp( + torch.sum(s ** 2, dim=-1, keepdim=True), + min=self.eps, + ) + ) + s = s / norm_denom + + return unnormalized_s, s + + +class PointProjection(nn.Module): + def __init__(self, + c_hidden: int, + num_points: int, + no_heads: int, + is_multimer: bool, + return_local_points: bool = False, + ): + super().__init__() + self.return_local_points = return_local_points + self.no_heads = no_heads + self.num_points = num_points + self.is_multimer = is_multimer + + # Multimer requires this to be run with fp32 precision during training + precision = torch.float32 if self.is_multimer else None + self.linear = Linear(c_hidden, no_heads * 3 * num_points, precision=precision) + + def forward(self, + activations: torch.Tensor, + rigids: Union[Rigid, Rigid3Array], + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + # TODO: Needs to run in high precision during training + points_local = self.linear(activations) + out_shape = points_local.shape[:-1] + (self.no_heads, self.num_points, 3) + + if self.is_multimer: + points_local = points_local.view( + points_local.shape[:-1] + (self.no_heads, -1) + ) + + points_local = torch.split( + points_local, points_local.shape[-1] // 3, dim=-1 + ) + + points_local = torch.stack(points_local, dim=-1).view(out_shape) + + points_global = rigids[..., None, None].apply(points_local) + + if(self.return_local_points): + return points_global, points_local + + return points_global + + +class InvariantPointAttention(nn.Module): + """ + Implements Algorithm 22. + """ + def __init__( + self, + c_s: int, + c_z: int, + c_hidden: int, + no_heads: int, + no_qk_points: int, + no_v_points: int, + inf: float = 1e5, + eps: float = 1e-8, + is_multimer: bool = False, + ): + """ + Args: + c_s: + Single representation channel dimension + c_z: + Pair representation channel dimension + c_hidden: + Hidden channel dimension + no_heads: + Number of attention heads + no_qk_points: + Number of query/key points to generate + no_v_points: + Number of value points to generate + """ + super(InvariantPointAttention, self).__init__() + + self.c_s = c_s + self.c_z = c_z + self.c_hidden = c_hidden + self.no_heads = no_heads + self.no_qk_points = no_qk_points + self.no_v_points = no_v_points + self.inf = inf + self.eps = eps + self.is_multimer = is_multimer + + # These linear layers differ from their specifications in the + # supplement. There, they lack bias and use Glorot initialization. + # Here as in the official source, they have bias and use the default + # Lecun initialization. + hc = self.c_hidden * self.no_heads + self.linear_q = Linear(self.c_s, hc, bias=(not is_multimer)) + + self.linear_q_points = PointProjection( + self.c_s, + self.no_qk_points, + self.no_heads, + self.is_multimer + ) + + if(is_multimer): + self.linear_k = Linear(self.c_s, hc, bias=False) + self.linear_v = Linear(self.c_s, hc, bias=False) + self.linear_k_points = PointProjection( + self.c_s, + self.no_qk_points, + self.no_heads, + self.is_multimer + ) + + self.linear_v_points = PointProjection( + self.c_s, + self.no_v_points, + self.no_heads, + self.is_multimer + ) + else: + self.linear_kv = Linear(self.c_s, 2 * hc) + self.linear_kv_points = PointProjection( + self.c_s, + self.no_qk_points + self.no_v_points, + self.no_heads, + self.is_multimer + ) + + self.linear_b = Linear(self.c_z, self.no_heads) + + self.head_weights = nn.Parameter(torch.zeros((no_heads))) + ipa_point_weights_init_(self.head_weights) + + concat_out_dim = self.no_heads * ( + self.c_z + self.c_hidden + self.no_v_points * 4 + ) + self.linear_out = Linear(concat_out_dim, self.c_s, init="final") + + self.softmax = nn.Softmax(dim=-1) + self.softplus = nn.Softplus() + + def forward( + self, + s: torch.Tensor, + z: torch.Tensor, + r: Union[Rigid, Rigid3Array], + mask: torch.Tensor, + inplace_safe: bool = False, + _offload_inference: bool = False, + _z_reference_list: Optional[Sequence[torch.Tensor]] = None, + ) -> torch.Tensor: + """ + Args: + s: + [*, N_res, C_s] single representation + z: + [*, N_res, N_res, C_z] pair representation + r: + [*, N_res] transformation object + mask: + [*, N_res] mask + Returns: + [*, N_res, C_s] single representation update + """ + if (_offload_inference and inplace_safe): + z = _z_reference_list + else: + z = [z] + + ####################################### + # Generate scalar and point activations + ####################################### + # [*, N_res, H * C_hidden] + q = self.linear_q(s) + + # [*, N_res, H, C_hidden] + q = q.view(q.shape[:-1] + (self.no_heads, -1)) + + # [*, N_res, H, P_qk] + q_pts = self.linear_q_points(s, r) + + # The following two blocks are equivalent + # They're separated only to preserve compatibility with old AF weights + if(self.is_multimer): + # [*, N_res, H * C_hidden] + k = self.linear_k(s) + v = self.linear_v(s) + + # [*, N_res, H, C_hidden] + k = k.view(k.shape[:-1] + (self.no_heads, -1)) + v = v.view(v.shape[:-1] + (self.no_heads, -1)) + + # [*, N_res, H, P_qk, 3] + k_pts = self.linear_k_points(s, r) + + # [*, N_res, H, P_v, 3] + v_pts = self.linear_v_points(s, r) + else: + # [*, N_res, H * 2 * C_hidden] + kv = self.linear_kv(s) + + # [*, N_res, H, 2 * C_hidden] + kv = kv.view(kv.shape[:-1] + (self.no_heads, -1)) + + # [*, N_res, H, C_hidden] + k, v = torch.split(kv, self.c_hidden, dim=-1) + + kv_pts = self.linear_kv_points(s, r) + + # [*, N_res, H, P_q/P_v, 3] + k_pts, v_pts = torch.split( + kv_pts, [self.no_qk_points, self.no_v_points], dim=-2 + ) + + ########################## + # Compute attention scores + ########################## + # [*, N_res, N_res, H] + b = self.linear_b(z[0]) + + if (_offload_inference): + assert (sys.getrefcount(z[0]) == 2) + z[0] = z[0].cpu() + + # [*, H, N_res, N_res] + if (is_fp16_enabled()): + with torch.cuda.amp.autocast(enabled=False): + a = torch.matmul( + permute_final_dims(q.float(), (1, 0, 2)), # [*, H, N_res, C_hidden] + permute_final_dims(k.float(), (1, 2, 0)), # [*, H, C_hidden, N_res] + ) + else: + a = torch.matmul( + permute_final_dims(q, (1, 0, 2)), # [*, H, N_res, C_hidden] + permute_final_dims(k, (1, 2, 0)), # [*, H, C_hidden, N_res] + ) + + a *= math.sqrt(1.0 / (3 * self.c_hidden)) + a += (math.sqrt(1.0 / 3) * permute_final_dims(b, (2, 0, 1))) + + # [*, N_res, N_res, H, P_q, 3] + pt_att = q_pts.unsqueeze(-4) - k_pts.unsqueeze(-5) + + if (inplace_safe): + pt_att *= pt_att + else: + pt_att = pt_att ** 2 + + pt_att = sum(torch.unbind(pt_att, dim=-1)) + + head_weights = self.softplus(self.head_weights).view( + *((1,) * len(pt_att.shape[:-2]) + (-1, 1)) + ) + head_weights = head_weights * math.sqrt( + 1.0 / (3 * (self.no_qk_points * 9.0 / 2)) + ) + + if (inplace_safe): + pt_att *= head_weights + else: + pt_att = pt_att * head_weights + + # [*, N_res, N_res, H] + pt_att = torch.sum(pt_att, dim=-1) * (-0.5) + + # [*, N_res, N_res] + square_mask = mask.unsqueeze(-1) * mask.unsqueeze(-2) + square_mask = self.inf * (square_mask - 1) + + # [*, H, N_res, N_res] + pt_att = permute_final_dims(pt_att, (2, 0, 1)) + + if (inplace_safe): + a += pt_att + del pt_att + a += square_mask.unsqueeze(-3) + # in-place softmax + attn_core_inplace_cuda.forward_( + a, + reduce(mul, a.shape[:-1]), + a.shape[-1], + ) + else: + a = a + pt_att + a = a + square_mask.unsqueeze(-3) + a = self.softmax(a) + + ################ + # Compute output + ################ + # [*, N_res, H, C_hidden] + o = torch.matmul( + a, v.transpose(-2, -3).to(dtype=a.dtype) + ).transpose(-2, -3) + + # [*, N_res, H * C_hidden] + o = flatten_final_dims(o, 2) + + # [*, H, 3, N_res, P_v] + if (inplace_safe): + v_pts = permute_final_dims(v_pts, (1, 3, 0, 2)) + o_pt = [ + torch.matmul(a, v.to(a.dtype)) + for v in torch.unbind(v_pts, dim=-3) + ] + o_pt = torch.stack(o_pt, dim=-3) + else: + o_pt = torch.sum( + ( + a[..., None, :, :, None] + * permute_final_dims(v_pts, (1, 3, 0, 2))[..., None, :, :] + ), + dim=-2, + ) + + # [*, N_res, H, P_v, 3] + o_pt = permute_final_dims(o_pt, (2, 0, 3, 1)) + o_pt = r[..., None, None].invert_apply(o_pt) + + # [*, N_res, H * P_v] + o_pt_norm = flatten_final_dims( + torch.sqrt(torch.sum(o_pt ** 2, dim=-1) + self.eps), 2 + ) + + # [*, N_res, H * P_v, 3] + o_pt = o_pt.reshape(*o_pt.shape[:-3], -1, 3) + o_pt = torch.unbind(o_pt, dim=-1) + + if (_offload_inference): + z[0] = z[0].to(o_pt.device) + + # [*, N_res, H, C_z] + o_pair = torch.matmul(a.transpose(-2, -3), z[0].to(dtype=a.dtype)) + + # [*, N_res, H * C_z] + o_pair = flatten_final_dims(o_pair, 2) + + # [*, N_res, C_s] + s = self.linear_out( + torch.cat( + (o, *o_pt, o_pt_norm, o_pair), dim=-1 + ).to(dtype=z[0].dtype) + ) + + return s + + +#TODO: This module follows the refactoring done in IPA for multimer. Running the regular IPA above +# in multimer mode should be equivalent, but tests do not pass unless using this version. Determine +# whether or not the increase in test error matters in practice. +class InvariantPointAttentionMultimer(nn.Module): + """ + Implements Algorithm 22. + """ + def __init__( + self, + c_s: int, + c_z: int, + c_hidden: int, + no_heads: int, + no_qk_points: int, + no_v_points: int, + inf: float = 1e5, + eps: float = 1e-8, + is_multimer: bool = True, + ): + """ + Args: + c_s: + Single representation channel dimension + c_z: + Pair representation channel dimension + c_hidden: + Hidden channel dimension + no_heads: + Number of attention heads + no_qk_points: + Number of query/key points to generate + no_v_points: + Number of value points to generate + """ + super(InvariantPointAttentionMultimer, self).__init__() + + self.c_s = c_s + self.c_z = c_z + self.c_hidden = c_hidden + self.no_heads = no_heads + self.no_qk_points = no_qk_points + self.no_v_points = no_v_points + self.inf = inf + self.eps = eps + + # These linear layers differ from their specifications in the + # supplement. There, they lack bias and use Glorot initialization. + # Here as in the official source, they have bias and use the default + # Lecun initialization. + hc = self.c_hidden * self.no_heads + self.linear_q = Linear(self.c_s, hc, bias=False) + + self.linear_q_points = PointProjection( + self.c_s, + self.no_qk_points, + self.no_heads, + is_multimer=True + ) + + self.linear_k = Linear(self.c_s, hc, bias=False) + self.linear_v = Linear(self.c_s, hc, bias=False) + self.linear_k_points = PointProjection( + self.c_s, + self.no_qk_points, + self.no_heads, + is_multimer=True + ) + + self.linear_v_points = PointProjection( + self.c_s, + self.no_v_points, + self.no_heads, + is_multimer=True + ) + + self.linear_b = Linear(self.c_z, self.no_heads) + + self.head_weights = nn.Parameter(torch.zeros((no_heads))) + ipa_point_weights_init_(self.head_weights) + + concat_out_dim = self.no_heads * ( + self.c_z + self.c_hidden + self.no_v_points * 4 + ) + self.linear_out = Linear(concat_out_dim, self.c_s, init="final") + + self.softmax = nn.Softmax(dim=-2) + + def forward( + self, + s: torch.Tensor, + z: Optional[torch.Tensor], + r: Union[Rigid, Rigid3Array], + mask: torch.Tensor, + inplace_safe: bool = False, + _offload_inference: bool = False, + _z_reference_list: Optional[Sequence[torch.Tensor]] = None, + ) -> torch.Tensor: + """ + Args: + s: + [*, N_res, C_s] single representation + z: + [*, N_res, N_res, C_z] pair representation + r: + [*, N_res] transformation object + mask: + [*, N_res] mask + Returns: + [*, N_res, C_s] single representation update + """ + if(_offload_inference and inplace_safe): + z = _z_reference_list + else: + z = [z] + + a = 0. + + point_variance = (max(self.no_qk_points, 1) * 9.0 / 2) + point_weights = math.sqrt(1.0 / point_variance) + + softplus = lambda x: torch.logaddexp(x, torch.zeros_like(x)) + + head_weights = softplus(self.head_weights) + point_weights = point_weights * head_weights + + ####################################### + # Generate scalar and point activations + ####################################### + + # [*, N_res, H, P_qk] + q_pts = Vec3Array.from_array(self.linear_q_points(s, r)) + + # [*, N_res, H, P_qk, 3] + k_pts = Vec3Array.from_array(self.linear_k_points(s, r)) + + pt_att = square_euclidean_distance(q_pts.unsqueeze(-3), k_pts.unsqueeze(-4), epsilon=0.) + pt_att = torch.sum(pt_att * point_weights[..., None], dim=-1) * (-0.5) + pt_att = pt_att.to(dtype=s.dtype) + a = a + pt_att + + scalar_variance = max(self.c_hidden, 1) * 1. + scalar_weights = math.sqrt(1.0 / scalar_variance) + + # [*, N_res, H * C_hidden] + q = self.linear_q(s) + k = self.linear_k(s) + + # [*, N_res, H, C_hidden] + q = q.view(q.shape[:-1] + (self.no_heads, -1)) + k = k.view(k.shape[:-1] + (self.no_heads, -1)) + + q = q * scalar_weights + a = a + torch.einsum('...qhc,...khc->...qkh', q, k) + + ########################## + # Compute attention scores + ########################## + # [*, N_res, N_res, H] + b = self.linear_b(z[0]) + + if (_offload_inference): + assert (sys.getrefcount(z[0]) == 2) + z[0] = z[0].cpu() + + a = a + b + + # [*, N_res, N_res] + square_mask = mask.unsqueeze(-1) * mask.unsqueeze(-2) + square_mask = self.inf * (square_mask - 1) + + a = a + square_mask.unsqueeze(-1) + a = a * math.sqrt(1. / 3) # Normalize by number of logit terms (3) + a = self.softmax(a) + + # [*, N_res, H * C_hidden] + v = self.linear_v(s) + + # [*, N_res, H, C_hidden] + v = v.view(v.shape[:-1] + (self.no_heads, -1)) + + o = torch.einsum('...qkh, ...khc->...qhc', a, v) + + # [*, N_res, H * C_hidden] + o = flatten_final_dims(o, 2) + + # [*, N_res, H, P_v, 3] + v_pts = Vec3Array.from_array(self.linear_v_points(s, r)) + + # [*, N_res, H, P_v] + o_pt = v_pts[..., None, :, :, :] * a.unsqueeze(-1) + o_pt = o_pt.sum(dim=-3) + # o_pt = Vec3Array( + # torch.sum(a.unsqueeze(-1) * v_pts[..., None, :, :, :].x, dim=-3), + # torch.sum(a.unsqueeze(-1) * v_pts[..., None, :, :, :].y, dim=-3), + # torch.sum(a.unsqueeze(-1) * v_pts[..., None, :, :, :].z, dim=-3), + # ) + + # [*, N_res, H * P_v, 3] + o_pt = o_pt.reshape(o_pt.shape[:-2] + (-1,)) + + # [*, N_res, H, P_v] + o_pt = r[..., None].apply_inverse_to_point(o_pt) + o_pt_flat = [o_pt.x, o_pt.y, o_pt.z] + o_pt_flat = [x.to(dtype=a.dtype) for x in o_pt_flat] + + # [*, N_res, H * P_v] + o_pt_norm = o_pt.norm(epsilon=1e-8) + + if (_offload_inference): + z[0] = z[0].to(o_pt.x.device) + + o_pair = torch.einsum('...ijh, ...ijc->...ihc', a, z[0].to(dtype=a.dtype)) + + # [*, N_res, H * C_z] + o_pair = flatten_final_dims(o_pair, 2) + + # [*, N_res, C_s] + s = self.linear_out( + torch.cat( + (o, *o_pt_flat, o_pt_norm, o_pair), dim=-1 + ).to(dtype=z[0].dtype) + ) + + return s + + +class BackboneUpdate(nn.Module): + """ + Implements part of Algorithm 23. + """ + + def __init__(self, c_s): + """ + Args: + c_s: + Single representation channel dimension + """ + super(BackboneUpdate, self).__init__() + + self.c_s = c_s + + self.linear = Linear(self.c_s, 6, init="final") + + def forward(self, s: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + [*, N_res, C_s] single representation + Returns: + [*, N_res, 6] update vector + """ + # [*, 6] + update = self.linear(s) + + return update + + +class StructureModuleTransitionLayer(nn.Module): + def __init__(self, c): + super(StructureModuleTransitionLayer, self).__init__() + + self.c = c + + self.linear_1 = Linear(self.c, self.c, init="relu") + self.linear_2 = Linear(self.c, self.c, init="relu") + self.linear_3 = Linear(self.c, self.c, init="final") + + self.relu = nn.ReLU() + + def forward(self, s): + s_initial = s + s = self.linear_1(s) + s = self.relu(s) + s = self.linear_2(s) + s = self.relu(s) + s = self.linear_3(s) + + s = s + s_initial + + return s + + +class StructureModuleTransition(nn.Module): + def __init__(self, c, num_layers, dropout_rate): + super(StructureModuleTransition, self).__init__() + + self.c = c + self.num_layers = num_layers + self.dropout_rate = dropout_rate + + self.layers = nn.ModuleList() + for _ in range(self.num_layers): + l = StructureModuleTransitionLayer(self.c) + self.layers.append(l) + + self.dropout = nn.Dropout(self.dropout_rate) + self.layer_norm = LayerNorm(self.c) + + def forward(self, s): + for l in self.layers: + s = l(s) + + s = self.dropout(s) + s = self.layer_norm(s) + + return s + + +class StructureModule(nn.Module): + def __init__( + self, + c_s, + c_z, + c_ipa, + c_resnet, + no_heads_ipa, + no_qk_points, + no_v_points, + dropout_rate, + no_blocks, + no_transition_layers, + no_resnet_blocks, + no_angles, + trans_scale_factor, + epsilon, + inf, + is_multimer=False, + **kwargs, + ): + """ + Args: + c_s: + Single representation channel dimension + c_z: + Pair representation channel dimension + c_ipa: + IPA hidden channel dimension + c_resnet: + Angle resnet (Alg. 23 lines 11-14) hidden channel dimension + no_heads_ipa: + Number of IPA heads + no_qk_points: + Number of query/key points to generate during IPA + no_v_points: + Number of value points to generate during IPA + dropout_rate: + Dropout rate used throughout the layer + no_blocks: + Number of structure module blocks + no_transition_layers: + Number of layers in the single representation transition + (Alg. 23 lines 8-9) + no_resnet_blocks: + Number of blocks in the angle resnet + no_angles: + Number of angles to generate in the angle resnet + trans_scale_factor: + Scale of single representation transition hidden dimension + epsilon: + Small number used in angle resnet normalization + inf: + Large number used for attention masking + """ + super(StructureModule, self).__init__() + + self.c_s = c_s + self.c_z = c_z + self.c_ipa = c_ipa + self.c_resnet = c_resnet + self.no_heads_ipa = no_heads_ipa + self.no_qk_points = no_qk_points + self.no_v_points = no_v_points + self.dropout_rate = dropout_rate + self.no_blocks = no_blocks + self.no_transition_layers = no_transition_layers + self.no_resnet_blocks = no_resnet_blocks + self.no_angles = no_angles + self.trans_scale_factor = trans_scale_factor + self.epsilon = epsilon + self.inf = inf + self.is_multimer = is_multimer + + # Buffers to be lazily initialized later + # self.default_frames + # self.group_idx + # self.atom_mask + # self.lit_positions + + self.layer_norm_s = LayerNorm(self.c_s) + self.layer_norm_z = LayerNorm(self.c_z) + + self.linear_in = Linear(self.c_s, self.c_s) + + ipa = InvariantPointAttention if not self.is_multimer else InvariantPointAttentionMultimer + self.ipa = ipa( + self.c_s, + self.c_z, + self.c_ipa, + self.no_heads_ipa, + self.no_qk_points, + self.no_v_points, + inf=self.inf, + eps=self.epsilon, + is_multimer=self.is_multimer, + ) + + self.ipa_dropout = nn.Dropout(self.dropout_rate) + self.layer_norm_ipa = LayerNorm(self.c_s) + + self.transition = StructureModuleTransition( + self.c_s, + self.no_transition_layers, + self.dropout_rate, + ) + + if self.is_multimer: + self.bb_update = QuatRigid(self.c_s, full_quat=False) + else: + self.bb_update = BackboneUpdate(self.c_s) + + self.angle_resnet = AngleResnet( + self.c_s, + self.c_resnet, + self.no_resnet_blocks, + self.no_angles, + self.epsilon, + ) + + def _forward_monomer( + self, + evoformer_output_dict, + aatype, + mask=None, + inplace_safe=False, + _offload_inference=False, + ): + """ + Args: + evoformer_output_dict: + Dictionary containing: + "single": + [*, N_res, C_s] single representation + "pair": + [*, N_res, N_res, C_z] pair representation + aatype: + [*, N_res] amino acid indices + mask: + Optional [*, N_res] sequence mask + Returns: + A dictionary of outputs + """ + s = evoformer_output_dict["single"] + + if mask is None: + # [*, N] + mask = s.new_ones(s.shape[:-1]) + + # [*, N, C_s] + s = self.layer_norm_s(s) + + # [*, N, N, C_z] + z = self.layer_norm_z(evoformer_output_dict["pair"]) + + z_reference_list = None + if (_offload_inference): + assert (sys.getrefcount(evoformer_output_dict["pair"]) == 2) + evoformer_output_dict["pair"] = evoformer_output_dict["pair"].cpu() + z_reference_list = [z] + z = None + + # [*, N, C_s] + s_initial = s + s = self.linear_in(s) + + # [*, N] + rigids = Rigid.identity( + s.shape[:-1], + s.dtype, + s.device, + self.training, + fmt="quat", + ) + outputs = [] + for i in range(self.no_blocks): + # [*, N, C_s] + s = s + self.ipa( + s, + z, + rigids, + mask, + inplace_safe=inplace_safe, + _offload_inference=_offload_inference, + _z_reference_list=z_reference_list + ) + s = self.ipa_dropout(s) + s = self.layer_norm_ipa(s) + s = self.transition(s) + + # [*, N] + rigids = rigids.compose_q_update_vec(self.bb_update(s)) + + # To hew as closely as possible to AlphaFold, we convert our + # quaternion-based transformations to rotation-matrix ones + # here + backb_to_global = Rigid( + Rotation( + rot_mats=rigids.get_rots().get_rot_mats(), + quats=None + ), + rigids.get_trans(), + ) + + backb_to_global = backb_to_global.scale_translation( + self.trans_scale_factor + ) + + # [*, N, 7, 2] + unnormalized_angles, angles = self.angle_resnet(s, s_initial) + + all_frames_to_global = self.torsion_angles_to_frames( + backb_to_global, + angles, + aatype, + ) + + pred_xyz = self.frames_and_literature_positions_to_atom14_pos( + all_frames_to_global, + aatype, + ) + + scaled_rigids = rigids.scale_translation(self.trans_scale_factor) + + preds = { + "frames": scaled_rigids.to_tensor_7(), + "sidechain_frames": all_frames_to_global.to_tensor_4x4(), + "unnormalized_angles": unnormalized_angles, + "angles": angles, + "positions": pred_xyz, + "states": s, + } + + outputs.append(preds) + + rigids = rigids.stop_rot_gradient() + + del z, z_reference_list + + if (_offload_inference): + evoformer_output_dict["pair"] = ( + evoformer_output_dict["pair"].to(s.device) + ) + + outputs = dict_multimap(torch.stack, outputs) + outputs["single"] = s + + return outputs + + def _forward_multimer( + self, + evoformer_output_dict, + aatype, + mask=None, + inplace_safe=False, + _offload_inference=False, + ): + s = evoformer_output_dict["single"] + + if mask is None: + # [*, N] + mask = s.new_ones(s.shape[:-1]) + + # [*, N, C_s] + s = self.layer_norm_s(s) + + # [*, N, N, C_z] + z = self.layer_norm_z(evoformer_output_dict["pair"]) + + z_reference_list = None + if (_offload_inference): + assert (sys.getrefcount(evoformer_output_dict["pair"]) == 2) + evoformer_output_dict["pair"] = evoformer_output_dict["pair"].cpu() + z_reference_list = [z] + z = None + + # [*, N, C_s] + s_initial = s + s = self.linear_in(s) + + # [*, N] + rigids = Rigid3Array.identity( + s.shape[:-1], + s.device, + ) + outputs = [] + for i in range(self.no_blocks): + # [*, N, C_s] + s = s + self.ipa( + s, + z, + rigids, + mask, + inplace_safe=inplace_safe, + _offload_inference=_offload_inference, + _z_reference_list=z_reference_list + ) + s = self.ipa_dropout(s) + s = self.layer_norm_ipa(s) + s = self.transition(s) + + # [*, N] + rigids = rigids @ self.bb_update(s) + + # [*, N, 7, 2] + unnormalized_angles, angles = self.angle_resnet(s, s_initial) + + all_frames_to_global = self.torsion_angles_to_frames( + rigids.scale_translation(self.trans_scale_factor), + angles, + aatype, + ) + + pred_xyz = self.frames_and_literature_positions_to_atom14_pos( + all_frames_to_global, + aatype, + ) + + preds = { + "frames": rigids.scale_translation(self.trans_scale_factor).to_tensor(), + "sidechain_frames": all_frames_to_global.to_tensor_4x4(), + "unnormalized_angles": unnormalized_angles, + "angles": angles, + "positions": pred_xyz, + } + + preds = {k: v.to(dtype=s.dtype) for k, v in preds.items()} + + outputs.append(preds) + + rigids = rigids.stop_rot_gradient() + + del z, z_reference_list + + if (_offload_inference): + evoformer_output_dict["pair"] = ( + evoformer_output_dict["pair"].to(s.device) + ) + + outputs = dict_multimap(torch.stack, outputs) + outputs["single"] = s + + return outputs + + def forward( + self, + evoformer_output_dict, + aatype, + mask=None, + inplace_safe=False, + _offload_inference=False, + ): + """ + Args: + s: + [*, N_res, C_s] single representation + z: + [*, N_res, N_res, C_z] pair representation + aatype: + [*, N_res] amino acid indices + mask: + Optional [*, N_res] sequence mask + Returns: + A dictionary of outputs + """ + if(self.is_multimer): + outputs = self._forward_multimer(evoformer_output_dict, aatype, mask, inplace_safe, _offload_inference) + else: + outputs = self._forward_monomer(evoformer_output_dict, aatype, mask, inplace_safe, _offload_inference) + + return outputs + + def _init_residue_constants(self, float_dtype, device): + if not hasattr(self, "default_frames"): + self.register_buffer( + "default_frames", + torch.tensor( + restype_rigid_group_default_frame, + dtype=float_dtype, + device=device, + requires_grad=False, + ), + persistent=False, + ) + if not hasattr(self, "group_idx"): + self.register_buffer( + "group_idx", + torch.tensor( + restype_atom14_to_rigid_group, + device=device, + requires_grad=False, + ), + persistent=False, + ) + if not hasattr(self, "atom_mask"): + self.register_buffer( + "atom_mask", + torch.tensor( + restype_atom14_mask, + dtype=float_dtype, + device=device, + requires_grad=False, + ), + persistent=False, + ) + if not hasattr(self, "lit_positions"): + self.register_buffer( + "lit_positions", + torch.tensor( + restype_atom14_rigid_group_positions, + dtype=float_dtype, + device=device, + requires_grad=False, + ), + persistent=False, + ) + + def torsion_angles_to_frames(self, r, alpha, f): + # Lazily initialize the residue constants on the correct device + self._init_residue_constants(alpha.dtype, alpha.device) + # Separated purely to make testing less annoying + return torsion_angles_to_frames(r, alpha, f, self.default_frames) + + def frames_and_literature_positions_to_atom14_pos( + self, r, f # [*, N, 8] # [*, N] + ): + # Lazily initialize the residue constants on the correct device + self._init_residue_constants(r.dtype, r.device) + return frames_and_literature_positions_to_atom14_pos( + r, + f, + self.default_frames, + self.group_idx, + self.atom_mask, + self.lit_positions, + ) diff --git a/model/openfold/template.py b/model/openfold/template.py new file mode 100644 index 0000000000000000000000000000000000000000..9be26c4216d4b7a44d3497cc8e6b0f525bad9734 --- /dev/null +++ b/model/openfold/template.py @@ -0,0 +1,693 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from functools import partial +import math +import sys +from typing import Optional, List + +import torch +import torch.nn as nn + +from model.openfold.primitives import LayerNorm, Attention +from model.openfold.dropout import ( + DropoutRowwise, + DropoutColumnwise, +) +from model.openfold.pair_transition import PairTransition +from model.openfold.triangular_attention import ( + TriangleAttentionStartingNode, + TriangleAttentionEndingNode, +) +from model.openfold.triangular_multiplicative_update import ( + TriangleMultiplicationOutgoing, + TriangleMultiplicationIncoming, + FusedTriangleMultiplicationOutgoing, + FusedTriangleMultiplicationIncoming +) +from onescience.utils.openfold.checkpointing import checkpoint_blocks +from onescience.utils.openfold.chunk_utils import ( + chunk_layer, + ChunkSizeTuner, +) +from onescience.utils.openfold.feats import ( + build_template_angle_feat, + build_template_pair_feat, +) +from onescience.utils.openfold.tensor_utils import ( + add, + permute_final_dims, + tensor_tree_map, +) + + +class TemplatePointwiseAttention(nn.Module): + """ + Implements Algorithm 17. + """ + + def __init__(self, c_t, c_z, c_hidden, no_heads, inf, **kwargs): + """ + Args: + c_t: + Template embedding channel dimension + c_z: + Pair embedding channel dimension + c_hidden: + Hidden channel dimension + """ + super(TemplatePointwiseAttention, self).__init__() + + self.c_t = c_t + self.c_z = c_z + self.c_hidden = c_hidden + self.no_heads = no_heads + self.inf = inf + + self.mha = Attention( + self.c_z, + self.c_t, + self.c_t, + self.c_hidden, + self.no_heads, + gating=False, + ) + + def _chunk(self, + z: torch.Tensor, + t: torch.Tensor, + biases: List[torch.Tensor], + chunk_size: int, + use_lma: bool = False, + ) -> torch.Tensor: + mha_inputs = { + "q_x": z, + "kv_x": t, + "biases": biases, + } + return chunk_layer( + partial(self.mha, use_lma=use_lma), + mha_inputs, + chunk_size=chunk_size, + no_batch_dims=len(z.shape[:-2]), + ) + + def forward(self, + t: torch.Tensor, + z: torch.Tensor, + template_mask: Optional[torch.Tensor] = None, + # This module suffers greatly from a small chunk size + chunk_size: Optional[int] = 256, + use_lma: bool = False, + ) -> torch.Tensor: + """ + Args: + t: + [*, N_templ, N_res, N_res, C_t] template embedding + z: + [*, N_res, N_res, C_t] pair embedding + template_mask: + [*, N_templ] template mask + Returns: + [*, N_res, N_res, C_z] pair embedding update + """ + if template_mask is None: + template_mask = t.new_ones(t.shape[:-3]) + + bias = self.inf * (template_mask[..., None, None, None, None, :] - 1) + + # [*, N_res, N_res, 1, C_z] + z = z.unsqueeze(-2) + + # [*, N_res, N_res, N_temp, C_t] + t = permute_final_dims(t, (1, 2, 0, 3)) + + # [*, N_res, N_res, 1, C_z] + biases = [bias] + if chunk_size is not None and not self.training: + z = self._chunk(z, t, biases, chunk_size, use_lma=use_lma) + else: + z = self.mha(q_x=z, kv_x=t, biases=biases, use_lma=use_lma) + + # [*, N_res, N_res, C_z] + z = z.squeeze(-2) + + return z + + +class TemplatePairStackBlock(nn.Module): + def __init__( + self, + c_t: int, + c_hidden_tri_att: int, + c_hidden_tri_mul: int, + no_heads: int, + pair_transition_n: int, + dropout_rate: float, + tri_mul_first: bool, + fuse_projection_weights: bool, + inf: float, + **kwargs, + ): + super(TemplatePairStackBlock, self).__init__() + + self.c_t = c_t + self.c_hidden_tri_att = c_hidden_tri_att + self.c_hidden_tri_mul = c_hidden_tri_mul + self.no_heads = no_heads + self.pair_transition_n = pair_transition_n + self.dropout_rate = dropout_rate + self.inf = inf + self.tri_mul_first = tri_mul_first + + self.dropout_row = DropoutRowwise(self.dropout_rate) + self.dropout_col = DropoutColumnwise(self.dropout_rate) + + self.tri_att_start = TriangleAttentionStartingNode( + self.c_t, + self.c_hidden_tri_att, + self.no_heads, + inf=inf, + ) + self.tri_att_end = TriangleAttentionEndingNode( + self.c_t, + self.c_hidden_tri_att, + self.no_heads, + inf=inf, + ) + + if fuse_projection_weights: + self.tri_mul_out = FusedTriangleMultiplicationOutgoing( + self.c_t, + self.c_hidden_tri_mul, + ) + self.tri_mul_in = FusedTriangleMultiplicationIncoming( + self.c_t, + self.c_hidden_tri_mul, + ) + else: + self.tri_mul_out = TriangleMultiplicationOutgoing( + self.c_t, + self.c_hidden_tri_mul, + ) + self.tri_mul_in = TriangleMultiplicationIncoming( + self.c_t, + self.c_hidden_tri_mul, + ) + + self.pair_transition = PairTransition( + self.c_t, + self.pair_transition_n, + ) + + def tri_att_start_end(self, + single: torch.Tensor, + _attn_chunk_size: Optional[int], + single_mask: torch.Tensor, + use_deepspeed_evo_attention: bool, + use_lma: bool, + inplace_safe: bool): + single = add(single, + self.dropout_row( + self.tri_att_start( + single, + chunk_size=_attn_chunk_size, + mask=single_mask, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe, + ) + ), + inplace_safe, + ) + + single = add(single, + self.dropout_col( + self.tri_att_end( + single, + chunk_size=_attn_chunk_size, + mask=single_mask, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe, + ) + ), + inplace_safe, + ) + + return single + + def tri_mul_out_in(self, + single: torch.Tensor, + single_mask: torch.Tensor, + inplace_safe: bool): + tmu_update = self.tri_mul_out( + single, + mask=single_mask, + inplace_safe=inplace_safe, + _add_with_inplace=True, + ) + if not inplace_safe: + single = single + self.dropout_row(tmu_update) + else: + single = tmu_update + + del tmu_update + + tmu_update = self.tri_mul_in( + single, + mask=single_mask, + inplace_safe=inplace_safe, + _add_with_inplace=True, + ) + if not inplace_safe: + single = single + self.dropout_row(tmu_update) + else: + single = tmu_update + + del tmu_update + + return single + + def forward(self, + z: torch.Tensor, + mask: torch.Tensor, + chunk_size: Optional[int] = None, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + inplace_safe: bool = False, + _mask_trans: bool = True, + _attn_chunk_size: Optional[int] = None, + ): + if _attn_chunk_size is None: + _attn_chunk_size = chunk_size + + single_templates = [ + t.unsqueeze(-4) for t in torch.unbind(z, dim=-4) + ] + single_templates_masks = [ + m.unsqueeze(-3) for m in torch.unbind(mask, dim=-3) + ] + + for i in range(len(single_templates)): + single = single_templates[i] + single_mask = single_templates_masks[i] + + if self.tri_mul_first: + single = self.tri_att_start_end(single=self.tri_mul_out_in(single=single, + single_mask=single_mask, + inplace_safe=inplace_safe), + _attn_chunk_size=_attn_chunk_size, + single_mask=single_mask, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe) + else: + single = self.tri_mul_out_in( + single=self.tri_att_start_end(single=single, + _attn_chunk_size=_attn_chunk_size, + single_mask=single_mask, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe), + single_mask=single_mask, + inplace_safe=inplace_safe) + + single = add(single, + self.pair_transition( + single, + mask=single_mask if _mask_trans else None, + chunk_size=chunk_size, + ), + inplace_safe, + ) + + if not inplace_safe: + single_templates[i] = single + + if not inplace_safe: + z = torch.cat(single_templates, dim=-4) + + return z + + +class TemplatePairStack(nn.Module): + """ + Implements Algorithm 16. + """ + + def __init__( + self, + c_t, + c_hidden_tri_att, + c_hidden_tri_mul, + no_blocks, + no_heads, + pair_transition_n, + dropout_rate, + tri_mul_first, + fuse_projection_weights, + blocks_per_ckpt, + tune_chunk_size: bool = False, + inf=1e9, + **kwargs, + ): + """ + Args: + c_t: + Template embedding channel dimension + c_hidden_tri_att: + Per-head hidden dimension for triangular attention + c_hidden_tri_att: + Hidden dimension for triangular multiplication + no_blocks: + Number of blocks in the stack + pair_transition_n: + Scale of pair transition (Alg. 15) hidden dimension + dropout_rate: + Dropout rate used throughout the stack + blocks_per_ckpt: + Number of blocks per activation checkpoint. None disables + activation checkpointing + """ + super(TemplatePairStack, self).__init__() + + self.blocks_per_ckpt = blocks_per_ckpt + + self.blocks = nn.ModuleList() + for _ in range(no_blocks): + block = TemplatePairStackBlock( + c_t=c_t, + c_hidden_tri_att=c_hidden_tri_att, + c_hidden_tri_mul=c_hidden_tri_mul, + no_heads=no_heads, + pair_transition_n=pair_transition_n, + dropout_rate=dropout_rate, + tri_mul_first=tri_mul_first, + fuse_projection_weights=fuse_projection_weights, + inf=inf, + ) + self.blocks.append(block) + + self.layer_norm = LayerNorm(c_t) + + self.tune_chunk_size = tune_chunk_size + self.chunk_size_tuner = None + if tune_chunk_size: + self.chunk_size_tuner = ChunkSizeTuner() + + def forward( + self, + t: torch.tensor, + mask: torch.tensor, + chunk_size: int, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + inplace_safe: bool = False, + _mask_trans: bool = True, + ): + """ + Args: + t: + [*, N_templ, N_res, N_res, C_t] template embedding + mask: + [*, N_templ, N_res, N_res] mask + Returns: + [*, N_templ, N_res, N_res, C_t] template embedding update + """ + if mask.shape[-3] == 1: + expand_idx = list(mask.shape) + expand_idx[-3] = t.shape[-4] + mask = mask.expand(*expand_idx) + + blocks = [ + partial( + b, + mask=mask, + chunk_size=chunk_size, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe, + _mask_trans=_mask_trans, + ) + for b in self.blocks + ] + + if chunk_size is not None and self.chunk_size_tuner is not None: + assert (not self.training) + tuned_chunk_size = self.chunk_size_tuner.tune_chunk_size( + representative_fn=blocks[0], + args=(t.clone(),), + min_chunk_size=chunk_size, + ) + blocks = [ + partial(b, + chunk_size=tuned_chunk_size, + _attn_chunk_size=max(chunk_size, tuned_chunk_size // 4), + ) for b in blocks + ] + + t, = checkpoint_blocks( + blocks=blocks, + args=(t,), + blocks_per_ckpt=self.blocks_per_ckpt if self.training else None, + ) + + t = self.layer_norm(t) + + return t + + +def embed_templates_offload( + model, + batch, + z, + pair_mask, + templ_dim, + template_chunk_size=256, + inplace_safe=False, +): + """ + Args: + model: + An AlphaFold model object + batch: + An AlphaFold input batch. See documentation of AlphaFold. + z: + A [*, N, N, C_z] pair embedding + pair_mask: + A [*, N, N] pair mask + templ_dim: + The template dimension of the template tensors in batch + template_chunk_size: + Integer value controlling how quickly the offloaded pair embedding + tensor is brought back into GPU memory. In dire straits, can be + lowered to reduce memory consumption of this function even more. + Returns: + A dictionary of template pair and angle embeddings. + + A version of the "embed_templates" method of the AlphaFold class that + offloads the large template pair tensor to CPU. Slower but more frugal + with GPU memory than the original. Useful for long-sequence inference. + """ + # Embed the templates one at a time (with a poor man's vmap) + pair_embeds_cpu = [] + n = z.shape[-2] + n_templ = batch["template_aatype"].shape[templ_dim] + for i in range(n_templ): + idx = batch["template_aatype"].new_tensor(i) + single_template_feats = tensor_tree_map( + lambda t: torch.index_select(t, templ_dim, idx).squeeze(templ_dim), + batch, + ) + + # [*, N, N, C_t] + t = build_template_pair_feat( + single_template_feats, + use_unit_vector=model.config.template.use_unit_vector, + inf=model.config.template.inf, + eps=model.config.template.eps, + **model.config.template.distogram, + ).to(z.dtype) + t = model.template_pair_embedder(t) + + # [*, 1, N, N, C_z] + t = model.template_pair_stack( + t.unsqueeze(templ_dim), + pair_mask.unsqueeze(-3).to(dtype=z.dtype), + chunk_size=model.globals.chunk_size, + use_deepspeed_evo_attention=model.globals.use_deepspeed_evo_attention, + use_lma=model.globals.use_lma, + inplace_safe=inplace_safe, + _mask_trans=model.config._mask_trans, + ) + + assert (sys.getrefcount(t) == 2) + + pair_embeds_cpu.append(t.cpu()) + + del t + + # Preallocate the output tensor + t = z.new_zeros(z.shape) + + for i in range(0, n, template_chunk_size): + pair_chunks = [ + p[..., i: i + template_chunk_size, :, :] for p in pair_embeds_cpu + ] + pair_chunk = torch.cat(pair_chunks, dim=templ_dim).to(device=z.device) + z_chunk = z[..., i: i + template_chunk_size, :, :] + att_chunk = model.template_pointwise_att( + pair_chunk, + z_chunk, + template_mask=batch["template_mask"].to(dtype=z.dtype), + use_lma=model.globals.use_lma, + ) + + t[..., i: i + template_chunk_size, :, :] = att_chunk + + del pair_chunks + + if inplace_safe: + t = t * (torch.sum(batch["template_mask"], dim=-1) > 0) + else: + t *= (torch.sum(batch["template_mask"], dim=-1) > 0) + + ret = {} + if model.config.template.embed_angles: + template_angle_feat = build_template_angle_feat( + batch, + ) + + # [*, N, C_m] + a = model.template_single_embedder(template_angle_feat) + + ret["template_single_embedding"] = a + + ret.update({"template_pair_embedding": t}) + + return ret + + +def embed_templates_average( + model, + batch, + z, + pair_mask, + templ_dim, + templ_group_size=2, + inplace_safe=False, +): + """ + Args: + model: + An AlphaFold model object + batch: + An AlphaFold input batch. See documentation of AlphaFold. + z: + A [*, N, N, C_z] pair embedding + pair_mask: + A [*, N, N] pair mask + templ_dim: + The template dimension of the template tensors in batch + templ_group_size: + Granularity of the approximation. Larger values trade memory for + greater proximity to the original function + Returns: + A dictionary of template pair and angle embeddings. + + A memory-efficient approximation of the "embed_templates" method of the + AlphaFold class. Instead of running pointwise attention over pair + embeddings for all of the templates at the same time, it splits templates + into groups of size templ_group_size, computes embeddings for each group + normally, and then averages the group embeddings. In our experiments, this + approximation has a minimal effect on the quality of the resulting + embedding, while its low memory footprint allows the number of templates + to scale almost indefinitely. + """ + # Embed the templates one at a time (with a poor man's vmap) + n = z.shape[-2] + n_templ = batch["template_aatype"].shape[templ_dim] + out_tensor = z.new_zeros(z.shape) + for i in range(0, n_templ, templ_group_size): + def slice_template_tensor(t): + s = [slice(None) for _ in t.shape] + s[templ_dim] = slice(i, i + templ_group_size) + return t[s] + + template_feats = tensor_tree_map( + slice_template_tensor, + batch, + ) + + # [*, N, N, C_t] + t = build_template_pair_feat( + template_feats, + use_unit_vector=model.config.template.use_unit_vector, + inf=model.config.template.inf, + eps=model.config.template.eps, + **model.config.template.distogram, + ).to(z.dtype) + + # [*, S_t, N, N, C_z] + t = model.template_pair_embedder(t) + t = model.template_pair_stack( + t, + pair_mask.unsqueeze(-3).to(dtype=z.dtype), + chunk_size=model.globals.chunk_size, + use_deepspeed_evo_attention=model.globals.use_deepspeed_evo_attention, + use_lma=model.globals.use_lma, + inplace_safe=inplace_safe, + _mask_trans=model.config._mask_trans, + ) + + t = model.template_pointwise_att( + t, + z, + template_mask=template_feats["template_mask"].to(dtype=z.dtype), + use_lma=model.globals.use_lma, + ) + + denom = math.ceil(n_templ / templ_group_size) + if inplace_safe: + t /= denom + else: + t = t / denom + + if inplace_safe: + out_tensor += t + else: + out_tensor = out_tensor + t + + del t + + if inplace_safe: + out_tensor *= (torch.sum(batch["template_mask"], dim=-1) > 0) + else: + out_tensor = out_tensor * (torch.sum(batch["template_mask"], dim=-1) > 0) + + ret = {} + if model.config.template.embed_angles: + template_angle_feat = build_template_angle_feat( + batch, + ) + + # [*, N, C_m] + a = model.template_single_embedder(template_angle_feat) + + ret["template_single_embedding"] = a + + ret.update({"template_pair_embedding": out_tensor}) + + return ret diff --git a/model/openfold/torchscript.py b/model/openfold/torchscript.py new file mode 100644 index 0000000000000000000000000000000000000000..ab255721eeb4bd83be84feceabaa0d7ca701df37 --- /dev/null +++ b/model/openfold/torchscript.py @@ -0,0 +1,215 @@ +# Copyright 2021 AlQuraishi Laboratory +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Sequence, Tuple + +import torch +import torch.nn as nn + +from model.openfold.dropout import ( + DropoutRowwise, + DropoutColumnwise, +) +from model.openfold.evoformer import ( + EvoformerBlock, + EvoformerStack, +) +from model.openfold.outer_product_mean import OuterProductMean +from model.openfold.msa import ( + MSARowAttentionWithPairBias, + MSAColumnAttention, + MSAColumnGlobalAttention, +) +from model.openfold.pair_transition import PairTransition +from model.openfold.primitives import Attention, GlobalAttention +from model.openfold.structure_module import ( + InvariantPointAttention, + BackboneUpdate, +) +from model.openfold.template import TemplatePairStackBlock +from model.openfold.triangular_attention import ( + TriangleAttentionStartingNode, + TriangleAttentionEndingNode, +) +from model.openfold.triangular_multiplicative_update import ( + TriangleMultiplicationOutgoing, + TriangleMultiplicationIncoming, +) + + +def script_preset_(model: torch.nn.Module): + """ + TorchScript a handful of low-level but frequently used submodule types + that are known to be scriptable. + + Args: + model: + A torch.nn.Module. It should contain at least some modules from + this repository, or this function won't do anything. + """ + script_submodules_( + model, + [ + nn.Dropout, + Attention, + GlobalAttention, + EvoformerBlock, + #TemplatePairStackBlock, + ], + attempt_trace=False, + batch_dims=None, + ) + + +def _get_module_device(module: torch.nn.Module) -> torch.device: + """ + Fetches the device of a module, assuming that all of the module's + parameters reside on a single device + + Args: + module: A torch.nn.Module + Returns: + The module's device + """ + return next(module.parameters()).device + + +def _trace_module(module, batch_dims=None): + if(batch_dims is None): + batch_dims = () + + # Stand-in values + n_seq = 10 + n_res = 10 + + device = _get_module_device(module) + + def msa(channel_dim): + return torch.rand( + (*batch_dims, n_seq, n_res, channel_dim), + device=device, + ) + + def pair(channel_dim): + return torch.rand( + (*batch_dims, n_res, n_res, channel_dim), + device=device, + ) + + if(isinstance(module, MSARowAttentionWithPairBias)): + inputs = { + "forward": ( + msa(module.c_in), # m + pair(module.c_z), # z + torch.randint( + 0, 2, + (*batch_dims, n_seq, n_res) + ), # mask + ), + } + elif(isinstance(module, MSAColumnAttention)): + inputs = { + "forward": ( + msa(module.c_in), # m + torch.randint( + 0, 2, + (*batch_dims, n_seq, n_res) + ), # mask + ), + } + elif(isinstance(module, OuterProductMean)): + inputs = { + "forward": ( + msa(module.c_m), + torch.randint( + 0, 2, + (*batch_dims, n_seq, n_res) + ) + ) + } + else: + raise TypeError( + f"tracing is not supported for modules of type {type(module)}" + ) + + return torch.jit.trace_module(module, inputs) + + +def _script_submodules_helper_( + model, + types, + attempt_trace, + to_trace, +): + for name, child in model.named_children(): + if(types is None or any(isinstance(child, t) for t in types)): + try: + scripted = torch.jit.script(child) + setattr(model, name, scripted) + continue + except (RuntimeError, torch.jit.frontend.NotSupportedError) as e: + if(attempt_trace): + to_trace.add(type(child)) + else: + raise e + + _script_submodules_helper_(child, types, attempt_trace, to_trace) + + +def _trace_submodules_( + model, + types, + batch_dims=None, +): + for name, child in model.named_children(): + if(any(isinstance(child, t) for t in types)): + traced = _trace_module(child, batch_dims=batch_dims) + setattr(model, name, traced) + else: + _trace_submodules_(child, types, batch_dims=batch_dims) + + +def script_submodules_( + model: nn.Module, + types: Optional[Sequence[type]] = None, + attempt_trace: Optional[bool] = True, + batch_dims: Optional[Tuple[int]] = None, +): + """ + Convert all submodules whose types match one of those in the input + list to recursively scripted equivalents in place. To script the entire + model, just call torch.jit.script on it directly. + + When types is None, all submodules are scripted. + + Args: + model: + A torch.nn.Module + types: + A list of types of submodules to script + attempt_trace: + Whether to attempt to trace specified modules if scripting + fails. Recall that tracing eliminates all conditional + logic---with great tracing comes the mild responsibility of + having to remember to ensure that the modules in question + perform the same computations no matter what. + """ + to_trace = set() + + # Aggressively script as much as possible first... + _script_submodules_helper_(model, types, attempt_trace, to_trace) + + # ... and then trace stragglers. + if(attempt_trace and len(to_trace) > 0): + _trace_submodules_(model, to_trace, batch_dims=batch_dims) diff --git a/model/openfold/triangular_attention.py b/model/openfold/triangular_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..9c92b86bdf77065d48b72e94c71b736f70f682e7 --- /dev/null +++ b/model/openfold/triangular_attention.py @@ -0,0 +1,168 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import partialmethod, partial +import math +from typing import Optional, List + +import torch +import torch.nn as nn + +from model.openfold.primitives import Linear, LayerNorm, Attention +from onescience.utils.openfold.chunk_utils import chunk_layer +from onescience.utils.openfold.tensor_utils import ( + permute_final_dims, + flatten_final_dims, +) + + +class TriangleAttention(nn.Module): + def __init__( + self, c_in, c_hidden, no_heads, starting=True, inf=1e9, bias: bool=True + ): + """ + Args: + c_in: + Input channel dimension + c_hidden: + Overall hidden channel dimension (not per-head) + no_heads: + Number of attention heads + """ + super(TriangleAttention, self).__init__() + + self.c_in = c_in + self.c_hidden = c_hidden + self.no_heads = no_heads + self.starting = starting + self.inf = inf + + self.layer_norm = LayerNorm(self.c_in) + + self.linear = Linear(c_in, self.no_heads, bias=False, init="normal") + if bias==False: + self.mha = Attention( + self.c_in, self.c_in, self.c_in, self.c_hidden, self.no_heads, bias=False + ) + else: + self.mha = Attention( + self.c_in, self.c_in, self.c_in, self.c_hidden, self.no_heads + ) + + @torch.jit.ignore + def _chunk(self, + x: torch.Tensor, + biases: List[torch.Tensor], + chunk_size: int, + use_memory_efficient_kernel: bool = False, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + inplace_safe: bool = False, + ) -> torch.Tensor: + "triangle! triangle!" + mha_inputs = { + "q_x": x, + "kv_x": x, + "biases": biases, + } + + return chunk_layer( + partial( + self.mha, + use_memory_efficient_kernel=use_memory_efficient_kernel, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma + ), + mha_inputs, + chunk_size=chunk_size, + no_batch_dims=len(x.shape[:-2]), + _out=x if inplace_safe else None, + ) + + def forward(self, + x: torch.Tensor, + mask: Optional[torch.Tensor] = None, + chunk_size: Optional[int] = None, + use_memory_efficient_kernel: bool = False, + use_deepspeed_evo_attention: bool = False, + use_lma: bool = False, + inplace_safe: bool = False, + ) -> torch.Tensor: + """ + Args: + x: + [*, I, J, C_in] input tensor (e.g. the pair representation) + Returns: + [*, I, J, C_in] output tensor + """ + if mask is None: + # [*, I, J] + mask = x.new_ones( + x.shape[:-1], + ) + + if(not self.starting): + x = x.transpose(-2, -3) + mask = mask.transpose(-1, -2) + + # [*, I, J, C_in] + x = self.layer_norm(x) + + # [*, I, 1, 1, J] + mask_bias = (self.inf * (mask - 1))[..., :, None, None, :] + + # [*, H, I, J] + triangle_bias = permute_final_dims(self.linear(x), (2, 0, 1)) + + # [*, 1, H, I, J] + triangle_bias = triangle_bias.unsqueeze(-4) + + biases = [mask_bias, triangle_bias] + + if chunk_size is not None: + x = self._chunk( + x, + biases, + chunk_size, + use_memory_efficient_kernel=use_memory_efficient_kernel, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma, + inplace_safe=inplace_safe, + ) + else: + x = self.mha( + q_x=x, + kv_x=x, + biases=biases, + use_memory_efficient_kernel=use_memory_efficient_kernel, + use_deepspeed_evo_attention=use_deepspeed_evo_attention, + use_lma=use_lma + ) + + if(not self.starting): + x = x.transpose(-2, -3) + + return x + + +# Implements Algorithm 13 +TriangleAttentionStartingNode = TriangleAttention + + +class TriangleAttentionEndingNode(TriangleAttention): + """ + Implements Algorithm 14. + """ + __init__ = partialmethod(TriangleAttention.__init__, starting=False) diff --git a/model/openfold/triangular_multiplicative_update.py b/model/openfold/triangular_multiplicative_update.py new file mode 100644 index 0000000000000000000000000000000000000000..336c964c3a9f06b72c494b67b09d705999ecf5d1 --- /dev/null +++ b/model/openfold/triangular_multiplicative_update.py @@ -0,0 +1,647 @@ +# Copyright 2021 AlQuraishi Laboratory +# Copyright 2021 DeepMind Technologies Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import partialmethod +from typing import Optional +from abc import ABC, abstractmethod + +import torch +import torch.nn as nn + +from model.openfold.primitives import Linear, LayerNorm +from onescience.utils.openfold.chunk_utils import chunk_layer +from onescience.utils.openfold.precision_utils import is_fp16_enabled +from onescience.utils.openfold.tensor_utils import add, permute_final_dims + + +class BaseTriangleMultiplicativeUpdate(nn.Module, ABC): + """ + Implements Algorithms 11 and 12. + """ + @abstractmethod + def __init__(self, c_z, c_hidden, _outgoing, bias=True): + """ + Args: + c_z: + Input channel dimension + c: + Hidden channel dimension + """ + super(BaseTriangleMultiplicativeUpdate, self).__init__() + self.c_z = c_z + self.c_hidden = c_hidden + self._outgoing = _outgoing + self.bias = bias + + self.linear_g = Linear(self.c_z, self.c_z, bias=bias, init="gating") + self.linear_z = Linear(self.c_hidden, self.c_z, bias=bias, init="final") + + self.layer_norm_in = LayerNorm(self.c_z) + self.layer_norm_out = LayerNorm(self.c_hidden) + + self.sigmoid = nn.Sigmoid() + + def _combine_projections(self, + a: torch.Tensor, + b: torch.Tensor, + _inplace_chunk_size: Optional[int] = None + ) -> torch.Tensor: + if(self._outgoing): + a = permute_final_dims(a, (2, 0, 1)) + b = permute_final_dims(b, (2, 1, 0)) + else: + a = permute_final_dims(a, (2, 1, 0)) + b = permute_final_dims(b, (2, 0, 1)) + + if(_inplace_chunk_size is not None): + # To be replaced by torch vmap + for i in range(0, a.shape[-3], _inplace_chunk_size): + a_chunk = a[..., i: i + _inplace_chunk_size, :, :] + b_chunk = b[..., i: i + _inplace_chunk_size, :, :] + a[..., i: i + _inplace_chunk_size, :, :] = ( + torch.matmul( + a_chunk, + b_chunk, + ) + ) + + p = a + else: + p = torch.matmul(a, b) + + return permute_final_dims(p, (1, 2, 0)) + + @abstractmethod + def forward(self, + z: torch.Tensor, + mask: Optional[torch.Tensor] = None, + inplace_safe: bool = False, + _add_with_inplace: bool = False + ) -> torch.Tensor: + """ + Args: + x: + [*, N_res, N_res, C_z] input tensor + mask: + [*, N_res, N_res] input mask + Returns: + [*, N_res, N_res, C_z] output tensor + """ + pass + + +class TriangleMultiplicativeUpdate(BaseTriangleMultiplicativeUpdate): + """ + Implements Algorithms 11 and 12. + """ + def __init__(self, c_z, c_hidden, _outgoing=True, bias: bool=True): + """ + Args: + c_z: + Input channel dimension + c: + Hidden channel dimension + """ + super(TriangleMultiplicativeUpdate, self).__init__(c_z=c_z, + c_hidden=c_hidden, + _outgoing=_outgoing, + bias=bias + ) + + self.linear_a_p = Linear(self.c_z, self.c_hidden, bias=bias) + self.linear_a_g = Linear(self.c_z, self.c_hidden, bias=bias, init="gating") + self.linear_b_p = Linear(self.c_z, self.c_hidden, bias=bias,) + self.linear_b_g = Linear(self.c_z, self.c_hidden, bias=bias, init="gating") + + def _inference_forward(self, + z: torch.Tensor, + mask: Optional[torch.Tensor] = None, + inplace_chunk_size: Optional[int] = None, + with_add: bool = True, + ): + """ + Args: + z: + A [*, N, N, C_z] pair representation + mask: + A [*, N, N] pair mask + inplace_chunk_size: + Size of chunks used in the main computation. Increase to trade + memory for speed. + with_add: + If True, z is overwritten with (z + update). Otherwise, it is + overwritten with (update). + Returns: + A reference to the overwritten z + + More memory-efficient, inference-only version of the forward function. + Uses in-place operations, fusion of the addition that happens after + this module in the Evoformer, a smidge of recomputation, and + a cache of overwritten values to lower peak memory consumption of this + module from 5x the size of the input tensor z to 2.5x its size. Useful + for inference on extremely long sequences. + + It works as follows. We will make reference to variables used in the + default forward implementation below. Naively, triangle multiplication + attention requires the manifestation of 5 tensors the size of z: + 1) z, the "square" input tensor, 2) a, the first projection of z, + 3) b, the second projection of b, 4) g, a z-sized mask, and 5) a + z-sized tensor for intermediate computations. For large N, this is + prohibitively expensive; for N=4000, for example, z is more than 8GB + alone. To avoid this problem, we compute b, g, and all intermediate + tensors in small chunks, noting that the chunks required to compute a + chunk of the output depend only on the tensor a and corresponding + vertical and horizontal chunks of z. This suggests an algorithm that + loops over pairs of chunks of z: hereafter "columns" and "rows" of + z, even though each "column" and "row" in fact contains + inplace_chunk_size contiguous true columns and rows of z. Writing + output chunks to a new tensor would bring total memory consumption + down to 3x the size of z. However, more memory can be saved by writing + output chunks directly to z in-place. WLOG, we choose to write output + chunks vertically, overwriting the ith "column" of z at the end of + the ith iteration of the main loop. Despite this overwriting, the + ith column is always one column ahead of previously overwritten columns + and can be recovered directly from z. After the first iteration, + however, the ith row of z is always at least partially overwritten. For + this reason, we introduce the z-cache, a tensor one-half the size of + z. The z-cache initially contains the left half (2nd and 3rd quadrants) + of z. For 0 < i < N/2, the missing left part of the ith row of z is + recovered from this cache at the beginning of the ith iteration. Once i + exceeds n/2, the cache is "reoriented" to encompass the 3rd and 4th + quadrants of z instead. Though the 3rd quadrant of the original z is + entirely overwritten at this point, it can be recovered from the z-cache + itself. Thereafter, the ith row of z can be recovered in its entirety + from the reoriented z-cache. After the final iteration, z has been + completely overwritten and contains the triangular multiplicative + update. If with_add is True, it instead contains the sum of z and the + triangular multiplicative update. In either case, peak memory + consumption is just 2.5x the size of z, disregarding memory used for + chunks and other small variables. + """ + if mask is None: + mask = z.new_ones(z.shape[:-1]) + + mask = mask.unsqueeze(-1) + + def compute_projection_helper(pair, mask, a=True): + if(a): + linear_g = self.linear_a_g + linear_p = self.linear_a_p + else: + linear_g = self.linear_b_g + linear_p = self.linear_b_p + + pair = self.layer_norm_in(pair) + p = linear_g(pair) + p.sigmoid_() + p *= linear_p(pair) + p *= mask + p = permute_final_dims(p, (2, 0, 1)) + return p + + def compute_projection(pair, mask, a=True, chunked=True): + need_transpose = self._outgoing ^ a + if(not chunked): + p = compute_projection_helper(pair, mask, a) + if(need_transpose): + p = p.transpose(-1, -2) + else: + # This computation is chunked so as not to exceed our 2.5x + # budget with a large intermediate tensor + linear_g = self.linear_a_g if a else self.linear_b_g + #c = linear_g.bias.shape[-1] + if self.bias: + c = linear_g.bias.shape[-1] + else: + c = linear_g.weight.shape[0] + out_shape = pair.shape[:-3] + (c,) + pair.shape[-3:-1] + p = pair.new_zeros(out_shape) + for i in range(0, pair.shape[-3], inplace_chunk_size): + pair_chunk = pair[..., i: i + inplace_chunk_size, :, :] + mask_chunk = mask[..., i: i + inplace_chunk_size, :, :] + pair_chunk = compute_projection_helper( + pair[..., i: i + inplace_chunk_size, :, :], + mask[..., i: i + inplace_chunk_size, :, :], + a, + ) + if(need_transpose): + pair_chunk = pair_chunk.transpose(-1, -2) + p[..., i: i + inplace_chunk_size] = pair_chunk + else: + p[..., i: i + inplace_chunk_size, :] = pair_chunk + + del pair_chunk + + return p + + # We start by fully manifesting a. In addition to the input, this + # brings total memory consumption to 2x z (disregarding size of chunks) + # [*, N, N, c] + a = compute_projection(z, mask, True, chunked=True) + #if bias==True: + # a = compute_projection(z, mask, True, True, chunked=True) + #else: + # a = compute_projection(z, mask, False, True, chunked=True) + + if(inplace_chunk_size is not None): + n = a.shape[-1] + half_n = n // 2 + n % 2 + row_dim = -3 + col_dim = -2 + b_chunk_dim = row_dim if self._outgoing else col_dim + + def empty_slicer(t): + return [slice(None) for _ in t.shape] + + def slice_tensor(t, start, end, dim): + # Slices start:end from the dim dimension of t + s = empty_slicer(t) + s[dim] = slice(start, end) + return t[s] + + def flip_z_cache_(z_cache, z): + # "Reorient" the z_cache (see below), filling it with quadrants + # 3---recovered from the z_cache---and 4---recovered from z--- + # of the input tensor z. + quadrant_3 = slice_tensor( + z_cache, half_n, None, row_dim + ) + z_cache = z_cache.transpose(row_dim, col_dim) + + # If n is odd, we need to shrink the z_cache by one row + z_cache = z_cache[..., :(n // 2), :, :] + + # Move the 3rd quadrant of z into the + first_half_slicer = empty_slicer(z_cache) + first_half_slicer[col_dim] = slice(0, half_n) + z_cache[first_half_slicer] = quadrant_3 + + # Get the fourth quadrant of z + quadrant_4 = slice_tensor(z, half_n, None, row_dim) + quadrant_4 = slice_tensor( + quadrant_4, half_n, None, col_dim + ) + + # Insert said quadrant into the rotated z-cache + quadrant_3_slicer = empty_slicer(z_cache) + quadrant_3_slicer[col_dim] = slice(half_n, None) + + z_cache[quadrant_3_slicer] = quadrant_4 + + return z_cache + + # Initialize the z cache to the left half of z. + z_cache_shape = list(z.shape) + z_cache_shape[col_dim] = half_n + z_cache = z.new_zeros(z_cache_shape) + z_cache_slicer = empty_slicer(z_cache) + z_cache_slicer[col_dim] = slice(0, half_n) + z_cache.copy_(z[z_cache_slicer]) + z_cache_rotated = False + + # We need to reorient the z-cache at the halfway point, and we + # don't want a single chunk to straddle that point. We contract one + # of the chunks in the middle to address that problem. + i_range = list(range(0, half_n, inplace_chunk_size)) + initial_offsets = [ + i_2 - i_1 for i_1, i_2 in zip(i_range, i_range[1:] + [half_n]) + ] + after_half = list(range(half_n, n, inplace_chunk_size)) + after_half_offsets = [inplace_chunk_size for _ in after_half] + combined_range_with_offsets = zip( + i_range + after_half, initial_offsets + after_half_offsets + ) + for i, offset in combined_range_with_offsets: + if(not z_cache_rotated and i >= half_n): + z_cache = flip_z_cache_(z_cache, z) + z_cache_rotated = True + + z_chunk_b = slice_tensor( + z, i, i + offset, b_chunk_dim, + ) + mask_chunk = slice_tensor( + mask, i, i + offset, b_chunk_dim, + ) + + z_chunk_b = z_chunk_b.clone() + if(b_chunk_dim == col_dim): + z_chunk_b = slice_tensor( + z, i, i + offset, col_dim + ) + else: # b_chunk_dim == row_dim + # In this case, the b-dimension (b_chunk_dim) is partially + # overwritten at the end of each iteration. We need to + # restore the missing component from the z-cache. + if(not z_cache_rotated): + z_chunk_slicer = empty_slicer(z_chunk_b) + z_chunk_slicer[col_dim] = slice(0, half_n) + z_chunk_b[z_chunk_slicer] = slice_tensor( + z_cache, i, i + offset, row_dim, + ) + else: + z_cache_offset = i - half_n + z_chunk_b = slice_tensor( + z_cache, + z_cache_offset, z_cache_offset + offset, + row_dim + ) + b_chunk = compute_projection(z_chunk_b, mask_chunk, a=False, chunked=False) + #if bias==True: + # b_chunk = compute_projection( + # z_chunk_b, mask_chunk, True, a=False, chunked=False + # ) + #else: + # b_chunk = compute_projection(z_chunk_b, mask_chunk, False, a=False, chunked=False) + del z_chunk_b + + x_chunk = torch.matmul( + a, + b_chunk, + ) + x_chunk = permute_final_dims(x_chunk, (1, 2, 0)) + x_chunk = self.layer_norm_out(x_chunk) + x_chunk = self.linear_z(x_chunk) + + # The g dimension (col_dim) is parallel to and ahead of the + # overwrites in z. We can extract the g chunk normally. + z_chunk_g = slice_tensor( + z, i, i + offset, col_dim + ) + g_chunk = self.linear_g(self.layer_norm_in(z_chunk_g)) + g_chunk.sigmoid_() + del z_chunk_g + + x_chunk *= g_chunk + + # Write the columns into z in-place + z_slicer = empty_slicer(z) + z_slicer[col_dim] = slice(i, i + offset) + if(with_add): + z[z_slicer] += x_chunk + else: + z[z_slicer] = x_chunk + else: + b = compute_projection(z, mask, False, False) + #if bias==True: + # b = compute_projection(z, mask, True, False, False) + #else: + # b = compute_projection(z, mask, False, False, False) + x = torch.matmul(a, b) + x = self.layer_norm_out(x) + x = self.linear_z(x) + g = self.linear_g(z) + g.sigmoid_() + x *= g + if(with_add): + z += x + else: + z = x + + return z + + def forward(self, + z: torch.Tensor, + mask: Optional[torch.Tensor] = None, + inplace_safe: bool = False, + _add_with_inplace: bool = False, + _inplace_chunk_size: Optional[int] = 256, + ) -> torch.Tensor: + """ + Args: + x: + [*, N_res, N_res, C_z] input tensor + mask: + [*, N_res, N_res] input mask + Returns: + [*, N_res, N_res, C_z] output tensor + """ + if(inplace_safe): + x = self._inference_forward( + z, + mask, + inplace_chunk_size=_inplace_chunk_size, + with_add=_add_with_inplace, + ) + return x + + if mask is None: + mask = z.new_ones(z.shape[:-1]) + + mask = mask.unsqueeze(-1) + + z = self.layer_norm_in(z) + a = mask + a = a * self.sigmoid(self.linear_a_g(z)) + a = a * self.linear_a_p(z) + b = mask + b = b * self.sigmoid(self.linear_b_g(z)) + b = b * self.linear_b_p(z) + + # Prevents overflow of torch.matmul in combine projections in + # reduced-precision modes + a_std = a.std() + b_std = b.std() + if(is_fp16_enabled() and a_std != 0. and b_std != 0.): + a = a / a.std() + b = b / b.std() + + if(is_fp16_enabled()): + with torch.cuda.amp.autocast(enabled=False): + x = self._combine_projections(a.float(), b.float()) + else: + x = self._combine_projections(a, b) + + del a, b + x = self.layer_norm_out(x) + x = self.linear_z(x) + g = self.sigmoid(self.linear_g(z)) + x = x * g + + return x + + +class TriangleMultiplicationOutgoing(TriangleMultiplicativeUpdate): + """ + Implements Algorithm 11. + """ + __init__ = partialmethod(TriangleMultiplicativeUpdate.__init__, _outgoing=True) + + +class TriangleMultiplicationIncoming(TriangleMultiplicativeUpdate): + """ + Implements Algorithm 12. + """ + __init__ = partialmethod(TriangleMultiplicativeUpdate.__init__, _outgoing=False) + +class ProtenixTriangleMultiplicationOutgoing(TriangleMultiplicativeUpdate): + """ + Implements Algorithm 11. + """ + __init__ = partialmethod(TriangleMultiplicativeUpdate.__init__, _outgoing=True, bias=False) + +class ProtenixTriangleMultiplicationIncoming(TriangleMultiplicativeUpdate): + """ + Implements Algorithm 12. + """ + __init__ = partialmethod(TriangleMultiplicativeUpdate.__init__, _outgoing=False, bias=False) + + +class FusedTriangleMultiplicativeUpdate(BaseTriangleMultiplicativeUpdate): + """ + Implements Algorithms 11 and 12. + """ + + def __init__(self, c_z, c_hidden, _outgoing=True): + """ + Args: + c_z: + Input channel dimension + c: + Hidden channel dimension + """ + super(FusedTriangleMultiplicativeUpdate, self).__init__(c_z=c_z, + c_hidden=c_hidden, + _outgoing=_outgoing) + + self.linear_ab_p = Linear(self.c_z, self.c_hidden * 2) + self.linear_ab_g = Linear(self.c_z, self.c_hidden * 2, init="gating") + + def _inference_forward(self, + z: torch.Tensor, + mask: Optional[torch.Tensor] = None, + _inplace_chunk_size: Optional[int] = None, + with_add: bool = True, + ): + """ + Args: + z: + A [*, N, N, C_z] pair representation + mask: + A [*, N, N] pair mask + with_add: + If True, z is overwritten with (z + update). Otherwise, it is + overwritten with (update). + Returns: + A reference to the overwritten z + """ + if mask is None: + mask = z.new_ones(z.shape[:-1]) + + mask = mask.unsqueeze(-1) + + def compute_projection_helper(pair, mask): + p = self.linear_ab_g(pair) + p.sigmoid_() + p *= self.linear_ab_p(pair) + p *= mask + + return p + + def compute_projection(pair, mask): + p = compute_projection_helper(pair, mask) + left = p[..., :self.c_hidden] + right = p[..., self.c_hidden:] + + return left, right + + z_norm_in = self.layer_norm_in(z) + a, b = compute_projection(z_norm_in, mask) + x = self._combine_projections(a, b, _inplace_chunk_size=_inplace_chunk_size) + x = self.layer_norm_out(x) + x = self.linear_z(x) + g = self.linear_g(z_norm_in) + g.sigmoid_() + x *= g + if (with_add): + z += x + else: + z = x + + return z + + def forward(self, + z: torch.Tensor, + mask: Optional[torch.Tensor] = None, + inplace_safe: bool = False, + _add_with_inplace: bool = False, + _inplace_chunk_size: Optional[int] = 256 + ) -> torch.Tensor: + """ + Args: + x: + [*, N_res, N_res, C_z] input tensor + mask: + [*, N_res, N_res] input mask + Returns: + [*, N_res, N_res, C_z] output tensor + """ + if (inplace_safe): + x = self._inference_forward( + z, + mask, + _inplace_chunk_size=_inplace_chunk_size, + with_add=_add_with_inplace, + ) + return x + + if mask is None: + mask = z.new_ones(z.shape[:-1]) + + mask = mask.unsqueeze(-1) + + z = self.layer_norm_in(z) + ab = mask + ab = ab * self.sigmoid(self.linear_ab_g(z)) + ab = ab * self.linear_ab_p(z) + + a = ab[..., :self.c_hidden] + b = ab[..., self.c_hidden:] + + # Prevents overflow of torch.matmul in combine projections in + # reduced-precision modes + a_std = a.std() + b_std = b.std() + if (is_fp16_enabled() and a_std != 0. and b_std != 0.): + a = a / a.std() + b = b / b.std() + + if (is_fp16_enabled()): + with torch.cuda.amp.autocast(enabled=False): + x = self._combine_projections(a.float(), b.float()) + else: + x = self._combine_projections(a, b) + + del a, b + x = self.layer_norm_out(x) + x = self.linear_z(x) + g = self.sigmoid(self.linear_g(z)) + x = x * g + + return x + + +class FusedTriangleMultiplicationOutgoing(FusedTriangleMultiplicativeUpdate): + """ + Implements Algorithm 11. + """ + __init__ = partialmethod(FusedTriangleMultiplicativeUpdate.__init__, _outgoing=True) + + +class FusedTriangleMultiplicationIncoming(FusedTriangleMultiplicativeUpdate): + """ + Implements Algorithm 12. + """ + __init__ = partialmethod(FusedTriangleMultiplicativeUpdate.__init__, _outgoing=False) + diff --git a/weight/esm-main/scripts/atlas/v0/full/tarballs/tm_.90_1_plddt_.70_.80.txt b/weight/esm-main/scripts/atlas/v0/full/tarballs/tm_.90_1_plddt_.70_.80.txt new file mode 100644 index 0000000000000000000000000000000000000000..60b6381df841ba7e3bd80503ed97948ef02aba91 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v0/full/tarballs/tm_.90_1_plddt_.70_.80.txt @@ -0,0 +1 @@ +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_.90_1_plddt_.70_.80_00.tar.gz diff --git a/weight/esm-main/scripts/atlas/v0/full/tarballs/tm_0_.40_plddt_0_.40.txt b/weight/esm-main/scripts/atlas/v0/full/tarballs/tm_0_.40_plddt_0_.40.txt new file mode 100644 index 0000000000000000000000000000000000000000..84818ce30f8993ada9af56788e5645a2cb73a737 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v0/full/tarballs/tm_0_.40_plddt_0_.40.txt @@ -0,0 +1,46 @@ +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_01.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_02.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_03.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_04.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_05.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_06.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_07.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_08.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_09.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_10.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_11.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_12.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_13.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_14.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_15.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_16.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_17.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_18.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_19.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_20.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_21.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_22.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_23.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_24.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_25.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_26.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_27.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_28.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_29.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_30.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_31.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_32.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_33.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_34.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_35.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_36.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_37.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_38.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_39.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_40.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_41.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_42.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_43.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_44.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v0/full/tarballs/tm_0_.40_plddt_0_.40_45.tar.gz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.70_.80_plddt_.70_.80.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.70_.80_plddt_.70_.80.txt new file mode 100644 index 0000000000000000000000000000000000000000..6eea04cf880d7776e7f26d5f85fb80392036060c --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.70_.80_plddt_.70_.80.txt @@ -0,0 +1,13 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_00.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_01.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_02.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_03.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_04.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_05.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_06.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_07.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_08.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_09.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_10.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_11.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.70_.80_12.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.70_.80_plddt_.80_.90.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.70_.80_plddt_.80_.90.txt new file mode 100644 index 0000000000000000000000000000000000000000..52722f57c7f80fba5c15ef0cf27f764dd7685946 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.70_.80_plddt_.80_.90.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.80_.90_00.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_.80_.90_01.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.70_.80_plddt_0_.40.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.70_.80_plddt_0_.40.txt new file mode 100644 index 0000000000000000000000000000000000000000..17a2ded59e97577bdc423c39995a3c9fd1819e04 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.70_.80_plddt_0_.40.txt @@ -0,0 +1 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.70_.80_plddt_0_.40_00.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.80_.90_plddt_.70_.80.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.80_.90_plddt_.70_.80.txt new file mode 100644 index 0000000000000000000000000000000000000000..0e9556a03fcbb3e98d886c024ce3652317d891bf --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.80_.90_plddt_.70_.80.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.70_.80_00.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.70_.80_01.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.80_.90_plddt_.80_.90.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.80_.90_plddt_.80_.90.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012820abc4143718522dbe5f73e92d88105c50e --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.80_.90_plddt_.80_.90.txt @@ -0,0 +1,25 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_00.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_01.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_02.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_03.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_04.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_05.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_06.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_07.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_08.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_09.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_10.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_11.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_12.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_13.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_14.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_15.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_16.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_17.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_18.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_19.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_20.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_21.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_22.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_23.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.80_.90_24.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.80_.90_plddt_.90_1.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.80_.90_plddt_.90_1.txt new file mode 100644 index 0000000000000000000000000000000000000000..24566e2afbcb40e424947691a4297f40196c4c4f --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.80_.90_plddt_.90_1.txt @@ -0,0 +1,4 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.90_1_00.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.90_1_01.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.90_1_02.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.80_.90_plddt_.90_1_03.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.90_1_plddt_.70_.80.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.90_1_plddt_.70_.80.txt new file mode 100644 index 0000000000000000000000000000000000000000..e5d43bd232b6259cfb7e0048e64dfa32ab4c0ddc --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.90_1_plddt_.70_.80.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.70_.80_00.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.70_.80_01.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.90_1_plddt_.80_.90.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.90_1_plddt_.80_.90.txt new file mode 100644 index 0000000000000000000000000000000000000000..ecbfcd86c743b1af4a191fb9a15733e239cc6173 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.90_1_plddt_.80_.90.txt @@ -0,0 +1,4 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.80_.90_00.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.80_.90_01.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.80_.90_02.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.80_.90_03.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.90_1_plddt_.90_1.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.90_1_plddt_.90_1.txt new file mode 100644 index 0000000000000000000000000000000000000000..a0e447fca68866c73513c6a1ad2c62e9274ce5ff --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_.90_1_plddt_.90_1.txt @@ -0,0 +1,17 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_00.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_01.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_02.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_03.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_04.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_05.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_06.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_07.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_08.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_09.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_10.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_11.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_12.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_13.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_14.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_15.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_.90_1_plddt_.90_1_16.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.40_.50.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.40_.50.txt new file mode 100644 index 0000000000000000000000000000000000000000..9661609e24f3dd3514bdeaf06ef592daba936d18 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.40_.50.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_0_.40_plddt_.40_.50_00.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_0_.40_plddt_.40_.50_01.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.50_.60.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.50_.60.txt new file mode 100644 index 0000000000000000000000000000000000000000..545baaf9aea25280c13d2e8913f5ef7d18299aca --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.50_.60.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_0_.40_plddt_.50_.60_00.npz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_0_.40_plddt_.50_.60_01.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.60_.70.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.60_.70.txt new file mode 100644 index 0000000000000000000000000000000000000000..91f85968304f646b58ba3fd85f8f6c55db5f3de9 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.60_.70.txt @@ -0,0 +1 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_0_.40_plddt_.60_.70_00.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.70_.80.txt b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.70_.80.txt new file mode 100644 index 0000000000000000000000000000000000000000..593ed6e9c4159e307f30e4e5168c37b314c29186 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/esm2_embeddings/tm_0_.40_plddt_.70_.80.txt @@ -0,0 +1 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/lm_reps/tm_0_.40_plddt_.70_.80_00.npz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40.txt b/weight/esm-main/scripts/atlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40.txt new file mode 100644 index 0000000000000000000000000000000000000000..e41a2c8ce608937573caeb030b34a58100a020f3 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40.txt @@ -0,0 +1,406 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_00.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_01.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_02.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_03.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_04.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_05.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_06.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_07.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_08.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_09.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_10.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_11.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_12.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_13.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_14.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_15.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_16.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_17.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_18.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_19.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_20.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_21.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_22.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_23.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_24.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_25.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_26.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_27.DB_ss.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB.lookup +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB.source +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB_ca +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB_ca.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB_ca.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB_h +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB_h.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB_h.index +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB_ss +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB_ss.dbtype +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/foldseekdb/tm_0_.40_plddt_0_.40_28.DB_ss.index diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.40_.50.txt b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.40_.50.txt new file mode 100644 index 0000000000000000000000000000000000000000..34c76c20362abb7aba1cd60b6049cbbad687db2b --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.40_.50.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.40_.50_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.40_.50_01.tar.gz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.50_.60.txt b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.50_.60.txt new file mode 100644 index 0000000000000000000000000000000000000000..cc1091e1ef4f5636705835ba1612364949da2076 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.50_.60.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.50_.60_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.50_.60_01.tar.gz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.60_.70.txt b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.60_.70.txt new file mode 100644 index 0000000000000000000000000000000000000000..b43f2fcd4e530258dc441ee6e228cf842f5b9ab4 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.60_.70.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.60_.70_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.40_.50_plddt_.60_.70_01.tar.gz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.50_.60_plddt_.60_.70.txt b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.50_.60_plddt_.60_.70.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6bcd3c19d02853b2f6c2eb840f2ab64748d4d2c --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.50_.60_plddt_.60_.70.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.50_.60_plddt_.60_.70_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.50_.60_plddt_.60_.70_01.tar.gz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.50_.60_plddt_.70_.80.txt b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.50_.60_plddt_.70_.80.txt new file mode 100644 index 0000000000000000000000000000000000000000..e147d519e547e97578caf48adffd8f425bc92829 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.50_.60_plddt_.70_.80.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.50_.60_plddt_.70_.80_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.50_.60_plddt_.70_.80_01.tar.gz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.50_.60_plddt_0_.40.txt b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.50_.60_plddt_0_.40.txt new file mode 100644 index 0000000000000000000000000000000000000000..8ea94eba50065c54d0e43b6eb0c646793d3cc4bf --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.50_.60_plddt_0_.40.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.50_.60_plddt_0_.40_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.50_.60_plddt_0_.40_01.tar.gz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.40_.50.txt b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.40_.50.txt new file mode 100644 index 0000000000000000000000000000000000000000..3c9a80630e590c7a3ea53a95089bb3044d17872d --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.40_.50.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.40_.50_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.40_.50_01.tar.gz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.50_.60.txt b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.50_.60.txt new file mode 100644 index 0000000000000000000000000000000000000000..e0951b5915d5880ef54661a271162a2d1da2ec6c --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.50_.60.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.50_.60_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.50_.60_01.tar.gz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.60_.70.txt b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.60_.70.txt new file mode 100644 index 0000000000000000000000000000000000000000..3cc472e6ff170791e5e6bb1484557f8fcafbbfaa --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.60_.70.txt @@ -0,0 +1,6 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.60_.70_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.60_.70_01.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.60_.70_02.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.60_.70_03.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.60_.70_04.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.60_.70_05.tar.gz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.80_.90.txt b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.80_.90.txt new file mode 100644 index 0000000000000000000000000000000000000000..f9771f1ae9ad69b21c9267437fff96ea358d114f --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.80_.90.txt @@ -0,0 +1,2 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.80_.90_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.60_.70_plddt_.80_.90_01.tar.gz diff --git a/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.90_1_plddt_.80_.90.txt b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.90_1_plddt_.80_.90.txt new file mode 100644 index 0000000000000000000000000000000000000000..c3dcd042d53209db2662177484642e0b547486b1 --- /dev/null +++ b/weight/esm-main/scripts/atlas/v2023_02/full/tarballs/tm_.90_1_plddt_.80_.90.txt @@ -0,0 +1,4 @@ +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.90_1_plddt_.80_.90_00.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.90_1_plddt_.80_.90_01.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.90_1_plddt_.80_.90_02.tar.gz +https://dl.fbaipublicfiles.com/esmatlas/v2023_02/full/tarballs/tm_.90_1_plddt_.80_.90_03.tar.gz diff --git a/weight/esm-main/tests/inverse_folding_test_example.json b/weight/esm-main/tests/inverse_folding_test_example.json new file mode 100644 index 0000000000000000000000000000000000000000..c1135d41e439af462daa76425d45c3a94de3894a --- /dev/null +++ b/weight/esm-main/tests/inverse_folding_test_example.json @@ -0,0 +1 @@ +[{"coords": [[[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[99.86399841308594, 52.58100128173828, -5.098999977111816], [98.87999725341797, 51.736000061035156, -5.841000080108643], [97.86199951171875, 51.09700012207031, -4.889999866485596]], [[98.34400177001953, 50.35900115966797, -3.8980000019073486], [97.46099853515625, 49.7239990234375, -2.9230000972747803], [97.66100311279297, 50.36199951171875, -1.5609999895095825]], [[96.5780029296875, 50.7869987487793, -0.9269999861717224], [96.68000030517578, 51.40399932861328, 0.3869999945163727], [95.85700225830078, 50.59600067138672, 1.3669999837875366]], [[96.27400207519531, 50.53300094604492, 2.619999885559082], [95.51699829101562, 49.762001037597656, 3.5959999561309814], [95.36799621582031, 50.46500015258789, 4.919000148773193]], [[94.1240005493164, 50.66699981689453, 5.302999973297119], [93.79000091552734, 51.32899856567383, 6.541999816894531], [93.00399780273438, 50.375, 7.421999931335449]], [[93.4520034790039, 50.18600082397461, 8.656999588012695], [92.74400329589844, 49.303001403808594, 9.571000099182129], [92.20800018310547, 50.15999984741211, 10.678999900817871]], [[90.90299987792969, 50.388999938964844, 10.65999984741211], [90.29000091552734, 51.19900131225586, 11.694000244140625], [90.60700225830078, 50.69200134277344, 13.093999862670898]], [[91.2959976196289, 49.553001403808594, 13.17199993133545], [91.66899871826172, 48.965999603271484, 14.458999633789062], [92.58999633789062, 49.9109992980957, 15.246000289916992]], [[93.50299835205078, 50.56700134277344, 14.538999557495117], [94.44300079345703, 51.46900177001953, 15.178999900817871], [94.13500213623047, 52.95899963378906, 15.083999633789062]], [[93.31800079345703, 53.36600112915039, 14.11299991607666], [92.97599792480469, 54.784000396728516, 13.963000297546387], [92.1449966430664, 55.22700119018555, 15.163999557495117]], [[92.83499908447266, 55.41999816894531, 16.281999588012695], [92.2030029296875, 55.84199905395508, 17.51799964904785], [91.78399658203125, 57.30799865722656, 17.433000564575195]], [[92.35199737548828, 58.04100036621094, 16.479000091552734], [92.03700256347656, 59.4640007019043, 16.322999954223633], [92.03199768066406, 59.95500183105469, 14.871999740600586]], [[91.41000366210938, 61.10300064086914, 14.647000312805176], [91.34300231933594, 61.6619987487793, 13.307999610900879], [92.72200012207031, 62.099998474121094, 12.82800006866455]], [[93.68900299072266, 62.13600158691406, 13.732000350952148], [95.03299713134766, 62.520999908447266, 13.336000442504883], [95.76300048828125, 61.24300003051758, 12.993000030517578]], [[95.30599975585938, 60.14899826049805, 13.597000122070312], [95.89199829101562, 58.84000015258789, 13.345000267028809], [95.61699676513672, 58.60499954223633, 11.86400032043457]], [[94.39199829101562, 58.94200134277344, 11.479999542236328], [93.91899871826172, 58.79399871826172, 10.112000465393066], [94.84300231933594, 59.444000244140625, 9.092000007629395]], [[95.21299743652344, 60.70199966430664, 9.348999977111816], [96.08200073242188, 61.47600173950195, 8.454000473022461], [97.53900146484375, 61.04199981689453, 8.505000114440918]], [[97.94999694824219, 60.553001403808594, 9.666000366210938], [99.30599975585938, 60.07099914550781, 9.82800006866455], [99.44599914550781, 58.83300018310547, 8.961000442504883]], [[98.3759994506836, 58.04600143432617, 8.907999992370605], [98.36000061035156, 56.808998107910156, 8.138999938964844], [98.2760009765625, 57.040000915527344, 6.644999980926514]], [[97.3740005493164, 57.91699981689453, 6.241000175476074], [97.21600341796875, 58.242000579833984, 4.835000038146973], [98.54100036621094, 58.7760009765625, 4.314000129699707]], [[99.14399719238281, 59.67399978637695, 5.083000183105469], [100.41699981689453, 60.25600051879883, 4.703999996185303], [101.40499877929688, 59.172000885009766, 4.293000221252441]], [[101.54299926757812, 58.124000549316406, 5.104000091552734], [102.48300170898438, 57.04399871826172, 4.781000137329102], [101.9800033569336, 56.09600067138672, 3.697999954223633]], [[100.7229995727539, 55.680999755859375, 3.821000099182129], [100.13400268554688, 54.73400115966797, 2.888000011444092], [100.06999969482422, 55.25600051879883, 1.4759999513626099]], [[99.70800018310547, 56.52399826049805, 1.3289999961853027], [99.60399627685547, 57.12300109863281, 0.009999999776482582], [100.88700103759766, 57.875999450683594, -0.3089999854564667]], [[101.85099792480469, 57.770999908447266, 0.5979999899864197], [103.14199829101562, 58.4379997253418, 0.45100000500679016], [102.96800231933594, 59.91699981689453, 0.07999999821186066]], [[102.04000091552734, 60.59199905395508, 0.7480000257492065], [101.78600311279297, 62.00299835205078, 0.49300000071525574], [103.0459976196289, 62.85499954223633, 0.7120000123977661]], [[102.99500274658203, 64.14900207519531, 0.36800000071525574], [104.18000030517578, 64.9800033569336, 0.5479999780654907], [104.55599975585938, 65.17400360107422, 1.99399995803833]], [[105.8270034790039, 65.48600006103516, 2.2290000915527344], [106.34500122070312, 65.71499633789062, 3.5810000896453857], [105.59600067138672, 66.8270034790039, 4.316999912261963]], [[105.09100341796875, 67.80799865722656, 3.5789999961853027], [104.36199951171875, 68.91100311279297, 4.178999900817871], [102.86499786376953, 68.66300201416016, 4.301000118255615]], [[102.40599822998047, 67.45500183105469, 3.9820001125335693], [100.97699737548828, 67.15799713134766, 4.076000213623047], [100.39099884033203, 67.81300354003906, 5.335999965667725]], [[99.48600006103516, 68.76799774169922, 5.125999927520752], [98.88600158691406, 69.48799896240234, 6.238999843597412], [97.8479995727539, 68.76200103759766, 7.071000099182129]], [[97.5979995727539, 67.49600219726562, 6.763999938964844], [96.62000274658203, 66.73400115966797, 7.51200008392334], [95.302001953125, 67.48999786376953, 7.5960001945495605]], [[94.66000366210938, 67.66000366210938, 6.447999954223633], [93.37100219726562, 68.31500244140625, 6.38100004196167], [92.72899627685547, 67.9489974975586, 5.054999828338623]], [[91.40799713134766, 67.8270034790039, 5.058000087738037], [90.65299987792969, 67.4540023803711, 3.865000009536743], [91.0780029296875, 68.2770004272461, 2.6589999198913574]], [[91.42500305175781, 69.52999877929688, 2.9040000438690186], [91.84300231933594, 70.37899780273438, 1.8109999895095825], [93.15899658203125, 69.88300323486328, 1.2389999628067017]], [[94.06999969482422, 69.4749984741211, 2.111999988555908], [95.36100006103516, 68.98400115966797, 1.6549999713897705], [95.24400329589844, 67.5780029296875, 1.0820000171661377]], [[94.2030029296875, 66.85600280761719, 1.4809999465942383], [94.02799987792969, 65.51200103759766, 0.9580000042915344], [93.50800323486328, 65.65799713134766, -0.4490000009536743]], [[92.49199676513672, 66.49400329589844, -0.6159999966621399], [91.9229965209961, 66.70999908447266, -1.9320000410079956], [92.98600006103516, 67.05699920654297, -2.9609999656677246]], [[93.7760009765625, 68.08300018310547, -2.6510000228881836], [94.84400177001953, 68.53500366210938, -3.5380001068115234], [95.79299926757812, 67.38300323486328, -3.8550000190734863]], [[96.18699645996094, 66.64099884033203, -2.818000078201294], [97.0790023803711, 65.48799896240234, -2.9570000171661377], [96.4489974975586, 64.43699645996094, -3.864000082015991]], [[95.1259994506836, 64.3270034790039, -3.7820000648498535], [94.37999725341797, 63.37300109863281, -4.584000110626221], [94.32599639892578, 63.895999908447266, -6.007999897003174]], [[94.33000183105469, 65.21600341796875, -6.144999980926514], [94.24500274658203, 65.83699798583984, -7.4629998207092285], [95.56700134277344, 66.2490005493164, -8.118000030517578]], [[96.6780014038086, 65.63899993896484, -7.7170000076293945], [97.93599700927734, 66.00399780273438, -8.329000473022461], [99.15299987792969, 65.21299743652344, -7.894999980926514]], [[99.25700378417969, 64.89099884033203, -6.607999801635742], [100.41899871826172, 64.16100311279297, -6.105999946594238], [100.32499694824219, 62.64799880981445, -6.260000228881836]], [[99.26399993896484, 62.0629997253418, -5.715000152587891], [99.04100036621094, 60.62200164794922, -5.76200008392334], [99.01499938964844, 60.06700134277344, -7.177000045776367]], [[99.5780029296875, 58.869998931884766, -7.330999851226807], [99.65299987792969, 58.176998138427734, -8.619999885559082], [98.63099670410156, 57.05500030517578, -8.713000297546387]], [[97.80000305175781, 57.12099838256836, -9.744999885559082], [96.77300262451172, 56.12300109863281, -9.97599983215332], [97.25800323486328, 55.11899948120117, -11.02299976348877]], [[96.63400268554688, 53.917999267578125, -11.104000091552734], [95.50800323486328, 53.39899826049805, -10.307000160217285], [95.80000305175781, 53.24399948120117, -8.815999984741211]], [[94.7509994506836, 53.395999908447266, -8.017000198364258], [94.85800170898438, 53.2599983215332, -6.574999809265137], [93.68399810791016, 52.433998107910156, -6.073999881744385]], [[93.98300170898438, 51.4109992980957, -5.289000034332275], [92.9469985961914, 50.57600021362305, -4.7129998207092285], [93.14299774169922, 50.74700164794922, -3.2070000171661377]], [[92.13200378417969, 51.33399963378906, -2.558000087738037], [92.15899658203125, 51.5989990234375, -1.11899995803833], [91.3030014038086, 50.63100051879883, -0.328000009059906]], [[91.927001953125, 49.87699890136719, 0.5770000219345093], [91.18800354003906, 48.93000030517578, 1.406000018119812], [91.04199981689453, 49.46799850463867, 2.8269999027252197]], [[89.7979965209961, 49.67100143432617, 3.25600004196167], [89.5199966430664, 50.17599868774414, 4.5920000076293945], [88.92400360107422, 49.06399917602539, 5.440000057220459]], [[89.78299713134766, 48.33700180053711, 6.1570000648498535], [89.3489990234375, 47.250999450683594, 7.0370001792907715], [88.66799926757812, 47.80500030517578, 8.298999786376953]], [[87.53199768066406, 47.22100067138672, 8.656999588012695], [86.79100036621094, 47.645999908447266, 9.838000297546387], [86.52999877929688, 49.14400100708008, 9.88599967956543]], [[85.87899780273438, 49.66999816894531, 8.855999946594238], [85.56099700927734, 51.09299850463867, 8.789999961853027], [84.7229995727539, 51.50199890136719, 9.99899959564209]], [[84.09100341796875, 50.52299880981445, 10.645999908447266], [83.23799896240234, 50.79600143432617, 11.79699993133545], [83.9540023803711, 51.51100158691406, 12.928000450134277]], [[84.96099853515625, 50.85200119018555, 13.482999801635742], [85.73300170898438, 51.444000244140625, 14.560999870300293], [86.18599700927734, 52.832000732421875, 14.133000373840332]], [[87.0270004272461, 52.86399841308594, 13.097000122070312], [87.58599853515625, 54.10499954223633, 12.562999725341797], [86.54199981689453, 55.18600082397461, 12.357999801635742]], [[85.2699966430664, 54.803001403808594, 12.35099983215332], [84.20099639892578, 55.775001525878906, 12.178000450134277], [83.77400207519531, 56.316001892089844, 13.529999732971191]], [[83.46900177001953, 55.39699935913086, 14.442000389099121], [83.02999877929688, 55.74800109863281, 15.791999816894531], [84.12799835205078, 56.36399841308594, 16.66900062561035]], [[85.31300354003906, 55.75899887084961, 16.679000854492188], [86.41200256347656, 56.27199935913086, 17.48900032043457], [87.06800079345703, 57.50600051879883, 16.87299919128418]], [[86.28099822998047, 58.33000183105469, 16.187000274658203], [86.80699920654297, 59.53900146484375, 15.555999755859375], [85.72699737548828, 60.604000091552734, 15.369999885559082]], [[84.6780014038086, 60.52899932861328, 16.18400001525879], [83.58799743652344, 61.494998931884766, 16.11400032043457], [82.96499633789062, 61.513999938964844, 14.727999687194824]], [[83.02799987792969, 60.37799835205078, 14.038999557495117], [82.45899963378906, 60.26499938964844, 12.70199966430664], [83.2490005493164, 61.06700134277344, 11.666999816894531]], [[84.49600219726562, 61.39699935913086, 11.993000030517578], [85.32499694824219, 62.17100143432617, 11.081000328063965], [85.96399688720703, 61.37699890136719, 9.951000213623047]], [[85.68900299072266, 60.07500076293945, 9.916000366210938], [86.23799896240234, 59.20600128173828, 8.88599967956543], [85.37200164794922, 59.2599983215332, 7.64900016784668]], [[84.11299896240234, 59.6349983215332, 7.820000171661377], [83.197998046875, 59.69499969482422, 6.695000171661377], [83.48899841308594, 60.8390007019043, 5.714000225067139]], [[84.03700256347656, 61.944000244140625, 6.204999923706055], [84.35399627685547, 63.07400131225586, 5.328000068664551], [85.56500244140625, 62.689998626708984, 4.505000114440918]], [[86.55799865722656, 62.082000732421875, 5.1570000648498535], [87.74800109863281, 61.63100051879883, 4.452000141143799], [87.30999755859375, 60.71200180053711, 3.309999942779541]], [[86.56900024414062, 59.6510009765625, 3.634999990463257], [86.08999633789062, 58.70899963378906, 2.615999937057495], [85.31300354003906, 59.40299987792969, 1.4919999837875366]], [[84.58899688720703, 60.46900177001953, 1.819000005722046], [83.8280029296875, 61.183998107910156, 0.8059999942779541], [84.76100158691406, 61.948001861572266, -0.12600000202655792]], [[85.93499755859375, 62.30799865722656, 0.37599998712539673], [86.88800048828125, 63.02399826049805, -0.4440000057220459], [87.36299896240234, 62.04100036621094, -1.4889999628067017]], [[87.5780029296875, 60.805999755859375, -1.0570000410079956], [88.03600311279297, 59.76599884033203, -1.965000033378601], [86.90499877929688, 59.446998596191406, -2.946000099182129]], [[85.6719970703125, 59.40299987792969, -2.433000087738037], [84.48799896240234, 59.12300109863281, -3.242000102996826], [84.25499725341797, 60.25699996948242, -4.228000164031982]], [[84.83300018310547, 61.41600036621094, -3.9240000247955322], [84.7040023803711, 62.59299850463867, -4.769999980926514], [85.75700378417969, 62.56700134277344, -5.867000102996826]], [[86.98799896240234, 62.22100067138672, -5.50600004196167], [88.07099914550781, 62.154998779296875, -6.478000164031982], [87.66100311279297, 61.19900131225586, -7.580999851226807]], [[87.1709976196289, 60.03900146484375, -7.163000106811523], [86.69200134277344, 59.005001068115234, -8.072999954223633], [85.65899658203125, 59.61899948120117, -9.026000022888184]], [[84.71600341796875, 60.36899948120117, -8.447999954223633], [83.6510009765625, 61.04600143432617, -9.187999725341797], [84.21299743652344, 61.979000091552734, -10.256999969482422]], [[85.00399780273438, 62.96699905395508, -9.83899974822998], [85.5999984741211, 63.89899826049805, -10.779000282287598], [86.2959976196289, 63.11800003051758, -11.876999855041504]], [[86.85600280761719, 61.96200180053711, -11.52400016784668], [87.52200317382812, 61.141998291015625, -12.520999908447266], [88.59100341796875, 60.16299819946289, -12.0600004196167]], [[89.3290023803711, 60.50699996948242, -10.998000144958496], [90.4219970703125, 59.67499923706055, -10.470000267028809], [90.13500213623047, 58.1870002746582, -10.54800033569336]], [[91.14399719238281, 57.391998291015625, -10.899999618530273], [90.9800033569336, 55.933998107910156, -10.982999801635742], [91.22799682617188, 55.327999114990234, -9.597000122070312]], [[90.18599700927734, 55.3489990234375, -8.779999732971191], [90.26799774169922, 54.8390007019043, -7.433000087738037], [89.29499816894531, 53.702999114990234, -7.234000205993652]], [[89.73100280761719, 52.68299865722656, -6.494999885559082], [88.87999725341797, 51.54600143432617, -6.160999774932861], [88.84100341796875, 51.51300048828125, -4.644999980926514]], [[87.63899993896484, 51.415000915527344, -4.090000152587891], [87.48600006103516, 51.42399978637695, -2.6559998989105225], [86.87100219726562, 50.150001525878906, -2.1019999980926514]], [[87.65799713134766, 49.38199996948242, -1.3489999771118164], [87.14600372314453, 48.15999984741211, -0.7329999804496765], [86.86000061035156, 48.42499923706055, 0.7390000224113464]], [[85.58100128173828, 48.37300109863281, 1.1019999980926514], [85.16100311279297, 48.60900115966797, 2.4790000915527344], [84.99700164794922, 47.27199935913086, 3.186000108718872]], [[86.1050033569336, 46.70500183105469, 3.6540000438690186], [86.06199645996094, 45.43000030517578, 4.355000019073486], [86.02899932861328, 45.665000915527344, 5.853000164031982]]], "seq": "MKKAVINGEQIRSISDLHQTLKKELALPEYYGENLDALWDCLTGWVEYPLVLEWRQFEQSKQLTENGAESVLQVFREAKAEGADITIILS"}, {"coords": [[[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[30.12700080871582, 13.123000144958496, 1.2970000505447388], [29.613000869750977, 12.701000213623047, -0.04600000008940697], [28.246999740600586, 13.321000099182129, -0.38199999928474426]], [[27.267000198364258, 13.14799976348877, 0.5019999742507935], [25.930999755859375, 13.6850004196167, 0.26499998569488525], [24.871999740600586, 12.588000297546387, 0.10999999940395355]], [[25.349000930786133, 11.352999687194824, -0.10000000149011612], [24.52400016784668, 10.156000137329102, -0.2800000011920929], [23.461999893188477, 9.958000183105469, 0.7839999794960022]], [[22.29199981689453, 9.512999534606934, 0.3370000123977661], [21.18600082397461, 9.256999969482422, 1.2339999675750732], [21.354999542236328, 7.870999813079834, 1.878999948501587]], [[20.711999893188477, 7.698999881744385, 3.0299999713897705], [20.743999481201172, 6.441999912261963, 3.7790000438690186], [19.36199951171875, 6.218999862670898, 4.425000190734863]], [[19.1200008392334, 4.989999771118164, 4.89300012588501], [17.844999313354492, 4.609000205993652, 5.53000020980835], [16.68600082397461, 4.574999809265137, 4.531000137329102]], [[16.937999725341797, 4.081999778747559, 3.315000057220459], [15.904999732971191, 3.9779999256134033, 2.2820000648498535], [14.9350004196167, 2.881999969482422, 2.694000005722046]], [[13.689000129699707, 3.255000114440918, 2.9690001010894775], [12.692999839782715, 2.2790000438690186, 3.390000104904175], [11.355999946594238, 2.4790000915527344, 2.687000036239624]], [[10.645999908447266, 1.3830000162124634, 2.440000057220459], [9.345000267028809, 1.4500000476837158, 1.7730000019073486], [8.192999839782715, 1.5449999570846558, 2.76200008392334]], [[7.2270002365112305, 2.4030001163482666, 2.4660000801086426], [6.052000045776367, 2.555999994277954, 3.315999984741211], [4.892000198364258, 2.249000072479248, 2.378999948501587]], [[4.196000099182129, 1.1460000276565552, 2.640000104904175], [3.1089999675750732, 0.6970000267028809, 1.7949999570846558], [1.9800000190734863, 0.05299999937415123, 2.6050000190734863]], [[0.7519999742507935, 0.421999990940094, 2.2679998874664307], [-0.45500001311302185, -0.08799999952316284, 2.9049999713897705], [-1.527999997138977, -0.024000000208616257, 1.8270000219345093]], [[-2.138000011444092, -1.1649999618530273, 1.5130000114440918], [-3.1649999618530273, -1.2640000581741333, 0.46399998664855957], [-2.7019999027252197, -0.8180000185966492, -0.9319999814033508]], [[-1.399999976158142, -0.9919999837875366, -1.1990000009536743], [-0.746999979019165, -0.6439999938011169, -2.4779999256134033], [-0.3370000123977661, 0.8289999961853027, -2.627000093460083]], [[-0.7360000014305115, 1.6619999408721924, -1.656000018119812], [-0.35499998927116394, 3.078000068664551, -1.6160000562667847], [1.090000033378601, 2.9700000286102295, -1.156000018119812]], [[2.0230000019073486, 3.255000114440918, -2.072000026702881], [3.4590001106262207, 3.0910000801086426, -1.8359999656677246], [4.3429999351501465, 4.331999778747559, -1.9279999732971191]], [[5.178999900817871, 4.495999813079834, -0.8989999890327454], [6.084000110626221, 5.632999897003174, -0.7639999985694885], [7.47599983215332, 5.224999904632568, -0.2980000078678131]], [[8.49899959564209, 5.830999851226807, -0.8960000276565552], [9.88700008392334, 5.578000068664551, -0.49900001287460327], [10.293000221252441, 6.75, 0.4099999964237213]], [[10.998000144958496, 6.453000068664551, 1.4930000305175781], [11.444999694824219, 7.491000175476074, 2.424999952316284], [12.932999610900879, 7.284999847412109, 2.694000005722046]], [[13.642999649047852, 8.355999946594238, 3.0230000019073486], [15.074999809265137, 8.239999771118164, 3.3010001182556152], [15.604000091552734, 9.369999885559082, 4.181000232696533]], [[16.926000595092773, 9.385000228881836, 4.363999843597412], [17.611000061035156, 10.4350004196167, 5.135000228881836], [18.916000366210938, 10.866999626159668, 4.440999984741211]], [[19.445999145507812, 12.0649995803833, 4.757999897003174], [18.94099998474121, 13.057999610900879, 5.715000152587891], [18.208999633789062, 14.22700023651123, 5.0970001220703125]], [[17.448999404907227, 14.935999870300293, 5.922999858856201], [16.731000900268555, 16.128999710083008, 5.489999771118164], [17.8799991607666, 17.115999221801758, 5.267000198364258]], [[17.941999435424805, 17.743999481201172, 4.0970001220703125], [19.04599952697754, 18.658000946044922, 3.816999912261963], [18.6560001373291, 20.003000259399414, 3.2139999866485596]], [[19.511999130249023, 20.993000030517578, 3.4619998931884766], [19.351999282836914, 22.356000900268555, 2.946000099182129], [20.448999404907227, 22.545000076293945, 1.8819999694824219]], [[21.43199920654297, 21.636999130249023, 1.9249999523162842], [22.58099937438965, 21.604999542236328, 1.0130000114440918], [22.371000289916992, 20.77400016784668, -0.2759999930858612]], [[21.42099952697754, 19.836000442504883, -0.2540000081062317], [21.176000595092773, 18.95800018310547, -1.4010000228881836], [19.715999603271484, 18.7549991607666, -1.8279999494552612]], [[19.531999588012695, 18.357999801635742, -3.0910000801086426], [18.215999603271484, 18.0049991607666, -3.6389999389648438], [18.356000900268555, 16.565000534057617, -4.1529998779296875]], [[17.23900032043457, 15.854000091552734, -4.283999919891357], [17.256999969482422, 14.434000015258789, -4.6479997634887695], [16.433000564575195, 14.00100040435791, -5.855999946594238]], [[16.981000900268555, 13.095000267028809, -6.660999774932861], [16.231000900268555, 12.581999778747559, -7.801000118255615], [15.996000289916992, 11.090999603271484, -7.572000026702881]], [[14.739999771118164, 10.677000045776367, -7.752999782562256], [14.33899974822998, 9.281999588012695, -7.5370001792907715], [14.020000457763672, 8.494000434875488, -8.810999870300293]], [[14.604000091552734, 7.296000003814697, -8.947999954223633], [14.345999717712402, 6.427999973297119, -10.112000465393066], [13.848999977111816, 5.031000137329102, -9.704000473022461]], [[13.11400032043457, 4.373000144958496, -10.607999801635742], [12.600000381469727, 3.015000104904175, -10.359999656677246], [12.939000129699707, 2.078000068664551, -11.550000190734863]], [[13.067000389099121, 0.777999997138977, -11.295000076293945], [13.36299991607666, -0.16200000047683716, -12.368000030517578], [12.946999549865723, -1.5700000524520874, -11.987000465393066]], [[12.815999984741211, -2.431999921798706, -12.994000434875488], [12.479999542236328, -3.8429999351501465, -12.807999610900879], [13.463000297546387, -4.598999977111816, -13.713000297546387]], [[13.291999816894531, -5.914999961853027, -13.805999755859375], [14.142999649047852, -6.754000186920166, -14.642999649047852], [13.711000442504883, -6.5960001945495605, -16.101999282836914]], [[12.567000389099121, -5.940999984741211, -16.308000564575195], [11.991999626159668, -5.691999912261963, -17.631999969482422], [12.229999542236328, -4.247000217437744, -18.100000381469727]], [[12.390000343322754, -3.319000005722046, -17.15399932861328], [12.567000389099121, -1.8949999809265137, -17.475000381469727], [13.725000381469727, -1.1749999523162842, -16.784000396728516]], [[14.461000442504883, -0.37599998712539673, -17.56399917602539], [15.61400032043457, 0.38999998569488525, -17.065000534057617], [15.223999977111816, 1.6019999980926514, -16.200000762939453]], [[16.172000885009766, 2.114000082015991, -15.402000427246094], [15.911999702453613, 3.253000020980835, -14.506999969482422], [15.156000137329102, 4.360000133514404, -15.210000038146973]], [[14.144000053405762, 4.881999969482422, -14.539999961853027], [13.35099983215332, 5.964000225067139, -15.086000442504883], [13.043000221252441, 6.909999847412109, -13.920000076293945]], [[13.303000450134277, 8.206000328063965, -14.107999801635742], [13.04800033569336, 9.211999893188477, -13.086999893188477], [11.565999984741211, 9.51099967956543, -12.88599967956543]], [[11.161999702453613, 9.614999771118164, -11.616000175476074], [9.786999702453613, 9.932000160217285, -11.232999801635742], [9.8100004196167, 11.25, -10.472000122070312]], [[10.58899974822998, 11.335000038146973, -9.401000022888184], [10.720999717712402, 12.57699966430664, -8.635000228881836], [11.98799991607666, 13.274999618530273, -9.159000396728516]], [[11.852999687194824, 14.531999588012695, -9.595999717712402], [12.967000007629395, 15.324999809265137, -10.13599967956543], [13.432000160217285, 16.44099998474121, -9.211000442504883]], [[14.677000045776367, 16.343000411987305, -8.76200008392334], [15.29800033569336, 17.350000381469727, -7.909999847412109], [14.444000244140625, 17.920000076293945, -6.763999938964844]], [[13.972000122070312, 17.055999755859375, -5.86899995803833], [13.170999526977539, 17.49799919128418, -4.73199987411499], [14.015999794006348, 17.687000274658203, -3.4730000495910645]], [[13.48799991607666, 18.459999084472656, -2.5360000133514404], [14.145000457763672, 18.724000930786133, -1.2549999952316284], [13.444000244140625, 17.89299964904785, -0.1599999964237213]], [[12.345999717712402, 17.240999221801758, -0.5370000004768372], [11.598999977111816, 16.374000549316406, 0.367000013589859], [12.41100025177002, 15.095000267028809, 0.6349999904632568]], [[11.986000061035156, 14.291999816894531, 1.6030000448226929], [12.692000389099121, 13.045000076293945, 1.8739999532699585], [11.774999618530273, 11.824000358581543, 1.7450000047683716]], [[10.845999717712402, 11.920000076293945, 0.7990000247955322], [9.918000221252441, 10.847000122070312, 0.49300000071525574], [9.390999794006348, 11.060999870300293, -0.9340000152587891]], [[9.083000183105469, 9.961000442504883, -1.6299999952316284], [8.565999984741211, 10.015000343322754, -2.992000102996826], [7.4770002365112305, 8.967000007629395, -3.178999900817871]], [[6.315999984741211, 9.404000282287598, -3.6510000228881836], [5.179999828338623, 8.529000282287598, -3.9000000953674316], [5.386000156402588, 7.785999774932861, -5.215000152587891]], [[5.49399995803833, 6.459000110626221, -5.14300012588501], [5.7129998207092285, 5.640999794006348, -6.335999965667725], [4.559000015258789, 4.736999988555908, -6.696000099182129]], [[3.4100000858306885, 4.966000080108643, -6.078000068664551], [2.2100000381469727, 4.166999816894531, -6.315000057220459], [1.8860000371932983, 3.8949999809265137, -7.796999931335449]], [[1.8890000581741333, 4.932000160217285, -8.630000114440918], [1.5369999408721924, 4.734000205993652, -10.029999732971191], [2.4809999465942383, 3.8320000171661377, -10.824999809265137]], [[3.7239999771118164, 3.746999979019165, -10.380999565124512], [4.665999889373779, 2.9010000228881836, -11.071999549865723], [4.497000217437744, 1.4479999542236328, -10.654999732971191]], [[4.372000217437744, 1.194000005722046, -9.355999946594238], [4.223999977111816, -0.17299999296665192, -8.90999984741211], [2.930999994277954, -0.8610000014305115, -9.333000183105469]], [[1.8669999837875366, -0.10599999874830246, -9.609999656677246], [0.5960000157356262, -0.7260000109672546, -10.01200008392334], [0.4819999933242798, -1.1440000534057617, -11.47599983215332]], [[1.4930000305175781, -0.8190000057220459, -12.282999992370605], [1.503000020980835, -1.2389999628067017, -13.694000244140625], [1.3860000371932983, -2.7760000228881836, -13.663000106811523]], [[2.0269999504089355, -3.380000114440918, -12.652999877929688], [1.9839999675750732, -4.810999870300293, -12.378999710083008], [2.5320000648498535, -5.073999881744385, -10.97599983215332]], [[1.6139999628067017, -5.26800012588501, -10.031999588012695], [1.9709999561309814, -5.547999858856201, -8.640999794006348], [2.6589999198913574, -6.9079999923706055, -8.402000427246094]], [[2.818000078201294, -7.710000038146973, -9.456000328063965], [3.484999895095825, -8.996000289916992, -9.331000328063965], [4.986999988555908, -8.95300006866455, -9.593999862670898]], [[5.4720001220703125, -7.827000141143799, -10.121999740600586], [6.900000095367432, -7.6539998054504395, -10.432999610900879], [7.704999923706055, -7.177999973297119, -9.225000381469727]], [[9.008000373840332, -7.439000129699707, -9.229999542236328], [9.88599967956543, -6.989999771118164, -8.145000457763672], [10.543000221252441, -5.672999858856201, -8.593999862670898]], [[10.404999732971191, -4.624000072479248, -7.776000022888184], [10.932999610900879, -3.2760000228881836, -8.065999984741211], [12.093999862670898, -2.809000015258789, -7.1620001792907715]], [[12.899999618530273, -1.8980000019073486, -7.703999996185303], [13.99899959564209, -1.284000039100647, -6.953000068664551], [14.04699993133545, 0.1940000057220459, -7.303999900817871]], [[14.305000305175781, 1.0169999599456787, -6.306000232696533], [14.392999649047852, 2.4509999752044678, -6.492000102996826], [15.800000190734863, 2.8940000534057617, -6.047999858856201]], [[16.18600082397461, 4.116000175476074, -6.413000106811523], [17.469999313354492, 4.660999774932861, -5.985000133514404], [17.378999710083008, 6.168000221252441, -5.8420000076293945]], [[18.110000610351562, 6.691999912261963, -4.861000061035156], [18.117000579833984, 8.12600040435791, -4.644000053405762], [19.468000411987305, 8.699000358581543, -5.0920000076293945]], [[19.40399932861328, 9.64799976348877, -6.0289998054504395], [20.575000762939453, 10.357999801635742, -6.546999931335449], [20.594999313354492, 11.727999687194824, -5.860000133514404]], [[21.768999099731445, 12.175999641418457, -5.428999900817871], [21.915000915527344, 13.473999977111816, -4.750999927520752], [22.635000228881836, 14.543999671936035, -5.611999988555908]], [[22.138999938964844, 15.782999992370605, -5.59499979019165], [22.746000289916992, 16.8700008392334, -6.375], [22.878000259399414, 18.115999221801758, -5.494999885559082]], [[24.052000045776367, 18.791000366210938, -5.505000114440918], [24.233999252319336, 20.009000778198242, -4.684000015258789], [23.187999725341797, 21.094999313354492, -5.019000053405762]], [[22.64699935913086, 21.738000869750977, -3.990999937057495], [21.62700080871582, 22.773000717163086, -4.171000003814697], [22.145999908447266, 24.208999633789062, -4.263999938964844]], [[23.295000076293945, 24.46500015258789, -3.631999969482422], [23.905000686645508, 25.79800033569336, -3.628000020980835], [25.420000076293945, 25.756000518798828, -3.859999895095825]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[28.145000457763672, 24.05299949645996, -10.020999908447266], [29.01300048828125, 25.089000701904297, -10.640999794006348], [30.28700065612793, 25.31999969482422, -9.791999816894531]], [[30.704999923706055, 24.329999923706055, -9.012999534606934], [31.886999130249023, 24.511999130249023, -8.178999900817871], [33.24800109863281, 24.340999603271484, -8.91100025177002]], [[33.32600021362305, 23.413999557495117, -9.883000373840332], [34.529998779296875, 23.149999618530273, -10.687999725341797], [34.090999603271484, 22.948999404907227, -12.133000373840332]], [[35.05099868774414, 22.729999542236328, -13.020000457763672], [34.74300003051758, 22.47100067138672, -14.420000076293945], [34.81100082397461, 20.966999053955078, -14.682000160217285]], [[34.9119987487793, 20.209999084472656, -13.590999603271484], [34.9640007019043, 18.759000778198242, -13.62399959564209], [33.874000549316406, 18.301000595092773, -12.652000427246094]], [[32.59299850463867, 18.424999237060547, -13.043000221252441], [31.538000106811523, 17.993999481201172, -12.12600040435791], [31.621000289916992, 16.483999252319336, -11.859000205993652]], [[31.381999969482422, 16.107999801635742, -10.609000205993652], [31.45599937438965, 14.710000038146973, -10.220000267028809], [30.20400047302246, 13.939000129699707, -10.607000350952148]], [[30.31399917602539, 12.619000434875488, -10.527000427246094], [29.204999923706055, 11.73799991607666, -10.817000389099121], [28.250999450683594, 11.845000267028809, -9.64900016784668]], [[26.933000564575195, 11.901000022888184, -9.91100025177002], [26.02400016784668, 12.003999710083008, -8.76200008392334], [26.086000442504883, 10.776000022888184, -7.828999996185303]], [[26.121999740600586, 11.005000114440918, -6.51200008392334], [26.179000854492188, 9.88700008392334, -5.567999839782715], [24.783000946044922, 9.244000434875488, -5.491000175476074]], [[24.718000411987305, 7.9120001792907715, -5.406000137329102], [23.43600082397461, 7.2210001945495605, -5.303999900817871], [23.344999313354492, 6.126999855041504, -4.244999885559082]], [[22.111000061035156, 5.809999942779541, -3.868000030517578], [21.832000732421875, 4.767000198364258, -2.8910000324249268], [20.67099952697754, 3.9149999618530273, -3.4010000228881836]], [[20.924999237060547, 2.625, -3.6010000705718994], [19.895000457763672, 1.7050000429153442, -4.077000141143799], [19.048999786376953, 1.2990000247955322, -2.8989999294281006]], [[17.759000778198242, 1.1100000143051147, -3.138000011444092], [16.843000411987305, 0.7070000171661377, -2.0739998817443848], [16.767000198364258, -0.824999988079071, -1.9910000562667847]], [[16.062999725341797, -1.3539999723434448, -0.9750000238418579], [15.942000389099121, -2.812999963760376, -0.8830000162124634], [14.96500015258789, -3.2200000286102295, -2.003000020980835]], [[14.807999610900879, -4.508999824523926, -2.256999969482422], [13.848999977111816, -4.947999954223633, -3.2660000324249268], [12.428000450134277, -4.8429999351501465, -2.6589999198913574]], [[11.42300033569336, -4.724999904632568, -3.5250000953674316], [10.031000137329102, -4.5929999351501465, -3.0920000076293945], [9.062000274658203, -5.294000148773193, -4.079999923706055]], [[8.29699993133545, -6.263999938964844, -3.5940001010894775], [7.3429999351501465, -6.982999801635742, -4.431000232696533], [5.98199987411499, -6.605999946594238, -3.8519999980926514]], [[5.2870001792907715, -5.64900016784668, -4.492000102996826], [3.9790000915527344, -5.1570000648498535, -4.034999847412109], [2.9609999656677246, -6.131999969482422, -3.4579999446868896]], [[2.621000051498413, -7.171000003814697, -4.211999893188477], [1.6269999742507935, -8.130999565124512, -3.753999948501587], [1.9789999723434448, -8.897000312805176, -2.4719998836517334]], [[3.260999917984009, -8.970000267028809, -2.121999979019165], [3.690999984741211, -9.666999816894531, -0.9079999923706055], [4.105999946594238, -8.706000328063965, 0.20900000631809235]], [[4.7270002365112305, -7.59499979019165, -0.1720000058412552], [5.223999977111816, -6.6539998054504395, 0.8119999766349792], [4.322999954223633, -5.558000087738037, 1.3350000381469727]], [[3.2639999389648438, -5.205999851226807, 0.6150000095367432], [2.359999895095825, -4.1519999504089355, 1.090999960899353], [1.5709999799728394, -4.605999946594238, 2.3499999046325684]], [[1.3899999856948853, -3.7100000381469727, 3.325000047683716], [0.640999972820282, -4.046000003814697, 4.545000076293945], [-0.8709999918937683, -3.992000102996826, 4.298999786376953]], [[-1.6239999532699585, -4.885000228881836, 4.935999870300293], [-3.0769999027252197, -4.857999801635742, 4.793000221252441], [-3.553999900817871, -3.763000011444092, 5.72599983215332]], [[-4.488999843597412, -2.9579999446868896, 5.260000228881836], [-4.974999904632568, -1.8650000095367432, 6.078000068664551], [-6.142000198364258, -2.2249999046325684, 6.961999893188477]], [[-6.343999862670898, -1.4390000104904175, 8.015999794006348], [-7.438000202178955, -1.6690000295639038, 8.958999633789062], [-8.779000282287598, -1.6920000314712524, 8.211000442504883]], [[-9.569000244140625, -2.7790000438690186, 8.35200023651123], [-10.869000434875488, -2.890000104904175, 7.675000190734863], [-11.90999984741211, -1.930999994277954, 8.253000259399414]], [[-13.097999572753906, -1.9010000228881836, 7.671999931335449], [-14.164999961853027, -1.027999997138977, 8.166999816894531], [-15.534000396728516, -1.6779999732971191, 8.050000190734863]], [[-16.208999633789062, -1.7940000295639038, 9.1899995803833], [-17.54800033569336, -2.3570001125335693, 9.229999542236328], [-18.46299934387207, -1.2760000228881836, 8.680000305175781]], [[-18.92300033569336, -1.534999966621399, 7.466000080108643], [-19.795000076293945, -0.6510000228881836, 6.72599983215332], [-21.263999938964844, -0.8399999737739563, 7.146999835968018]], [[-21.59600067138672, -2.0190000534057617, 7.666999816894531], [-22.96299934387207, -2.302000045776367, 8.097000122070312], [-23.076000213623047, -3.5859999656677246, 8.918000221252441]], [[-24.113000869750977, -3.6630001068115234, 9.755000114440918], [-24.37299919128418, -4.822999954223633, 10.616000175476074], [-25.80299949645996, -4.763999938964844, 11.138999938964844]], [[-26.32900047302246, -5.919000148773193, 11.555999755859375], [-27.68199920654297, -6.035999774932861, 12.119000434875488], [-28.003999710083008, -7.466000080108643, 12.541000366210938]], [[-28.770999908447266, -7.5879998207092285, 13.619000434875488], [-29.16200065612793, -8.883999824523926, 14.163000106811523], [-30.660999298095703, -9.119999885559082, 13.937999725341797]], [[-30.969999313354492, -10.04800033569336, 13.034000396728516], [-32.345001220703125, -10.428000450134277, 12.680000305175781], [-32.84000015258789, -11.571999549865723, 13.593000411987305]], [[-33.04600143432617, -11.26200008392334, 14.871000289916992], [-33.50699996948242, -12.269000053405762, 15.807999610900879], [-32.39699935913086, -13.21500015258789, 16.218000411987305]], [[-32.18199920654297, -14.258999824523926, 15.428000450134277], [-31.128999710083008, -15.222999572753906, 15.717000007629395], [-30.04800033569336, -15.145000457763672, 14.62399959564209]], [[-30.13800048828125, -14.11400032043457, 13.78499984741211], [-29.187999725341797, -13.923999786376953, 12.684000015258789], [-28.35700035095215, -12.62600040435791, 12.699000358581543]], [[-27.09600067138672, -12.73799991607666, 13.11400032043457], [-26.20599937438965, -11.583999633789062, 13.126999855041504], [-25.565000534057617, -11.494000434875488, 11.744999885559082]], [[-25.63800048828125, -10.305000305175781, 11.156999588012695], [-25.0939998626709, -10.050000190734863, 9.829999923706055], [-24.166000366210938, -8.821999549865723, 9.805000305175781]], [[-22.961000442504883, -9.006999969482422, 9.258000373840332], [-21.964000701904297, -7.941999912261963, 9.170999526977539], [-21.434999465942383, -7.806000232696533, 7.739999771118164]], [[-21.225000381469727, -6.567999839782715, 7.315000057220459], [-20.739999771118164, -6.2729997634887695, 5.986000061035156], [-19.51300048828125, -5.374000072479248, 6.034999847412109]], [[-18.45199966430664, -5.85099983215332, 5.389999866485596], [-17.173999786376953, -5.165999889373779, 5.315000057220459], [-17.107999801635742, -4.258999824523926, 4.093999862670898]], [[-16.746999740600586, -3.005000114440918, 4.316999912261963], [-16.621000289916992, -2.0160000324249268, 3.256999969482422], [-15.494999885559082, -2.440000057220459, 2.2880001068115234]], [[-15.711000442504883, -2.2730000019073486, 0.9829999804496765], [-14.706999778747559, -2.6570000648498535, -0.00800000037997961], [-13.468999862670898, -1.7419999837875366, -0.03999999910593033]], [[-12.329999923706055, -2.2880001068115234, 0.3799999952316284], [-11.076000213623047, -1.5470000505447388, 0.40799999237060547], [-10.529999732971191, -1.2619999647140503, -0.9860000014305115]], [[-10.111000061035156, -0.01600000075995922, -1.1959999799728394], [-9.536999702453613, 0.44699999690055847, -2.4570000171661377], [-8.303000450134277, 1.277999997138977, -2.119999885559082]], [[-7.64300012588501, 1.7869999408721924, -3.1570000648498535], [-6.499000072479248, 2.6730000972747803, -2.9739999771118164], [-6.519000053405762, 3.7300000190734863, -4.072999954223633]], [[-6.334000110626221, 4.982999801635742, -3.6559998989105225], [-6.306000232696533, 6.122000217437744, -4.567999839782715], [-4.947999954223633, 6.1519999504089355, -5.269999980926514]], [[-4.964000225067139, 5.935999870300293, -6.584000110626221], [-3.7320001125335693, 5.935999870300293, -7.367000102996826], [-3.4609999656677246, 7.314000129699707, -7.965000152587891]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[-7.020999908447266, 6.611000061035156, -11.567999839782715], [-8.026000022888184, 5.585000038146973, -11.149999618530273], [-7.679999828338623, 5.116000175476074, -9.755999565124512]], [[-8.4350004196167, 4.132999897003174, -9.281000137329102], [-8.21399974822998, 3.5450000762939453, -7.965000152587891], [-7.9120001792907715, 2.0480000972747803, -8.192000389099121]], [[-7.377999782562256, 1.371000051498413, -7.177999973297119], [-7.079999923706055, -0.0560000017285347, -7.298999786376953], [-7.7129998207092285, -0.7950000166893005, -6.123000144958496]], [[-8.670999526977539, -1.6770000457763672, -6.431000232696533], [-9.383000373840332, -2.486999988555908, -5.423999786376953], [-8.590999603271484, -3.700000047683716, -4.900000095367432]], [[-8.772000312805176, -3.993000030517578, -3.609999895095825], [-8.095000267028809, -5.10099983215332, -2.9179999828338623], [-8.600000381469727, -6.455999851226807, -3.4210000038146973]], [[-9.89799976348877, -6.501999855041504, -3.691999912261963], [-10.534000396728516, -7.703999996185303, -4.190000057220459], [-10.163999557495117, -7.916999816894531, -5.672999858856201]], [[-9.996000289916992, -6.821000099182129, -6.414999961853027], [-9.633999824523926, -6.879000186920166, -7.823999881744385], [-8.248000144958496, -7.449999809265137, -8.027999877929688]], [[-7.400000095367432, -7.309999942779541, -7.021999835968018], [-6.038000106811523, -7.809000015258789, -7.086999893188477], [-5.916999816894531, -9.239999771118164, -6.563000202178955]], [[-6.39300012588501, -9.437000274658203, -5.330999851226807], [-6.300000190734863, -10.711999893188477, -4.619999885559082], [-7.360000133514404, -11.776000022888184, -4.8979997634887695]], [[-8.520999908447266, -11.385000228881836, -5.415999889373779], [-9.557000160217285, -12.371999740600586, -5.671999931335449], [-9.864999771118164, -13.24899959564209, -4.458000183105469]], [[-9.845999717712402, -14.557000160217285, -4.6579999923706055], [-10.12399959564209, -15.5, -3.5940001010894775], [-9.09000015258789, -15.572999954223633, -2.4609999656677246]], [[-7.9039998054504395, -15.020000457763672, -2.674999952316284], [-6.855000019073486, -15.031999588012695, -1.6440000534057617], [-7.1529998779296875, -14.15999984741211, -0.41200000047683716]], [[-8.02299976348877, -13.163999557495117, -0.578000009059906], [-8.390999794006348, -12.26099967956543, 0.5120000243186951], [-9.597000122070312, -12.789999961853027, 1.2970000505447388]], [[-9.394000053405762, -12.954999923706055, 2.5989999771118164], [-10.413999557495117, -13.454000473022461, 3.5], [-10.527000427246094, -12.460000038146973, 4.632999897003174]], [[-11.593000411987305, -12.593999862670898, 5.421000003814697], [-11.869999885559082, -11.74899959564209, 6.5970001220703125], [-12.241000175476074, -12.609000205993652, 7.827000141143799]], [[-11.809000015258789, -12.173999786376953, 9.003999710083008], [-12.079999923706055, -12.88599967956543, 10.239999771118164], [-12.942999839782715, -11.98900032043457, 11.10099983215332]], [[-13.95300006866455, -12.569000244140625, 11.748000144958496], [-14.85200023651123, -11.822999954223633, 12.630000114440918], [-14.602999687194824, -12.286999702453613, 14.04800033569336]], [[-14.407999992370605, -11.326000213623047, 14.947999954223633], [-14.190999984741211, -11.595000267028809, 16.364999771118164], [-15.47700023651123, -11.140000343322754, 17.025999069213867]], [[-16.082000732421875, -11.996999740600586, 17.85300064086914], [-17.322999954223633, -11.642000198364258, 18.551000595092773], [-17.47599983215332, -12.25100040435791, 19.94700050354004]], [[-18.00200080871582, -11.416000366210938, 20.8439998626709], [-18.253999710083008, -11.734999656677246, 22.2549991607666], [-19.58300018310547, -11.057999610900879, 22.660999298095703]], [[-20.114999771118164, -11.399999618530273, 23.83799934387207], [-21.39699935913086, -10.843000411987305, 24.315000534057617], [-21.285999298095703, -9.682000160217285, 25.315000534057617]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[-14.067999839782715, -14.552000045776367, 23.367000579833984], [-14.081000328063965, -14.126999855041504, 21.937999725341797], [-14.11299991607666, -15.347000122070312, 21.011999130249023]], [[-15.133000373840332, -15.432999610900879, 20.160999298095703], [-15.234999656677246, -16.527999877929688, 19.187000274658203], [-14.9399995803833, -15.920999526977539, 17.79400062561035]], [[-14.468999862670898, -16.73699951171875, 16.8439998626709], [-14.137999534606934, -16.246000289916992, 15.496999740600586], [-14.72599983215332, -16.94300079345703, 14.267999649047852]], [[-15.274999618530273, -16.142000198364258, 13.359999656677246], [-15.829999923706055, -16.638999938964844, 12.102999687194824], [-14.868000030517578, -16.20400047302246, 10.986000061035156]], [[-15.036999702453613, -16.768999099731445, 9.791000366210938], [-14.185999870300293, -16.44099998474121, 8.640000343322754], [-15.017000198364258, -16.464000701904297, 7.364999771118164]], [[-14.60099983215332, -15.6899995803833, 6.366000175476074], [-15.324000358581543, -15.635000228881836, 5.10099983215332], [-14.411999702453613, -15.335000038146973, 3.9159998893737793]], [[-14.838000297546387, -15.765999794006348, 2.7339999675750732], [-14.079999923706055, -15.564000129699707, 1.503000020980835], [-14.579999923706055, -14.359000205993652, 0.7149999737739563]], [[-15.685999870300293, -13.774999618530273, 1.1729999780654907], [-16.28499984741211, -12.595000267028809, 0.546999990940094], [-16.211000442504883, -11.383000373840332, 1.5049999952316284]], [[-17.170000076293945, -10.472999572753906, 1.4249999523162842], [-17.176000595092773, -9.312000274658203, 2.312999963760376], [-18.3700008392334, -9.394000053405762, 3.257999897003174]], [[-19.058000564575195, -10.538000106811523, 3.259999990463257], [-20.246999740600586, -10.732000350952148, 4.0980000495910645], [-20.121999740600586, -11.803000450134277, 5.178999900817871]], [[-20.61199951171875, -11.484999656677246, 6.372000217437744], [-20.613000869750977, -12.414999961853027, 7.504000186920166], [-22.073999404907227, -12.642000198364258, 7.925000190734863]], [[-22.416000366210938, -13.881999969482422, 8.279999732971191], [-23.77400016784668, -14.227999687194824, 8.720999717712402], [-23.740999221801758, -15.42199993133545, 9.666000366210938]], [[-23.551000595092773, -15.156999588012695, 10.95300006866455], [-23.5, -16.235000610351562, 11.939000129699707], [-24.729000091552734, -16.22599983215332, 12.831000328063965]], [[-24.89299964904785, -17.28700065612793, 13.616000175476074], [-26.03499984741211, -17.375, 14.513999938964844], [-25.780000686645508, -16.87700080871582, 15.932999610900879]], [[-26.763999938964844, -16.125, 16.41900062561035], [-26.756000518798828, -15.508999824523926, 17.740999221801758], [-28.101999282836914, -15.755000114440918, 18.430999755859375]], [[-28.139999389648438, -15.59000015258789, 19.750999450683594], [-29.381000518798828, -15.77299976348877, 20.506000518798828], [-30.0, -14.390999794006348, 20.667999267578125]], [[-31.26300048828125, -14.234999656677246, 20.267000198364258], [-31.957000732421875, -12.944999694824219, 20.375999450683594], [-31.951000213623047, -12.442000389099121, 21.81999969482422]], [[-32.132999420166016, -11.130999565124512, 21.981000900268555], [-32.13399887084961, -10.527999877929688, 23.30299949645996], [-30.715999603271484, -10.229999542236328, 23.76099967956543]], [[-29.777000427246094, -11.067999839782715, 23.31399917602539], [-28.354000091552734, -10.947999954223633, 23.625], [-27.71299934387207, -9.770999908447266, 22.889999389648438]], [[-26.652999877929688, -9.234000205993652, 23.482999801635742], [-25.944000244140625, -8.119000434875488, 22.88599967956543], [-24.518999099731445, -8.550999641418457, 22.554000854492188]], [[-24.166000366210938, -8.413000106811523, 21.2810001373291], [-22.8439998626709, -8.788999557495117, 20.79400062561035], [-21.94099998474121, -7.620999813079834, 20.408000946044922]], [[-20.64299964904785, -7.84499979019165, 20.575000762939453], [-19.618999481201172, -6.882999897003174, 20.201000213623047], [-18.690000534057617, -7.693999767303467, 19.31800079345703]], [[-18.246000289916992, -7.076000213623047, 18.23200035095215], [-17.385000228881836, -7.73799991607666, 17.264999389648438], [-16.48699951171875, -6.744999885559082, 16.5310001373291]], [[-15.496000289916992, -7.289999961853027, 15.840999603271484], [-14.54800033569336, -6.51200008392334, 15.055999755859375], [-14.050999641418457, -7.427000045776367, 13.932000160217285]], [[-13.682000160217285, -6.839000225067139, 12.789999961853027], [-13.175000190734863, -7.622000217437744, 11.66100025177002], [-11.704000473022461, -7.339000225067139, 11.383999824523926]], [[-11.081000328063965, -8.272000312805176, 10.67199993133545], [-9.67300033569336, -8.204999923706055, 10.329000473022461], [-9.446999549865723, -8.876999855041504, 8.970999717712402]], [[-8.616000175476074, -8.258999824523926, 8.121000289916992], [-8.258000373840332, -8.803999900817871, 6.804999828338623], [-7.080999851226807, -9.791000366210938, 6.953999996185303]], [[-7.116000175476074, -10.87600040435791, 6.184999942779541], [-6.072000026702881, -11.902000427246094, 6.218999862670898], [-5.803999900817871, -12.527999877929688, 4.8460001945495605]], [[-4.5370001792907715, -12.682000160217285, 4.498000144958496], [-4.171000003814697, -13.345999717712402, 3.253000020980835], [-3.306999921798706, -14.512999534606934, 3.73799991607666]], [[-3.941999912261963, -15.663999557495117, 4.002999782562256], [-3.315000057220459, -16.900999069213867, 4.489999771118164], [-2.115999937057495, -17.464000701904297, 3.7100000381469727]], [[-1.9819999933242798, -17.13599967956543, 2.4260001182556152], [-0.8370000123977661, -17.64900016784668, 1.6679999828338623], [0.49900001287460327, -16.923999786376953, 1.968000054359436]], [[0.42399999499320984, -15.814000129699707, 2.7009999752044678], [1.6169999837875366, -15.050999641418457, 3.069999933242798], [2.380000114440918, -15.668999671936035, 4.251999855041504]], [[3.7009999752044678, -15.526000022888184, 4.234000205993652], [4.570000171661377, -16.06800079345703, 5.275000095367432], [4.769000053405762, -15.09000015258789, 6.416999816894531]], [[4.603000164031982, -13.800999641418457, 6.127999782562256], [4.671999931335449, -12.732999801635742, 7.129000186920166], [3.736999988555908, -11.631999969482422, 6.609000205993652]], [[3.3369998931884766, -10.704999923706055, 7.4770002365112305], [2.3919999599456787, -9.649999618530273, 7.0920000076293945], [1.0499999523162842, -10.291999816894531, 6.671000003814697]], [[0.652999997138977, -11.331999778747559, 7.4029998779296875], [-0.5770000219345093, -12.055999755859375, 7.140999794006348], [-1.8609999418258667, -11.329000473022461, 7.513999938964844]], [[-1.805999994277954, -10.527000427246094, 8.576000213623047], [-2.989000082015991, -9.831999778747559, 9.079999923706055], [-2.9049999713897705, -8.326000213623047, 9.173999786376953]], [[-4.070000171661377, -7.693999767303467, 9.20199966430664], [-4.142000198364258, -6.247000217437744, 9.32699966430664], [-4.565000057220459, -5.923999786376953, 10.765000343322754]], [[-4.554999828338623, -4.626999855041504, 11.152000427246094], [-4.99399995803833, -4.3379998207092285, 12.524999618530273], [-6.5229997634887695, -4.6479997634887695, 12.592000007629395]], [[-7.066999912261963, -4.86299991607666, 13.788000106811523], [-8.496000289916992, -5.135000228881836, 13.909000396728516], [-9.1850004196167, -3.818000078201294, 13.675000190734863]], [[-10.402000427246094, -3.8469998836517334, 13.145999908447266], [-11.13599967956543, -2.5969998836517334, 12.920999526977539], [-11.713000297546387, -2.0959999561309814, 14.255000114440918]], [[-12.361000061035156, -0.9300000071525574, 14.208000183105469], [-13.029999732971191, -0.36800000071525574, 15.383000373840332], [-14.156999588012695, -1.3530000448226929, 15.739999771118164]], [[-14.40999984741211, -1.50600004196167, 17.031999588012695], [-15.444000244140625, -2.4159998893737793, 17.503999710083008], [-16.836999893188477, -1.7960000038146973, 17.36199951171875]], [[-17.82699966430664, -2.6679999828338623, 17.149999618530273], [-19.242000579833984, -2.303999900817871, 17.027000427246094], [-20.014999389648438, -3.3289999961853027, 17.868999481201172]], [[-21.07900047302246, -2.890000104904175, 18.53700065612793], [-21.906999588012695, -3.7950000762939453, 19.347999572753906], [-23.351999282836914, -3.61299991607666, 18.909000396728516]], [[-24.101999282836914, -4.704999923706055, 18.893999099731445], [-25.510000228881836, -4.677999973297119, 18.492000579833984], [-26.343000411987305, -3.6519999504089355, 19.275999069213867]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]]], "seq": "ADTTGRAYNLTWKSTNFKTILEWEPKSIDHVYTVQISTRLENWKSKCFLTAETECDLTDEVVKDVGQTYMARVLSYPARNGNTTGFPEEPPFRNSPEFTPYLDTNLGQPTIQSFEQVGTKLNVTVQDARTLVRRNGTFLSLRAVFGKDLNYTLYYWRASSTGKKTATTNTNEFLIDVDKGENYCFSVQAVIPSRKRKQRSPESLTECTSREQGRAREMF"}, {"coords": [[[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[-27.249000549316406, 32.00699996948242, 36.51499938964844], [-27.124000549316406, 31.660999298095703, 37.965999603271484], [-25.827999114990234, 32.1870002746582, 38.5620002746582]], [[-24.71500015258789, 31.972000122070312, 37.86899948120117], [-23.433000564575195, 32.43299865722656, 38.382999420166016], [-22.660999298095703, 31.291000366210938, 39.034000396728516]], [[-23.02899932861328, 30.06800079345703, 38.66299819946289], [-22.395000457763672, 28.86199951171875, 39.176998138427734], [-21.084999084472656, 28.636999130249023, 38.40399932861328]], [[-20.006000518798828, 28.343000411987305, 39.11399841308594], [-18.72100067138672, 28.152999877929688, 38.44499969482422], [-18.625, 26.902000427246094, 37.55799865722656]], [[-17.722999572753906, 26.94499969482422, 36.58300018310547], [-17.503999710083008, 25.80500030517578, 35.70899963378906], [-16.016000747680664, 25.65999984741211, 35.292999267578125]], [[-15.690999984741211, 24.520000457763672, 34.69599914550781], [-14.335000038146973, 24.215999603271484, 34.25699996948242], [-13.371000289916992, 24.086999893188477, 35.435001373291016]], [[-13.833000183105469, 23.483999252319336, 36.5260009765625], [-12.970999717712402, 23.273000717163086, 37.6879997253418], [-11.916000366210938, 22.235000610351562, 37.28200149536133]], [[-10.640000343322754, 22.59600067138672, 37.39400100708008], [-9.562000274658203, 21.68600082397461, 37.00699996948242], [-8.406999588012695, 21.767000198364258, 38.00400161743164]], [[-7.684999942779541, 20.677000045776367, 38.202999114990234], [-6.559000015258789, 20.684999465942383, 39.1349983215332], [-5.214000225067139, 20.83099937438965, 38.435001373291016]], [[-4.491000175476074, 21.90999984741211, 38.74100112915039], [-3.1489999294281006, 22.152999877929688, 38.18299865722656], [-2.1480000019073486, 21.709999084472656, 39.275001525878906]], [[-1.3830000162124634, 20.650999069213867, 39.00899887084961], [-0.48500001430511475, 20.10700035095215, 40.020999908447266], [0.8149999976158142, 19.542999267578125, 39.44599914550781]], [[1.9639999866485596, 20.020999908447266, 39.939998626708984], [3.322999954223633, 19.58099937438965, 39.500999450683594], [4.2729997634887695, 19.395000457763672, 40.70500183105469]], [[4.747000217437744, 18.18000030517578, 40.926998138427734], [5.619999885559082, 17.913000106811523, 42.077999114990234], [4.966000080108643, 18.298999786376953, 43.42100143432617]], [[3.6619999408721924, 18.03499984741211, 43.516998291015625], [2.861999988555908, 18.301000595092773, 44.70399856567383], [2.3910000324249268, 19.72100067138672, 44.952999114990234]], [[2.882999897003174, 20.665000915527344, 44.15299987792969], [2.4579999446868896, 22.062999725341797, 44.220001220703125], [1.0410000085830688, 22.0, 43.560001373291016]], [[0.0020000000949949026, 22.320999145507812, 44.3390007019043], [-1.3930000066757202, 22.190000534057617, 43.9109992980957], [-2.2799999713897705, 23.40999984741211, 44.029998779296875]], [[-2.994999885559082, 23.67300033569336, 42.93600082397461], [-3.9079999923706055, 24.80699920654297, 42.78799819946289], [-5.190999984741211, 24.392000198364258, 42.03200149536133]], [[-6.328999996185303, 24.89699935913086, 42.472999572753906], [-7.576000213623047, 24.635000228881836, 41.777000427246094], [-7.908999919891357, 25.891000747680664, 40.96500015258789]], [[-8.392000198364258, 25.70199966430664, 39.74800109863281], [-8.767000198364258, 26.805999755859375, 38.867000579833984], [-10.182000160217285, 26.601999282836914, 38.32899856567383]], [[-10.901000022888184, 27.70199966430664, 38.119998931884766], [-12.272000312805176, 27.621999740600586, 37.61000061035156], [-12.64799976348877, 28.867000579833984, 36.80099868774414]], [[-13.883999824523926, 28.88800048828125, 36.30699920654297], [-14.416999816894531, 30.017000198364258, 35.555999755859375], [-15.833000183105469, 30.30900001525879, 36.10300064086914]], [[-16.349000930786133, 31.534000396728516, 35.89099884033203], [-15.708000183105469, 32.645999908447266, 35.180999755859375], [-15.173999786376953, 33.76599884033203, 36.0629997253418]], [[-14.45300006866455, 34.67100143432617, 35.41899871826172], [-13.899999618530273, 35.849998474121094, 36.07899856567383], [-15.053000450134277, 36.84600067138672, 35.9640007019043]], [[-15.53499984741211, 37.36899948120117, 37.0880012512207], [-16.642000198364258, 38.321998596191406, 37.025001525878906], [-16.55900001525879, 39.417999267578125, 38.066001892089844]], [[-16.961000442504883, 40.608001708984375, 37.65999984741211], [-16.920000076293945, 41.73500061035156, 38.5620002746582], [-18.066999435424805, 41.691001892089844, 39.59199905395508]], [[-19.10300064086914, 40.887001037597656, 39.33399963378906], [-20.25, 40.80799865722656, 40.249000549316406], [-20.374000549316406, 39.60100173950195, 41.18199920654297]], [[-19.302000045776367, 38.816001892089844, 41.28900146484375], [-19.29400062561035, 37.637001037597656, 42.14899826049805], [-17.976999282836914, 37.40299987792969, 42.88800048828125]], [[-18.094999313354492, 36.85900115966797, 44.095001220703125], [-16.930999755859375, 36.46500015258789, 44.880001068115234], [-17.121999740600586, 34.97100067138672, 45.152000427246094]], [[-16.030000686645508, 34.275001525878906, 45.433998107910156], [-16.05699920654297, 32.84199905395508, 45.61800003051758], [-15.406999588012695, 32.375, 46.90800094604492]], [[-15.86400032043457, 31.23200035095215, 47.415000915527344], [-15.289999961853027, 30.597000122070312, 48.597999572753906], [-14.982000350952148, 29.170000076293945, 48.165000915527344]], [[-13.79699993133545, 28.69300079345703, 48.52000045776367], [-13.394000053405762, 27.363000869750977, 48.13800048828125], [-13.220000267028809, 26.552000045776367, 49.40299987792969]], [[-13.920000076293945, 25.41900062561035, 49.47600173950195], [-13.795000076293945, 24.511999130249023, 50.62099838256836], [-13.244999885559082, 23.183000564575195, 50.07899856567383]], [[-12.64799976348877, 22.39299964904785, 50.96500015258789], [-12.090999603271484, 21.08799934387207, 50.617000579833984], [-12.510000228881836, 20.066999435424805, 51.70100021362305]], [[-12.654999732971191, 18.799999237060547, 51.33100128173828], [-13.017000198364258, 17.759000778198242, 52.30799865722656], [-12.359999656677246, 16.46500015258789, 51.91699981689453]], [[-12.418000221252441, 15.508000373840332, 52.832000732421875], [-11.92199993133545, 14.154000282287598, 52.606998443603516], [-13.031999588012695, 13.305999755859375, 53.2859992980957]], [[-12.986000061035156, 11.984000205993652, 53.20600128173828], [-14.027999877929688, 11.232000350952148, 53.87699890136719], [-13.861000061035156, 11.22599983215332, 55.41400146484375]], [[-12.755000114440918, 11.793000221252441, 55.89799880981445], [-12.46500015258789, 11.890000343322754, 57.327999114990234], [-12.586999893188477, 13.33899974822998, 57.792999267578125]], [[-12.543999671936035, 14.295999526977539, 56.86199951171875], [-12.630999565124512, 15.720999717712402, 57.21200180053711], [-13.859999656677246, 16.319000244140625, 56.58100128173828]], [[-14.567999839782715, 17.132999420166016, 57.35499954223633], [-15.767999649047852, 17.805999755859375, 56.867000579833984], [-15.347999572753906, 19.007999420166016, 56.05400085449219]], [[-16.19499969482422, 19.444000244140625, 55.12699890136719], [-15.895999908447266, 20.593000411987305, 54.27399826049805], [-15.196000099182129, 21.711000442504883, 55.071998596191406]], [[-14.093999862670898, 22.253999710083008, 54.551998138427734], [-13.361000061035156, 23.319000244140625, 55.244998931884766], [-12.989999771118164, 24.452999114990234, 54.297000885009766]], [[-13.178999900817871, 25.70199966430664, 54.74300003051758], [-12.899999618530273, 26.892000198364258, 53.933998107910156], [-11.442999839782715, 27.281999588012695, 53.941001892089844]], [[-10.914999961853027, 27.540000915527344, 52.74599838256836], [-9.508999824523926, 27.90999984741211, 52.558998107910156], [-9.36400032043457, 29.267000198364258, 51.87799835205078]], [[-10.152999877929688, 29.541000366210938, 50.84199905395508], [-10.119999885559082, 30.841999053955078, 50.16899871826172], [-11.508000373840332, 31.39299964904785, 50.51499938964844]], [[-11.560999870300293, 32.52799987792969, 51.2130012512207], [-12.821999549865723, 33.154998779296875, 51.63399887084961], [-13.244000434875488, 34.39699935913086, 50.854000091552734]], [[-14.395000457763672, 34.29899978637695, 50.20600128173828], [-14.979000091552734, 35.393001556396484, 49.439998626708984], [-13.958000183105469, 36.20000076293945, 48.64500045776367]], [[-13.246999740600586, 35.54199981689453, 47.74399948120117], [-12.22700023651123, 36.21200180053711, 46.94599914550781], [-12.734999656677246, 36.499000549316406, 45.52799987792969]], [[-12.095000267028809, 37.44900131225586, 44.847999572753906], [-12.444999694824219, 37.816001892089844, 43.48099899291992], [-11.57800006866455, 37.02000045776367, 42.499000549316406]], [[-10.545999526977539, 36.374000549316406, 43.0359992980957], [-9.602999687194824, 35.55699920654297, 42.263999938964844], [-10.255000114440918, 34.257999420166016, 41.80500030517578]], [[-9.84000015258789, 33.72100067138672, 40.65700149536133], [-10.375, 32.439998626708984, 40.16699981689453], [-9.35099983215332, 31.270000457763672, 40.27899932861328]], [[-8.531000137329102, 31.308000564575195, 41.32699966430664], [-7.5920000076293945, 30.249000549316406, 41.612998962402344], [-7.294000148773193, 30.197999954223633, 43.1150016784668]], [[-7.104000091552734, 28.988000869750977, 43.6349983215332], [-6.874000072479248, 28.815000534057617, 45.060001373291016], [-5.7829999923706055, 27.785999298095703, 45.36600112915039]], [[-4.663000106811523, 28.211000442504883, 45.9370002746582], [-3.5820000171661377, 27.284000396728516, 46.26599884033203], [-3.99399995803833, 26.378999710083008, 47.43899917602539]], [[-3.986999988555908, 25.069000244140625, 47.20199966430664], [-4.36299991607666, 24.093000411987305, 48.22800064086914], [-3.2130000591278076, 23.167999267578125, 48.64099884033203]], [[-1.9930000305175781, 23.520000457763672, 48.24599838256836], [-0.7730000019073486, 22.773000717163086, 48.58000183105469], [-0.609000027179718, 22.44499969482422, 50.082000732421875]], [[-0.7940000295639038, 23.430999755859375, 50.948001861572266], [-0.6190000176429749, 23.198999404907227, 52.39500045776367], [-1.5169999599456787, 22.115999221801758, 52.983001708984375]], [[-2.687000036239624, 21.948999404907227, 52.375999450683594], [-3.683000087738037, 20.979999542236328, 52.81100082397461], [-3.428999900817871, 19.56999969482422, 52.26100158691406]], [[-3.2249999046325684, 19.461000442504883, 50.94599914550781], [-2.997999906539917, 18.176000595092773, 50.3120002746582], [-1.7100000381469727, 17.4689998626709, 50.71500015258789]], [[-0.6539999842643738, 18.22100067138672, 51.04100036621094], [0.6060000061988831, 17.57200050354004, 51.43899917602539], [0.6209999918937683, 17.035999298095703, 52.85499954223633]], [[-0.460999995470047, 17.266000747680664, 53.5989990234375], [-0.5820000171661377, 16.746000289916992, 54.965999603271484], [-0.7730000019073486, 15.222999572753906, 54.922000885009766]], [[-1.2690000534057617, 14.718999862670898, 53.797000885009766], [-1.4529999494552612, 13.289999961853027, 53.61199951171875], [-1.6059999465942383, 13.04800033569336, 52.0989990234375]], [[-0.4869999885559082, 12.821999549865723, 51.40700149536133], [-0.5120000243186951, 12.62399959564209, 49.95500183105469], [-1.2130000591278076, 11.359999656677246, 49.49399948120117]], [[-1.5670000314712524, 10.498000144958496, 50.446998596191406], [-2.244999885559082, 9.256999969482422, 50.11199951171875], [-3.7639999389648438, 9.359999656677246, 50.138999938964844]], [[-4.2729997634887695, 10.467000007629395, 50.65599822998047], [-5.708000183105469, 10.687999725341797, 50.779998779296875], [-6.329999923706055, 11.244000434875488, 49.49100112915039]], [[-7.63700008392334, 11.005999565124512, 49.29100036621094], [-8.406000137329102, 11.529999732971191, 48.132999420166016], [-9.206999778747559, 12.751999855041504, 48.641998291015625]], [[-9.09000015258789, 13.866999626159668, 47.91899871826172], [-9.718999862670898, 15.128999710083008, 48.26599884033203], [-10.730999946594238, 15.579999923706055, 47.220001220703125]], [[-11.677000045776367, 16.402000427246094, 47.6609992980957], [-12.638999938964844, 17.006000518798828, 46.76599884033203], [-12.79699993133545, 18.45400047302246, 47.22600173950195]], [[-12.918000221252441, 19.368999481201172, 46.27199935913086], [-13.107000350952148, 20.777000427246094, 46.56700134277344], [-14.456999778747559, 21.20800018310547, 45.95800018310547]], [[-14.968999862670898, 22.364999771118164, 46.38999938964844], [-16.19300079345703, 22.94099998474121, 45.79600143432617], [-16.094999313354492, 24.488000869750977, 45.71699905395508]], [[-16.683000564575195, 25.073999404907227, 44.685001373291016], [-16.672000885009766, 26.52400016784668, 44.54600143432617], [-18.082000732421875, 27.077999114990234, 44.858001708984375]], [[-18.167999267578125, 27.950000762939453, 45.85300064086914], [-19.434999465942383, 28.579999923706055, 46.236000061035156], [-19.41900062561035, 29.97599983215332, 45.65999984741211]], [[-20.542999267578125, 30.393999099731445, 45.07099914550781], [-20.69099998474121, 31.732999801635742, 44.48899841308594], [-21.527999877929688, 32.667999267578125, 45.375]], [[-21.055999755859375, 33.90299987792969, 45.54899978637695], [-21.76300048828125, 34.92300033569336, 46.319000244140625], [-21.663000106811523, 36.20899963378906, 45.51900100708008]], [[-22.604999542236328, 37.138999938964844, 45.73099899291992], [-22.542999267578125, 38.40800094604492, 44.98099899291992], [-21.44300079345703, 39.29199981689453, 45.57899856567383]], [[-20.923999786376953, 40.23099899291992, 44.78499984741211], [-19.854000091552734, 41.14500045776367, 45.23400115966797], [-20.275999069213867, 42.19200134277344, 46.292999267578125]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[-29.514999389648438, 42.95899963378906, 50.729000091552734], [-29.913000106811523, 44.305999755859375, 50.23699951171875], [-30.913000106811523, 44.22700119018555, 49.05099868774414]], [[-31.52400016784668, 43.04499816894531, 48.86000061035156], [-32.50400161743164, 42.849998474121094, 47.77799987792969], [-33.814998626708984, 42.257999420166016, 48.31700134277344]], [[-33.67900085449219, 41.30400085449219, 49.24399948120117], [-34.7760009765625, 40.608001708984375, 49.930999755859375], [-34.183998107910156, 40.18299865722656, 51.25699996948242]], [[-35.047000885009766, 39.983001708984375, 52.250999450683594], [-34.60300064086914, 39.516998291015625, 53.555999755859375], [-34.512001037597656, 38.0, 53.540000915527344]], [[-34.941001892089844, 37.42300033569336, 52.40999984741211], [-34.96500015258789, 35.97800064086914, 52.13399887084961], [-34.07500076293945, 35.63800048828125, 50.91400146484375]], [[-32.733001708984375, 35.597999572753906, 51.08399963378906], [-31.836000442504883, 35.28499984741211, 49.96099853515625], [-31.77899932861328, 33.81100082397461, 49.54800033569336]], [[-31.558000564575195, 33.60100173950195, 48.25600051879883], [-31.46500015258789, 32.26599884033203, 47.6619987487793], [-30.141000747680664, 31.600000381469727, 48.01300048828125]], [[-30.152000427246094, 30.27199935913086, 48.073001861572266], [-28.944000244140625, 29.500999450683594, 48.400001525878906], [-27.812000274658203, 29.726999282836914, 47.38999938964844]], [[-26.54800033569336, 29.740999221801758, 47.86600112915039], [-25.3700008392334, 29.950000762939453, 47.01599884033203], [-25.143999099731445, 28.79400062561035, 46.04899978637695]], [[-25.016000747680664, 29.082000732421875, 44.749000549316406], [-24.7810001373291, 27.999000549316406, 43.79999923706055], [-23.44700050354004, 27.351999282836914, 44.20100021362305]], [[-23.284000396728516, 26.07200050354004, 43.88399887084961], [-22.062000274658203, 25.357999801635742, 44.20000076293945], [-21.69300079345703, 24.533000946044922, 42.97100067138672]], [[-20.48699951171875, 23.983999252319336, 42.9900016784668], [-19.965999603271484, 23.1560001373291, 41.90999984741211], [-18.849000930786133, 22.301000595092773, 42.527000427246094]], [[-19.02400016784668, 20.979000091552734, 42.52899932861328], [-18.02400016784668, 20.072999954223633, 43.082000732421875], [-16.891000747680664, 19.86400032043457, 42.07500076293945]], [[-15.678000450134277, 19.69499969482422, 42.584999084472656], [-14.520999908447266, 19.4689998626709, 41.71699905395508], [-14.307999610900879, 17.9689998626709, 41.4630012512207]], [[-13.406000137329102, 17.625, 40.50400161743164], [-13.15999984741211, 16.19499969482422, 40.27199935913086], [-12.428000450134277, 15.713000297546387, 41.55799865722656]], [[-12.460000038146973, 14.427000045776367, 41.88199996948242], [-11.732999801635742, 13.947999954223633, 43.055999755859375], [-10.246999740600586, 14.119999885559082, 42.755001068115234]], [[-9.41100025177002, 14.177000045776367, 43.79399871826172], [-7.965000152587891, 14.37600040435791, 43.606998443603516], [-7.129000186920166, 13.57699966430664, 44.5989990234375]], [[-6.230000019073486, 12.739999771118164, 44.097999572753906], [-5.343999862670898, 11.942999839782715, 44.96500015258789], [-3.934999942779541, 12.37600040435791, 44.54499816894531]], [[-3.2290000915527344, 13.156000137329102, 45.38999938964844], [-1.8739999532699585, 13.682000160217285, 45.12200164794922], [-0.8220000267028809, 12.706999778747559, 44.60100173950195]], [[-0.6639999747276306, 11.60099983215332, 45.3120002746582], [0.26899999380111694, 10.538000106811523, 44.981998443603516], [0.07999999821186066, 10.041999816894531, 43.542999267578125]], [[-1.1690000295639038, 9.956000328063965, 43.095001220703125], [-1.4819999933242798, 9.480999946594238, 41.74800109863281], [-1.7089999914169312, 10.515999794006348, 40.665000915527344]], [[-2.0850000381469727, 11.739999771118164, 41.0369987487793], [-2.427000045776367, 12.777999877929688, 40.05799865722656], [-1.4869999885559082, 13.947999954223633, 39.84299850463867]], [[-0.5289999842643738, 14.163000106811523, 40.74399948120117], [0.39500001072883606, 15.265999794006348, 40.51300048828125], [1.343999981880188, 14.90999984741211, 39.33300018310547]], [[1.5950000286102295, 15.861000061035156, 38.439998626708984], [2.503999948501587, 15.649999618530273, 37.31399917602539], [3.9549999237060547, 15.654000282287598, 37.82899856567383]], [[4.813000202178955, 14.82699966430664, 37.231998443603516], [6.2270002365112305, 14.758000373840332, 37.6150016784668], [6.942999839782715, 15.89799976348877, 36.89099884033203]], [[7.809999942779541, 16.634000778198242, 37.59700012207031], [8.515999794006348, 17.761999130249023, 36.99100112915039], [9.765999794006348, 17.347999572753906, 36.237998962402344]], [[10.196999549865723, 18.170000076293945, 35.26900100708008], [11.37600040435791, 17.969999313354492, 34.430999755859375], [12.621000289916992, 17.716999053955078, 35.29399871826172]], [[13.343999862670898, 16.60700035095215, 35.040000915527344], [14.54699993133545, 16.292999267578125, 35.82600021362305], [15.699999809265137, 17.18600082397461, 35.42100143432617]], [[16.79599952697754, 17.034000396728516, 36.14699935913086], [18.02400016784668, 17.760000228881836, 35.887001037597656], [19.23900032043457, 16.792999267578125, 35.926998138427734]], [[20.038999557495117, 16.804000854492188, 34.86399841308594], [21.2549991607666, 15.986000061035156, 34.77199935913086], [22.3700008392334, 16.618000030517578, 35.63100051879883]], [[22.823999404907227, 15.883999824523926, 36.645999908447266], [23.874000549316406, 16.3439998626709, 37.55699920654297], [25.308000564575195, 16.242000579833984, 36.98500061035156]], [[25.594999313354492, 15.119999885559082, 36.32699966430664], [26.888999938964844, 14.861000061035156, 35.71099853515625], [26.83799934387207, 13.690999984741211, 34.71500015258789]], [[27.91900062561035, 13.57699966430664, 33.94200134277344], [28.106000900268555, 12.531000137329102, 32.94200134277344], [29.577999114990234, 12.406999588012695, 32.55699920654297]], [[29.889999389648438, 11.326000213623047, 31.854999542236328], [31.243999481201172, 11.050000190734863, 31.3799991607666], [31.305999755859375, 9.697999954223633, 30.667999267578125]], [[32.040000915527344, 9.678000450134277, 29.565000534057617], [32.189998626708984, 8.475000381469727, 28.763999938964844], [33.619998931884766, 7.955999851226807, 28.85700035095215]], [[33.7869987487793, 6.775000095367432, 29.43899917602539], [35.11000061035156, 6.177000045776367, 29.54199981689453], [35.2239990234375, 4.839000225067139, 28.80500030517578]], [[35.81999969482422, 4.900000095367432, 27.621000289916992], [36.01499938964844, 3.7269999980926514, 26.79800033569336], [34.7239990234375, 3.2139999866485596, 26.204999923706055]], [[34.29499816894531, 2.0810000896453857, 26.740999221801758], [33.09400177001953, 1.3700000047683716, 26.33099937438965], [31.88599967956543, 1.6799999475479126, 27.260000228881836]], [[32.1150016784668, 2.510999917984009, 28.274999618530273], [31.06999969482422, 2.867000102996826, 29.235000610351562], [30.62700080871582, 4.328999996185303, 29.21299934387207]], [[29.325000762939453, 4.546000003814697, 29.378999710083008], [28.783000946044922, 5.90500020980835, 29.469999313354492], [28.069000244140625, 5.979000091552734, 30.822999954223633]], [[28.420000076293945, 6.9720001220703125, 31.64299964904785], [27.756000518798828, 7.139999866485596, 32.94599914550781], [26.996999740600586, 8.460000038146973, 33.047000885009766]], [[25.77400016784668, 8.390999794006348, 33.573001861572266], [24.957000732421875, 9.57800006866455, 33.73899841308594], [24.315000534057617, 9.552000045776367, 35.11000061035156]], [[24.336999893188477, 10.710000038146973, 35.77000045776367], [23.736000061035156, 10.885000228881836, 37.08599853515625], [22.69099998474121, 12.0, 37.07500076293945]], [[21.523000717163086, 11.71399974822998, 37.64899826049805], [20.41900062561035, 12.677000045776367, 37.750999450683594], [20.458999633789062, 13.331999778747559, 39.14400100708008]], [[20.26099967956543, 14.644000053405762, 39.215999603271484], [20.27199935913086, 15.345999717712402, 40.50199890136719], [19.100000381469727, 14.890999794006348, 41.39500045776367]], [[19.349000930786133, 14.718999862670898, 42.70399856567383], [18.30299949645996, 14.288999557495117, 43.63600158691406], [17.250999450683594, 15.401000022888184, 43.71099853515625]], [[16.05299949645996, 15.109000205993652, 43.20199966430664], [14.96500015258789, 16.090999603271484, 43.167999267578125], [14.199999809265137, 16.277000427246094, 44.46200180053711]], [[13.777999877929688, 17.518999099731445, 44.70100021362305], [12.993000030517578, 17.94300079345703, 45.87900161743164], [11.529000282287598, 18.288999557495117, 45.52799987792969]], [[10.590999603271484, 17.929000854492188, 46.40399932861328], [9.168999671936035, 18.19300079345703, 46.183998107910156], [8.67199993133545, 19.33099937438965, 47.07600021362305]], [[7.38700008392334, 19.666000366210938, 46.95600128173828], [6.769000053405762, 20.715999603271484, 47.777000427246094], [6.163000106811523, 20.158000946044922, 49.07500076293945]], [[6.359000205993652, 18.868000030517578, 49.321998596191406], [5.8480000495910645, 18.236000061035156, 50.53200149536133], [6.656000137329102, 18.701000213623047, 51.75199890136719]], [[5.942999839782715, 19.177000045776367, 52.7599983215332], [6.579999923706055, 19.658000946044922, 53.96699905395508], [5.872000217437744, 19.082000732421875, 55.176998138427734]], [[6.604000091552734, 18.459999084472656, 56.08599853515625], [5.947999954223633, 17.961999893188477, 57.28799819946289], [6.328999996185303, 18.958999633789062, 58.400001525878906]], [[6.5970001220703125, 20.19499969482422, 57.941001892089844], [6.960000038146973, 21.40399932861328, 58.70500183105469], [8.196999549865723, 22.197999954223633, 58.29499816894531]], [[9.392999649047852, 21.732999801635742, 58.63600158691406], [10.57699966430664, 22.485000610351562, 58.250999450683594], [11.144000053405762, 22.219999313354492, 56.856998443603516]], [[11.678000450134277, 21.016000747680664, 56.667999267578125], [12.300999641418457, 20.608999252319336, 55.41299819946289], [11.420999526977539, 20.06399917602539, 54.27199935913086]], [[11.845999717712402, 20.3439998626709, 53.04899978637695], [11.175000190734863, 19.85300064086914, 51.86600112915039], [11.758000373840332, 18.474000930786133, 51.645999908447266]], [[10.902000427246094, 17.457000732421875, 51.62699890136719], [11.352999687194824, 16.075000762939453, 51.43600082397461], [11.559000015258789, 15.760000228881836, 49.952999114990234]], [[12.458999633789062, 14.835000038146973, 49.641998291015625], [12.710000038146973, 14.501999855041504, 48.250999450683594], [11.664999961853027, 13.649999618530273, 47.5629997253418]], [[11.737000465393066, 13.647000312805176, 46.233001708984375], [10.86400032043457, 12.854999542236328, 45.388999938964844], [11.133999824523926, 11.366999626159668, 45.72200012207031]], [[12.401000022888184, 11.017000198364258, 45.93199920654297], [12.751999855041504, 9.654999732971191, 46.30099868774414], [12.163000106811523, 9.300999641418457, 47.6870002746582]], [[12.0, 10.284000396728516, 48.58000183105469], [11.4350004196167, 10.02299976348877, 49.922000885009766], [9.954000473022461, 9.704999923706055, 49.88999938964844]], [[9.234999656677246, 10.295000076293945, 48.93600082397461], [7.796000003814697, 10.043000221252441, 48.779998779296875], [7.51800012588501, 8.758000373840332, 48.000999450683594]], [[8.151000022888184, 8.605999946594238, 46.84199905395508], [7.929999828338623, 7.439000129699707, 45.981998443603516], [8.756999969482422, 6.176000118255615, 46.233001708984375]], [[9.892999649047852, 6.285999774932861, 46.91299819946289], [10.708000183105469, 5.111000061035156, 47.17300033569336], [11.074999809265137, 4.370999813079834, 45.893001556396484]], [[10.868000030517578, 3.052999973297119, 45.89099884033203], [11.164999961853027, 2.1440000534057617, 44.77399826049805], [10.279999732971191, 2.377000093460083, 43.52799987792969]], [[9.138999938964844, 3.0409998893737793, 43.71200180053711], [8.208000183105469, 3.3469998836517334, 42.61600112915039], [8.805999755859375, 4.263000011444092, 41.53900146484375]], [[9.67300033569336, 5.170000076293945, 41.974998474121094], [10.295999526977539, 6.140999794006348, 41.095001220703125], [11.423999786376953, 5.553999900817871, 40.270999908447266]], [[11.366000175476074, 5.715000152587891, 38.941001892089844], [12.428000450134277, 5.2270002365112305, 38.02299880981445], [12.815999984741211, 6.3429999351501465, 37.04999923706055]], [[13.83899974822998, 6.0879998207092285, 36.23400115966797], [14.303999900817871, 7.044000148773193, 35.23400115966797], [14.67199993133545, 6.330999851226807, 33.91600036621094]], [[14.381999969482422, 6.986999988555908, 32.790000915527344], [14.66100025177002, 6.48799991607666, 31.441999435424805], [15.666999816894531, 7.448999881744385, 30.825000762939453]], [[16.628999710083008, 6.901000022888184, 30.077999114990234], [17.652999877929688, 7.679999828338623, 29.35700035095215], [17.39900016784668, 7.5370001792907715, 27.833999633789062]], [[17.375, 8.652000427246094, 27.11400032043457], [17.20400047302246, 8.609999656677246, 25.670000076293945], [18.586000442504883, 8.88700008392334, 25.14699935913086]], [[19.174999237060547, 7.964000225067139, 24.37299919128418], [20.51799964904785, 8.20199966430664, 23.82900047302246], [20.69700050354004, 7.857999801635742, 22.357999801635742]], [[21.584999084472656, 8.618000030517578, 21.729999542236328], [21.88800048828125, 8.48900032043457, 20.315000534057617], [23.280000686645508, 9.083999633789062, 20.01099967956543]], [[23.819000244140625, 8.781999588012695, 18.832000732421875], [25.132999420166016, 9.29699993133545, 18.410999298095703], [24.94300079345703, 10.704999923706055, 17.805999755859375]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[17.28700065612793, 6.52400016784668, 17.940000534057617], [17.218000411987305, 6.703000068664551, 19.42300033569336], [16.95599937438965, 5.377999782562256, 20.11199951171875]], [[17.77199935913086, 5.059999942779541, 21.121999740600586], [17.59600067138672, 3.8420000076293945, 21.933000564575195], [17.37700080871582, 4.3420000076293945, 23.349000930786133]], [[16.905000686645508, 3.4670000076293945, 24.23200035095215], [16.64900016784668, 3.864000082015991, 25.611000061035156], [17.20199966430664, 2.8589999675750732, 26.631999969482422]], [[17.36400032043457, 3.3269999027252197, 27.874000549316406], [17.822999954223633, 2.513000011444092, 29.00200080871582], [16.964000701904297, 2.937000036239624, 30.19499969482422]], [[16.93899917602539, 2.13700008392334, 31.253000259399414], [16.13599967956543, 2.4489998817443848, 32.426998138427734], [16.908000946044922, 2.1480000019073486, 33.72200012207031]], [[16.535999298095703, 2.7799999713897705, 34.83000183105469], [17.195999145507812, 2.513000011444092, 36.10599899291992], [16.235000610351562, 2.743000030517578, 37.242000579833984]], [[16.49799919128418, 2.0910000801086426, 38.36800003051758], [15.64799976348877, 2.253000020980835, 39.54800033569336], [16.270999908447266, 3.2209999561309814, 40.55099868774414]], [[17.525999069213867, 3.609999895095825, 40.319000244140625], [18.2189998626709, 4.565999984741211, 41.18600082397461], [18.325000762939453, 5.89900016784668, 40.4119987487793]], [[19.361000061035156, 6.684999942779541, 40.69499969482422], [19.60300064086914, 7.958000183105469, 39.999000549316406], [20.80500030517578, 7.906000137329102, 39.02899932861328]], [[21.374000549316406, 6.7230000495910645, 38.81999969482422], [22.538999557495117, 6.563000202178955, 37.9370002746582], [22.32200050354004, 5.546000003814697, 36.810001373291016]], [[22.92300033569336, 5.8460001945495605, 35.6619987487793], [22.878000259399414, 4.984000205993652, 34.48699951171875], [24.327999114990234, 4.626999855041504, 34.1619987487793]], [[24.577999114990234, 3.3459999561309814, 33.895999908447266], [25.895999908447266, 2.8399999141693115, 33.499000549316406], [25.600000381469727, 1.878999948501587, 32.36000061035156]], [[25.805999755859375, 2.3340001106262207, 31.131000518798828], [25.520000457763672, 1.5219999551773071, 29.948999404907227], [26.722999572753906, 1.3580000400543213, 29.027999877929688]], [[26.63800048828125, 0.33399999141693115, 28.180999755859375], [27.677000045776367, 0.009999999776482582, 27.19499969482422], [27.466999053955078, 0.7829999923706055, 25.885000228881836]], [[28.538000106811523, 1.406999945640564, 25.405000686645508], [28.52400016784668, 2.180999994277954, 24.17099952697754], [29.77899932861328, 1.875, 23.360000610351562]], [[29.784000396728516, 2.3310000896453857, 22.11199951171875], [30.915000915527344, 2.121999979019165, 21.2189998626709], [31.98699951171875, 3.1589999198913574, 21.49799919128418]], [[33.21900177001953, 2.697999954223633, 21.69099998474121], [34.361000061035156, 3.5739998817443848, 21.972000122070312], [34.6609992980957, 4.579999923706055, 20.85099983215332]], [[35.13100051879883, 5.765999794006348, 21.231000900268555], [35.45800018310547, 6.775000095367432, 20.236000061035156], [34.270999908447266, 7.39300012588501, 19.514999389648438]], [[33.071998596191406, 6.9070000648498535, 19.836000442504883], [31.812999725341797, 7.400000095367432, 19.263999938964844], [31.23699951171875, 8.42300033569336, 20.246999740600586]], [[30.89699935913086, 9.614999771118164, 19.76799964904785], [30.320999145507812, 10.638999938964844, 20.641000747680664], [28.77199935913086, 10.567000389099121, 20.79800033569336]], [[28.2549991607666, 10.79699993133545, 22.018999099731445], [26.80699920654297, 10.711000442504883, 22.288000106811523], [26.06800079345703, 11.949999809265137, 22.784000396728516]], [[24.756000518798828, 11.942999839782715, 22.576000213623047], [23.861000061035156, 13.001999855041504, 23.041000366210938], [22.673999786376953, 12.295999526977539, 23.704999923706055]], [[22.136999130249023, 12.904999732971191, 24.76099967956543], [21.04800033569336, 12.291999816894531, 25.518999099731445], [20.23200035095215, 13.26099967956543, 26.395000457763672]], [[19.160999298095703, 12.727999687194824, 26.96500015258789], [18.273000717163086, 13.456000328063965, 27.86400032043457], [17.674999237060547, 12.390999794006348, 28.799999237060547]], [[17.173999786376953, 12.782999992370605, 29.969999313354492], [16.559999465942383, 11.795999526977539, 30.868999481201172], [15.098999977111816, 12.118000030517578, 31.142000198364258]], [[14.390999794006348, 11.173999786376953, 31.760000228881836], [12.979000091552734, 11.345999717712402, 32.04199981689453], [12.592000007629395, 10.555999755859375, 33.257999420166016]], [[11.791999816894531, 11.163000106811523, 34.138999938964844], [11.29800033569336, 10.52400016784668, 35.362998962402344], [10.086999893188477, 9.659000396728516, 34.98400115966797]], [[10.003000259399414, 8.439000129699707, 35.51100158691406], [8.899999618530273, 7.5320000648498535, 35.180999755859375], [8.368000030517578, 6.741000175476074, 36.38600158691406]], [[7.048999786376953, 6.697000026702881, 36.555999755859375], [6.478000164031982, 5.877999782562256, 37.62799835205078], [5.64300012588501, 4.864999771118164, 36.85300064086914]], [[6.236999988555908, 3.7060000896453857, 36.534000396728516], [5.60699987411499, 2.618000030517578, 35.779998779296875], [4.271999835968018, 2.0840001106262207, 36.29999923706055]], [[4.034999847412109, 2.2070000171661377, 37.597999572753906], [2.7829999923706055, 1.7660000324249268, 38.189998626708984], [1.5299999713897705, 2.615000009536743, 37.84700012207031]], [[1.7050000429153442, 3.8889999389648438, 37.5260009765625], [0.5609999895095825, 4.736999988555908, 37.20199966430664], [-0.10199999809265137, 4.284999847412109, 35.893001556396484]], [[-1.3550000190734863, 4.673999786376953, 35.654998779296875], [-2.0510001182556152, 4.290999889373779, 34.41600036621094], [-2.053999900817871, 5.454999923706055, 33.426998138427734]], [[-1.8669999837875366, 6.666999816894531, 33.94599914550781], [-1.8009999990463257, 7.88100004196167, 33.13100051879883], [-0.8679999709129333, 8.842000007629395, 33.89400100708008]], [[-0.31200000643730164, 9.845999717712402, 33.20600128173828], [0.6420000195503235, 10.812000274658203, 33.79399871826172], [1.8880000114440918, 10.057000160217285, 34.3129997253418]], [[2.2939999103546143, 9.027000427246094, 33.582000732421875], [3.4230000972747803, 8.189000129699707, 33.9630012512207], [4.808000087738037, 8.800000190734863, 33.790000915527344]], [[4.922999858856201, 9.736000061035156, 32.856998443603516], [6.210999965667725, 10.354999542236328, 32.5620002746582], [6.229000091552734, 11.857000350952148, 32.72999954223633]], [[7.419000148773193, 12.395000457763672, 32.94599914550781], [7.618000030517578, 13.82699966430664, 33.07400131225586], [8.145999908447266, 14.331999778747559, 31.738000869750977]], [[8.307000160217285, 15.654000282287598, 31.5939998626709], [8.857999801635742, 16.114999771118164, 30.315000534057617], [10.362000465393066, 15.673999786376953, 30.344999313354492]], [[10.994999885559082, 15.640999794006348, 29.172000885009766], [12.411999702453613, 15.281000137329102, 29.062999725341797], [13.27299976348877, 16.4689998626709, 29.476999282836914]], [[14.461000442504883, 16.17300033569336, 29.99799919128418], [15.357000350952148, 17.222999572753906, 30.43400001525879], [16.016000747680664, 17.843000411987305, 29.218000411987305]], [[16.83099937438965, 18.865999221801758, 29.464000701904297], [17.604999542236328, 19.496000289916992, 28.413000106811523], [18.56999969482422, 18.40999984741211, 27.913999557495117]], [[19.01300048828125, 18.52199935913086, 26.66699981689453], [19.94300079345703, 17.56599998474121, 26.069000244140625], [21.42099952697754, 17.89299964904785, 26.374000549316406]], [[22.225000381469727, 16.847000122070312, 26.583999633789062], [23.656999588012695, 16.999000549316406, 26.854999542236328], [24.400999069213867, 16.106000900268555, 25.882999420166016]], [[25.679000854492188, 16.39299964904785, 25.66900062561035], [26.493999481201172, 15.602999687194824, 24.75], [27.885000228881836, 15.390000343322754, 25.336999893188477]], [[28.444000244140625, 14.206999778747559, 25.108999252319336], [29.775999069213867, 13.89900016784668, 25.618000030517578], [30.87700080871582, 14.531000137329102, 24.76300048828125]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]]], "seq": "ADTTGRAYNLTWKSTNFKTILEWEPKSIDHVYTVQISTRLENWKSKCFLTAETECDLTDEVVKDVGQTYMARVLSYPARNGNTTGFPEEPPFRNSPEFTPYLDTNLGQPTIQSFEQVGTKLNVTVQDARTLVRRNGTFLSLRAVFGKDLNYTLYYWRASSTGKKTATTNTNEFLIDVDKGENYCFSVQAVIPSRKRKQRSPESLTECTSREQGRAREMF"}, {"coords": [[[-5.833000183105469, 72.58899688720703, 45.643001556396484], [-5.9019999504089355, 73.70700073242188, 44.64699935913086], [-5.849999904632568, 73.22599792480469, 43.202999114990234]], [[-6.245999813079834, 74.0530014038086, 42.20500183105469], [-6.263999938964844, 73.74299621582031, 40.763999938964844], [-4.922999858856201, 73.4729995727539, 40.07699966430664]], [[-4.895999908447266, 72.39900207519531, 39.28799819946289], [-3.7049999237060547, 71.98799896240234, 38.54199981689453], [-4.122000217437744, 71.3759994506836, 37.196998596191406]], [[-3.247999906539917, 71.48699951171875, 36.198001861572266], [-3.49399995803833, 70.89800262451172, 34.880001068115234], [-2.812999963760376, 69.52100372314453, 34.84199905395508]], [[-3.4010000228881836, 68.5790023803711, 34.104000091552734], [-2.8429999351501465, 67.24099731445312, 34.0], [-3.0390000343322754, 66.75199890136719, 32.58100128173828]], [[-2.01200008392334, 66.11299896240234, 32.029998779296875], [-2.072999954223633, 65.57099914550781, 30.665000915527344], [-2.871000051498413, 64.25599670410156, 30.68600082397461]], [[-4.105999946594238, 64.2979965209961, 30.17799949645996], [-5.0, 63.13800048828125, 30.106000900268555], [-4.534999847412109, 62.108001708984375, 29.062000274658203]], [[-3.9089999198913574, 62.60499954223633, 27.992000579833984], [-3.371000051498413, 61.79800033569336, 26.891000747680664], [-2.1480000019073486, 62.507999420166016, 26.31999969482422]], [[-0.9649999737739563, 61.95199966430664, 26.55500030517578], [0.2980000078678131, 62.5260009765625, 26.06800079345703], [0.46700000762939453, 62.4640007019043, 24.53499984741211]], [[1.3760000467300415, 63.28099822998047, 23.969999313354492], [1.5479999780654907, 63.20000076293945, 22.516000747680664], [2.3359999656677246, 61.902000427246094, 22.211000442504883]], [[2.247999906539917, 61.39899826049805, 20.98200035095215], [2.9709999561309814, 60.18199920654297, 20.6200008392334], [4.419000148773193, 60.53799819946289, 20.43899917602539]], [[5.284999847412109, 59.880001068115234, 21.207000732421875], [6.73199987411499, 60.11399841308594, 21.21299934387207], [7.486999988555908, 59.87699890136719, 19.89900016784668]], [[6.9720001220703125, 58.970001220703125, 19.076000213623047], [7.5879998207092285, 58.66400146484375, 17.781999588012695], [6.571000099182129, 58.54600143432617, 16.655000686645508]], [[7.070000171661377, 58.72700119018555, 15.444000244140625], [6.244999885559082, 58.61199951171875, 14.265999794006348], [5.6570000648498535, 59.88199996948242, 13.684000015258789]], [[5.7729997634887695, 61.013999938964844, 14.38599967956543], [5.23199987411499, 62.27899932861328, 13.869999885559082], [6.24399995803833, 63.10300064086914, 13.107000350952148]], [[5.826000213623047, 63.57099914550781, 11.942000389099121], [6.703999996185303, 64.36399841308594, 11.104999542236328], [6.546000003814697, 65.87699890136719, 11.331000328063965]], [[7.682000160217285, 66.54199981689453, 11.543000221252441], [7.776000022888184, 67.99199676513672, 11.704999923706055], [7.8429999351501465, 68.50900268554688, 10.262999534606934]], [[7.047999858856201, 69.52899932861328, 9.95300006866455], [7.017000198364258, 70.08300018310547, 8.593999862670898], [7.697999954223633, 71.4520034790039, 8.442999839782715]], [[8.390999794006348, 71.65399932861328, 7.328000068664551], [9.026000022888184, 72.9229965209961, 7.063000202178955], [7.941999912261963, 73.875, 6.546999931335449]], [[8.173999786376953, 75.18299865722656, 6.670000076293945], [7.211999893188477, 76.16699981689453, 6.206999778747559], [6.820000171661377, 75.99600219726562, 4.752999782562256]], [[7.714000225067139, 75.42500305175781, 3.9549999237060547], [7.46999979019165, 75.22599792480469, 2.5290000438690186], [6.52400016784668, 74.0739974975586, 2.177000045776367]], [[6.310999870300293, 73.14199829101562, 3.0969998836517334], [5.4629998207092285, 71.98200225830078, 2.7909998893737793], [3.944999933242798, 72.24299621582031, 2.7709999084472656]], [[3.4769999980926514, 72.9749984741211, 3.7760000228881836], [2.066999912261963, 73.27799987792969, 3.865000009536743], [1.5859999656677246, 73.26899719238281, 5.296999931335449]], [[0.38100001215934753, 73.78900146484375, 5.5269999504089355], [-0.14900000393390656, 73.81700134277344, 6.88700008392334], [-1.152999997138977, 72.69100189208984, 7.191999912261963]], [[-1.5959999561309814, 71.98100280761719, 6.1579999923706055], [-2.5810000896453857, 70.90299987792969, 6.311999797821045], [-1.9880000352859497, 69.49700164794922, 6.078999996185303]], [[-1.062000036239624, 69.10299682617188, 6.934000015258789], [-0.42899999022483826, 67.80500030517578, 6.798999786376953], [-0.7170000076293945, 66.84300231933594, 7.936999797821045]], [[-1.628999948501587, 67.22599792480469, 8.831000328063965], [-1.9850000143051147, 66.37999725341797, 9.961999893188477], [-1.5549999475479126, 66.90499877929688, 11.319999694824219]], [[-2.3919999599456787, 66.65499877929688, 12.326000213623047], [-2.111999988555908, 67.10099792480469, 13.680999755859375], [-1.3660000562667847, 66.0999984741211, 14.54800033569336]], [[-0.49000000953674316, 66.58799743652344, 15.423999786376953], [0.2709999978542328, 65.75800323486328, 16.34000015258789], [-0.7519999742507935, 64.95500183105469, 17.134000778198242]], [[-0.7820000052452087, 63.617000579833984, 16.926000595092773], [-1.7039999961853027, 62.676998138427734, 17.590999603271484], [-1.3589999675750732, 62.28099822998047, 19.05299949645996]], [[-2.2799999713897705, 61.58700180053711, 19.719999313354492], [-2.0799999237060547, 61.172000885009766, 21.099000930786133], [-1.2699999809265137, 59.89899826049805, 21.218000411987305]], [[-0.8830000162124634, 59.5629997253418, 22.441999435424805], [-0.09700000286102295, 58.36399841308594, 22.709999084472656], [-0.6700000166893005, 57.060001373291016, 22.128000259399414]], [[-1.9880000352859497, 56.935001373291016, 22.06999969482422], [-2.6080000400543213, 55.73400115966797, 21.533000946044922], [-3.5439999103546143, 56.047000885009766, 20.368000030517578]], [[-3.0350000858306885, 55.96799850463867, 19.145999908447266], [-3.8559999465942383, 56.279998779296875, 17.981000900268555], [-3.7679998874664307, 55.24100112915039, 16.85700035095215]], [[-4.234000205993652, 54.01900100708008, 17.104000091552734], [-4.190000057220459, 52.98400115966797, 16.06999969482422], [-5.355999946594238, 53.16299819946289, 15.097000122070312]], [[-5.3420000076293945, 52.415000915527344, 13.994000434875488], [-6.364999771118164, 52.49300003051758, 12.9399995803833], [-7.822999954223633, 52.821998596191406, 13.331999778747559]], [[-8.432000160217285, 53.755001068115234, 12.597999572753906], [-9.822999954223633, 54.11600112915039, 12.842000007629395], [-10.189000129699707, 55.03799819946289, 13.998000144958496]], [[-9.473999977111816, 54.94300079345703, 15.121000289916992], [-9.727999687194824, 55.79899978637695, 16.290000915527344], [-8.446000099182129, 56.47999954223633, 16.785999298095703]], [[-8.232999801635742, 57.70399856567383, 16.31100082397461], [-7.066999912261963, 58.48500061035156, 16.684999465942383], [-7.3979997634887695, 59.39400100708008, 17.860000610351562]], [[-6.406000137329102, 59.58700180053711, 18.711000442504883], [-6.53000020980835, 60.459999084472656, 19.863000869750977], [-5.513999938964844, 61.56399917602539, 19.60099983215332]], [[-5.669000148773193, 62.691001892089844, 20.316999435424805], [-4.809999942779541, 63.882999420166016, 20.209999084472656], [-4.39300012588501, 64.22599792480469, 21.634000778198242]], [[-3.263000011444092, 64.95099639892578, 21.823999404907227], [-2.865000009536743, 65.27799987792969, 23.19700050354004], [-4.063000202178955, 65.95099639892578, 23.91200065612793]], [[-4.328000068664551, 65.55799865722656, 25.156999588012695], [-5.445000171661377, 66.08200073242188, 25.94099998474121], [-4.958000183105469, 66.37999725341797, 27.3439998626709]], [[-5.501999855041504, 67.447998046875, 27.908000946044922], [-5.14900016784668, 67.91899871826172, 29.242000579833984], [-6.446000099182129, 68.33599853515625, 29.93600082397461]], [[-6.427000045776367, 68.4020004272461, 31.256000518798828], [-7.630000114440918, 68.75299835205078, 31.97599983215332], [-7.250999927520752, 69.5719985961914, 33.1879997253418]], [[-8.095999717712402, 70.55000305175781, 33.52399826049805], [-7.906000137329102, 71.37799835205078, 34.71699905395508], [-8.737000465393066, 70.69999694824219, 35.82099914550781]], [[-8.0649995803833, 69.91500091552734, 36.66299819946289], [-8.718999862670898, 69.19400024414062, 37.749000549316406], [-8.852999687194824, 70.08499908447266, 38.970001220703125]], [[-10.062000274658203, 70.09100341796875, 39.54199981689453], [-10.404000282287598, 70.90499877929688, 40.715999603271484], [-10.435999870300293, 72.37799835205078, 40.24599838256836]], [[-11.062000274658203, 72.5510025024414, 39.082000732421875], [-11.208999633789062, 73.8219985961914, 38.37300109863281], [-11.607999801635742, 75.02799987792969, 39.18199920654297]], [[-11.092000007629395, 76.18800354003906, 38.784000396728516], [-11.458000183105469, 77.4530029296875, 39.415000915527344], [-11.652999877929688, 78.45500183105469, 38.28200149536133]], [[-12.906999588012695, 78.88099670410156, 38.0989990234375], [-13.284000396728516, 79.8239974975586, 37.051998138427734], [-13.074000358581543, 79.30999755859375, 35.63199996948242]], [[-13.79800033569336, 79.83699798583984, 34.62099838256836], [-13.557000160217285, 79.3290023803711, 33.266998291015625], [-12.057000160217285, 79.58200073242188, 33.04999923706055]], [[-11.388999938964844, 78.83599853515625, 32.176998138427734], [-9.954000473022461, 79.072998046875, 32.029998779296875], [-9.357000350952148, 79.30000305175781, 30.648000717163086]], [[-8.053999900817871, 79.54399871826172, 30.652000427246094], [-7.298999786376953, 79.76799774169922, 29.43899917602539], [-6.076000213623047, 78.83499908447266, 29.469999313354492]], [[-5.968999862670898, 77.96399688720703, 28.457000732421875], [-4.861000061035156, 77.00499725341797, 28.304000854492188], [-4.052000045776367, 77.39700317382812, 27.062000274658203]], [[-2.7330000400543213, 77.41699981689453, 27.20400047302246], [-1.8459999561309814, 77.7300033569336, 26.094999313354492], [-0.75, 76.64800262451172, 25.982999801635742]], [[-0.37400001287460327, 76.30699920654297, 24.759000778198242], [0.6600000262260437, 75.3010025024414, 24.56100082397461], [1.7330000400543213, 75.9209976196289, 23.68199920654297]], [[2.9649999141693115, 75.47699737548828, 23.86400032043457], [4.077000141143799, 75.9219970703125, 23.035999298095703], [5.206999778747559, 74.91799926757812, 23.176000595092773]], [[6.110000133514404, 74.93299865722656, 22.211999893188477], [7.228000164031982, 74.01699829101562, 22.190000534057617], [8.442999839782715, 74.41699981689453, 23.034000396728516]], [[8.812000274658203, 73.54299926757812, 23.975000381469727], [9.965999603271484, 73.74700164794922, 24.854999542236328], [11.050000190734863, 72.6989974975586, 24.566999435424805]], [[12.289999961853027, 73.03500366210938, 24.875], [13.38599967956543, 72.12899780273438, 24.60099983215332], [13.269000053405762, 70.8280029296875, 25.368999481201172]], [[14.02400016784668, 69.83699798583984, 24.892000198364258], [14.112000465393066, 68.50599670410156, 25.486000061035156], [14.925999641418457, 68.61699676513672, 26.773000717163086]], [[15.840999603271484, 69.58599853515625, 26.784000396728516], [16.708999633789062, 69.86900329589844, 27.92300033569336], [15.824999809265137, 70.23100280761719, 29.121000289916992]], [[16.2549991607666, 69.83499908447266, 30.320999145507812], [15.437999725341797, 70.03500366210938, 31.506999969482422], [14.781000137329102, 71.35399627685547, 31.802000045776367]], [[15.543000221252441, 72.41699981689453, 32.18199920654297], [14.854999542236328, 73.69599914550781, 32.44200134277344], [14.184000015258789, 74.00199890136719, 31.084999084472656]], [[12.86400032043457, 73.7490005493164, 30.969999313354492], [12.178000450134277, 73.98699951171875, 29.700000762939453], [12.270000457763672, 75.38400268554688, 29.158000946044922]], [[13.072999954223633, 75.54900360107422, 28.118999481201172], [13.17199993133545, 76.85299682617188, 27.500999450683594], [12.576000213623047, 76.71499633789062, 26.093000411987305]], [[11.944999694824219, 77.79100036621094, 25.618000030517578], [11.307999610900879, 77.85900115966797, 24.30299949645996], [12.17199993133545, 77.31300354003906, 23.163999557495117]], [[11.565999984741211, 76.50199890136719, 22.29599952697754], [12.26099967956543, 75.90699768066406, 21.14299964904785], [12.168000221252441, 76.84600067138672, 19.917999267578125]], [[13.168000221252441, 76.75700378417969, 19.03499984741211], [13.204000473022461, 77.56300354003906, 17.826000213623047], [12.043000221252441, 77.16500091552734, 16.937000274658203]], [[11.5600004196167, 75.92400360107422, 17.083999633789062], [10.406999588012695, 75.4000015258789, 16.31399917602539], [9.109000205993652, 75.98600006103516, 16.89900016784668]], [[8.102999687194824, 76.1760025024414, 16.049999237060547], [6.815000057220459, 76.71199798583984, 16.479999542236328], [5.614999771118164, 75.79900360107422, 16.17099952697754]], [[4.614999771118164, 75.83899688720703, 17.041000366210938], [3.4159998893737793, 75.05400085449219, 16.81599998474121], [2.687999963760376, 75.74099731445312, 15.659000396728516]], [[2.059999942779541, 74.94200134277344, 14.815999984741211], [1.340000033378601, 75.47100067138672, 13.670999526977539], [-0.07599999755620956, 74.88899993896484, 13.720000267028809]], [[-1.059999942779541, 75.58300018310547, 13.121999740600586], [-2.446000099182129, 75.10399627685547, 13.09000015258789], [-3.5840001106262207, 75.76799774169922, 13.82699966430664]], [[-4.5929999351501465, 74.95500183105469, 14.109000205993652], [-5.815000057220459, 75.3949966430664, 14.807999610900879], [-5.545000076293945, 76.19100189208984, 16.104000091552734]], [[-6.059999942779541, 77.4260025024414, 16.17099952697754], [-5.960999965667725, 78.35700225830078, 17.27899932861328], [-4.5, 78.68199920654297, 17.743000030517578]], [[-3.5280001163482666, 78.5999984741211, 16.829999923706055], [-2.119999885559082, 78.87799835205078, 17.12299919128418], [-1.7619999647140503, 80.23600006103516, 16.510000228881836]], [[-1.1030000448226929, 81.0199966430664, 17.29400062561035], [-0.5889999866485596, 82.38600158691406, 16.999000549316406], [0.6990000009536743, 82.22200012207031, 16.21299934387207]], [[1.0160000324249268, 83.29399871826172, 15.52400016784668], [2.1459999084472656, 83.35700225830078, 14.619999885559082], [3.489000082015991, 83.23100280761719, 15.244999885559082]], [[3.510999917984009, 82.92500305175781, 16.534000396728516], [4.788000106811523, 82.75499725341797, 17.229999542236328], [4.927999973297119, 81.31300354003906, 17.70599937438965]], [[3.8610000610351562, 80.55799865722656, 17.549999237060547], [3.8299999237060547, 79.1510009765625, 17.95800018310547], [3.1989998817443848, 78.9520034790039, 19.315000534057617]], [[2.3529999256134033, 79.88400268554688, 19.76300048828125], [1.6699999570846558, 79.73200225830078, 21.042999267578125], [0.2549999952316284, 79.38899993896484, 20.660999298095703]], [[-0.3050000071525574, 78.37300109863281, 21.29599952697754], [-1.6720000505447388, 77.95099639892578, 21.03499984741211], [-2.4000000953674316, 78.43699645996094, 22.28700065612793]], [[-3.5420000553131104, 79.08699798583984, 22.121000289916992], [-4.276000022888184, 79.58399963378906, 23.267000198364258], [-5.73199987411499, 79.26799774169922, 23.062000274658203]], [[-6.308000087738037, 78.6259994506836, 24.065000534057617], [-7.716000080108643, 78.24500274658203, 24.030000686645508], [-8.411999702453613, 78.552001953125, 25.364999771118164]], [[-9.734000205993652, 78.51499938964844, 25.351999282836914], [-10.487000465393066, 78.74199676513672, 26.569000244140625], [-11.293999671936035, 77.46900177001953, 26.881000518798828]], [[-11.135000228881836, 76.90299987792969, 28.082000732421875], [-11.92300033569336, 75.71499633789062, 28.466999053955078], [-13.17300033569336, 76.14800262451172, 29.257999420166016]], [[-14.345999717712402, 75.93299865722656, 28.642000198364258], [-15.64799976348877, 76.2959976196289, 29.200000762939453], [-15.859999656677246, 76.23200225830078, 30.704999923706055]], [[-16.957000732421875, 76.82499694824219, 31.211000442504883], [-17.219999313354492, 76.8030014038086, 32.65700149536133], [-17.506999969482422, 75.39600372314453, 33.2130012512207]], [[-17.8799991607666, 74.4800033569336, 32.314998626708984], [-18.204999923706055, 73.0989990234375, 32.65800094604492], [-17.0939998626709, 72.09500122070312, 32.33300018310547]], [[-17.031999588012695, 71.66699981689453, 31.076000213623047], [-16.010000228881836, 70.72100067138672, 30.631999969482422], [-14.62399959564209, 71.36599731445312, 30.719999313354492]], [[-13.812000274658203, 70.85900115966797, 31.635000228881836], [-12.47599983215332, 71.38300323486328, 31.856000900268555], [-11.392000198364258, 70.59200286865234, 31.14900016784668]], [[-11.718999862670898, 69.95999908447266, 30.02899932861328], [-10.706999778747559, 69.20899963378906, 29.288999557495117], [-10.479000091552734, 69.8550033569336, 27.90999984741211]], [[-9.218999862670898, 69.93399810791016, 27.49799919128418], [-8.883999824523926, 70.49400329589844, 26.197999954223633], [-8.173999786376953, 69.4229965209961, 25.371000289916992]], [[-8.779000282287598, 69.08300018310547, 24.236000061035156], [-8.229000091552734, 68.09700012207031, 23.312000274658203], [-7.6479997634887695, 68.89199829101562, 22.13800048828125]], [[-6.323999881744385, 69.06600189208984, 22.111000061035156], [-5.664000034332275, 69.82499694824219, 21.04400062561035], [-5.59499979019165, 69.00700378417969, 19.770000457763672]], [[-6.716000080108643, 69.072998046875, 19.069000244140625], [-7.001999855041504, 68.39199829101562, 17.826000213623047], [-6.02400016784668, 68.60099792480469, 16.663000106811523]], [[-5.578999996185303, 69.83000183105469, 16.417999267578125], [-4.685999870300293, 70.03900146484375, 15.291000366210938], [-3.431999921798706, 70.84100341796875, 15.550999641418457]], [[-2.433000087738037, 70.1989974975586, 16.1299991607666], [-1.1729999780654907, 70.85199737548828, 16.445999145507812], [-0.03999999910593033, 70.34700012207031, 15.550000190734863]], [[0.6330000162124634, 71.24700164794922, 14.843000411987305], [1.7450000047683716, 70.83200073242188, 14.00100040435791], [3.0409998893737793, 71.41100311279297, 14.543000221252441]], [[4.168000221252441, 71.02200317382812, 13.967000007629395], [5.461999893188477, 71.54900360107422, 14.380000114440918], [6.0960001945495605, 72.09600067138672, 13.105999946594238]], [[6.249000072479248, 73.41300201416016, 13.031000137329102], [6.881999969482422, 74.03099822998047, 11.869999885559082], [8.390999794006348, 73.94100189208984, 12.11400032043457]], [[9.081000328063965, 73.26100158691406, 11.229000091552734], [10.50100040435791, 73.04100036621094, 11.347999572753906], [11.371000289916992, 74.21199798583984, 10.850000381469727]], [[12.161999702453613, 74.81500244140625, 11.730999946594238], [13.043999671936035, 75.90399932861328, 11.295999526977539], [14.258999824523926, 75.31099700927734, 10.605999946594238]], [[14.720000267028809, 75.98500061035156, 9.555000305175781], [15.901000022888184, 75.52999877929688, 8.833000183105469], [17.084999084472656, 75.91999816894531, 9.725000381469727]], [[18.250999450683594, 75.33100128173828, 9.494999885559082], [19.433000564575195, 75.65499877929688, 10.28499984741211], [19.868999481201172, 77.10900115966797, 9.979999542236328]], [[19.482999801635742, 77.59600067138672, 8.805000305175781], [19.80500030517578, 78.94300079345703, 8.383000373840332], [18.732999801635742, 79.89800262451172, 8.89799976348877]], [[17.687000274658203, 79.33100128173828, 9.47700023651123], [16.54800033569336, 80.072998046875, 10.015000343322754], [16.67799949645996, 80.26000213623047, 11.51200008392334]], [[17.17300033569336, 79.2030029296875, 12.144000053405762], [17.316999435424805, 79.10900115966797, 13.579999923706055], [17.672000885009766, 80.33899688720703, 14.392999649047852]], [[18.75200080871582, 81.01499938964844, 14.010000228881836], [19.235000610351562, 82.21800231933594, 14.6899995803833], [18.219999313354492, 83.35099792480469, 14.708000183105469]], [[17.7450008392334, 83.73699951171875, 13.527999877929688], [16.743000030517578, 84.78199768066406, 13.42199993133545], [15.45199966430664, 84.47200012207031, 14.178999900817871]], [[14.991999626159668, 83.2229995727539, 14.081000328063965], [13.784000396728516, 82.75900268554688, 14.770000457763672], [14.053999900817871, 82.81099700927734, 16.28700065612793]], [[15.277000427246094, 82.4520034790039, 16.652000427246094], [15.697999954223633, 82.47599792480469, 18.034000396728516], [15.836999893188477, 83.91699981689453, 18.597999572753906]], [[16.538000106811523, 84.84600067138672, 17.922000885009766], [16.634000778198242, 86.2249984741211, 18.472999572753906], [15.23900032043457, 86.86599731445312, 18.55900001525879]], [[14.362000465393066, 86.51699829101562, 17.6200008392334], [12.996000289916992, 87.04000091552734, 17.57900047302246], [12.093000411987305, 86.5770034790039, 18.732999801635742]], [[12.265000343322754, 85.33499908447266, 19.18000030517578], [11.470000267028809, 84.83399963378906, 20.29400062561035], [12.10099983215332, 85.30400085449219, 21.594999313354492]], [[13.40999984741211, 85.56700134277344, 21.551000595092773], [14.178000450134277, 86.00499725341797, 22.726999282836914], [13.954000473022461, 87.48200225830078, 23.042999267578125]], [[13.666000366210938, 88.27200317382812, 22.003000259399414], [13.369000434875488, 89.7040023803711, 22.12299919128418], [11.994000434875488, 89.75399780273438, 22.760000228881836]], [[11.055999755859375, 89.0780029296875, 22.110000610351562], [9.699000358581543, 88.98799896240234, 22.586999893188477], [9.680000305175781, 88.58799743652344, 24.05699920654297]], [[10.59000015258789, 87.70800018310547, 24.45599937438965], [10.682000160217285, 87.28600311279297, 25.840999603271484], [11.175000190734863, 88.44300079345703, 26.716999053955078]], [[12.170000076293945, 89.17400360107422, 26.224000930786133], [12.753000259399414, 90.30599975585938, 26.945999145507812], [11.704000473022461, 91.32499694824219, 27.299999237060547]], [[11.074000358581543, 91.87799835205078, 26.280000686645508], [10.04800033569336, 92.85900115966797, 26.525999069213867], [8.739999771118164, 92.1760025024414, 26.915000915527344]], [[8.829000473022461, 91.10600280761719, 27.691999435424805], [7.623000144958496, 90.44499969482422, 28.155000686645508], [7.4120001792907715, 91.13800048828125, 29.493999481201172]], [[7.660999774932861, 90.43199920654297, 30.5939998626709], [7.480000019073486, 90.99099731445312, 31.93600082397461], [8.09000015258789, 92.40299987792969, 32.1150016784668]], [[9.345000267028809, 92.55699920654297, 31.70400047302246], [10.08899974822998, 93.81099700927734, 31.834999084472656], [9.720999717712402, 94.83499908447266, 30.711000442504883]], [[10.234999656677246, 96.0999984741211, 30.7810001373291], [10.057999610900879, 97.25199890136719, 29.8700008392334], [10.253000259399414, 97.08799743652344, 28.347999572753906]], [[9.699999809265137, 98.04199981689453, 27.607999801635742], [9.652000427246094, 98.05699920654297, 26.142000198364258], [10.746000289916992, 97.85399627685547, 25.101999282836914]], [[11.982000350952148, 98.26499938964844, 25.343000411987305], [12.949000358581543, 98.0790023803711, 24.274999618530273], [14.29800033569336, 97.46399688720703, 24.554000854492188]], [[15.097999572753906, 97.36199951171875, 23.49799919128418], [16.441999435424805, 96.81800079345703, 23.593000411987305], [17.5049991607666, 97.81900024414062, 23.1200008392334]], [[18.45599937438965, 98.0989990234375, 23.9950008392334], [19.555999755859375, 98.97799682617188, 23.65999984741211], [20.486000061035156, 98.06600189208984, 22.836000442504883]], [[21.466999053955078, 98.63600158691406, 22.145999908447266], [22.375, 97.822998046875, 21.343000411987305], [23.12299919128418, 96.7979965209961, 22.155000686645508]], [[23.437999725341797, 97.14399719238281, 23.399999618530273], [24.145000457763672, 96.23600006103516, 24.281999588012695], [23.195999145507812, 95.0790023803711, 24.57200050354004]], [[21.92799949645996, 95.38999938964844, 24.819000244140625], [20.91200065612793, 94.38800048828125, 25.118999481201172], [20.66699981689453, 93.51200103759766, 23.899999618530273]], [[20.804000854492188, 94.08399963378906, 22.711999893188477], [20.60700035095215, 93.33100128173828, 21.481000900268555], [21.83799934387207, 92.4729995727539, 21.18000030517578]], [[23.024999618530273, 92.96399688720703, 21.523000717163086], [24.261999130249023, 92.20600128173828, 21.301000595092773], [24.270000457763672, 90.93699645996094, 22.201000213623047]], [[23.819000244140625, 91.05599975585938, 23.45599937438965], [23.78700065612793, 89.91200256347656, 24.374000549316406], [22.604000091552734, 88.95700073242188, 24.150999069213867]], [[21.60099983215332, 89.44200134277344, 23.43199920654297], [20.40999984741211, 88.68000030517578, 23.08300018310547], [20.881999969482422, 87.77100372314453, 21.93600082397461]], [[21.670000076293945, 88.33000183105469, 21.017000198364258], [22.21500015258789, 87.55599975585938, 19.909000396728516], [23.240999221801758, 86.5479965209961, 20.441999435424805]], [[23.78700065612793, 86.81500244140625, 21.62700080871582], [24.75, 85.9020004272461, 22.229000091552734], [24.055999755859375, 84.65599822998047, 22.756000518798828]], [[22.816999435424805, 84.78299713134766, 23.22100067138672], [22.082000732421875, 83.60299682617188, 23.680999755859375], [21.849000930786133, 82.74199676513672, 22.461999893188477]], [[21.29199981689453, 83.35600280761719, 21.426000595092773], [20.996000289916992, 82.677001953125, 20.179000854492188], [22.118000030517578, 81.75700378417969, 19.68000030517578]], [[23.351999282836914, 82.2509994506836, 19.618000030517578], [24.44300079345703, 81.40899658203125, 19.128000259399414], [24.725000381469727, 80.26399993896484, 20.08799934387207]], [[24.841999053955078, 80.56800079345703, 21.375], [25.10099983215332, 79.51699829101562, 22.35300064086914], [23.864999771118164, 78.71099853515625, 22.763999938964844]], [[22.733999252319336, 78.97699737548828, 22.11400032043457], [21.496000289916992, 78.24700164794922, 22.381000518798828], [21.349000930786133, 77.28600311279297, 21.207000732421875]], [[21.81999969482422, 77.72200012207031, 20.038000106811523], [21.766000747680664, 76.91200256347656, 18.82699966430664], [22.636999130249023, 75.67500305175781, 19.065000534057617]], [[23.916000366210938, 75.89800262451172, 19.39900016784668], [24.899999618530273, 74.8219985961914, 19.618999481201172], [24.47599983215332, 73.66899871826172, 20.545000076293945]], [[23.499000549316406, 73.89399719238281, 21.429000854492188], [23.010000228881836, 72.8239974975586, 22.322999954223633], [21.517000198364258, 72.50199890136719, 22.187000274658203]], [[20.902000427246094, 72.89600372314453, 21.07699966430664], [19.48200035095215, 72.63099670410156, 20.886999130249023], [19.226999282836914, 71.5459976196289, 19.84600067138672]], [[18.485000610351562, 70.5270004272461, 20.263999938964844], [18.13599967956543, 69.37899780273438, 19.42099952697754], [16.937999725341797, 69.71299743652344, 18.533000946044922]], [[17.156999588012695, 69.85700225830078, 17.229000091552734], [16.045000076293945, 70.18399810791016, 16.34000015258789], [15.170999526977539, 68.98799896240234, 15.98799991607666]], [[15.49899959564209, 67.81099700927734, 16.533000946044922], [14.729000091552734, 66.58899688720703, 16.288999557495117], [13.800999641418457, 66.23100280761719, 17.43400001525879]], [[13.95199966430664, 66.91100311279297, 18.558000564575195], [13.135000228881836, 66.61799621582031, 19.72599983215332], [12.526000022888184, 67.85600280761719, 20.365999221801758]], [[11.21500015258789, 67.83000183105469, 20.599000930786133], [10.524999618530273, 68.947998046875, 21.256999969482422], [9.61299991607666, 68.38099670410156, 22.33300018310547]], [[9.118000030517578, 69.24700164794922, 23.207000732421875], [8.190999984741211, 68.84500122070312, 24.2549991607666], [7.081999778747559, 69.8949966430664, 24.29800033569336]], [[5.839000225067139, 69.4530029296875, 24.452999114990234], [4.7230000495910645, 70.38300323486328, 24.547000885009766], [4.728000164031982, 70.88700103759766, 25.972000122070312]], [[4.427000045776367, 72.16799926757812, 26.15399932861328], [4.343999862670898, 72.75599670410156, 27.479000091552734], [2.9730000495910645, 73.41799926757812, 27.535999298095703]], [[2.128000020980835, 72.875, 28.406999588012695], [0.7870000004768372, 73.38300323486328, 28.63800048828125], [0.824999988079071, 74.33000183105469, 29.82900047302246]], [[0.32499998807907104, 75.53700256347656, 29.61199951171875], [0.29600000381469727, 76.55799865722656, 30.645000457763672], [-1.159999966621399, 76.9260025024414, 30.816999435424805]], [[-1.746999979019165, 76.5009994506836, 31.93600082397461], [-3.1500000953674316, 76.79399871826172, 32.2239990234375], [-3.2980000972747803, 78.08300018310547, 33.06999969482422]], [[-4.335000038146973, 78.86900329589844, 32.77399826049805], [-4.5879998207092285, 80.13099670410156, 33.46500015258789], [-6.02400016784668, 80.28900146484375, 33.88999938964844]], [[-6.204999923706055, 80.48999786376953, 35.1870002746582], [-7.519000053405762, 80.69300079345703, 35.77199935913086], [-7.8480000495910645, 82.16799926757812, 35.59299850463867]], [[-8.87399959564209, 82.44100189208984, 34.78300094604492], [-9.333000183105469, 83.80899810791016, 34.51300048828125], [-9.623000144958496, 84.5780029296875, 35.82400131225586]], [[-3.7829999923706055, 86.88700103759766, 33.0629997253418], [-4.553999900817871, 86.05599975585938, 33.99399948120117], [-3.7119998931884766, 85.36000061035156, 35.097999572753906]], [[-4.386000156402588, 84.5260009765625, 35.88999938964844], [-3.7850000858306885, 83.78500366210938, 37.00199890136719], [-3.1389999389648438, 82.47000122070312, 36.540000915527344]], [[-1.8329999446868896, 82.27999877929688, 36.810001373291016], [-1.1749999523162842, 81.03900146484375, 36.38600158691406], [-1.4229999780654907, 79.84200286865234, 37.308998107910156]], [[-1.2280000448226929, 78.64700317382812, 36.74800109863281], [-1.3109999895095825, 77.35299682617188, 37.45000076293945], [0.08299999684095383, 76.77999877929688, 37.12099838256836]], [[0.531000018119812, 75.69499969482422, 37.75600051879883], [1.8669999837875366, 75.2300033569336, 37.37799835205078], [1.8969999551773071, 74.62799835205078, 35.96099853515625]], [[2.947000026702881, 74.93499755859375, 35.17900085449219], [3.065000057220459, 74.41699981689453, 33.81800079345703], [3.4839999675750732, 72.947998046875, 33.77799987792969]], [[2.75600004196167, 72.177001953125, 32.97200012207031], [2.984999895095825, 70.74500274658203, 32.80699920654297], [3.6449999809265137, 70.40899658203125, 31.465999603271484]], [[4.85699987411499, 69.85600280761719, 31.527000427246094], [5.618000030517578, 69.44999694824219, 30.340999603271484], [5.436999797821045, 67.95600128173828, 29.94499969482422]], [[5.077000141143799, 67.73200225830078, 28.687999725341797], [4.876999855041504, 66.38899993896484, 28.187999725341797], [6.209000110626221, 65.6969985961914, 27.895999908447266]], [[6.116000175476074, 64.43000030517578, 27.531999588012695], [7.290999889373779, 63.641998291015625, 27.201000213623047], [7.811999797821045, 64.13200378417969, 25.847999572753906]], [[9.065999984741211, 63.79899978637695, 25.49799919128418], [9.670000076293945, 64.21299743652344, 24.22800064086914], [8.864999771118164, 63.805999755859375, 23.003000259399414]], [[9.052000045776367, 64.50399780273438, 21.900999069213867], [8.345999717712402, 64.19300079345703, 20.665000915527344], [9.446999549865723, 64.15799713134766, 19.625999450683594]], [[9.696999549865723, 62.99300003051758, 19.045000076293945], [10.76200008392334, 62.90599822998047, 18.059999465942383], [10.286999702453613, 63.119998931884766, 16.642000198364258]], [[11.098999977111816, 63.82400131225586, 15.869999885559082], [10.791000366210938, 64.09500122070312, 14.467000007629395], [11.031000137329102, 62.792999267578125, 13.6899995803833]], [[9.972000122070312, 62.262001037597656, 13.076000213623047], [10.060999870300293, 61.0359992980957, 12.282999992370605], [10.942000389099121, 61.21699905395508, 11.064000129699707]], [[11.192000389099121, 62.46500015258789, 10.680999755859375], [12.032999992370605, 62.736000061035156, 9.520000457763672], [13.505000114440918, 62.85300064086914, 9.871999740600586]], [[13.821999549865723, 62.957000732421875, 11.152000427246094], [15.215999603271484, 62.99700164794922, 11.541999816894531], [15.642999649047852, 61.55099868774414, 11.331000328063965]], [[16.74799919128418, 61.30699920654297, 10.585000038146973], [17.2450008392334, 59.95000076293945, 10.317000389099121], [17.14299964904785, 59.005001068115234, 11.527000427246094]], [[17.50200080871582, 59.51900100708008, 12.708000183105469], [17.437999725341797, 58.72700119018555, 13.939000129699707], [16.29199981689453, 58.94300079345703, 14.947999954223633]], [[15.043000221252441, 59.055999755859375, 14.47700023651123], [13.95199966430664, 59.27899932861328, 15.418000221252441], [12.611000061035156, 58.60499954223633, 15.187999725341797]], [[12.416999816894531, 57.946998596191406, 14.045000076293945], [11.119000434875488, 57.327999114990234, 13.781999588012695], [10.809000015258789, 56.005001068115234, 14.454999923706055]], [[9.604000091552734, 55.53900146484375, 14.14900016784668], [9.027000427246094, 54.290000915527344, 14.618000030517578], [9.810999870300293, 53.191001892089844, 13.98900032043457]], [[10.369000434875488, 52.30400085449219, 14.805999755859375], [11.098999977111816, 51.143001556396484, 14.312999725341797], [9.99899959564209, 50.25699996948242, 13.741999626159668]], [[10.071999549865723, 49.88800048828125, 12.479999542236328], [9.026000022888184, 49.05400085449219, 11.906999588012695], [9.602999687194824, 47.91899871826172, 11.050000190734863]], [[9.083000183105469, 46.71099853515625, 11.239999771118164], [9.505999565124512, 45.573001861572266, 10.446000099182129], [8.432000160217285, 45.45800018310547, 9.361000061035156]], [[8.82699966430664, 45.74700164794922, 8.12399959564209], [7.955999851226807, 45.698001861572266, 6.956999778747559], [7.697000026702881, 44.233001708984375, 6.63100004196167]], [[8.77299976348877, 43.52299880981445, 6.3429999351501465], [8.694000244140625, 42.10100173950195, 6.047999858856201], [10.0600004196167, 41.42599868774414, 6.177000045776367]], [[10.043000221252441, 40.11600112915039, 6.336999893188477], [11.300000190734863, 39.415000915527344, 6.458000183105469], [11.286999702453613, 38.14099884033203, 5.678999900817871]], [[12.491999626159668, 37.76599884033203, 5.301000118255615], [12.873000144958496, 36.58599853515625, 4.550000190734863], [12.244999885559082, 35.28200149536133, 4.938000202178955]], [[12.538999557495117, 34.880001068115234, 6.177999973297119], [12.076000213623047, 33.624000549316406, 6.744999885559082], [11.559000015258789, 33.849998474121094, 8.12399959564209]], [[10.645000457763672, 32.974998474121094, 8.553000450134277], [10.020000457763672, 33.04399871826172, 9.878999710083008], [10.262999534606934, 31.77899932861328, 10.680999755859375]], [[11.17199993133545, 30.94700050354004, 10.211999893188477], [11.498000144958496, 29.718000411987305, 10.916000366210938], [12.902999877929688, 29.315000534057617, 10.574000358581543]], [[13.53499984741211, 28.631000518798828, 11.512999534606934], [14.890999794006348, 28.174999237060547, 11.319000244140625], [15.26200008392334, 27.22800064086914, 12.432000160217285]], [[16.450000762939453, 26.659000396728516, 12.312000274658203], [16.964000701904297, 25.70400047302246, 13.281000137329102], [17.21500015258789, 26.224000930786133, 14.682999610900879]], [[16.93000030517578, 25.360000610351562, 15.652999877929688], [17.14299964904785, 25.68000030517578, 17.052000045776367], [18.639999389648438, 25.95800018310547, 17.320999145507812]], [[19.510000228881836, 25.589000701904297, 16.384000778198242], [20.96299934387207, 25.79400062561035, 16.510000228881836], [21.386999130249023, 27.250999450683594, 16.308000564575195]], [[20.503000259399414, 28.06100082397461, 15.72700023651123], [20.823999404907227, 29.46500015258789, 15.506999969482422], [21.913999557495117, 29.628999710083008, 14.470000267028809]], [[21.847000122070312, 30.69499969482422, 13.689000129699707], [22.847999572753906, 30.895999908447266, 12.661999702453613], [22.224000930786133, 30.958999633789062, 11.291999816894531]], [[20.902999877929688, 30.95199966430664, 11.232999801635742], [20.17799949645996, 31.069000244140625, 9.967000007629395], [20.090999603271484, 32.55699920654297, 9.583000183105469]], [[20.518999099731445, 32.8849983215332, 8.362000465393066], [20.493000030517578, 34.24300003051758, 7.8470001220703125], [19.111000061035156, 34.70100021362305, 7.335000038146973]], [[18.632999420166016, 35.792999267578125, 7.939000129699707], [17.354000091552734, 36.45199966430664, 7.6539998054504395], [17.56599998474121, 37.88100051879883, 7.0960001945495605]], [[16.85300064086914, 38.21200180053711, 6.014999866485596], [16.847999572753906, 39.53200149536133, 5.394000053405762], [15.59000015258789, 40.21799850463867, 5.958000183105469]], [[15.765999794006348, 41.29600143432617, 6.705999851226807], [14.657999992370605, 42.018001556396484, 7.295000076293945], [14.538999557495117, 43.37699890136719, 6.625]], [[13.352999687194824, 43.71799850463867, 6.124000072479248], [13.119000434875488, 45.02399826049805, 5.485000133514404], [12.503000259399414, 45.9119987487793, 6.546000003814697]], [[13.069000244140625, 47.09000015258789, 6.7729997634887695], [12.550999641418457, 47.98699951171875, 7.815000057220459], [12.461000442504883, 49.44599914550781, 7.379000186920166]], [[11.89799976348877, 50.305999755859375, 8.239999771118164], [11.835000038146973, 51.74100112915039, 7.948999881744385], [13.229000091552734, 52.21200180053711, 8.291999816894531]], [[13.637999534606934, 53.393001556396484, 7.8480000495910645], [14.99899959564209, 53.87699890136719, 8.12600040435791], [15.562000274658203, 53.68299865722656, 9.538999557495117]], [[16.70400047302246, 53.013999938964844, 9.649999618530273], [17.326000213623047, 52.834999084472656, 10.965999603271484], [18.79599952697754, 53.19300079345703, 10.89799976348877]], [[19.374000549316406, 53.48099899291992, 12.057000160217285], [20.79599952697754, 53.805999755859375, 12.13700008392334], [21.655000686645508, 52.55699920654297, 12.357999801635742]], [[22.631000518798828, 52.33599853515625, 11.47599983215332], [23.533000946044922, 51.17499923706055, 11.54699993133545], [24.03700065612793, 50.8489990234375, 12.947999954223633]], [[24.527000427246094, 51.86000061035156, 13.645999908447266], [25.093000411987305, 51.638999938964844, 14.954000473022461], [24.19499969482422, 51.90800094604492, 16.132999420166016]], [[22.891000747680664, 51.99800109863281, 15.923999786376953], [22.000999450683594, 52.24300003051758, 17.05299949645996], [20.82200050354004, 51.25400161743164, 17.07200050354004]], [[20.930999755859375, 50.215999603271484, 16.256000518798828], [19.858999252319336, 49.237998962402344, 16.14299964904785], [20.23699951171875, 47.816001892089844, 16.565000534057617]], [[19.22800064086914, 47.0359992980957, 16.93000030517578], [19.426000595092773, 45.654998779296875, 17.326000213623047], [18.225000381469727, 44.810001373291016, 16.927000045776367]], [[18.374000549316406, 43.49800109863281, 17.03700065612793], [17.308000564575195, 42.5629997253418, 16.716999053955078], [17.16200065612793, 41.64899826049805, 17.92799949645996]], [[16.232999801635742, 42.042999267578125, 18.78499984741211], [15.951000213623047, 41.369998931884766, 20.04400062561035], [14.965999603271484, 40.22200012207031, 19.94099998474121]], [[15.442999839782715, 39.01300048828125, 20.215999603271484], [14.592000007629395, 37.81999969482422, 20.22100067138672], [14.225000381469727, 37.57899856567383, 21.707000732421875]], [[13.038999557495117, 37.04899978637695, 21.951000213623047], [12.630999565124512, 36.84199905395508, 23.32200050354004], [11.33899974822998, 36.047000885009766, 23.340999603271484]], [[10.984000205993652, 35.56100082397461, 24.525999069213867], [9.74899959564209, 34.819000244140625, 24.78700065612793], [9.449999809265137, 35.12699890136719, 26.256999969482422]], [[8.364999771118164, 35.8489990234375, 26.52199935913086], [8.050999641418457, 36.2400016784668, 27.89900016784668], [7.067999839782715, 35.321998596191406, 28.687000274658203]], [[7.639999866485596, 34.34299850463867, 29.402999877929688], [6.886000156402588, 33.39400100708008, 30.253999710083008], [7.111000061035156, 33.92300033569336, 31.687000274658203]], [[7.2870001792907715, 33.042999267578125, 32.676998138427734], [7.553999900817871, 33.50600051879883, 34.055999755859375], [9.08899974822998, 33.540000915527344, 34.25400161743164]], [[9.795000076293945, 33.46900177001953, 33.12099838256836], [11.26200008392334, 33.45199966430664, 33.047000885009766], [11.83899974822998, 34.65399932861328, 32.26300048828125]], [[11.755999565124512, 34.60900115966797, 30.926000595092773], [12.251999855041504, 35.70100021362305, 30.10300064086914], [13.595999717712402, 35.54600143432617, 29.402000427246094]], [[13.663999557495117, 34.66899871826172, 28.398000717163086], [14.890000343322754, 34.45600128173828, 27.60300064086914], [14.989999771118164, 35.56100082397461, 26.55699920654297]], [[16.20800018310547, 36.025001525878906, 26.30900001525879], [16.441999435424805, 37.01900100708008, 25.277999877929688], [17.797000885009766, 36.76300048828125, 24.632999420166016]], [[17.885000228881836, 37.073001861572266, 23.350000381469727], [19.101999282836914, 36.90800094604492, 22.576000213623047], [19.038999557495117, 37.93199920654297, 21.451000213623047]], [[20.094999313354492, 37.98500061035156, 20.649999618530273], [20.20400047302246, 38.97800064086914, 19.59000015258789], [20.628999710083008, 38.492000579833984, 18.194000244140625]], [[20.25200080871582, 39.270999908447266, 17.180999755859375], [20.62700080871582, 38.97700119018555, 15.814000129699707], [22.111000061035156, 39.2599983215332, 15.675000190734863]], [[22.735000610351562, 38.630001068115234, 14.701000213623047], [24.1560001373291, 38.79899978637695, 14.470000267028809], [24.378000259399414, 39.46699905395508, 13.109000205993652]], [[24.85700035095215, 40.69599914550781, 13.142999649047852], [25.10099983215332, 41.45100021362305, 11.92199993133545], [26.086000442504883, 42.58300018310547, 12.222999572753906]], [[26.851999282836914, 43.00600051879883, 11.223999977111816], [27.78700065612793, 44.112998962402344, 11.383999824523926], [27.128000259399414, 45.297000885009766, 10.657999992370605]], [[27.632999420166016, 46.53900146484375, 10.86299991607666], [26.996999740600586, 47.66400146484375, 10.15999984741211], [26.902000427246094, 47.49300003051758, 8.62600040435791]], [[27.79599952697754, 46.6879997253418, 8.048999786376953], [27.81100082397461, 46.422000885009766, 6.610000133514404], [26.56599998474121, 45.638999938964844, 6.247000217437744]], [[26.01099967956543, 44.95399856567383, 7.244999885559082], [24.81399917602539, 44.143001556396484, 7.078000068664551], [23.552000045776367, 44.957000732421875, 7.083000183105469]], [[23.645000457763672, 46.19900131225586, 7.556000232696533], [22.503999710083008, 47.112998962402344, 7.550000190734863], [22.656999588012695, 47.827999114990234, 6.202000141143799]], [[21.781999588012695, 47.48099899291992, 5.26800012588501], [21.81800079345703, 48.00899887084961, 3.9110000133514404], [21.0310001373291, 49.308998107910156, 3.6549999713897705]], [[21.760000228881836, 50.400001525878906, 3.3559999465942383], [21.167999267578125, 51.7140007019043, 3.0429999828338623], [20.02899932861328, 52.108001708984375, 4.000999927520752]], [[20.201000213623047, 51.7859992980957, 5.289999961853027], [19.246000289916992, 52.07099914550781, 6.375999927520752], [17.83799934387207, 51.483001708984375, 6.270999908447266]], [[17.601999282836914, 50.632999420166016, 5.2769999504089355], [16.2810001373291, 50.02899932861328, 5.0370001792907715], [16.19499969482422, 48.481998443603516, 5.040999889373779]], [[17.30699920654297, 47.82099914550781, 5.321000099182129], [17.351999282836914, 46.37699890136719, 5.348999977111816], [18.475000381469727, 45.98099899291992, 6.2870001792907715]], [[18.27199935913086, 44.85300064086914, 6.9629998207092285], [19.238000869750977, 44.27199935913086, 7.869999885559082], [19.386999130249023, 42.78099822998047, 7.5269999504089355]], [[20.60300064086914, 42.34000015258789, 7.244999885559082], [20.85099983215332, 40.93299865722656, 6.960000038146973], [21.527000427246094, 40.4379997253418, 8.220000267028809]], [[20.82900047302246, 39.58100128173828, 8.954999923706055], [21.363000869750977, 39.04899978637695, 10.190999984741211], [21.240999221801758, 37.53300094604492, 10.312999725341797]], [[22.07900047302246, 36.9630012512207, 11.178999900817871], [22.066999435424805, 35.534000396728516, 11.46500015258789], [21.367000579833984, 35.334999084472656, 12.829999923706055]], [[20.308000564575195, 34.52299880981445, 12.83899974822998], [19.507999420166016, 34.279998779296875, 14.043999671936035], [20.357999801635742, 33.77199935913086, 15.210000038146973]], [[19.91699981689453, 34.01900100708008, 16.444000244140625], [20.740999221801758, 33.51300048828125, 17.538000106811523], [20.323999404907227, 32.0620002746582, 17.900999069213867]], [[21.166000366210938, 31.347000122070312, 18.6560001373291], [20.764999389648438, 29.989999771118164, 19.007999420166016], [19.687999725341797, 30.006000518798828, 20.10099983215332]], [[18.761999130249023, 29.05699920654297, 20.062000274658203], [17.733999252319336, 28.966999053955078, 21.0939998626709], [18.381000518798828, 28.510000228881836, 22.41699981689453]], [[17.760000228881836, 28.809999465942383, 23.551000595092773], [18.35300064086914, 28.429000854492188, 24.836999893188477], [18.371999740600586, 26.930999755859375, 25.200000762939453]], [[17.597999572753906, 26.110000610351562, 24.5], [17.55900001525879, 24.665000915527344, 24.766000747680664], [17.857999801635742, 23.900999069213867, 23.466999053955078]], [[19.09000015258789, 23.415000915527344, 23.304000854492188], [19.47100067138672, 22.66200065612793, 22.094999313354492], [18.64699935913086, 21.389999389648438, 21.93199920654297]], [[18.166000366210938, 20.860000610351562, 23.04599952697754], [17.38599967956543, 19.628000259399414, 23.051000595092773], [15.873000144958496, 19.79599952697754, 22.858999252319336]], [[15.402999877929688, 20.60700035095215, 21.917999267578125], [13.946000099182129, 20.69499969482422, 21.788999557495117], [13.41100025177002, 19.604999542236328, 20.878000259399414]], [[12.282999992370605, 19.034000396728516, 21.275999069213867], [11.678000450134277, 17.961000442504883, 20.506999969482422], [10.482000350952148, 18.42300033569336, 19.684999465942383]], [[9.970999717712402, 19.601999282836914, 20.007999420166016], [8.821999549865723, 20.158000946044922, 19.31100082397461], [9.164999961853027, 21.591999053955078, 18.908000946044922]], [[8.715999603271484, 22.058000564575195, 17.718000411987305], [9.074000358581543, 23.44700050354004, 17.402999877929688], [8.503999710083008, 24.44300079345703, 18.45199966430664]], [[9.227999687194824, 25.527999877929688, 18.67099952697754], [8.840999603271484, 26.55500030517578, 19.639999389648438], [8.845000267028809, 27.90399932861328, 18.954999923706055]], [[7.796999931335449, 28.673999786376953, 19.202999114990234], [7.65500020980835, 30.009000778198242, 18.645000457763672], [8.118000030517578, 31.023000717163086, 19.702999114990234]], [[9.006999969482422, 31.898000717163086, 19.267000198364258], [9.550000190734863, 32.96900177001953, 20.076000213623047], [9.22599983215332, 34.2239990234375, 19.2450008392334]], [[9.699999809265137, 35.37099838256836, 19.68600082397461], [9.432999610900879, 36.59299850463867, 18.957000732421875], [10.73799991607666, 37.31100082397461, 18.711000442504883]], [[10.72700023651123, 38.185001373291016, 17.714000701904297], [11.878999710083008, 39.03099822998047, 17.45199966430664], [11.291000366210938, 40.4119987487793, 17.299999237060547]], [[12.14900016784668, 41.402000427246094, 17.46500015258789], [11.741999626159668, 42.784000396728516, 17.347000122070312], [12.972000122070312, 43.59299850463867, 17.14699935913086]], [[12.784000396728516, 44.75, 16.55299949645996], [13.866999626159668, 45.69499969482422, 16.363000869750977], [13.821999549865723, 46.520999908447266, 17.648000717163086]], [[15.0, 46.874000549316406, 18.135000228881836], [15.128000259399414, 47.654998779296875, 19.341999053955078], [16.242000579833984, 48.6609992980957, 19.13800048828125]], [[16.04199981689453, 49.847999572753906, 19.69700050354004], [17.013999938964844, 50.926998138427734, 19.625], [17.945999145507812, 50.75600051879883, 20.82200050354004]], [[19.253000259399414, 50.79199981689453, 20.575000762939453], [20.261999130249023, 50.60599899291992, 21.625999450683594], [20.277999877929688, 51.689998626708984, 22.697999954223633]], [[20.417999267578125, 52.93600082397461, 22.267000198364258], [20.461999893188477, 54.055999755859375, 23.187999725341797], [19.06100082397461, 54.66999816894531, 23.479999542236328]], [[18.017000198364258, 53.854000091552734, 23.326000213623047], [16.656999588012695, 54.29399871826172, 23.60099983215332], [15.800000190734863, 53.22999954223633, 24.292999267578125]], [[15.694999694824219, 52.060001373291016, 23.67300033569336], [14.906000137329102, 50.99700164794922, 24.261999130249023], [13.527999877929688, 50.816001892089844, 23.65999984741211]], [[13.178000450134277, 51.630001068115234, 22.66699981689453], [11.869999885559082, 51.51900100708008, 22.010000228881836], [12.020000457763672, 50.4109992980957, 21.007999420166016]], [[10.930999755859375, 49.68199920654297, 20.81800079345703], [10.92199993133545, 48.525001525878906, 19.941999435424805], [9.892000198364258, 48.66999816894531, 18.854000091552734]], [[10.019000053405762, 47.82099914550781, 17.8439998626709], [9.069000244140625, 47.8120002746582, 16.756000518798828], [8.029000282287598, 46.79100036621094, 17.141000747680664]], [[6.946000099182129, 46.68899917602539, 16.374000549316406], [5.929999828338623, 45.667999267578125, 16.632999420166016], [6.666999816894531, 44.362998962402344, 16.371999740600586]], [[6.520999908447266, 43.40299987792969, 17.280000686645508], [7.199999809265137, 42.112998962402344, 17.16900062561035], [6.585999965667725, 41.12799835205078, 16.17799949645996]], [[7.464000225067139, 40.41299819946289, 15.477999687194824], [7.071000099182129, 39.374000549316406, 14.520000457763672], [7.502999782562256, 38.07600021362305, 15.222000122070312]], [[6.834000110626221, 36.96699905395508, 14.895999908447266], [7.144999980926514, 35.654998779296875, 15.454999923706055], [8.208000183105469, 34.91299819946289, 14.633000373840332]], [[8.906000137329102, 34.007999420166016, 15.29699993133545], [9.902999877929688, 33.18600082397461, 14.63700008392334], [9.810999870300293, 31.840999603271484, 15.359000205993652]], [[9.862000465393066, 30.756000518798828, 14.595000267028809], [9.76200008392334, 29.445999145507812, 15.180999755859375], [11.017000198364258, 28.638999938964844, 15.02299976348877]], [[11.50100040435791, 28.177000045776367, 16.156999588012695], [12.666999816894531, 27.35700035095215, 16.208999633789062], [12.251999855041504, 25.92300033569336, 15.951000213623047]], [[13.003999710083008, 25.231000900268555, 15.093999862670898], [12.734999656677246, 23.833999633789062, 14.772000312805176], [13.954000473022461, 23.000999450683594, 15.145999908447266]], [[13.729000091552734, 21.690000534057617, 15.4399995803833], [14.682000160217285, 20.628999710083008, 15.824999809265137], [15.729000091552734, 20.27899932861328, 14.744999885559082]]], "seq": "GPYLVIVEQPKQRGFRFRYGCEGPSHGGLPGASSEKGRKTYPTVKICNYEGPAKIEVDLVTHSDPPRAHAHSLVGKQCSELGICAVSVGPKDMTAQFNNLGVLHVTKKNMMGTMIQKLQRQRLRSRPQGLTEAEQRELEQEAKELKKVMDLSIVRLRFSAFLRSLPLKPVISQPIHDSKSPGASNLKISRMDKTAGSVRGGDEVYLLCDKVQKDDIEVRFYEDDENGWQAFGDFSPTDVHKQYAIVFRTPPYHKMKIERPVTVFLQLKRKRGGDVSDSKQFTYYP"}, {"coords": [[[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[-13.663000106811523, -0.6499999761581421, 26.72100067138672], [-12.8149995803833, 0.1809999942779541, 27.625999450683594], [-11.343999862670898, -0.22599999606609344, 27.55299949645996]], [[-10.508999824523926, 0.4970000088214874, 28.288000106811523], [-9.07699966430664, 0.24300000071525574, 28.30500030517578], [-8.60099983215332, -0.6359999775886536, 29.451000213623047]], [[-7.798999786376953, -1.6430000066757202, 29.124000549316406], [-7.251999855041504, -2.5360000133514404, 30.135000228881836], [-5.7170000076293945, -2.5250000953674316, 30.073999404907227]], [[-5.071000099182129, -3.1489999294281006, 31.05299949645996], [-3.615999937057495, -3.1619999408721924, 31.083999633789062], [-3.065999984741211, -4.561999797821045, 31.191999435424805]], [[-1.9190000295639038, -4.781000137329102, 30.562999725341797], [-1.2960000038146973, -6.086999893188477, 30.600000381469727], [0.21899999678134918, -5.992000102996826, 30.65399932861328]], [[0.8220000267028809, -7.005000114440918, 31.260000228881836], [2.26200008392334, -7.080999851226807, 31.384000778198242], [2.7820000648498535, -7.73199987411499, 30.111000061035156]], [[4.079999923706055, -7.605999946594238, 29.857999801635742], [4.681000232696533, -8.168999671936035, 28.652999877929688], [4.375999927520752, -9.642000198364258, 28.408000946044922]], [[4.448999881744385, -10.480999946594238, 29.45400047302246], [4.15500020980835, -11.899999618530273, 29.22100067138672], [2.7139999866485596, -12.074000358581543, 28.739999771118164]], [[1.843000054359436, -11.180000305175781, 29.195999145507812], [0.4410000145435333, -11.225000381469727, 28.81599998474121], [0.2939999997615814, -10.8100004196167, 27.36199951171875]], [[0.9350000023841858, -9.70300006866455, 26.990999221801758], [0.8999999761581421, -9.21399974822998, 25.621000289916992], [1.3459999561309814, -10.329000473022461, 24.68000030517578]], [[2.500999927520752, -10.913999557495117, 24.976999282836914], [3.0409998893737793, -12.00100040435791, 24.172000885009766], [2.0439999103546143, -13.142000198364258, 23.986000061035156]], [[1.281000018119812, -13.437999725341797, 25.030000686645508], [0.3059999942779541, -14.517000198364258, 24.983999252319336], [-0.9390000104904175, -14.147000312805176, 24.167999267578125]], [[-1.2690000534057617, -12.86400032043457, 24.12700080871582], [-2.431999921798706, -12.428999900817871, 23.371999740600586], [-2.1059999465942383, -12.41100025177002, 21.87700080871582]], [[-0.8799999952316284, -12.017999649047852, 21.54199981689453], [-0.46000000834465027, -11.956000328063965, 20.14699935913086], [-0.5960000157356262, -13.295999526977539, 19.434999465942383]], [[-0.5230000019073486, -14.380999565124512, 20.19300079345703], [-0.6460000276565552, -15.710000038146973, 19.618999481201172], [-2.109999895095825, -16.099000930786133, 19.4689998626709]], [[-3.01200008392334, -15.262999534606934, 19.969999313354492], [-4.434000015258789, -15.576000213623047, 19.889999389648438], [-5.059999942779541, -15.206000328063965, 18.541000366210938]], [[-4.269999980926514, -14.63700008392334, 17.632999420166016], [-4.7779998779296875, -14.27299976348877, 16.308000564575195], [-4.955999851226807, -15.576000213623047, 15.52400016784668]], [[-6.175000190734863, -15.869000434875488, 15.07699966430664], [-6.392000198364258, -17.124000549316406, 14.371999740600586], [-6.432000160217285, -17.121000289916992, 12.845999717712402]], [[-5.7789998054504395, -16.143999099731445, 12.22700023651123], [-5.734000205993652, -16.111000061035156, 10.776000022888184], [-4.5229997634887695, -16.881000518798828, 10.281999588012695]], [[-4.446000099182129, -17.118999481201172, 8.975000381469727], [-3.2990000247955322, -17.81599998474121, 8.38599967956543], [-2.5899999141693115, -16.804000854492188, 7.50600004196167]], [[-1.2660000324249268, -16.780000686645508, 7.539999961853027], [-0.5630000233650208, -15.8149995803833, 6.7179999351501465], [-0.10400000214576721, -16.42300033569336, 5.4029998779296875]], [[-0.10000000149011612, -15.585000038146973, 4.373000144958496], [0.3199999928474426, -15.956000328063965, 3.0280001163482666], [0.7820000052452087, -14.654999732971191, 2.4010000228881836]], [[2.0840001106262207, -14.526000022888184, 2.178999900817871], [2.5920000076293945, -13.286999702453613, 1.6269999742507935], [2.247999906539917, -12.246999740600586, 2.6730000972747803]], [[1.9160000085830688, -11.029999732971191, 2.253000020980835], [1.5499999523162842, -9.98799991607666, 3.2070000171661377], [0.03200000151991844, -9.977999687194824, 3.374000072479248]], [[-0.5429999828338623, -11.168999671936035, 3.496999979019165], [-1.9830000400543213, -11.29699993133545, 3.6429998874664307], [-2.381999969482422, -12.312999725341797, 4.715000152587891]], [[-3.309000015258789, -11.925000190734863, 5.585999965667725], [-3.7829999923706055, -12.829999923706055, 6.631999969482422], [-5.239999771118164, -13.182000160217285, 6.361000061035156]], [[-5.5229997634887695, -14.475000381469727, 6.239999771118164], [-6.86899995803833, -14.947999954223633, 5.955999851226807], [-7.552000045776367, -15.592000007629395, 7.1479997634887695]], [[-8.746999740600586, -15.10099983215332, 7.460000038146973], [-9.538999557495117, -15.605999946594238, 8.571999549865723], [-10.727999687194824, -16.445999145507812, 8.095000267028809]], [[-11.138999938964844, -17.408000946044922, 8.914999961853027], [-12.251999855041504, -18.274999618530273, 8.564000129699707], [-13.02299976348877, -18.79400062561035, 9.781000137329102]], [[-14.347000122070312, -18.856000900268555, 9.651000022888184], [-15.185999870300293, -19.3439998626709, 10.732000350952148], [-16.283000946044922, -18.371000289916992, 11.107999801635742]], [[-17.141000747680664, -18.7810001373291, 12.043999671936035], [-18.256999969482422, -17.950000762939453, 12.508000373840332], [-17.763999938964844, -16.722999572753906, 13.265000343322754]], [[-16.78499984741211, -16.924999237060547, 14.140000343322754], [-16.209999084472656, -15.812999725341797, 14.888999938964844], [-14.704000473022461, -15.855999946594238, 14.673999786376953]], [[-14.128999710083008, -14.718000411987305, 14.3100004196167], [-12.692999839782715, -14.640999794006348, 14.097999572753906], [-12.08899974822998, -13.640000343322754, 15.074999809265137]], [[-10.847999572753906, -13.907999992370605, 15.46500015258789], [-10.138999938964844, -13.053999900817871, 16.400999069213867], [-8.937999725341797, -12.416000366210938, 15.744000434875488]], [[-9.001999855041504, -11.10099983215332, 15.626999855041504], [-7.933000087738037, -10.329999923706055, 15.041999816894531], [-7.171000003814697, -9.732999801635742, 16.219999313354492]], [[-5.855000019073486, -9.89799976348877, 16.25], [-5.074999809265137, -9.3149995803833, 17.336000442504883], [-3.9800000190734863, -8.45300006866455, 16.729999542236328]], [[-4.119999885559082, -7.138999938964844, 16.871999740600586], [-3.1519999504089355, -6.203000068664551, 16.31999969482422], [-2.375999927520752, -5.434000015258789, 17.378999710083008]], [[-1.1419999599456787, -5.076000213623047, 17.04199981689453], [-0.29899999499320984, -4.322999954223633, 17.950000762939453], [0.2800000011920929, -3.125, 17.2189998626709]], [[0.39399999380111694, -2.009999990463257, 17.933000564575195], [0.9580000042915344, -0.7929999828338623, 17.388999938964844], [2.4700000286102295, -0.7910000085830688, 17.61199951171875]], [[3.2119998931884766, -1.0219999551773071, 16.52899932861328], [4.676000118255615, -1.0440000295639038, 16.524999618530273], [5.317999839782715, -1.9919999837875366, 17.531999588012695]], [[5.3520002365112305, -3.2780001163482666, 17.208999633789062], [5.984000205993652, -4.239999771118164, 18.08799934387207], [7.447000026702881, -3.812000036239624, 18.04800033569336]], [[8.244999885559082, -4.190000057220459, 19.055999755859375], [9.657999992370605, -3.7850000858306885, 19.03700065612793], [10.440999984741211, -4.426000118255615, 17.888999938964844]], [[11.189000129699707, -3.6010000705718994, 17.163000106811523], [11.970000267028809, -4.109000205993652, 16.051000595092773], [11.317000389099121, -3.805999994277954, 14.71500015258789]], [[10.006999969482422, -3.6019999980926514, 14.729000091552734], [9.288000106811523, -3.303999900817871, 13.503999710083008], [9.38700008392334, -1.8179999589920044, 13.204000473022461]], [[9.21500015258789, -1.437000036239624, 11.928999900817871], [9.288000106811523, -0.03400000184774399, 11.508999824523926], [8.258000373840332, 0.8539999723434448, 12.185999870300293]], [[8.586000442504883, 2.13100004196167, 12.310999870300293], [7.673999786376953, 3.0840001106262207, 12.918000221252441], [7.2870001792907715, 4.105999946594238, 11.857000350952148]], [[5.98199987411499, 4.421999931335449, 11.744999885559082], [4.869999885559082, 3.874000072479248, 12.538000106811523], [4.191999912261963, 2.746999979019165, 11.769000053405762]], [[3.753999948501587, 1.7029999494552612, 12.46399974822998], [3.0989999771118164, 0.6010000109672546, 11.779000282287598], [2.253999948501587, -0.30000001192092896, 12.687999725341797]], [[1.3940000534057617, -1.1119999885559082, 12.072999954223633], [0.5419999957084656, -2.0510001182556152, 12.79699993133545], [1.0870000123977661, -3.437999963760376, 12.49899959564209]], [[1.0490000247955322, -4.331999778747559, 13.479999542236328], [1.50600004196167, -5.690999984741211, 13.255000114440918], [0.3400000035762787, -6.593999862670898, 13.58899974822998]], [[0.0689999982714653, -7.558000087738037, 12.711999893188477], [-1.0089999437332153, -8.517999649047852, 12.913999557495117], [-0.4099999964237213, -9.843000411987305, 12.503000259399414]], [[-0.45500001311302185, -10.815999984741211, 13.406000137329102], [0.12600000202655792, -12.135000228881836, 13.166999816894531], [1.6039999723434448, -12.027000427246094, 12.781999588012695]], [[2.2899999618530273, -11.095999717712402, 13.4350004196167], [3.7139999866485596, -10.86400032043457, 13.22700023651123], [4.0920000076293945, -10.265000343322754, 11.878000259399414]], [[3.1019999980926514, -9.793999671936035, 11.128999710083008], [3.3550000190734863, -9.194000244140625, 9.826000213623047], [3.065000057220459, -7.699999809265137, 9.894000053405762]], [[3.7869999408721924, -6.921999931335449, 9.097999572753906], [3.614000082015991, -5.4770002365112305, 9.062999725341797], [2.4779999256134033, -5.059999942779541, 8.118000030517578]], [[1.5399999618530273, -4.270999908447266, 8.644000053405762], [0.39800000190734863, -3.7669999599456787, 7.881999969482422], [-0.11500000208616257, -4.757999897003174, 6.839000225067139]], [[-0.4569999873638153, -5.986999988555908, 7.252999782562256], [-0.9340000152587891, -6.921000003814697, 6.236000061035156], [-2.368000030517578, -6.645999908447266, 5.822000026702881]], [[-2.7720000743865967, -7.27400016784668, 4.728000164031982], [-4.126999855041504, -7.139999866485596, 4.230999946594238], [-4.8979997634887695, -8.265000343322754, 4.9019999504089355]], [[-6.026000022888184, -7.927000045776367, 5.513999938964844], [-6.829999923706055, -8.9350004196167, 6.199999809265137], [-8.003999710083008, -9.402999877929688, 5.353000164031982]], [[-8.152000427246094, -10.717000007629395, 5.230000019073486], [-9.241999626159668, -11.307999610900879, 4.453999996185303], [-10.28600025177002, -11.883999824523926, 5.40500020980835]], [[-11.444000244140625, -11.239999771118164, 5.48199987411499], [-12.5, -11.682999610900879, 6.38700008392334], [-13.769000053405762, -12.196999549865723, 5.695000171661377]], [[-14.295000076293945, -13.345000267028809, 6.1529998779296875], [-15.508999824523926, -13.892000198364258, 5.539000034332275], [-16.70800018310547, -13.020000457763672, 5.86299991607666]], [[-17.44099998474121, -12.611000061035156, 4.831999778747559], [-18.625, -11.795000076293945, 5.046000003814697], [-19.545000076293945, -12.571000099182129, 5.999000072479248]], [[-20.19099998474121, -11.861000061035156, 6.916999816894531], [-21.094999313354492, -12.515999794006348, 7.849999904632568], [-20.51300048828125, -13.027999877929688, 9.159000396728516]], [[-19.19099998474121, -13.067999839782715, 9.263999938964844], [-18.554000854492188, -13.541000366210938, 10.491000175476074], [-18.576000213623047, -12.454000473022461, 11.564000129699707]], [[-18.479000091552734, -12.86400032043457, 12.821999549865723], [-18.42799949645996, -11.895999908447266, 13.902999877929688], [-16.93400001525879, -11.692999839782715, 14.08899974822998]], [[-16.485000610351562, -10.446000099182129, 14.001999855041504], [-15.0649995803833, -10.128999710083008, 14.12600040435791], [-14.692000389099121, -9.586999893188477, 15.501999855041504]], [[-13.92199993133545, -10.369999885559082, 16.242000579833984], [-13.480999946594238, -9.954000473022461, 17.562000274658203], [-12.081999778747559, -9.369000434875488, 17.43600082397461]], [[-11.979999542236328, -8.055000305175781, 17.590999603271484], [-10.704999923706055, -7.35699987411499, 17.485000610351562], [-10.071999549865723, -6.979000091552734, 18.82900047302246]], [[-8.807000160217285, -7.357999801635742, 19.0049991607666], [-8.043000221252441, -7.020999908447266, 20.20800018310547], [-6.882999897003174, -6.142000198364258, 19.746999740600586]], [[-6.809000015258789, -4.921999931335449, 20.273000717163086], [-5.749000072479248, -3.986999988555908, 19.91900062561035], [-4.840000152587891, -3.6760001182556152, 21.121000289916992]], [[-3.5320000648498535, -3.8369998931884766, 20.920000076293945], [-2.5329999923706055, -3.5889999866485596, 21.96299934387207], [-1.4980000257492065, -2.565999984741211, 21.516000747680664]], [[-1.3350000381469727, -1.5010000467300415, 22.297000885009766], [-0.34200000762939453, -0.48399999737739563, 21.97599983215332], [0.9750000238418579, -0.7870000004768372, 22.70199966430664]], [[1.9220000505447388, -1.3860000371932983, 21.98699951171875], [3.2109999656677246, -1.7319999933242798, 22.55299949645996], [4.293000221252441, -0.6980000138282776, 22.242000579833984]], [[3.8910000324249268, 0.453000009059906, 21.718000411987305], [4.841000080108643, 1.4910000562667847, 21.34600067138672], [5.000999927520752, 2.552000045776367, 22.413999557495117]], [[6.177000045776367, 2.5850000381469727, 23.0310001373291], [6.456999778747559, 3.5450000762939453, 24.086999893188477], [6.427000045776367, 4.982999801635742, 23.57900047302246]], [[5.7789998054504395, 5.874000072479248, 24.323999404907227], [5.732999801635742, 7.27400016784668, 23.930999755859375], [4.585000038146973, 7.763999938964844, 23.062000274658203]], [[3.8420000076293945, 6.8429999351501465, 22.447999954223633], [2.7230000495910645, 7.210000038146973, 21.569000244140625], [1.440999984741211, 6.440000057220459, 21.84000015258789]], [[0.32100000977516174, 7.1479997634887695, 21.739999771118164], [-0.9779999852180481, 6.539999961853027, 21.966999053955078], [-1.7130000591278076, 6.298999786376953, 20.659000396728516]], [[-1.2039999961853027, 5.357999801635742, 19.878000259399414], [-1.7979999780654907, 5.021999835968018, 18.59000015258789], [-3.256999969482422, 4.617000102996826, 18.732999801635742]], [[-4.019999980926514, 4.750999927520752, 17.64900016784668], [-5.422999858856201, 4.3379998207092285, 17.638999938964844], [-5.577000141143799, 3.2780001163482666, 16.549999237060547]], [[-6.702000141143799, 2.5759999752044678, 16.55500030517578], [-6.968999862670898, 1.5529999732971191, 15.545999526977539], [-8.418000221252441, 1.7630000114440918, 15.11400032043457]], [[-8.597000122070312, 2.5360000133514404, 14.048999786376953], [-9.92300033569336, 2.8559999465942383, 13.545999526977539], [-10.204999923706055, 2.325000047683716, 12.149999618530273]], [[-11.38599967956543, 1.7359999418258667, 11.97700023651123], [-11.77299976348877, 1.194000005722046, 10.684000015258789], [-12.619999885559082, 2.190999984741211, 9.920000076293945]], [[-12.21399974822998, 2.4579999446868896, 8.6850004196167], [-12.925000190734863, 3.3910000324249268, 7.824999809265137], [-13.074000358581543, 2.815999984741211, 6.410999774932861]], [[-13.82800006866455, 3.5199999809265137, 5.573999881744385], [-14.062000274658203, 3.0920000076293945, 4.195000171661377], [-13.180999755859375, 3.8940000534057617, 3.249000072479248]], [[-12.732000350952148, 5.059999942779541, 3.7039999961853027], [-11.878000259399414, 5.9079999923706055, 2.890000104904175], [-10.484999656677246, 5.289999961853027, 2.816999912261963]], [[-9.949999809265137, 5.184000015258789, 1.6069999933242798], [-8.63599967956543, 4.590000152587891, 1.4479999542236328], [-7.51200008392334, 5.586999893188477, 1.2680000066757202]], [[-6.281000137329102, 5.0879998207092285, 1.1089999675750732], [-5.0980000495910645, 5.931000232696533, 0.9190000295639038], [-5.182000160217285, 6.708000183105469, -0.38999998569488525]], [[-4.418000221252441, 7.798999786376953, -0.5070000290870667], [-3.502000093460083, 8.317000389099121, 0.5130000114440918], [-4.216000080108643, 9.152999877929688, 1.5729999542236328]], [[-3.6540000438690186, 9.1899995803833, 2.7790000438690186], [-4.251999855041504, 9.958999633789062, 3.86899995803833], [-3.500999927520752, 11.255999565124512, 4.099999904632568]], [[-4.216000080108643, 12.293000221252441, 4.519000053405762], [-3.5829999446868896, 13.583000183105469, 4.781000137329102], [-2.953000068664551, 13.588000297546387, 6.172999858856201]], [[-2.2809998989105225, 14.682000160217285, 6.51200008392334], [-1.6369999647140503, 14.8149995803833, 7.815000057220459], [-2.7119998931884766, 14.598999977111816, 8.871000289916992]], [[-3.859999895095825, 15.223999977111816, 8.638999938964844], [-5.019000053405762, 15.118000030517578, 9.508000373840332], [-6.057000160217285, 14.390999794006348, 8.659000396728516]], [[-6.03000020980835, 13.052000045776367, 8.666999816894531], [-6.974999904632568, 12.265999794006348, 7.874000072479248], [-8.442000389099121, 12.63700008392334, 8.079000473022461]], [[-9.215999603271484, 12.572999954223633, 6.998000144958496], [-10.651000022888184, 12.848999977111816, 7.019999980926514], [-11.24899959564209, 11.486000061035156, 6.701000213623047]], [[-11.527000427246094, 10.696999549865723, 7.735000133514404], [-12.03499984741211, 9.343000411987305, 7.544000148773193], [-13.420000076293945, 9.04699993133545, 8.107000350952148]], [[-14.17199993133545, 10.086000442504883, 8.444999694824219], [-15.513999938964844, 9.892999649047852, 8.982999801635742], [-16.433000564575195, 9.388999938964844, 7.880000114440918]], [[-17.371999740600586, 8.50100040435791, 8.225000381469727], [-17.59000015258789, 7.978000164031982, 9.574999809265137], [-16.79599952697754, 6.723999977111816, 9.883999824523926]], [[-16.27199935913086, 6.638000011444092, 11.10099983215332], [-15.522000312805176, 5.460999965667725, 11.49899959564209], [-16.552000045776367, 4.359000205993652, 11.663999557495117]], [[-16.274999618530273, 3.2100000381469727, 11.071999549865723], [-17.180999755859375, 2.0850000381469727, 11.12600040435791], [-17.097000122070312, 1.402999997138977, 12.482999801635742]], [[-15.873000144958496, 1.3040000200271606, 12.986000061035156], [-15.583000183105469, 0.6809999942779541, 14.269000053405762], [-14.161999702453613, 1.1440000534057617, 14.597000122070312]], [[-13.899999618530273, 1.4559999704360962, 15.86299991607666], [-12.57800006866455, 1.934999942779541, 16.236000061035156], [-12.345000267028809, 2.0360000133514404, 17.733999252319336]], [[-11.10200023651123, 2.312999963760376, 18.1200008392334], [-10.73799991607666, 2.378000020980835, 19.530000686645508], [-10.47599983215332, 3.759000062942505, 20.121000289916992]], [[-10.194999694824219, 4.739999771118164, 19.277999877929688], [-9.868000030517578, 6.039000034332275, 19.827999114990234], [-8.411999702453613, 5.922999858856201, 20.27400016784668]], [[-7.821000099182129, 7.0320000648498535, 20.716999053955078], [-6.421000003814697, 7.052999973297119, 21.152000427246094], [-6.175000190734863, 6.230999946594238, 22.415000915527344]], [[-5.146999835968018, 5.39300012588501, 22.368999481201172], [-4.820000171661377, 4.511000156402588, 23.483999252319336], [-3.8369998931884766, 5.144000053405762, 24.447999954223633]], [[-3.736999988555908, 4.585000038146973, 25.665000915527344], [-2.7980000972747803, 5.125, 26.650999069213867], [-1.3830000162124634, 4.682000160217285, 26.256999969482422]], [[-0.38100001215934753, 5.498000144958496, 26.55900001525879], [1.0, 5.142000198364258, 26.240999221801758], [1.4479999542236328, 4.093999862670898, 27.261999130249023]], [[2.3959999084472656, 3.2130000591278076, 26.89699935913086], [2.822000026702881, 2.2100000381469727, 27.885000228881836], [3.3489999771118164, 2.9000000953674316, 29.149999618530273]], [[3.1500000953674316, 2.2780001163482666, 30.31100082397461], [3.5950000286102295, 2.877000093460083, 31.565000534057617], [3.7190001010894775, 1.8589999675750732, 32.696998596191406]], [[4.64300012588501, 2.121999979019165, 33.61899948120117], [4.877999782562256, 1.2510000467300415, 34.7760009765625], [5.02400016784668, -0.21400000154972076, 34.38999938964844]], [[5.656000137329102, -0.47200000286102295, 33.25], [5.840000152587891, -1.8450000286102295, 32.80799865722656], [4.576000213623047, -2.5450000762939453, 32.327999114990234]], [[3.5480000972747803, -1.7680000066757202, 31.9950008392334], [2.2929999828338623, -2.3320000171661377, 31.500999450683594], [1.840999960899353, -1.6629999876022339, 30.187000274658203]], [[1.1549999713897705, -2.433000087738037, 29.340999603271484], [0.6819999814033508, -1.9520000219345093, 28.035999298095703], [-0.8370000123977661, -1.9589999914169312, 27.94700050354004]], [[-1.4079999923706055, -0.8640000224113464, 27.466999053955078], [-2.8540000915527344, -0.7929999828338623, 27.354999542236328], [-3.4170000553131104, -1.565999984741211, 26.170000076293945]], [[-4.620999813079834, -2.1010000705718994, 26.33300018310547], [-5.257999897003174, -2.8369998931884766, 25.26300048828125], [-6.761000156402588, -2.7939999103546143, 25.422000885009766]], [[-7.46999979019165, -3.003999948501587, 24.32200050354004], [-8.923999786376953, -2.997999906539917, 24.360000610351562], [-9.479999542236328, -4.005000114440918, 23.374000549316406]], [[-10.765000343322754, -4.290999889373779, 23.496999740600586], [-11.42199993133545, -5.23799991607666, 22.614999771118164], [-12.704999923706055, -4.63100004196167, 22.107999801635742]], [[-13.07800006866455, -4.9770002365112305, 20.885000228881836], [-14.336000442504883, -4.501999855041504, 20.3439998626709], [-14.812999725341797, -5.519999980926514, 19.332000732421875]], [[-16.111000061035156, -5.514999866485596, 19.054000854492188], [-16.670000076293945, -6.459000110626221, 18.104000091552734], [-17.09600067138672, -5.7230000495910645, 16.826000213623047]], [[-17.179000854492188, -6.459000110626221, 15.72599983215332], [-17.507999420166016, -5.855000019073486, 14.4399995803833], [-18.209999084472656, -6.882999897003174, 13.581000328063965]], [[-19.336000442504883, -6.488999843597412, 12.986000061035156], [-20.108999252319336, -7.360000133514404, 12.090999603271484], [-20.14299964904785, -6.560999870300293, 10.791000366210938]], [[-19.121000289916992, -6.736000061035156, 9.940999984741211], [-19.06599998474121, -5.994999885559082, 8.680999755859375], [-19.784000396728516, -6.6620001792907715, 7.513999938964844]], [[-20.195999145507812, -5.84499979019165, 6.548999786376953], [-20.84000015258789, -6.3480000495910645, 5.3420000076293945], [-19.738000869750977, -6.392000198364258, 4.284999847412109]], [[-19.895999908447266, -7.257999897003174, 3.2850000858306885], [-18.905000686645508, -7.389999866485596, 2.2160000801086426], [-18.499000549316406, -6.011000156402588, 1.7109999656677246]], [[-17.216999053955078, -5.861999988555908, 1.3890000581741333], [-16.718000411987305, -4.585999965667725, 0.9100000262260437], [-15.25100040435791, -4.425000190734863, 1.2510000467300415]], [[-14.72599983215332, -3.2260000705718994, 1.062999963760376], [-13.32699966430664, -2.9839999675750732, 1.3609999418258667], [-13.194999694824219, -1.8919999599456787, 2.4030001163482666]], [[-12.444999694824219, -2.184000015258789, 3.4649999141693115], [-12.244999885559082, -1.2259999513626099, 4.553999900817871], [-10.762999534606934, -1.0440000295639038, 4.828999996185303]], [[-10.435999870300293, -0.014999999664723873, 5.605000019073486], [-9.048999786376953, 0.28200000524520874, 5.948999881744385], [-8.883000373840332, 0.5529999732971191, 7.443999767303467]], [[-7.90500020980835, -0.09399999678134918, 8.072999954223633], [-7.664000034332275, 0.18199999630451202, 9.479999542236328], [-6.531000137329102, 1.1890000104904175, 9.524999618530273]], [[-6.748000144958496, 2.2929999828338623, 10.234000205993652], [-5.750999927520752, 3.3440001010894775, 10.319000244140625], [-5.585000038146973, 3.9539999961853027, 11.715999603271484]], [[-4.442999839782715, 4.593999862670898, 11.939000129699707], [-4.196000099182129, 5.267000198364258, 13.204000473022461], [-4.4120001792907715, 6.752999782562256, 12.925000190734863]], [[-5.183000087738037, 7.433000087738037, 13.772000312805176], [-5.465000152587891, 8.855999946594238, 13.569000244140625], [-4.486999988555908, 9.878000259399414, 14.144000053405762]], [[-3.3580000400543213, 9.432000160217285, 14.687999725341797], [-2.4110000133514404, 10.404999732971191, 15.215999603271484], [-1.843000054359436, 11.152000427246094, 14.019000053405762]], [[-1.9600000381469727, 12.479999542236328, 14.022000312805176], [-1.4529999494552612, 13.291999816894531, 12.918000221252441], [-0.11699999868869781, 12.833000183105469, 12.324999809265137]], [[-0.1289999932050705, 12.614999771118164, 11.015000343322754], [1.065999984741211, 12.197999954223633, 10.319999694824219], [1.2860000133514404, 10.706000328063965, 10.25100040435791]], [[0.675000011920929, 9.942000389099121, 11.142999649047852], [0.8980000019073486, 8.505999565124512, 11.125], [0.38100001215934753, 7.77400016784668, 9.888999938964844]], [[-0.9089999794960022, 7.914999961853027, 9.592000007629395], [-1.4869999885559082, 7.256999969482422, 8.42300033569336], [-0.7720000147819519, 7.789999961853027, 7.188000202178955]], [[-0.5370000004768372, 9.098999977111816, 7.185999870300293], [0.15199999511241913, 9.769000053405762, 6.0929999351501465], [1.5, 9.116999626159668, 5.784999847412109]], [[2.127000093460083, 8.49899959564209, 6.784999847412109], [3.421999931335449, 7.869999885559082, 6.559000015258789], [3.374000072479248, 6.3520002365112305, 6.534999847412109]], [[2.184000015258789, 5.7820000648498535, 6.368000030517578], [2.0969998836517334, 4.336999893188477, 6.2779998779296875], [1.4529999494552612, 3.5230000019073486, 7.375999927520752]], [[0.878000020980835, 4.1529998779296875, 8.39799976348877], [0.2549999952316284, 3.3499999046325684, 9.45300006866455], [-1.2050000429153442, 3.0260000228881836, 9.12600040435791]], [[-1.3919999599456787, 2.1089999675750732, 8.184000015258789], [-2.7149999141693115, 1.6950000524520874, 7.750999927520752], [-2.5880000591278076, 0.3569999933242798, 7.056000232696533]], [[-3.691999912261963, -0.37299999594688416, 6.995999813079834], [-3.7049999237060547, -1.6649999618530273, 6.3460001945495605], [-5.059999942779541, -1.812000036239624, 5.685999870300293]], [[-5.210000038146973, -2.8320000171661377, 4.853000164031982], [-6.465000152587891, -3.052999973297119, 4.144000053405762], [-7.2179999351501465, -4.2779998779296875, 4.646999835968018]], [[-8.541000366210938, -4.165999889373779, 4.677000045776367], [-9.378999710083008, -5.2789998054504395, 5.1020002365112305], [-10.326000213623047, -5.605999946594238, 3.9549999237060547]], [[-10.392000198364258, -6.875, 3.5850000381469727], [-11.288999557495117, -7.281000137329102, 2.51200008392334], [-12.331000328063965, -8.258000373840332, 3.0269999504089355]], [[-13.559000015258789, -7.7779998779296875, 3.130000114440918], [-14.645000457763672, -8.616999626159668, 3.5999999046325684], [-15.401000022888184, -9.14900016784668, 2.3980000019073486]], [[-15.329999923706055, -10.45300006866455, 2.190000057220459], [-16.023000717163086, -11.043999671936035, 1.065000057220459], [-16.29199981689453, -12.510000228881836, 1.3279999494552612]]], "seq": "GTLDSTWKEATLPQVKAMLEKDDGKVSGDTVTYSGKTVHVVAAAVLPGFPFPSFEVHDKKNPTLEIPAGATVDVTFINTNKGFGHSFDITKKGPPYAVMPVIDPIVAGTGFSPVPKDGKFGYTDFTWHPTAGTYYYVCQIPGHAATGMFGKIVVK"}, {"coords": [[[27.70400047302246, 6.3470001220703125, 33.91600036621094], [28.832000732421875, 5.552999973297119, 33.356998443603516], [28.323999404907227, 4.589000225067139, 32.28799819946289]], [[28.785999298095703, 4.890999794006348, 31.094999313354492], [28.514999389648438, 4.057000160217285, 29.913000106811523], [29.676000595092773, 3.0739998817443848, 29.849000930786133]], [[29.388999938964844, 1.809999942779541, 30.134000778198242], [30.465999603271484, 0.8209999799728394, 30.280000686645508], [30.673999786376953, 0.07900000363588333, 28.9689998626709]], [[31.698999404907227, 0.5090000033378601, 28.211000442504883], [31.933000564575195, -0.1720000058412552, 26.929000854492188], [32.492000579833984, -1.5720000267028809, 27.117000579833984]], [[32.895999908447266, -2.0220000743865967, 28.30699920654297], [33.25699996948242, -3.444000005722046, 28.448999404907227], [32.05799865722656, -4.359000205993652, 28.235000610351562]], [[30.83799934387207, -3.8420000076293945, 28.277999877929688], [29.624000549316406, -4.552999973297119, 27.98699951171875], [29.15999984741211, -4.425000190734863, 26.525999069213867]], [[29.965999603271484, -3.9130001068115234, 25.618999481201172], [29.58300018310547, -3.7829999923706055, 24.211000442504883], [29.08099937438965, -5.0980000495910645, 23.625999450683594]], [[29.81100082397461, -6.171000003814697, 23.924999237060547], [29.472999572753906, -7.49399995803833, 23.415000915527344], [28.840999603271484, -8.359999656677246, 24.510000228881836]], [[28.18199920654297, -7.723999977111816, 25.45599937438965], [27.424999237060547, -8.432000160217285, 26.51099967956543], [26.038000106811523, -8.711999893188477, 25.941999435424805]], [[25.003999710083008, -8.71399974822998, 26.773000717163086], [23.63599967956543, -8.967000007629395, 26.343000411987305], [23.128999710083008, -8.069000244140625, 25.240999221801758]], [[23.576000213623047, -6.813000202178955, 25.155000686645508], [23.179000854492188, -5.9029998779296875, 24.09600067138672], [23.722000122070312, -6.315000057220459, 22.724000930786133]], [[24.74799919128418, -7.1579999923706055, 22.695999145507812], [25.238000869750977, -7.732999801635742, 21.434999465942383], [25.413999557495117, -6.617000102996826, 20.393999099731445]], [[26.17099952697754, -5.585000038146973, 20.770999908447266], [26.34600067138672, -4.435999870300293, 19.87700080871582], [27.36199951171875, -4.665999889373779, 18.788000106811523]], [[28.093000411987305, -5.793000221252441, 18.851999282836914], [29.038999557495117, -6.160999774932861, 17.797000885009766], [28.28499984741211, -6.617000102996826, 16.56800079345703]], [[26.958999633789062, -6.796999931335449, 16.68600082397461], [26.06399917602539, -7.033999919891357, 15.579000473022461], [25.950000762939453, -5.770999908447266, 14.704999923706055]], [[26.327999114990234, -4.618000030517578, 15.227999687194824], [26.246999740600586, -3.3529999256134033, 14.51200008392334], [27.645999908447266, -2.9019999504089355, 14.105999946594238]], [[28.55299949645996, -2.7720000743865967, 15.069000244140625], [29.95199966430664, -2.4820001125335693, 14.713000297546387], [30.834999084472656, -2.819000005722046, 15.89900016784668]], [[32.150001525878906, -2.822000026702881, 15.652000427246094], [33.07400131225586, -3.121000051498413, 16.75], [33.07600021362305, -2.053999900817871, 17.836999893188477]], [[33.57500076293945, -2.4769999980926514, 19.016000747680664], [33.53900146484375, -1.5989999771118164, 20.19099998474121], [34.2869987487793, -0.31200000643730164, 19.89299964904785]], [[33.7130012512207, 0.8339999914169312, 20.246999740600586], [34.345001220703125, 2.1059999465942383, 20.010000228881836], [35.520999908447266, 2.3359999656677246, 20.961000442504883]], [[36.30500030517578, 3.378999948501587, 20.652999877929688], [37.43600082397461, 3.6630001068115234, 21.552000045776367], [36.94200134277344, 4.281000137329102, 22.84000015258789]], [[35.89899826049805, 5.124000072479248, 22.729000091552734], [35.39799880981445, 5.938000202178955, 23.83799934387207], [33.874000549316406, 5.953000068664551, 23.86400032043457]], [[33.27799987792969, 5.953999996185303, 25.049999237060547], [31.843000411987305, 6.076000213623047, 25.179000854492188], [31.409000396728516, 7.453999996185303, 24.697999954223633]], [[30.344999313354492, 7.511000156402588, 23.908000946044922], [29.774999618530273, 8.777999877929688, 23.458999633789062], [28.722000122070312, 9.256999969482422, 24.45400047302246]], [[28.801000595092773, 10.472999572753906, 24.961999893188477], [27.77899932861328, 10.972999572753906, 25.886999130249023], [26.71299934387207, 11.656000137329102, 25.03499984741211]], [[25.452999114990234, 11.241999626159668, 25.149999618530273], [24.437999725341797, 11.86400032043457, 24.29400062561035], [23.371999740600586, 12.494999885559082, 25.20599937438965]], [[23.02199935913086, 13.732999801635742, 24.940000534057617], [22.02199935913086, 14.461999893188477, 25.711000442504883], [20.861000061035156, 14.845999717712402, 24.79599952697754]], [[19.660999298095703, 14.810999870300293, 25.3700008392334], [18.450000762939453, 15.215999603271484, 24.655000686645508], [17.834999084472656, 16.371999740600586, 25.466999053955078]], [[16.760000228881836, 16.98200035095215, 25.009000778198242], [16.23900032043457, 18.166000366210938, 25.708999633789062], [15.852999687194824, 17.827999114990234, 27.145999908447266]], [[15.359999656677246, 16.615999221801758, 27.363000869750977], [14.840999603271484, 16.211000442504883, 28.65399932861328], [15.335000038146973, 14.836999893188477, 29.07200050354004]], [[16.47100067138672, 14.37399959564209, 28.569000244140625], [16.961999893188477, 13.038999557495117, 28.937999725341797], [18.437000274658203, 12.892000198364258, 28.583999633789062]], [[19.12299919128418, 11.854999542236328, 29.048999786376953], [20.52400016784668, 11.71500015258789, 28.60300064086914], [20.792999267578125, 10.199000358581543, 28.589000701904297]], [[21.687999725341797, 9.782999992370605, 27.704999923706055], [22.08099937438965, 8.371999740600586, 27.69099998474121], [23.30500030517578, 8.194999694824219, 28.59000015258789]], [[22.996000289916992, 7.848999977111816, 29.822999954223633], [23.768999099731445, 7.714000225067139, 31.014999389648438], [24.444000244140625, 6.394000053405762, 31.299999237060547]], [[23.93600082397461, 5.381999969482422, 30.608999252319336], [24.19099998474121, 3.9630000591278076, 30.83799934387207], [24.11400032043457, 3.256999969482422, 29.488000869750977]], [[25.06100082397461, 2.364000082015991, 29.17799949645996], [24.93600082397461, 1.6019999980926514, 27.916000366210938], [23.617000579833984, 0.8309999704360962, 27.878999710083008]], [[22.76799964904785, 1.0950000286102295, 26.895000457763672], [21.489999771118164, 0.38100001215934753, 26.781999588012695], [20.39299964904785, 1.065000057220459, 27.58799934387207]], [[20.67300033569336, 2.2699999809265137, 28.086999893188477], [19.660999298095703, 2.950000047683716, 28.91699981689453], [18.33099937438965, 3.00600004196167, 28.200000762939453]], [[17.222999572753906, 2.7269999980926514, 28.882999420166016], [15.927000045776367, 2.818000078201294, 28.233999252319336], [15.399999618530273, 4.24399995803833, 28.256999969482422]], [[15.130000114440918, 4.748000144958496, 27.062999725341797], [14.513999938964844, 6.031000137329102, 26.863000869750977], [13.163999557495117, 5.815999984741211, 26.17300033569336]], [[12.315999984741211, 6.875999927520752, 26.20400047302246], [11.071000099182129, 6.736000061035156, 25.440000534057617], [11.182999610900879, 7.428999900817871, 24.09600067138672]], [[10.307999610900879, 7.053999900817871, 23.163000106811523], [10.164999961853027, 7.741000175476074, 21.878999710083008], [9.887999534606934, 9.215999603271484, 22.089000701904297]], [[9.059000015258789, 9.553999900817871, 23.086999893188477], [8.847999572753906, 10.970000267028809, 23.381000518798828], [10.152999877929688, 11.675999641418457, 23.679000854492188]], [[10.98799991607666, 11.017999649047852, 24.482999801635742], [12.253000259399414, 11.63599967956543, 24.885000228881836], [13.211000442504883, 11.904999732971191, 23.733999252319336]], [[13.199999809265137, 11.024999618530273, 22.743000030517578], [14.13599967956543, 11.151000022888184, 21.617000579833984], [13.470000267028809, 11.5600004196167, 20.325000762939453]], [[12.329999923706055, 12.253000259399414, 20.385000228881836], [11.656999588012695, 12.64799976348877, 19.150999069213867], [12.442999839782715, 13.706999778747559, 18.382999420166016]], [[13.267000198364258, 14.489999771118164, 19.06800079345703], [14.123000144958496, 15.461000442504883, 18.409000396728516], [15.593000411987305, 15.053000450134277, 18.541000366210938]], [[16.422000885009766, 15.692000389099121, 17.725000381469727], [17.854999542236328, 15.418000221252441, 17.753999710083008], [18.416000366210938, 15.635000228881836, 19.14900016784668]], [[19.506999969482422, 14.949999809265137, 19.445999145507812], [20.22599983215332, 15.21399974822998, 20.67300033569336], [20.58099937438965, 16.69499969482422, 20.695999145507812]], [[20.54400062561035, 17.253999710083008, 21.89900016784668], [20.982999801635742, 18.643999099731445, 22.07900047302246], [22.500999450683594, 18.726999282836914, 22.121000289916992]], [[23.180999755859375, 17.634000778198242, 22.45800018310547], [24.643999099731445, 17.650999069213867, 22.493000030517578], [25.21299934387207, 16.253000259399414, 22.47100067138672]], [[26.3799991607666, 16.059999465942383, 21.878999710083008], [27.06100082397461, 14.760000228881836, 21.91900062561035], [28.54599952697754, 15.104000091552734, 22.204999923706055]], [[29.20199966430664, 14.305999755859375, 23.024999618530273], [30.5939998626709, 14.508000373840332, 23.29400062561035], [31.266000747680664, 13.13599967956543, 23.547000885009766]], [[32.60900115966797, 13.26200008392334, 23.5939998626709], [33.39699935913086, 12.098999977111816, 24.010000228881836], [34.821998596191406, 12.590999603271484, 24.325000762939453]], [[35.474998474121094, 11.840999603271484, 25.18000030517578], [36.875, 11.98799991607666, 25.513999938964844], [37.70500183105469, 11.781000137329102, 24.25200080871582]], [[38.606998443603516, 12.684000015258789, 23.909000396728516], [39.35900115966797, 12.458999633789062, 22.666000366210938], [38.80500030517578, 13.303999900817871, 21.54199981689453]], [[37.58300018310547, 13.868000030517578, 21.71299934387207], [37.00299835205078, 14.640999794006348, 20.611000061035156], [37.99700164794922, 15.6850004196167, 20.107999801635742]], [[38.249000549316406, 15.727999687194824, 18.808000564575195], [39.16600036621094, 16.724000930786133, 18.246000289916992], [38.375999450683594, 17.770999908447266, 17.490999221801758]], [[38.35900115966797, 19.000999450683594, 18.000999450683594], [37.62099838256836, 20.093000411987305, 17.409000396728516], [38.18600082397461, 20.476999282836914, 16.06599998474121]], [[39.42399978637695, 20.11199951171875, 15.77400016784668], [40.0, 20.334999084472656, 14.454999923706055], [39.80500030517578, 19.229000091552734, 13.447999954223633]], [[39.071998596191406, 18.17099952697754, 13.76099967956543], [38.85900115966797, 17.076000213623047, 12.817000389099121], [37.422000885009766, 16.992000579833984, 12.298999786376953]], [[37.24700164794922, 16.163000106811523, 11.281000137329102], [35.952999114990234, 15.902000427246094, 10.673999786376953], [35.49800109863281, 14.479000091552734, 11.029000282287598]], [[34.17399978637695, 14.36299991607666, 11.223999977111816], [33.63999938964844, 13.093000411987305, 11.704999923706055], [32.3489990234375, 12.732000350952148, 10.989999771118164]], [[32.040000915527344, 11.444000244140625, 11.043999671936035], [30.77899932861328, 10.895000457763672, 10.550999641418457], [29.916000366210938, 10.45199966430664, 11.73799991607666]], [[28.62299919128418, 10.800000190734863, 11.73900032043457], [27.808000564575195, 10.505999565124512, 12.923999786376953], [26.652999877929688, 9.631999969482422, 12.46399974822998]], [[26.371000289916992, 8.524999618530273, 13.14799976348877], [25.31599998474121, 7.6620001792907715, 12.644000053405762], [24.395999908447266, 7.252999782562256, 13.803000450134277]], [[23.0939998626709, 7.195000171661377, 13.519000053405762], [22.22800064086914, 6.535999774932861, 14.529999732971191], [21.509000778198242, 5.389999866485596, 13.800999641418457]], [[21.552000045776367, 4.179999828338623, 14.324999809265137], [20.96299934387207, 3.0910000801086426, 13.52299976348877], [20.243000030517578, 2.0869998931884766, 14.425999641418457]], [[19.190000534057617, 1.4700000286102295, 13.932999610900879], [18.368000030517578, 0.49000000953674316, 14.645999908447266], [18.506999969482422, -0.8690000176429749, 13.961999893188477]], [[19.21500015258789, -1.7929999828338623, 14.597000122070312], [19.40399932861328, -3.125999927520752, 14.020000457763672], [18.156999588012695, -3.994999885559082, 14.182000160217285]], [[17.20400047302246, -3.5250000953674316, 14.982000350952148], [16.05900001525879, -4.315999984741211, 15.40999984741211], [14.760000228881836, -3.8480000495910645, 14.788000106811523]], [[14.795999526977539, -3.5769999027252197, 13.472000122070312], [13.574999809265137, -3.243000030517578, 12.769000053405762], [12.973999977111816, -4.4120001792907715, 12.043999671936035]], [[11.730999946594238, -4.765999794006348, 12.184000015258789], [10.706999778747559, -4.2789998054504395, 13.043999671936035], [10.854999542236328, -4.73199987411499, 14.491999626159668]], [[11.62600040435791, -5.765999794006348, 14.781999588012695], [11.82800006866455, -6.209000110626221, 16.14900016784668], [13.190999984741211, -6.876999855041504, 16.253999710083008]], [[13.692999839782715, -6.98199987411499, 17.475000381469727], [14.96500015258789, -7.660999774932861, 17.676000595092773], [14.848999977111816, -9.147000312805176, 17.39299964904785]], [[13.680000305175781, -9.739999771118164, 17.67300033569336], [13.506999969482422, -11.173999786376953, 17.44700050354004], [13.355999946594238, -11.489999771118164, 15.96399974822998]], [[12.843999862670898, -10.543999671936035, 15.187000274658203], [12.715999603271484, -10.777999877929688, 13.741000175476074], [13.178999900817871, -9.543999671936035, 13.006999969482422]], [[14.487000465393066, -9.319999694824219, 12.95300006866455], [15.052000045776367, -8.059000015258789, 12.458999633789062], [15.187999725341797, -8.064000129699707, 10.954999923706055]], [[14.053000450134277, -8.256999969482422, 10.279999732971191], [14.152999877929688, -8.447999954223633, 8.82699966430664], [14.53499984741211, -7.183000087738037, 8.083999633789062]], [[14.38700008392334, -5.9670000076293945, 8.600000381469727], [14.888999938964844, -4.767000198364258, 7.932000160217285], [16.187000274658203, -4.265999794006348, 8.572999954223633]], [[16.878999710083008, -5.138999938964844, 9.277000427246094], [18.1560001373291, -4.835999965667725, 9.871999740600586], [19.066999435424805, -4.064000129699707, 8.906999588012695]], [[19.768999099731445, -2.944000005722046, 9.020999908447266], [19.475000381469727, -1.8680000305175781, 9.914999961853027], [18.500999450683594, -0.8379999995231628, 9.312999725341797]], [[17.656999588012695, -0.2460000067949295, 10.170000076293945], [16.802000045776367, 0.8650000095367432, 9.75100040435791], [17.525999069213867, 2.0950000286102295, 10.309000015258789]], [[18.187999725341797, 2.8570001125335693, 9.437000274658203], [19.06999969482422, 3.941999912261963, 9.866000175476074], [18.326000213623047, 5.269000053405762, 10.038999557495117]], [[18.545000076293945, 5.881999969482422, 11.197999954223633], [17.790000915527344, 7.0320000648498535, 11.663000106811523], [18.5049991607666, 8.343999862670898, 11.378000259399414]], [[19.8439998626709, 8.347999572753906, 11.359999656677246], [20.492000579833984, 9.675000190734863, 11.180999755859375], [21.868000030517578, 9.388999938964844, 10.576000213623047]], [[22.270999908447266, 10.138999938964844, 9.5649995803833], [23.554000854492188, 9.878000259399414, 8.916999816894531], [24.125999450683594, 11.253999710083008, 8.541000366210938]], [[25.214000701904297, 11.628999710083008, 9.182000160217285], [25.819000244140625, 12.937000274658203, 8.961999893188477], [27.308000564575195, 12.75, 8.66100025177002]], [[27.816999435424805, 13.437999725341797, 7.630000114440918], [29.26099967956543, 13.345999717712402, 7.375], [29.878000259399414, 14.73900032043457, 7.408999919891357]], [[31.218000411987305, 14.786999702453613, 7.482999801635742], [31.882999420166016, 16.093000411987305, 7.491000175476074], [31.426000595092773, 16.964000701904297, 8.645000457763672]], [[31.138999938964844, 16.365999221801758, 9.803999900817871], [30.799999237060547, 17.152999877929688, 10.982000350952148], [32.09400177001953, 17.69499969482422, 11.579999923706055]], [[32.185001373291016, 19.009000778198242, 11.833000183105469], [33.36800003051758, 19.5310001373291, 12.513999938964844], [33.308998107910156, 19.288999557495117, 14.01200008392334]], [[34.27299880981445, 18.527000427246094, 14.527000427246094], [34.2869987487793, 18.222999572753906, 15.963000297546387], [32.933998107910156, 17.558000564575195, 16.29199981689453]], [[32.321998596191406, 18.009000778198242, 17.37299919128418], [30.996999740600586, 17.499000549316406, 17.761999130249023], [29.863000869750977, 18.450000762939453, 17.40999984741211]], [[30.05699920654297, 19.361000061035156, 16.44700050354004], [29.0310001373291, 20.336999893188477, 16.076000213623047], [28.11199951171875, 19.73900032043457, 15.029999732971191]], [[27.058000564575195, 19.041000366210938, 15.475000381469727], [26.177000045776367, 18.333999633789062, 14.550999641418457], [25.687999725341797, 19.243999481201172, 13.4350004196167]], [[25.327999114990234, 20.493000030517578, 13.758000373840332], [24.805999755859375, 21.42099952697754, 12.789999961853027], [25.812999725341797, 21.74799919128418, 11.704000473022461]], [[27.110000610351562, 21.520999908447266, 11.885000228881836], [28.048999786376953, 21.80299949645996, 10.8100004196167], [28.124000549316406, 20.711999893188477, 9.753999710083008]], [[27.61400032043457, 19.518999099731445, 10.059000015258789], [27.777000427246094, 18.40399932861328, 9.13700008392334], [26.864999771118164, 18.53700065612793, 7.926000118255615]], [[27.108999252319336, 17.62299919128418, 6.994999885559082], [26.277999877929688, 17.42799949645996, 5.816999912261963], [25.34000015258789, 16.26300048828125, 6.110000133514404]], [[24.052000045776367, 16.541000366210938, 6.183000087738037], [23.049999237060547, 15.527000427246094, 6.53000020980835], [22.660999298095703, 14.670000076293945, 5.339000225067139]], [[23.19300079345703, 13.4350004196167, 5.26200008392334], [22.795000076293945, 12.503999710083008, 4.22599983215332], [21.413999557495117, 11.880000114440918, 4.492000102996826]], [[21.084999084472656, 11.63599967956543, 5.757999897003174], [19.788000106811523, 11.067999839782715, 6.13100004196167], [19.26799964904785, 11.812000274658203, 7.376999855041504]], [[18.30299949645996, 12.680999755859375, 7.181000232696533], [17.857999801635742, 13.571999549865723, 8.244000434875488], [17.229000091552734, 12.774999618530273, 9.38599967956543]], [[17.27400016784668, 13.395999908447266, 10.564000129699707], [16.920000076293945, 12.704999923706055, 11.788999557495117], [15.465999603271484, 12.256999969482422, 11.887999534606934]], [[15.336000442504883, 10.970999717712402, 12.178999900817871], [14.036999702453613, 10.333000183105469, 12.437000274658203], [14.128000259399414, 9.760000228881836, 13.843000411987305]], [[13.262999534606934, 10.175000190734863, 14.767000198364258], [13.274999618530273, 9.595999717712402, 16.094999313354492], [12.989999771118164, 8.109999656677246, 16.163999557495117]], [[12.977999687194824, 7.579999923706055, 17.374000549316406], [12.654000282287598, 6.1620001792907715, 17.575000762939453], [11.170000076293945, 5.934000015258789, 17.358999252319336]], [[10.784000396728516, 5.0, 16.479999542236328], [9.414999961853027, 4.6020002365112305, 16.242000579833984], [9.208000183105469, 3.0899999141693115, 16.28499984741211]], [[9.61400032043457, 2.4649999141693115, 17.386999130249023], [9.461000442504883, 1.0140000581741333, 17.5], [7.991000175476074, 0.6370000243186951, 17.483999252319336]], [[7.611000061035156, -0.3149999976158142, 16.645000457763672], [6.2129998207092285, -0.7020000219345093, 16.514999389648438], [5.677000045776367, -1.246000051498413, 17.826000213623047]], [[4.434000015258789, -0.8980000019073486, 18.1560001373291], [3.809999942779541, -1.4299999475479126, 19.3700008392334], [3.8540000915527344, -2.947999954223633, 19.381000518798828]], [[4.239999771118164, -3.5399999618530273, 20.511999130249023], [4.2820000648498535, -4.993000030517578, 20.643999099731445], [5.607999801635742, -5.613999843597412, 20.214000701904297]], [[6.548999786376953, -4.798999786376953, 19.729000091552734], [7.84499979019165, -5.3420000076293945, 19.332000732421875], [8.89900016784668, -5.2129998207092285, 20.41900062561035]], [[8.5600004196167, -4.709000110626221, 21.600000381469727], [9.541000366210938, -4.63700008392334, 22.676000595092773], [10.647000312805176, -3.622999906539917, 22.45199966430664]], [[11.72599983215332, -3.7909998893737793, 23.23900032043457], [12.843999862670898, -2.8559999465942383, 23.170000076293945], [13.717000007629395, -3.046999931335449, 21.941999435424805]], [[13.958999633789062, -1.9479999542236328, 21.21299934387207], [14.890000343322754, -1.9869999885559082, 20.111000061035156], [16.180999755859375, -1.3079999685287476, 20.604000091552734]], [[17.29800033569336, -1.7430000305175781, 20.05500030517578], [18.56999969482422, -1.0779999494552612, 20.301000595092773], [18.719999313354492, 0.08500000089406967, 19.31599998474121]], [[19.107999801635742, 1.2610000371932983, 19.81599998474121], [19.405000686645508, 2.3889999389648438, 18.933000564575195], [20.86199951171875, 2.743000030517578, 19.156999588012695]], [[21.67799949645996, 2.569999933242798, 18.107999801635742], [23.12700080871582, 2.638000011444092, 18.31399917602539], [23.68199920654297, 3.8929998874664307, 17.67099952697754]], [[24.378999710083008, 4.692999839782715, 18.468000411987305], [25.038999557495117, 5.906000137329102, 18.03700065612793], [26.517000198364258, 5.586999893188477, 17.798999786376953]], [[27.048999786376953, 6.0279998779296875, 16.660999298095703], [28.451000213623047, 5.752999782562256, 16.349000930786133], [29.104999542236328, 6.9730000495910645, 15.729999542236328]], [[30.35300064086914, 7.21999979019165, 16.07699966430664], [31.118000030517578, 8.342000007629395, 15.494999885559082], [32.4109992980957, 7.796000003814697, 14.876999855041504]], [[32.70199966430664, 8.189000129699707, 13.642000198364258], [33.888999938964844, 7.730999946594238, 12.935999870300293], [34.7760009765625, 8.920999526977539, 12.536999702453613]], [[36.0629997253418, 8.791999816894531, 12.812999725341797], [36.98699951171875, 9.88700008392334, 12.48799991607666], [37.29999923706055, 9.84000015258789, 11.006999969482422]], [[37.23400115966797, 10.986000061035156, 10.34000015258789], [37.55500030517578, 11.02400016784668, 8.928000450134277], [38.99599838256836, 11.543000221252441, 8.767000198364258]], [[39.54600143432617, 11.29699993133545, 7.580999851226807], [40.89099884033203, 11.798999786376953, 7.271999835968018], [40.85900115966797, 13.180000305175781, 6.656000137329102]], [[39.75899887084961, 13.538999557495117, 6.051000118255615], [39.483001708984375, 14.788999557495117, 5.353000164031982], [37.98899841308594, 14.723999977111816, 4.979000091552734]], [[37.53099822998047, 15.684000015258789, 4.210000038146973], [36.15599822998047, 15.71500015258789, 3.759000062942505], [35.82099914550781, 14.442000389099121, 3.000999927520752]], [[34.6349983215332, 13.883000373840332, 3.2049999237060547], [34.1619987487793, 12.777999877929688, 2.3929998874664307], [33.33000183105469, 13.305000305175781, 1.215999960899353]], [[33.40599822998047, 12.63700008392334, 0.0820000022649765], [32.52899932861328, 12.902000427246094, -1.0549999475479126], [31.68600082397461, 11.628000259399414, -1.2259999513626099]], [[30.423999786376953, 11.682999610900879, -0.8090000152587891], [29.611000061035156, 10.472000122070312, -0.7910000085830688], [28.639999389648438, 10.432999610900879, -1.9620000123977661]], [[28.229000091552734, 9.21399974822998, -2.321000099182129], [27.27199935913086, 9.095000267028809, -3.4049999713897705], [25.933000564575195, 8.590999603271484, -2.9049999713897705]], [[25.71500015258789, 8.59000015258789, -1.5729999542236328], [24.43199920654297, 8.154999732971191, -1.0160000324249268], [23.31999969482422, 9.147000312805176, -1.3309999704360962]], [[22.099000930786133, 8.6850004196167, -1.496000051498413], [20.961999893188477, 9.581000328063965, -1.7259999513626099], [20.743000030517578, 10.449999809265137, -0.4959999918937683]], [[20.191999435424805, 11.647000312805176, -0.6740000247955322], [19.884000778198242, 12.498000144958496, 0.47699999809265137], [18.466999053955078, 12.1850004196167, 0.9350000023841858]], [[18.31999969482422, 11.642999649047852, 2.134999990463257], [16.979000091552734, 11.260000228881836, 2.5959999561309814], [16.398000717163086, 12.272000312805176, 3.569999933242798]], [[15.14799976348877, 12.67300033569336, 3.3570001125335693], [14.463000297546387, 13.543999671936035, 4.306000232696533], [13.918000221252441, 12.704000473022461, 5.453000068664551]], [[13.404999732971191, 13.395999908447266, 6.47599983215332], [12.786999702453613, 12.640000343322754, 7.563000202178955], [11.262999534606934, 12.633000373840332, 7.357999801635742]], [[10.793999671936035, 12.854999542236328, 6.130000114440918], [9.342000007629395, 12.8100004196167, 5.886000156402588], [8.930000305175781, 11.581000328063965, 5.1020002365112305]], [[9.817999839782715, 10.586000442504883, 5.098999977111816], [9.60099983215332, 9.321000099182129, 4.431000232696533], [10.47599983215332, 8.25100040435791, 5.073999881744385]], [[9.956999778747559, 7.019000053405762, 5.050000190734863], [10.755999565124512, 5.86899995803833, 5.513999938964844], [11.48900032043457, 5.243000030517578, 4.331999778747559]], [[11.34000015258789, 5.843999862670898, 3.134000062942505], [11.996000289916992, 5.315999984741211, 1.940000057220459], [13.503000259399414, 5.2129998207092285, 2.0850000381469727]], [[14.0600004196167, 4.063000202178955, 1.6610000133514404], [15.482000350952148, 3.7739999294281006, 1.659000039100647], [16.150999069213867, 3.7720000743865967, 3.0230000019073486]], [[15.38599967956543, 3.627000093460083, 4.114999771118164], [16.035999298095703, 3.638000011444092, 5.434000015258789], [16.474000930786133, 2.246000051498413, 5.843999862670898]], [[15.675999641418457, 1.2319999933242798, 5.466000080108643], [15.954000473022461, -0.12300000339746475, 5.9670000076293945], [17.042999267578125, -0.8220000267028809, 5.159999847412109]], [[17.594999313354492, -1.899999976158142, 5.7179999351501465], [18.670000076293945, -2.677999973297119, 5.1620001792907715], [19.96299934387207, -1.8969999551773071, 5.015999794006348]], [[20.159000396728516, -0.9350000023841858, 5.914999961853027], [21.39299964904785, -0.16699999570846558, 6.001999855041504], [22.398000717163086, -0.9929999709129333, 6.789999961853027]], [[23.69099998474121, -0.8880000114440918, 6.501999855041504], [24.66900062561035, -1.5820000171661377, 7.328000068664551], [25.759000778198242, -0.5680000185966492, 7.664999961853027]], [[25.711000442504883, -0.027000000700354576, 8.890000343322754], [26.691999435424805, 1.0230000019073486, 9.22599983215332], [28.143999099731445, 0.6079999804496765, 9.059000015258789]], [[28.459999084472656, -0.6439999938011169, 9.381999969482422], [29.816999435424805, -1.1610000133514404, 9.211000442504883], [30.17799949645996, -1.2309999465942383, 7.72599983215332]], [[29.21500015258789, -1.4850000143051147, 6.8420000076293945], [29.554000854492188, -1.440000057220459, 5.418000221252441], [29.756999969482422, -0.010999999940395355, 4.943999767303467]], [[28.868000030517578, 0.9039999842643738, 5.353000164031982], [29.024999618530273, 2.309999942779541, 4.999000072479248], [30.399999618530273, 2.7860000133514404, 5.442999839782715]], [[30.777000427246094, 2.552000045776367, 6.7270002365112305], [32.01300048828125, 3.1730000972747803, 7.182000160217285], [33.2239990234375, 2.5390000343322754, 6.495999813079834]], [[33.12799835205078, 1.2410000562667847, 6.176000118255615], [34.25199890136719, 0.578000009059906, 5.5269999504089355], [34.35900115966797, 1.0199999809265137, 4.070000171661377]], [[33.24399948120117, 1.312999963760376, 3.443000078201294], [33.16899871826172, 1.8450000286102295, 2.0880000591278076], [33.99300003051758, 3.128999948501587, 2.005000114440918]], [[33.94300079345703, 3.990999937057495, 3.0199999809265137], [34.71500015258789, 5.224999904632568, 3.0220000743865967], [36.018001556396484, 5.109000205993652, 3.7679998874664307]], [[36.492000579833984, 3.871000051498413, 3.947999954223633], [37.790000915527344, 3.566999912261963, 4.510000228881836], [37.95600128173828, 4.164999961853027, 5.894000053405762]], [[36.8650016784668, 4.156000137329102, 6.670000076293945], [36.95100021362305, 4.630000114440918, 8.04699993133545], [37.21500015258789, 3.4110000133514404, 8.937999725341797]], [[37.944000244140625, 3.630000114440918, 10.015999794006348], [38.308998107910156, 2.5940001010894775, 10.960000038146973], [37.19200134277344, 2.311000108718872, 11.961000442504883]], [[37.597999572753906, 1.6330000162124634, 13.027000427246094], [36.65599822998047, 1.343999981880188, 14.126999855041504], [36.117000579833984, 2.671999931335449, 14.645000457763672]], [[34.891998291015625, 2.7009999752044678, 15.178000450134277], [34.34299850463867, 3.930000066757202, 15.708000183105469], [35.23699951171875, 4.534999847412109, 16.763999938964844]], [[35.40700149536133, 5.864999771118164, 16.774999618530273], [36.125999450683594, 6.495999813079834, 17.86400032043457], [35.220001220703125, 6.669000148773193, 19.07900047302246]], [[33.895999908447266, 6.74399995803833, 18.836999893188477], [32.987998962402344, 6.875, 19.983999252319336], [31.621999740600586, 6.254000186920166, 19.679000854492188]], [[30.95800018310547, 5.758999824523926, 20.743999481201172], [29.631999969482422, 5.13100004196167, 20.41200065612793], [28.871999740600586, 4.89300012588501, 21.722999572753906]], [[27.566999435424805, 4.689000129699707, 21.60700035095215], [26.738000869750977, 4.340000152587891, 22.746000289916992], [25.481000900268555, 3.6489999294281006, 22.207000732421875]], [[24.60700035095215, 3.249000072479248, 23.152000427246094], [23.35099983215332, 2.6670000553131104, 22.739999771118164], [22.26799964904785, 3.046999931335449, 23.76099967956543]], [[21.107999801635742, 3.4739999771118164, 23.270999908447266], [19.933000564575195, 3.5480000972747803, 24.13800048828125], [18.922000885009766, 2.5439999103546143, 23.593000411987305]], [[17.986000061035156, 2.1110000610351562, 24.44700050354004], [16.916000366210938, 1.246999979019165, 23.96500015258789], [15.595999717712402, 1.9900000095367432, 24.077999114990234]], [[14.58899974822998, 1.597000002861023, 23.298999786376953], [13.29800033569336, 2.2290000915527344, 23.389999389648438], [12.220999717712402, 1.274999976158142, 22.865999221801758]], [[11.050999641418457, 1.3359999656677246, 23.507999420166016], [9.887999534606934, 0.6330000162124634, 22.972999572753906], [8.739999771118164, 1.6339999437332153, 22.85099983215332]], [[7.685999870300293, 1.180999994277954, 22.19300079345703], [6.60099983215332, 2.0390000343322754, 21.739999771118164], [6.090000152587891, 2.9609999656677246, 22.833999633789062]], [[5.625999927520752, 4.13700008392334, 22.427000045776367], [4.872000217437744, 5.004000186920166, 23.319000244140625], [3.821000099182129, 5.734000205993652, 22.47100067138672]], [[3.0220000743865967, 6.572000026702881, 23.128999710083008], [1.8930000066757202, 7.199999809265137, 22.433000564575195], [2.2820000648498535, 8.34000015258789, 21.511999130249023]], [[3.5739998817443848, 8.678000450134277, 21.42300033569336], [4.02400016784668, 9.590999603271484, 20.395000457763672], [4.2769999504089355, 8.868000030517578, 19.082000732421875]], [[4.534999847412109, 7.560999870300293, 19.090999603271484], [4.941999912261963, 6.828999996185303, 17.89699935913086], [3.996999979019165, 6.936999797821045, 16.72599983215332]], [[2.677999973297119, 6.867000102996826, 16.899999618530273], [1.7710000276565552, 7.079999923706055, 15.779999732971191], [2.052000045776367, 8.392000198364258, 15.086000442504883]], [[2.3580000400543213, 9.46399974822998, 15.803999900817871], [2.5969998836517334, 10.753999710083008, 15.128000259399414], [3.8550000190734863, 10.725000381469727, 14.28600025177002]], [[4.859000205993652, 9.972999572753906, 14.727999687194824], [6.073999881744385, 9.791999816894531, 13.928999900817871], [5.77400016784668, 9.026000022888184, 12.656999588012695]], [[4.980000019073486, 7.935999870300293, 12.706999778747559], [4.589000225067139, 7.291999816894531, 11.4399995803833], [3.7850000858306885, 8.210000038146973, 10.538000106811523]], [[2.946000099182129, 9.083999633789062, 11.07800006866455], [2.256999969482422, 10.104000091552734, 10.276000022888184], [3.2639999389648438, 10.993000030517578, 9.529999732971191]], [[4.304999828338623, 11.38700008392334, 10.244000434875488], [5.410999774932861, 12.163000106811523, 9.687999725341797], [6.0879998207092285, 11.4399995803833, 8.524999618530273]], [[6.340000152587891, 10.147000312805176, 8.670999526977539], [7.004000186920166, 9.312000274658203, 7.703000068664551], [6.10699987411499, 8.972000122070312, 6.519000053405762]], [[4.815999984741211, 9.267000198364258, 6.631999969482422], [3.875, 8.991999626159668, 5.550000190734863], [3.7899999618530273, 10.239999771118164, 4.673999786376953]], [[4.480999946594238, 11.29699993133545, 5.0980000495910645], [4.570000171661377, 12.562000274658203, 4.406000137329102], [3.3989999294281006, 13.482000350952148, 4.764999866485596]]], "seq": "PVDLSKWSGPLSLQEVDERPQHPLQVKYGGAEVDELGKVLTPTQVKNRPTSITWDGLDPGKLYTLVLTDPDAPSRKDPKYREWHHFLVVNMKGNNISSGTVLSDYVGSGPPKGTGLHRYVWLVYEQEGPLKCDEPILSNRSGDHRGKFKVASFRKKYELGAPVAGTCYQAEWDDYVPKLYEQLSG"}, {"coords": [[[-19.0310001373291, 16.69499969482422, 3.7079999446868896], [-20.281999588012695, 16.902000427246094, 4.4039998054504395], [-20.575000762939453, 18.393999099731445, 4.215000152587891]], [[-21.00200080871582, 18.711000442504883, 2.9809999465942383], [-21.135000228881836, 20.030000686645508, 2.3310000896453857], [-19.672000885009766, 20.413000106811523, 2.0820000171661377]], [[-19.124000549316406, 21.57699966430664, 2.3499999046325684], [-17.743000030517578, 21.878000259399414, 2.069999933242798], [-16.88599967956543, 21.628999710083008, 3.305999994277954]], [[-16.988000869750977, 20.42300033569336, 3.8380000591278076], [-16.305999755859375, 19.99799919128418, 5.052999973297119], [-16.058000564575195, 18.51300048828125, 5.03000020980835]], [[-15.064000129699707, 18.038000106811523, 5.761000156402588], [-14.76200008392334, 16.6299991607666, 5.888999938964844], [-14.065999984741211, 16.45400047302246, 7.223999977111816]], [[-14.211999893188477, 15.286999702453613, 7.826000213623047], [-13.54800033569336, 14.96399974822998, 9.072999954223633], [-12.699999809265137, 13.753000259399414, 8.734999656677246]], [[-11.496000289916992, 13.739999771118164, 9.277000427246094], [-10.548999786376953, 12.682999610900879, 9.003000259399414], [-9.881999969482422, 12.371000289916992, 10.335000038146973]], [[-9.638999938964844, 11.083999633789062, 10.5600004196167], [-9.154999732971191, 10.593000411987305, 11.836999893188477], [-7.824999809265137, 9.904000282287598, 11.633999824523926]], [[-6.968999862670898, 10.090999603271484, 12.625], [-5.71999979019165, 9.380999565124512, 12.706000328063965], [-5.665999889373779, 8.699000358581543, 14.069000244140625]], [[-5.114999771118164, 7.513000011444092, 14.10099983215332], [-5.068999767303467, 6.730000019073486, 15.321999549865723], [-3.622999906539917, 6.5320000648498535, 15.777000427246094]], [[-3.3239998817443848, 5.525000095367432, 16.583999633789062], [-1.9600000381469727, 5.318999767303467, 16.996999740600586], [-1.1319999694824219, 4.49399995803833, 16.006999969482422]], [[-1.5440000295639038, 4.421000003814697, 14.748000144958496], [-0.7440000176429749, 3.8310000896453857, 13.699999809265137], [-0.628000020980835, 4.982999801635742, 12.701000213623047]], [[0.2280000001192093, 4.932000160217285, 11.682999610900879], [0.35499998927116394, 6.085999965667725, 10.791999816894531], [-0.6909999847412109, 5.96999979019165, 9.670999526977539]], [[-1.9850000143051147, 6.068999767303467, 10.008000373840332], [-3.058000087738037, 5.817999839782715, 9.079999923706055], [-4.046999931335449, 6.951000213623047, 9.098999977111816]], [[-4.552000045776367, 7.436999797821045, 7.9770002365112305], [-5.739999771118164, 8.263999938964844, 8.065999984741211], [-6.86299991607666, 7.2729997634887695, 7.796000003814697]], [[-8.052000045776367, 7.4710001945495605, 8.336999893188477], [-9.151000022888184, 6.603000164031982, 7.9629998207092285], [-9.6850004196167, 6.926000118255615, 6.564000129699707]], [[-9.404000282287598, 8.104000091552734, 6.0], [-9.87399959564209, 8.446000099182129, 4.669000148773193], [-8.635000228881836, 8.482000350952148, 3.819000005722046]], [[-8.638999938964844, 7.728000164031982, 2.73799991607666], [-7.485000133514404, 7.703000068664551, 1.8700000047683716], [-7.681000232696533, 8.642000198364258, 0.6790000200271606]], [[-8.904999732971191, 9.097999572753906, 0.4410000145435333], [-9.206000328063965, 9.98799991607666, -0.652999997138977], [-10.274999618530273, 10.944000244140625, -0.14900000393390656]], [[-10.17199993133545, 12.220999717712402, -0.5], [-11.142000198364258, 13.239999771118164, -0.13899999856948853], [-11.541999816894531, 13.9399995803833, -1.437000036239624]], [[-12.831000328063965, 14.222000122070312, -1.621000051498413], [-13.3149995803833, 14.920000076293945, -2.805000066757202], [-13.904999732971191, 16.257999420166016, -2.359999895095825]], [[-13.477999687194824, 17.339000701904297, -2.993000030517578], [-13.958999633789062, 18.670000076293945, -2.694000005722046], [-15.001999855041504, 18.927000045776367, -3.7690000534057617]], [[-16.23900032043457, 19.235000610351562, -3.4010000228881836], [-17.27899932861328, 19.492000579833984, -4.374000072479248], [-17.003000259399414, 20.819000244140625, -5.065000057220459]], [[-17.28499984741211, 20.829999923706055, -6.363999843597412], [-17.069000244140625, 22.020000457763672, -7.171999931335449], [-17.94099998474121, 23.226999282836914, -6.854000091552734]], [[-18.945999145507812, 23.132999420166016, -5.993000030517578], [-19.670000076293945, 24.31100082397461, -5.560999870300293], [-18.958999633789062, 24.99799919128418, -4.388000011444092]], [[-17.881999969482422, 24.40999984741211, -3.8589999675750732], [-17.219999313354492, 24.961999893188477, -2.700000047683716], [-16.207000732421875, 26.031999588012695, -3.01200008392334]], [[-16.3799991607666, 27.187999725341797, -2.36299991607666], [-15.414999961853027, 28.256000518798828, -2.4560000896453857], [-14.39900016784668, 28.04400062561035, -1.3250000476837158]], [[-14.817999839782715, 27.566999435424805, -0.1459999978542328], [-13.937999725341797, 27.30299949645996, 0.9940000176429749], [-14.10200023651123, 25.820999145507812, 1.319000005722046]], [[-13.116000175476074, 25.155000686645508, 1.9170000553131104], [-13.28499984741211, 23.78499984741211, 2.378000020980835], [-12.597999572753906, 23.67099952697754, 3.736999988555908]], [[-13.236000061035156, 22.93000030517578, 4.642000198364258], [-12.758000373840332, 22.750999450683594, 5.992000102996826], [-12.515999794006348, 21.27400016784668, 6.288000106811523]], [[-11.322999954223633, 20.98200035095215, 6.828999996185303], [-10.946999549865723, 19.643999099731445, 7.26200008392334], [-10.869000434875488, 19.643999099731445, 8.782999992370605]], [[-11.541000366210938, 18.702999114990234, 9.40999984741211], [-11.482999801635742, 18.54599952697754, 10.850000381469727], [-10.628999710083008, 17.31100082397461, 11.04699993133545]], [[-9.437999725341797, 17.458999633789062, 11.619000434875488], [-8.586000442504883, 16.326000213623047, 11.923999786376953], [-8.8149995803833, 15.881999969482422, 13.361000061035156]], [[-9.050000190734863, 14.60200023651123, 13.616000175476074], [-9.182999610900879, 14.142999649047852, 14.982000350952148], [-8.126999855041504, 13.083999633789062, 15.178000450134277]], [[-7.492000102996826, 13.071000099182129, 16.340999603271484], [-6.5329999923706055, 12.03499984741211, 16.645999908447266], [-7.28000020980835, 11.187999725341797, 17.663000106811523]], [[-7.73199987411499, 10.015999794006348, 17.229000091552734], [-8.513999938964844, 9.135000228881836, 18.072999954223633], [-7.6529998779296875, 8.137999534606934, 18.847000122070312]], [[-6.322000026702881, 8.217000007629395, 18.702999114990234], [-5.443999767303467, 7.426000118255615, 19.552000045776367], [-5.309999942779541, 7.9770002365112305, 20.981000900268555]], [[-4.415999889373779, 7.363999843597412, 21.7450008392334], [-4.178999900817871, 7.771999835968018, 23.118000030517578], [-2.7269999980926514, 8.128999710083008, 23.395000457763672]], [[-1.909999966621399, 8.253000259399414, 22.357999801635742], [-0.5080000162124634, 8.538999557495117, 22.601999282836914], [-0.1509999930858612, 9.996000289916992, 22.49799919128418]], [[0.9190000295639038, 10.321000099182129, 23.224000930786133], [1.3589999675750732, 11.682000160217285, 23.399999618530273], [1.812999963760376, 12.321999549865723, 22.108999252319336]], [[1.809000015258789, 13.647000312805176, 22.03700065612793], [2.127000093460083, 14.276000022888184, 20.775999069213867], [3.6089999675750732, 14.154999732971191, 20.486000061035156]], [[4.51800012588501, 14.121000289916992, 21.476999282836914], [5.9120001792907715, 13.9399995803833, 21.121000289916992], [6.205999851226807, 12.498000144958496, 20.743999481201172]], [[5.361000061035156, 11.532999992370605, 21.11400032043457], [5.5269999504089355, 10.13700008392334, 20.725000381469727], [4.894999980926514, 9.767000198364258, 19.3799991607666]], [[3.6710000038146973, 10.25100040435791, 19.134000778198242], [2.878999948501587, 9.888999938964844, 17.965999603271484], [1.8910000324249268, 10.991999626159668, 17.572999954223633]], [[2.3510000705718994, 12.241999626159668, 17.549999237060547], [1.5269999504089355, 13.347000122070312, 17.100000381469727], [1.2740000486373901, 13.217000007629395, 15.604000091552734]], [[0.2540000081062317, 13.86299991607666, 15.062000274658203], [-0.05400000140070915, 13.78499984741211, 13.630000114440918], [-0.5950000286102295, 15.138999938964844, 13.208000183105469]], [[-0.48100000619888306, 15.494000434875488, 11.939000129699707], [-1.0720000267028809, 16.72800064086914, 11.418000221252441], [-1.6399999856948853, 16.368000030517578, 10.04800033569336]], [[-2.0799999237060547, 17.299999237060547, 9.210000038146973], [-2.5929999351501465, 17.014999389648438, 7.877999782562256], [-2.053999900817871, 18.200000762939453, 7.0970001220703125]], [[-1.3359999656677246, 17.920000076293945, 6.015999794006348], [-0.6589999794960022, 18.92099952697754, 5.215000152587891], [-1.0959999561309814, 18.621999740600586, 3.7960000038146973]], [[-1.524999976158142, 19.632999420166016, 3.0420000553131104], [-1.9520000219345093, 19.469999313354492, 1.659999966621399], [-0.9629999995231628, 20.19099998474121, 0.7409999966621399]], [[-0.4569999873638153, 19.50200080871582, -0.2800000011920929], [0.47600001096725464, 20.066999435424805, -1.2549999952316284], [0.15399999916553497, 19.440000534057617, -2.615000009536743]], [[0.7170000076293945, 19.95800018310547, -3.7139999866485596], [0.6420000195503235, 19.263999938964844, -5.001999855041504], [1.4570000171661377, 18.003000259399414, -4.789999961853027]], [[1.055999994277954, 16.878999710083008, -5.382999897003174], [1.7319999933242798, 15.590999603271484, -5.2230000495910645], [3.244999885559082, 15.618000030517578, -5.455999851226807]], [[3.6570000648498535, 16.58799934387207, -6.275000095367432], [5.046000003814697, 16.867000579833984, -6.564000129699707], [5.820000171661377, 17.31399917602539, -5.329999923706055]], [[5.294000148773193, 18.326000213623047, -4.632999897003174], [5.978000164031982, 18.959999084472656, -3.5199999809265137], [6.0289998054504395, 18.21299934387207, -2.196000099182129]], [[5.519000053405762, 16.988000869750977, -2.1679999828338623], [5.390999794006348, 16.235000610351562, -0.9419999718666077], [6.699999809265137, 15.947999954223633, -0.22499999403953552]], [[7.677000045776367, 15.293000221252441, -0.8679999709129333], [8.937999725341797, 14.977999687194824, -0.21699999272823334], [9.680999755859375, 16.208999633789062, 0.23100000619888306]], [[9.661999702453613, 17.26799964904785, -0.5699999928474426], [10.331000328063965, 18.514999389648438, -0.25099998712539673], [9.741999626159668, 19.216999053955078, 0.9549999833106995]], [[8.404999732971191, 19.243000030517578, 1.0099999904632568], [7.730999946594238, 19.854999542236328, 2.1389999389648438], [7.947999954223633, 18.985000610351562, 3.372999906539917]], [[7.868000030517578, 17.666000366210938, 3.260999917984009], [8.048999786376953, 16.773000717163086, 4.388000011444092], [9.437999725341797, 16.908000946044922, 5.013000011444092]], [[10.510000228881836, 16.91200065612793, 4.205999851226807], [11.873000144958496, 17.079999923706055, 4.691999912261963], [12.064000129699707, 18.43600082397461, 5.3520002365112305]], [[11.706000328063965, 19.558000564575195, 4.7129998207092285], [11.824999809265137, 20.857999801635742, 5.35699987411499], [10.914999961853027, 21.003000259399414, 6.563000202178955]], [[9.791000366210938, 20.288000106811523, 6.538000106811523], [8.836000442504883, 20.256999969482422, 7.627999782562256], [9.458000183105469, 19.57900047302246, 8.847999572753906]], [[10.081000328063965, 18.40999984741211, 8.642999649047852], [10.758999824523926, 17.68000030517578, 9.710000038146973], [11.843999862670898, 18.562000274658203, 10.3100004196167]], [[12.57699966430664, 19.261999130249023, 9.458999633789062], [13.595999717712402, 20.18600082397461, 9.930000305175781], [13.031999588012695, 21.41200065612793, 10.64799976348877]], [[11.74899959564209, 21.773000717163086, 10.498000144958496], [11.175000190734863, 22.93199920654297, 11.178999900817871], [10.890000343322754, 22.7810001373291, 12.680999755859375]], [[10.736000061035156, 21.554000854492188, 13.17199993133545], [10.51200008392334, 21.336999893188477, 14.588000297546387], [9.039999961853027, 21.39900016784668, 14.989999771118164]], [[8.751999855041504, 20.822999954223633, 16.15999984741211], [7.4019999504089355, 20.75200080871582, 16.71500015258789], [6.7729997634887695, 22.117000579833984, 16.95599937438965]], [[7.520999908447266, 23.139999389648438, 17.389999389648438], [6.965000152587891, 24.47100067138672, 17.591999053955078], [6.381999969482422, 25.06599998474121, 16.301000595092773]], [[6.829999923706055, 24.558000564575195, 15.142000198364258], [6.322999954223633, 24.958999633789062, 13.842000007629395], [5.302000045776367, 23.98200035095215, 13.267000198364258]], [[4.860000133514404, 23.010000228881836, 14.067999839782715], [3.9709999561309814, 21.937999725341797, 13.647000312805176], [4.507999897003174, 21.138999938964844, 12.479000091552734]], [[5.830999851226807, 21.020999908447266, 12.526000022888184], [6.609000205993652, 20.367000579833984, 11.503999710083008], [6.281000137329102, 20.785999298095703, 10.069999694824219]], [[6.074999809265137, 22.084999084472656, 9.880000114440918], [5.853000164031982, 22.65399932861328, 8.564000129699707], [6.724999904632568, 23.916000366210938, 8.53499984741211]], [[7.585999965667725, 24.082000732421875, 7.531000137329102], [8.401000022888184, 25.275999069213867, 7.363999843597412], [7.534999847412109, 26.52899932861328, 7.315000057220459]], [[7.817999839782715, 27.545000076293945, 8.133999824523926], [7.046999931335449, 28.780000686645508, 8.140999794006348], [7.054999828338623, 29.45800018310547, 6.769000053405762]], [[5.889999866485596, 29.902999877929688, 6.296999931335449], [5.769000053405762, 30.594999313354492, 5.019999980926514], [6.11899995803833, 29.736000061035156, 3.815000057220459]], [[6.052000045776367, 28.402000427246094, 3.9749999046325684], [6.314000129699707, 27.486000061035156, 2.885999917984009], [5.218999862670898, 27.659000396728516, 1.8420000076293945]], [[5.686999797821045, 28.201000213623047, 0.7300000190734863], [4.831999778747559, 28.535999298095703, -0.3930000066757202], [4.1479997634887695, 27.351999282836914, -1.0360000133514404]], [[4.71999979019165, 26.158000946044922, -0.8479999899864197], [4.206999778747559, 24.929000854492188, -1.4359999895095825], [3.061000108718872, 24.315000534057617, -0.640999972820282]], [[2.818000078201294, 24.790000915527344, 0.5820000171661377], [1.8009999990463257, 24.238000869750977, 1.468000054359436], [0.5099999904632568, 25.010000228881836, 1.2259999513626099]], [[-0.4699999988079071, 24.325000762939453, 0.6380000114440918], [-1.774999976158142, 24.922000885009766, 0.4090000092983246], [-2.4839999675750732, 25.118999481201172, 1.75]], [[-2.433000087738037, 24.145999908447266, 2.6649999618530273], [-3.065999984741211, 24.288000106811523, 3.9639999866485596], [-2.4760000705718994, 23.236000061035156, 4.886000156402588]], [[-2.5399999618530273, 23.448999404907227, 6.202000141143799], [-2.0299999713897705, 22.45800018310547, 7.13700008392334], [-2.6989998817443848, 22.66200065612793, 8.486000061035156]], [[-2.8550000190734863, 21.6200008392334, 9.291999816894531], [-3.3340001106262207, 21.740999221801758, 10.666000366210938], [-2.0959999561309814, 21.858999252319336, 11.574999809265137]], [[-2.372999906539917, 21.94700050354004, 12.86400032043457], [-1.371999979019165, 21.871999740600586, 13.91100025177002], [-1.0729999542236328, 20.386999130249023, 14.154999732971191]], [[-0.020999999716877937, 20.086999893188477, 14.918999671936035], [0.28299999237060547, 18.722000122070312, 15.279999732971191], [-0.5210000276565552, 18.520999908447266, 16.54199981689453]], [[-1.2640000581741333, 17.43000030517578, 16.604000091552734], [-2.075000047683716, 17.150999069213867, 17.768999099731445], [-1.7009999752044678, 15.798999786376953, 18.322999954223633]], [[-1.934000015258789, 15.618000030517578, 19.60700035095215], [-1.7610000371932983, 14.32800006866455, 20.250999450683594], [-3.0899999141693115, 13.586999893188477, 20.424999237060547]], [[-3.0320000648498535, 12.416000366210938, 21.06399917602539], [-4.197999954223633, 11.574000358581543, 21.280000686645508], [-5.289999961853027, 12.291999816894531, 22.062999725341797]], [[-6.500999927520752, 12.119000434875488, 21.54400062561035], [-7.684999942779541, 12.694000244140625, 22.145000457763672], [-7.951000213623047, 14.12600040435791, 21.718000411987305]], [[-7.117000102996826, 14.692999839782715, 20.860000610351562], [-7.2729997634887695, 16.069000244140625, 20.413000106811523], [-7.824999809265137, 16.15399932861328, 19.003999710083008]], [[-8.08899974822998, 17.385000228881836, 18.552000045776367], [-8.52299976348877, 17.635000228881836, 17.19300079345703], [-8.086000442504883, 19.020000457763672, 16.73699951171875]], [[-8.067999839782715, 19.309999465942383, 15.4399995803833], [-7.744999885559082, 20.628000259399414, 14.961000442504883], [-8.444000244140625, 20.791000366210938, 13.621999740600586]], [[-8.795000076293945, 21.996999740600586, 13.192999839782715], [-9.404999732971191, 22.235000610351562, 11.89900016784668], [-8.637999534606934, 23.29800033569336, 11.145999908447266]], [[-8.758000373840332, 23.295000076293945, 9.819999694824219], [-8.170000076293945, 24.281999588012695, 8.935999870300293], [-9.190999984741211, 24.393999099731445, 7.791999816894531]], [[-9.404999732971191, 25.634000778198242, 7.363999843597412], [-10.310999870300293, 25.95599937438965, 6.2820000648498535], [-9.42199993133545, 26.586999893188477, 5.22599983215332]], [[-9.496000289916992, 26.216999053955078, 3.9549999237060547], [-8.675000190734863, 26.825000762939453, 2.934000015258789], [-9.524999618530273, 27.222999572753906, 1.746000051498413]], [[-8.993000030517578, 28.09000015258789, 0.8989999890327454], [-9.704000473022461, 28.51300048828125, -0.2919999957084656], [-9.527999877929688, 27.527000427246094, -1.437999963760376]], [[-10.640000343322754, 27.06800079345703, -1.996999979019165], [-10.605999946594238, 26.09600067138672, -3.072999954223633], [-10.006999969482422, 26.656999588012695, -4.361999988555908]], [[-9.958999633789062, 27.96500015258789, -4.630000114440918], [-9.336000442504883, 28.429000854492188, -5.866000175476074], [-7.816999912261963, 28.23900032043457, -5.894000053405762]], [[-7.2170000076293945, 27.80699920654297, -4.77400016784668], [-5.820000171661377, 27.393999099731445, -4.755000114440918], [-5.668000221252441, 26.048999786376953, -5.464000225067139]], [[-6.771999835968018, 25.347000122070312, -5.732999801635742], [-6.767000198364258, 24.059999465942383, -6.394999980926514], [-7.251999855041504, 24.15399932861328, -7.834000110626221]], [[-6.454999923706055, 23.56599998474121, -8.706999778747559], [-6.801000118255615, 23.398000717163086, -10.107000350952148], [-7.656000137329102, 22.131000518798828, -10.223999977111816]], [[-8.769000053405762, 22.163999557495117, -10.977999687194824], [-9.668999671936035, 21.033000946044922, -11.187000274658203], [-8.975000381469727, 19.829999923706055, -11.802000045776367]], [[-9.475000381469727, 18.636999130249023, -11.482999801635742], [-8.949000358581543, 17.393999099731445, -12.027000427246094], [-7.678999900817871, 16.92300033569336, -11.326000213623047]], [[-6.703999996185303, 17.840999603271484, -11.25], [-5.39900016784668, 17.663000106811523, -10.63599967956543], [-5.433000087738037, 17.006000518798828, -9.26200008392334]], [[-4.51200008392334, 16.079999923706055, -8.99899959564209], [-4.492000102996826, 15.404000282287598, -7.710999965667725], [-3.4630000591278076, 16.02899932861328, -6.78000020980835]], [[-3.937999963760376, 16.118999481201172, -5.545000076293945], [-3.1589999198913574, 16.691999435424805, -4.482999801635742], [-2.937999963760376, 15.611000061035156, -3.4590001106262207]], [[-1.8289999961853027, 15.807999610900879, -2.7829999923706055], [-1.3960000276565552, 14.880999565124512, -1.7769999504089355], [-1.7089999914169312, 15.494000434875488, -0.4269999861717224]], [[-2.3329999446868896, 14.755000114440918, 0.4779999852180481], [-2.3510000705718994, 15.232000350952148, 1.8370000123977661], [-1.565000057220459, 14.196000099182129, 2.634999990463257]], [[-0.796999990940094, 14.604000091552734, 3.632999897003174], [0.061000000685453415, 13.675999641418457, 4.3460001945495605], [0.3790000081062317, 14.272000312805176, 5.699999809265137]], [[1.0219999551773071, 13.479000091552734, 6.554999828338623], [1.4140000343322754, 13.881999969482422, 7.881999969482422], [2.9100000858306885, 14.13599967956543, 7.875999927520752]], [[3.3970000743865967, 15.293000221252441, 8.307999610900879], [4.810999870300293, 15.579999923706055, 8.265000343322754], [5.495999813079834, 15.402999877929688, 9.61400032043457]], [[4.822999954223633, 14.78499984741211, 10.597999572753906], [5.445000171661377, 14.454999923706055, 11.87600040435791], [6.636000156402588, 13.543000221252441, 11.531000137329102]], [[7.830999851226807, 13.64900016784668, 12.131999969482422], [9.012999534606934, 12.861000061035156, 11.79800033569336], [8.751999855041504, 11.402000427246094, 11.553000450134277]], [[9.09000015258789, 10.920000076293945, 10.373000144958496], [8.95199966430664, 9.527000427246094, 10.055000305175781], [7.586999893188477, 9.190999984741211, 9.515000343322754]], [[6.620999813079834, 10.109999656677246, 9.437000274658203], [5.275000095367432, 9.720000267028809, 9.032999992370605], [4.913000106811523, 9.958000183105469, 7.579999923706055]], [[5.560999870300293, 10.894000053405762, 6.882999897003174], [5.184999942779541, 11.291999816894531, 5.525000095367432], [4.966000080108643, 10.206999778747559, 4.486999988555908]], [[5.7820000648498535, 9.152999877929688, 4.422999858856201], [5.605000019073486, 8.137999534606934, 3.4049999713897705], [4.459000110626221, 7.21999979019165, 3.742000102996826]], [[4.077000141143799, 7.065999984741211, 4.995999813079834], [3.013000011444092, 6.138000011444092, 5.322999954223633], [1.6490000486373901, 6.76200008392334, 5.51800012588501]], [[1.6629999876022339, 7.932000160217285, 6.163000106811523], [0.453000009059906, 8.619000434875488, 6.578999996185303], [0.18700000643730164, 9.631999969482422, 5.4720001220703125]], [[-0.6309999823570251, 9.227999687194824, 4.5], [-0.8830000162124634, 10.01099967956543, 3.2909998893737793], [-2.2249999046325684, 9.708000183105469, 2.5989999771118164]], [[-2.6600000858306885, 10.541000366210938, 1.6519999504089355], [-3.888000011444092, 10.319000244140625, 0.906000018119812], [-3.9509999752044678, 11.251999855041504, -0.30000001192092896]], [[-5.072999954223633, 11.218999862670898, -1.0269999504089355], [-5.281000137329102, 11.980999946594238, -2.256999969482422], [-6.48799991607666, 12.89799976348877, -2.111999988555908]], [[-6.394000053405762, 14.130000114440918, -2.575000047683716], [-7.51800012588501, 15.041999816894531, -2.5360000133514404], [-7.76200008392334, 15.484000205993652, -3.9709999561309814]], [[-9.022000312805176, 15.506999969482422, -4.394999980926514], [-9.333999633789062, 16.0, -5.71999979019165], [-10.649999618530273, 16.749000549316406, -5.703999996185303]], [[-10.789999961853027, 17.6560001373291, -6.651000022888184], [-11.973999977111816, 18.4689998626709, -6.840000152587891], [-12.87600040435791, 17.7189998626709, -7.831999778747559]], [[-14.180999755859375, 17.55900001525879, -7.625999927520752], [-15.057000160217285, 16.843000411987305, -8.543000221252441], [-16.475000381469727, 17.329999923706055, -8.23799991607666]], [[-17.490999221801758, 17.04800033569336, -9.043000221252441], [-18.836000442504883, 17.448999404907227, -8.6850004196167], [-19.638999938964844, 16.1560001373291, -8.657999992370605]]], "seq": "AQCEATIESNDAMQYDLKEMVVDKSCKQFTVHLKHVGKMAKSAMGHNWVLTKEADKEGVATDGMNAGLAQDYVKAGDTRVIAHTKVIGGGESDSVTFDVSKLTPGEAYAYFCSFPGHWAMHKGTLKLSN"}, {"coords": [[[1.059000015258789, 0.42399999499320984, 0.26899999380111694], [1.944000005722046, 0.00800000037997961, -0.8629999756813049], [3.388000011444092, 0.4729999899864197, -0.6050000190734863]], [[3.561000108718872, 1.593999981880188, 0.06300000101327896], [4.935999870300293, 2.1029999256134033, 0.34599998593330383], [4.888000011444092, 3.0889999866485596, 1.5180000066757202]], [[5.827000141143799, 2.9769999980926514, 2.427000045776367], [5.861999988555908, 3.8959999084472656, 3.6040000915527344], [7.303999900817871, 4.359000205993652, 3.8380000591278076]], [[7.730999946594238, 5.370999813079834, 3.117000102996826], [9.118000030517578, 5.889999866485596, 3.2780001163482666], [9.041000366210938, 7.328999996185303, 3.8399999141693115]], [[8.0600004196167, 7.598999977111816, 4.677000045776367], [7.925000190734863, 8.960000038146973, 5.271999835968018], [7.696000099182129, 8.829999923706055, 6.7820000648498535]], [[8.663999557495117, 8.277999877929688, 7.47599983215332], [8.557000160217285, 8.114999771118164, 8.967000007629395], [9.734999656677246, 7.26200008392334, 9.493000030517578]], [[10.937999725341797, 7.796000003814697, 9.378000259399414], [12.163999557495117, 7.109000205993652, 9.83899974822998], [12.336000442504883, 7.204999923706055, 11.359000205993652]], [[13.17199993133545, 6.35699987411499, 11.904999732971191], [13.418999671936035, 6.363999843597412, 13.380999565124512], [14.786999702453613, 6.99399995803833, 13.692000389099121]], [[15.6899995803833, 7.011000156402588, 12.729999542236328], [17.05699920654297, 7.616000175476074, 12.946999549865723], [17.833999633789062, 6.8460001945495605, 14.032999992370605]], [[19.124000549316406, 7.091000080108643, 14.135000228881836], [19.952999114990234, 6.40500020980835, 15.173999786376953], [20.961999893188477, 7.414000034332275, 15.744000434875488]], [[20.4689998626709, 8.434000015258789, 16.413999557495117], [21.371000289916992, 9.479999542236328, 16.990999221801758], [22.06800079345703, 10.22700023651123, 15.845000267028809]], [[23.371000289916992, 10.36400032043457, 15.954999923706055], [24.18899917602539, 11.039999961853027, 14.928000450134277], [24.47599983215332, 10.092000007629395, 13.743000030517578]], [[24.000999450683594, 8.859999656677246, 13.800999641418457], [24.256999969482422, 7.89300012588501, 12.690999984741211], [22.961999893188477, 7.642000198364258, 11.906999588012695]], [[23.079999923706055, 7.193999767303467, 10.675000190734863], [21.863000869750977, 6.933000087738037, 9.83899974822998], [22.15399932861328, 5.815999984741211, 8.826000213623047]], [[21.534000396728516, 4.669000148773193, 8.989999771118164], [21.75200080871582, 3.5409998893737793, 8.027999877929688], [20.538999557495117, 3.4570000171661377, 7.085999965667725]], [[20.714000701904297, 3.8269999027252197, 5.835000038146973], [19.575000762939453, 3.7980000972747803, 4.867000102996826], [20.009000778198242, 3.193000078201294, 3.5209999084472656]], [[19.04800033569336, 2.7739999294281006, 2.7260000705718994], [19.349000930786133, 2.177000045776367, 1.3860000371932983], [19.43000030517578, 3.3010001182556152, 0.3409999907016754]], [[20.417999267578125, 3.255000114440918, -0.5239999890327454], [20.56399917602539, 4.320000171661377, -1.565999984741211], [19.621999740600586, 4.025000095367432, -2.753000020980835]], [[20.145000457763672, 3.618000030517578, -3.8970000743865967], [19.259000778198242, 3.3289999961853027, -5.076000213623047], [18.277000427246094, 2.2090001106262207, -4.7129998207092285]], [[16.99799919128418, 2.4560000896453857, -4.880000114440918], [15.968000411987305, 1.4229999780654907, -4.540999889373779], [15.37600040435791, 0.8320000171661377, -5.828999996185303]], [[15.173999786376953, -0.4699999988079071, -5.822000026702881], [14.60099983215332, -1.1979999542236328, -6.973999977111816], [13.072999954223633, -1.0219999551773071, -7.021999835968018]], [[12.420999526977539, -1.6779999732971191, -7.953000068664551], [10.9350004196167, -1.5570000410079956, -8.067999839782715], [10.25, -2.622999906539917, -7.203000068664551]], [[8.97599983215332, -2.444999933242798, -6.933000087738037], [8.225000381469727, -3.443000078201294, -6.104000091552734], [7.624000072479248, -4.5329999923706055, -7.008999824523926]], [[8.12600040435791, -4.672999858856201, -8.217000007629395], [7.5879998207092285, -5.710000038146973, -9.144000053405762], [8.71500015258789, -6.666999816894531, -9.543000221252441]], [[9.378999710083008, -7.245999813079834, -8.569000244140625], [10.482999801635742, -8.208000183105469, -8.873000144958496], [9.91100025177002, -9.366999626159668, -9.706000328063965]], [[10.73900032043457, -10.07800006866455, -10.434000015258789], [10.211999893188477, -11.213000297546387, -11.25100040435791], [10.769000053405762, -12.52400016784668, -10.711999893188477]], [[10.019000053405762, -13.175000190734863, -9.864999771118164], [10.479999542236328, -14.46399974822998, -9.291999816894531], [9.987000465393066, -15.607999801635742, -10.178000450134277]], [[10.899999618530273, -16.365999221801758, -10.734999656677246], [10.505999565124512, -17.499000549316406, -11.626999855041504], [10.07800006866455, -18.70400047302246, -10.77299976348877]], [[10.097000122070312, -19.886999130249023, -11.329000473022461], [9.670999526977539, -21.09600067138672, -10.560999870300293], [10.807999610900879, -22.117000579833984, -10.522000312805176]], [[10.751999855041504, -23.04199981689453, -9.593999862670898], [11.809000015258789, -24.094999313354492, -9.519000053405762], [11.717000007629395, -24.968000411987305, -10.779999732971191]], [[10.534000396728516, -25.073999404907227, -11.350000381469727], [10.33899974822998, -25.892000198364258, -12.588000297546387], [10.857999801635742, -25.128999710083008, -13.819999694824219]], [[10.829999923706055, -23.809999465942383, -13.793000221252441], [11.312000274658203, -23.014999389648438, -14.972000122070312], [12.531999588012695, -22.152999877929688, -14.592000007629395]], [[13.239999771118164, -22.53499984741211, -13.557000160217285], [14.440999984741211, -21.757999420166016, -13.092000007629395], [15.355999946594238, -21.392000198364258, -14.270999908447266]], [[15.399999618530273, -20.124000549316406, -14.62399959564209], [16.268999099731445, -19.67099952697754, -15.760000228881836], [16.466999053955078, -18.148000717163086, -15.694999694824219]], [[15.390999794006348, -17.40399932861328, -15.579999923706055], [15.489999771118164, -15.91100025177002, -15.513999938964844], [15.069999694824219, -15.4399995803833, -14.107999801635742]], [[14.572999954223633, -14.22700023651123, -13.991999626159668], [14.118000030517578, -13.699000358581543, -12.66100025177002], [15.307000160217285, -13.467000007629395, -11.718999862670898]], [[15.484999656677246, -12.24899959564209, -11.248000144958496], [16.61199951171875, -11.963000297546387, -10.312000274658203], [16.506000518798828, -10.534000396728516, -9.73900032043457]], [[17.631999969482422, -9.907999992370605, -9.442999839782715], [17.61199951171875, -8.52299976348877, -8.871000289916992], [17.57900047302246, -7.508999824523926, -10.020000457763672]], [[16.450000762939453, -6.874000072479248, -10.21399974822998], [16.320999145507812, -5.880000114440918, -11.317999839782715], [15.12399959564209, -4.953000068664551, -11.057999610900879]], [[14.883000373840332, -4.045000076293945, -11.965999603271484], [13.739999771118164, -3.1089999675750732, -11.82699966430664], [13.11400032043457, -2.878000020980835, -13.211000442504883]], [[11.92300033569336, -2.316999912261963, -13.248000144958496], [11.217000007629395, -2.062000036239624, -14.555000305175781], [12.197999954223633, -1.5329999923706055, -15.619000434875488]], [[13.109999656677246, -0.6650000214576721, -15.23900032043457], [14.104000091552734, -0.12200000137090683, -16.216999053955078], [15.515999794006348, -0.41499999165534973, -15.713000297546387]], [[15.932000160217285, 0.26899999380111694, -14.67300033569336], [17.29199981689453, 0.04399999976158142, -14.112000465393066], [17.4060001373291, 0.7540000081062317, -12.76099967956543]], [[17.615999221801758, 0.012000000104308128, -11.699000358581543], [17.73200035095215, 0.6499999761581421, -10.35200023651123], [17.996999740600586, -0.4090000092983246, -9.277000427246094]], [[19.073999404907227, -1.1549999713897705, -9.414999961853027], [19.42799949645996, -2.2070000171661377, -8.397000312805176], [19.457000732421875, -1.5750000476837158, -6.994999885559082]], [[19.121999740600586, -2.367000102996826, -5.999000072479248], [19.094999313354492, -1.9079999923706055, -4.5960001945495605], [20.503000259399414, -1.6990000009536743, -4.040999889373779]], [[20.663000106811523, -0.6840000152587891, -3.2309999465942383], [22.000999450683594, -0.3869999945163727, -2.624000072479248], [21.839000701904297, -0.17800000309944153, -1.1139999628067017]], [[22.843000411987305, -0.5230000019073486, -0.34700000286102295], [22.76300048828125, -0.3400000035762787, 1.1349999904632568], [24.049999237060547, 0.33399999141693115, 1.628000020980835]], [[23.940000534057617, 1.5449999570846558, 2.121000051498413], [25.145000457763672, 2.2760000228881836, 2.619999885559082], [24.84600067138672, 2.874000072479248, 4.0]], [[25.750999450683594, 2.7149999141693115, 4.938000202178955], [25.527999877929688, 3.2760000228881836, 6.306000232696533], [26.264999389648438, 4.611000061035156, 6.439000129699707]], [[25.559999465942383, 5.63700008392334, 6.8470001220703125], [26.187999725341797, 6.9730000495910645, 7.0279998779296875], [26.010000228881836, 7.385000228881836, 8.498000144958496]], [[26.948999404907227, 7.025000095367432, 9.34000015258789], [26.83300018310547, 7.373000144958496, 10.78499984741211], [28.075000762939453, 6.88700008392334, 11.527000427246094]], [[28.219999313354492, 5.5879998207092285, 11.668000221252441], [29.402000427246094, 5.01800012588501, 12.380999565124512], [30.70400047302246, 5.460000038146973, 11.687000274658203]], [[31.31399917602539, 6.52400016784668, 12.163000106811523], [32.5880012512207, 7.013999938964844, 11.54699993133545], [33.75299835205078, 6.74399995803833, 12.513999938964844]], [[33.79399871826172, 5.557000160217285, 13.072999954223633], [34.88199996948242, 5.184000015258789, 14.022000312805176], [35.85900115966797, 4.232999801635742, 13.312000274658203]], [[36.979000091552734, 3.930000066757202, 13.928000450134277], [37.96200180053711, 2.996999979019165, 13.291000366210938], [37.55500030517578, 1.5369999408721924, 13.571999549865723]], [[36.402000427246094, 1.3320000171661377, 14.175999641418457], [35.91600036621094, -0.04800000041723251, 14.47599983215332], [35.05799865722656, -0.5389999747276306, 13.295000076293945]], [[35.178001403808594, -1.812000036239624, 12.973999977111816], [34.41699981689453, -2.4200000762939453, 11.861000061035156], [32.944000244140625, -2.618000030517578, 12.253999710083008]], [[32.125999450683594, -3.046999931335449, 11.319999694824219], [30.674999237060547, -3.250999927520752, 11.616000175476074], [30.156999588012695, -4.451000213623047, 10.812999725341797]], [[29.090999603271484, -5.058000087738037, 11.272000312805176], [28.506999969482422, -6.218999862670898, 10.543999671936035], [27.000999450683594, -5.9770002365112305, 10.355999946594238]], [[26.552000045776367, -5.948999881744385, 9.12600040435791], [25.106000900268555, -5.701000213623047, 8.854999542236328], [24.4060001373291, -7.033999919891357, 8.57800006866455]], [[23.207000732421875, -7.192999839782715, 9.07699966430664], [22.45400047302246, -8.458999633789062, 8.838000297546387], [21.107999801635742, -8.128000259399414, 8.192999839782715]], [[20.63800048828125, -9.0, 7.348999977111816], [19.333999633789062, -8.779000282287598, 6.669000148773193], [18.277000427246094, -9.682000160217285, 7.306000232696533]], [[17.070999145507812, -9.192999839782715, 7.40500020980835], [15.96399974822998, -9.994999885559082, 7.993000030517578], [14.77299976348877, -9.942000389099121, 7.0329999923706055]], [[14.451000213623047, -11.04699993133545, 6.414000034332275], [13.305000305175781, -11.062000274658203, 5.461999893188477], [12.041000366210938, -11.539999961853027, 6.188000202178955]], [[10.934000015258789, -10.902000427246094, 5.920000076293945], [9.64799976348877, -11.300999641418457, 6.558000087738037], [8.503000259399414, -10.980999946594238, 5.5970001220703125]], [[7.703000068664551, -11.970000267028809, 5.265999794006348], [6.552999973297119, -11.770999908447266, 4.314000129699707], [5.797999858856201, -10.454000473022461, 4.60099983215332]], [[5.084000110626221, -9.958999633789062, 3.61299991607666], [4.327000141143799, -8.685999870300293, 3.765000104904175], [2.9769999980926514, -8.954999923706055, 4.436999797821045]], [[2.986999988555908, -9.104000091552734, 5.735000133514404], [1.7170000076293945, -9.359000205993652, 6.482999801635742], [1.1299999952316284, -8.017000198364258, 6.9679999351501465]], [[0.2590000033378601, -8.045999526977539, 7.953999996185303], [-0.3540000021457672, -6.7779998779296875, 8.46399974822998], [0.5059999823570251, -6.197999954223633, 9.600000381469727]], [[0.21299999952316284, -6.534999847412109, 10.840999603271484], [1.0160000324249268, -6.000999927520752, 11.984000205993652], [2.180999994277954, -6.958000183105469, 12.267000198364258]], [[1.937999963760376, -8.017999649047852, 13.008999824523926], [3.0220000743865967, -8.994000434875488, 13.321000099182129], [3.4769999980926514, -9.687000274658203, 12.036999702453613]], [[4.769000053405762, -9.765999794006348, 11.836000442504883], [5.302000045776367, -10.42300033569336, 10.605999946594238], [5.133999824523926, -11.944999694824219, 10.706999778747559]], [[5.335000038146973, -12.630999565124512, 9.611000061035156], [5.203000068664551, -14.121000289916992, 9.60200023651123], [6.551000118255615, -14.73799991607666, 9.184000015258789]], [[6.6570000648498535, -16.049999237060547, 9.272000312805176], [7.894999980926514, -16.756000518798828, 8.902000427246094], [8.0649995803833, -16.7450008392334, 7.381999969482422]], [[9.151000022888184, -16.16900062561035, 6.923999786376953], [9.432999610900879, -16.062000274658203, 5.456999778747559], [9.041000366210938, -17.343000411987305, 4.711999893188477]], [[8.517999649047852, -17.187999725341797, 3.5169999599456787], [8.104999542236328, -18.364999771118164, 2.7049999237060547], [9.350000381469727, -19.12700080871582, 2.2139999866485596]], [[9.777999877929688, -18.916000366210938, 0.9810000061988831], [10.972000122070312, -19.650999069213867, 0.45899999141693115], [11.684000015258789, -18.81399917602539, -0.6230000257492065]], [[11.741000175476074, -17.503000259399414, -0.47099998593330383], [12.435999870300293, -16.652000427246094, -1.4980000257492065], [13.954999923706055, -16.836999893188477, -1.3459999561309814]], [[14.446999549865723, -18.01300048828125, -1.656999945640564], [15.916999816894531, -18.2810001373291, -1.5240000486373901], [16.479999542236328, -18.86400032043457, -2.822999954223633]], [[17.350000381469727, -18.121999740600586, -3.4670000076293945], [17.981000900268555, -18.593000411987305, -4.744999885559082], [19.312000274658203, -17.850000381469727, -4.966000080108643]], [[20.19300079345703, -18.437999725341797, -5.736000061035156], [21.520999908447266, -17.80900001525879, -6.025000095367432], [21.426000595092773, -16.976999282836914, -7.320000171661377]], [[22.545000076293945, -16.674999237060547, -7.951000213623047], [22.5, -15.866000175476074, -9.210000038146973], [23.70400047302246, -16.19300079345703, -10.111000061035156]], [[23.9060001373291, -15.402999877929688, -11.14900016784668], [25.04199981689453, -15.63700008392334, -12.093000411987305], [25.045000076293945, -14.555999755859375, -13.177000045776367]], [[26.101999282836914, -13.78499984741211, -13.21500015258789], [26.26099967956543, -12.70300006866455, -14.232000350952148], [27.45400047302246, -11.836000442504883, -13.8100004196167]], [[28.429000854492188, -11.675999641418457, -14.680000305175781], [29.649999618530273, -10.847999572753906, -14.354000091552734], [29.270000457763672, -9.579000473022461, -13.565999984741211]], [[28.128999710083008, -9.003000259399414, -13.871000289916992], [27.635000228881836, -7.77400016784668, -13.152999877929688], [27.292999267578125, -8.090999603271484, -11.67199993133545]], [[27.525999069213867, -9.309000015258789, -11.222999572753906], [27.20599937438965, -9.699999809265137, -9.819000244140625], [28.343000411987305, -9.239999771118164, -8.894000053405762]], [[28.46500015258789, -7.947000026702881, -8.746999740600586], [29.523000717163086, -7.3480000495910645, -7.869999885559082], [29.312000274658203, -7.818999767303467, -6.416999816894531]], [[29.783000946044922, -9.003999710083008, -6.089000225067139], [29.60700035095215, -9.543999671936035, -4.703999996185303], [30.702999114990234, -8.991999626159668, -3.7730000019073486]], [[30.86199951171875, -7.685999870300293, -3.7290000915527344], [31.891000747680664, -7.076000213623047, -2.8289999961853027], [31.46500015258789, -7.314000129699707, -1.3730000257492065]], [[30.179000854492188, -7.242000102996826, -1.1100000143051147], [29.659000396728516, -7.46999979019165, 0.2639999985694885], [28.288999557495117, -8.15999984741211, 0.16300000250339508]], [[28.267000198364258, -9.468000411987305, 0.27399998903274536], [26.976999282836914, -10.222999572753906, 0.17499999701976776], [27.111000061035156, -11.585000038146973, 0.8690000176429749]], [[26.065000534057617, -12.020999908447266, 1.531000018119812], [26.100000381469727, -13.335000038146973, 2.249000072479248], [24.667999267578125, -13.833999633789062, 2.492000102996826]], [[24.48900032043457, -15.128000259399414, 2.6410000324249268], [23.115999221801758, -15.682000160217285, 2.884000062942505], [23.100000381469727, -16.535999298095703, 4.163000106811523]], [[21.933000564575195, -16.732999801635742, 4.736000061035156], [21.81800079345703, -17.55299949645996, 5.980999946594238], [20.391000747680664, -18.111000061035156, 6.0970001220703125]], [[20.211999893188477, -19.35700035095215, 5.736999988555908], [18.86199951171875, -19.996999740600586, 5.811999797821045], [18.503000259399414, -20.26300048828125, 7.281000137329102]], [[17.277000427246094, -19.97800064086914, 7.644000053405762], [16.81399917602539, -20.183000564575195, 9.048999786376953], [15.498000144958496, -20.986000061035156, 9.03499984741211]], [[15.071999549865723, -21.481000900268555, 10.180999755859375], [13.798999786376953, -22.277000427246094, 10.267999649047852], [12.619999885559082, -21.479999542236328, 9.675000190734863]], [[12.364999771118164, -21.663000106811523, 8.39799976348877], [11.244999885559082, -20.94300079345703, 7.703999996185303], [11.52299976348877, -19.427000045776367, 7.669000148773193]], [[12.779000282287598, -19.04400062561035, 7.552000045776367], [13.147000312805176, -17.591999053955078, 7.502999782562256], [14.473999977111816, -17.437999725341797, 6.739999771118164]], [[14.60200023651123, -16.408000946044922, 5.938000202178955], [15.868000030517578, -16.198999404907227, 5.165999889373779], [16.548999786376953, -14.907999992370605, 5.635000228881836]], [[17.82900047302246, -14.972999572753906, 5.928999900817871], [18.56399917602539, -13.746000289916992, 6.38100004196167], [19.886999130249023, -13.62399959564209, 5.61299991607666]], [[20.44499969482422, -12.437000274658203, 5.559999942779541], [21.731000900268555, -12.229999542236328, 4.821000099182129], [22.722999572753906, -11.484999656677246, 5.7179999351501465]], [[23.707000732421875, -12.182999610900879, 6.230000019073486], [24.715999603271484, -11.527000427246094, 7.11899995803833], [25.799999237060547, -10.866000175476074, 6.254000186920166]], [[25.641000747680664, -9.595999717712402, 5.956999778747559], [26.648000717163086, -8.87399959564209, 5.111999988555908], [27.472000122070312, -7.914999961853027, 5.991000175476074]], [[28.725000381469727, -8.265000343322754, 6.199999809265137], [29.652999877929688, -7.464000225067139, 7.0289998054504395], [30.131999969482422, -6.216000080108643, 6.27400016784668]], [[30.55699920654297, -5.201000213623047, 6.994999885559082], [31.03700065612793, -3.947999954223633, 6.336999893188477], [32.53200149536133, -3.759999990463257, 6.624000072479248]], [[33.30699920654297, -3.4489998817443848, 5.609000205993652], [34.775001525878906, -3.246999979019165, 5.800000190734863], [35.02299880981445, -1.944000005722046, 6.589000225067139]], [[36.270999908447266, -1.7009999752044678, 6.935999870300293], [36.66299819946289, -0.5070000290870667, 7.716000080108643], [36.60900115966797, 0.7789999842643738, 6.880000114440918]], [[37.540000915527344, 0.9670000076293945, 5.97599983215332], [37.547000885009766, 2.2100000381469727, 5.1529998779296875], [37.209999084472656, 1.878999948501587, 3.697000026702881]], [[36.047000885009766, 1.3250000476837158, 3.4779999256134033], [35.61000061035156, 0.9990000128746033, 2.0929999351501465], [34.42399978637695, 1.909999966621399, 1.7510000467300415]], [[34.422000885009766, 3.11299991607666, 2.302000045776367], [33.30500030517578, 4.078999996185303, 2.065000057220459], [31.979000091552734, 3.4639999866485596, 2.5399999618530273]], [[32.04399871826172, 2.4709999561309814, 3.4179999828338623], [30.798999786376953, 1.8079999685287476, 3.933000087738037], [29.86400032043457, 1.5149999856948853, 2.747999906539917]], [[30.368999481201172, 0.8220000267028809, 1.75], [29.545000076293945, 0.5210000276565552, 0.5410000085830688], [29.030000686645508, -0.921999990940094, 0.5699999928474426]], [[27.83099937438965, -1.1100000143051147, 0.07999999821186066], [27.211999893188477, -2.4670000076293945, 0.03099999949336052], [26.121999740600586, -2.4679999351501465, -1.055999994277954]], [[26.524999618530273, -2.561000108718872, -2.305000066757202], [25.538000106811523, -2.553999900817871, -3.433000087738037], [24.91200065612793, -3.947999954223633, -3.5850000381469727]], [[23.65399932861328, -4.000999927520752, -3.9590001106262207], [22.961999893188477, -5.317999839782715, -4.125], [22.65999984741211, -5.574999809265137, -5.604000091552734]], [[23.367000579833984, -6.505000114440918, -6.185999870300293], [23.143999099731445, -6.868000030517578, -7.620999813079834], [23.594999313354492, -8.326000213623047, -7.835000038146973]], [[23.517000198364258, -9.142000198364258, -6.801000118255615], [23.95800018310547, -10.574000358581543, -6.914999961853027], [22.926000595092773, -11.515000343322754, -6.2729997634887695]], [[21.708999633789062, -11.060999870300293, -6.0960001945495605], [20.656999588012695, -11.927000045776367, -5.489999771118164], [19.448999404907227, -11.96500015258789, -6.427000045776367]], [[18.97599983215332, -13.13700008392334, -6.765999794006348], [17.79800033569336, -13.229999542236328, -7.679999828338623], [16.527999877929688, -13.430000305175781, -6.855000019073486]], [[15.444999694824219, -13.793000221252441, -7.492000102996826], [14.17300033569336, -13.982999801635742, -6.748000144958496], [13.428999900817871, -15.210000038146973, -7.294000148773193]], [[12.993000030517578, -16.076000213623047, -6.413000106811523], [12.253000259399414, -17.301000595092773, -6.839000225067139], [10.805000305175781, -17.17300033569336, -6.3470001220703125]], [[9.845000267028809, -17.576000213623047, -7.140999794006348], [8.418999671936035, -17.457000732421875, -6.704999923706055], [8.045000076293945, -18.69300079345703, -5.895999908447266]], [[7.308000087738037, -18.5, -4.833000183105469], [6.913000106811523, -19.650999069213867, -3.9730000495910645], [5.5970001220703125, -20.219999313354492, -4.482999801635742]], [[5.651000022888184, -21.32699966430664, -5.185999870300293], [4.395999908447266, -21.954999923706055, -5.7129998207092285], [3.3440001010894775, -22.017000198364258, -4.5879998207092285]], [[2.3940000534057617, -21.101999282836914, -4.605999946594238], [1.3259999752044678, -21.048999786376953, -3.5490000247955322], [0.6769999861717224, -22.424999237060547, -3.3550000190734863]], [[0.3100000023841858, -23.094999313354492, -4.429999828338623], [-0.328000009059906, -24.448999404907227, -4.303999900817871], [0.5619999766349792, -25.361000061035156, -3.441999912261963]], [[1.8609999418258667, -25.347000122070312, -3.671999931335449], [2.7920000553131104, -26.19499969482422, -2.8559999465942383], [2.7829999923706055, -25.714000701904297, -1.3949999809265137]], [[2.6070001125335693, -24.427000045776367, -1.1770000457763672], [2.5739998817443848, -23.893999099731445, 0.22300000488758087], [1.156000018119812, -24.048999786376953, 0.8050000071525574]], [[0.2029999941587448, -24.47800064086914, 0.0010000000474974513], [-1.1920000314712524, -24.674999237060547, 0.5009999871253967], [-1.25, -26.017000198364258, 1.2350000143051147]], [[-0.628000020980835, -27.027999877929688, 0.6669999957084656], [-0.5960000157356262, -28.374000549316406, 1.3109999895095825], [0.36800000071525574, -28.35700035095215, 2.51200008392334]], [[1.1749999523162842, -27.31599998474121, 2.6470000743865967], [2.1480000019073486, -27.23200035095215, 3.7890000343322754], [1.4240000247955322, -27.43000030517578, 5.126999855041504]], [[0.24799999594688416, -26.854999542236328, 5.267000198364258], [-0.5799999833106995, -26.98699951171875, 6.52400016784668], [0.11900000274181366, -26.393999099731445, 7.763999938964844]], [[-0.675000011920929, -25.865999221801758, 8.677000045776367], [-0.13600000739097595, -25.270000457763672, 9.942000389099121], [0.5440000295639038, -23.93000030517578, 9.65999984741211]], [[1.649999976158142, -23.96500015258789, 8.961000442504883], [2.4019999504089355, -22.722999572753906, 8.637999534606934], [1.4839999675750732, -21.700000762939453, 7.961999893188477]], [[1.8489999771118164, -20.44700050354004, 8.029000282287598], [1.0399999618530273, -19.381000518798828, 7.375999927520752], [1.9390000104904175, -18.650999069213867, 6.36899995803833]], [[2.809999942779541, -19.395000457763672, 5.718999862670898], [3.7249999046325684, -18.801000595092773, 4.697000026702881], [2.888000011444092, -18.42099952697754, 3.4679999351501465]], [[3.5199999809265137, -18.016000747680664, 2.3910000324249268], [2.7639999389648438, -17.631999969482422, 1.1490000486373901], [1.7929999828338623, -18.77199935913086, 0.7580000162124634]], [[0.6039999723434448, -18.773000717163086, 1.3270000219345093], [-0.3970000147819519, -19.850000381469727, 1.0369999408721924], [-0.9399999976158142, -19.69300079345703, -0.3779999911785126]], [[-2.0869998931884766, -19.066999435424805, -0.5270000100135803], [-2.6760001182556152, -18.87299919128418, -1.8949999809265137], [-3.181999921798706, -17.430999755859375, -2.0840001106262207]], [[-3.2790000438690186, -16.652000427246094, -1.0290000438690186], [-3.7720000743865967, -15.243000030517578, -1.1640000343322754], [-2.7049999237060547, -14.24899959564209, -0.6639999747276306]], [[-1.4509999752044678, -14.652000427246094, -0.6140000224113464], [-0.3840000033378601, -13.720999717712402, -0.125], [0.8769999742507935, -13.838000297546387, -0.9980000257492065]], [[0.8270000219345093, -13.319999694824219, -2.2049999237060547], [2.0179998874664307, -13.371999740600586, -3.1080000400543213], [2.8459999561309814, -12.086999893188477, -2.9200000762939453]], [[3.2070000171661377, -11.770999908447266, -1.6920000314712524], [4.007999897003174, -10.53600025177002, -1.430999994277954], [4.849999904632568, -10.711999893188477, -0.15600000321865082]], [[6.133999824523926, -10.435999870300293, -0.23800000548362732], [7.025000095367432, -10.569000244140625, 0.9670000076293945], [7.622000217437744, -9.192999839782715, 1.312000036239624]], [[8.185999870300293, -9.034000396728516, 2.494999885559082], [8.765000343322754, -7.697000026702881, 2.878999948501587], [10.222999572753906, -7.8379998207092285, 3.3469998836517334]], [[11.026000022888184, -6.815999984741211, 3.122999906539917], [12.46500015258789, -6.86299991607666, 3.5409998893737793], [12.6899995803833, -6.006999969482422, 4.794000148773193]], [[13.496000289916992, -6.492000102996826, 5.714000225067139], [13.791000366210938, -5.716000080108643, 6.960999965667725], [15.3100004196167, -5.742000102996826, 7.2179999351501465]], [[15.987000465393066, -4.644999980926514, 6.961999893188477], [17.472000122070312, -4.585999965667725, 7.177000045776367], [17.761999130249023, -4.25, 8.645000457763672]], [[18.700000762939453, -4.936999797821045, 9.265000343322754], [19.018999099731445, -4.63700008392334, 10.699000358581543], [20.5310001373291, -4.715000152587891, 10.942999839782715]], [[21.091999053955078, -3.677999973297119, 11.519000053405762], [22.559999465942383, -3.6619999408721924, 11.819000244140625], [22.759000778198242, -3.6410000324249268, 13.33899974822998]], [[23.902999877929688, -4.090000152587891, 13.805000305175781], [24.172000885009766, -4.104000091552734, 15.269000053405762], [25.652000427246094, -3.755000114440918, 15.517000198364258]], [[25.93000030517578, -2.5169999599456787, 15.85200023651123], [27.341999053955078, -2.0999999046325684, 16.128000259399414], [27.422000885009766, -1.5829999446868896, 17.573999404907227]], [[28.08799934387207, -2.3459999561309814, 18.416000366210938], [28.25, -2.007999897003174, 19.8439998626709], [29.291000366210938, -0.8949999809265137, 20.027999877929688]], [[28.90399932861328, 0.1809999942779541, 20.672000885009766], [29.843000411987305, 1.315000057220459, 20.905000686645508], [30.68600082397461, 1.024999976158142, 22.15399932861328]], [[31.989999771118164, 1.1490000486373901, 22.016000747680664], [32.930999755859375, 0.9110000133514404, 23.128000259399414], [32.89899826049805, 2.0989999771118164, 24.106000900268555]], [[31.816999435424805, 2.2239999771118164, 24.84000015258789], [31.67099952697754, 3.3420000076293945, 25.82200050354004], [30.479999542236328, 3.0420000553131104, 26.742000579833984]], [[29.323999404907227, 2.7720000743865967, 26.17300033569336], [28.121999740600586, 2.4600000381469727, 27.009000778198242], [26.840999603271484, 2.7160000801086426, 26.209999084472656]], [[26.711000442504883, 2.0850000381469727, 25.065000534057617], [25.490999221801758, 2.2639999389648438, 24.211999893188477], [25.639999389648438, 1.4279999732971191, 22.930999755859375]], [[24.604000091552734, 0.7200000286102295, 22.548999786376953], [24.666000366210938, -0.12700000405311584, 21.31399917602539], [23.968000411987305, 0.5989999771118164, 20.156999588012695]], [[24.538999557495117, 0.5320000052452087, 18.97800064086914], [23.929000854492188, 1.2089999914169312, 17.792999267578125], [23.25200080871582, 0.15299999713897705, 16.902999877929688]], [[21.969999313354492, -0.052000001072883606, 17.086999893188477], [21.235000610351562, -1.062000036239624, 16.26300048828125], [20.27400016784668, -0.33799999952316284, 15.312000274658203]], [[20.16900062561035, -0.8029999732971191, 14.086000442504883], [19.259000778198242, -0.1340000033378601, 13.098999977111816], [18.200000762939453, -1.1200000047683716, 12.583000183105469]], [[17.14299964904785, -0.6000000238418579, 11.994999885559082], [16.054000854492188, -1.4759999513626099, 11.454000473022461], [15.340999603271484, -0.7599999904632568, 10.293000221252441]], [[15.692000389099121, -1.093999981880188, 9.072999954223633], [15.041000366210938, -0.4519999921321869, 7.88700008392334], [14.041999816894531, -1.437999963760376, 7.264999866485596]], [[13.154000282287598, -0.9490000009536743, 6.434999942779541], [12.149999618530273, -1.840999960899353, 5.7820000648498535], [12.112000465393066, -1.5390000343322754, 4.28000020980835]], [[12.498000144958496, -2.490999937057495, 3.4590001106262207], [12.487000465393066, -2.265000104904175, 1.9809999465942383], [11.074000358581543, -2.5390000343322754, 1.437999963760376]], [[10.802000045776367, -2.11299991607666, 0.22499999403953552], [9.454999923706055, -2.3389999866485596, -0.375], [9.177000045776367, -3.8489999771118164, -0.4620000123977661]], [[7.908999919891357, -4.195000171661377, -0.4399999976158142], [7.465000152587891, -5.6020002365112305, -0.4950000047683716], [7.613999843597412, -6.179999828338623, -1.9110000133514404]], [[8.194000244140625, -7.355000019073486, -2.0139999389648438], [8.366999626159668, -8.008000373840332, -3.3489999771118164], [7.025000095367432, -8.61299991607666, -3.7739999294281006]], [[6.572000026702881, -8.303000450134277, -4.9670000076293945], [5.26800012588501, -8.854000091552734, -5.448999881744385], [5.434999942779541, -9.425999641418457, -6.861999988555908]], [[5.039999961853027, -10.663999557495117, -7.057000160217285], [5.163000106811523, -11.305000305175781, -8.404000282287598], [3.7980000972747803, -11.307999610900879, -9.109000205993652]], [[2.7179999351501465, -11.416999816894531, -8.361000061035156], [1.3530000448226929, -11.425000190734863, -8.982999801635742], [1.1069999933242798, -10.095999717712402, -9.71500015258789]], [[0.621999979019165, -10.161999702453613, -10.9350004196167], [0.3610000014305115, -8.914999961853027, -11.72599983215332], [-1.1069999933242798, -8.479999542236328, -11.565999984741211]], [[-2.0, -9.416999816894531, -11.333000183105469], [-3.444999933242798, -9.074999809265137, -11.170000076293945], [-3.688999891281128, -8.548999786376953, -9.74899959564209]], [[-4.409999847412109, -7.456999778747559, -9.628000259399414], [-4.695000171661377, -6.874000072479248, -8.276000022888184], [-5.585000038146973, -7.835000038146973, -7.4679999351501465]]], "seq": "MKDWQLPSHSGPYELRIEVQPKSHHRARYETEGSRGAVKASAGGHPIVQLHGYLENEPLMLQLFIGTADDRLLRPHAFYQVHRITGKTVSTTSHEAILSNTKVLEIPLLPENSMRAVIDCAGILKLRNSDIELRKGETDIGRKNTRVRLVFRVHVPQPSGRTLSLQVASNPIECSQRS"}, {"coords": [[[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[-13.70300006866455, 52.731998443603516, 32.26499938964844], [-12.859000205993652, 51.499000549316406, 32.23400115966797], [-12.218000411987305, 51.27399826049805, 30.860000610351562]], [[-10.906000137329102, 51.49100112915039, 30.801000595092773], [-10.133000373840332, 51.33300018310547, 29.570999145507812], [-9.531000137329102, 49.9370002746582, 29.42300033569336]], [[-9.041000366210938, 49.64099884033203, 28.22599983215332], [-8.42300033569336, 48.35100173950195, 27.961000442504883], [-7.03000020980835, 48.487998962402344, 27.371999740600586]], [[-6.205999851226807, 47.47700119018555, 27.604999542236328], [-4.8420000076293945, 47.516998291015625, 27.12299919128418], [-4.34499979019165, 46.20800018310547, 26.51799964904785]], [[-3.364000082015991, 46.35900115966797, 25.628999710083008], [-2.6740000247955322, 45.25400161743164, 24.975000381469727], [-1.2330000400543213, 45.737998962402344, 24.867000579833984]], [[-0.29600000381469727, 44.930999755859375, 25.349000930786133], [1.1030000448226929, 45.292999267578125, 25.268999099731445], [1.9730000495910645, 44.06100082397461, 25.218000411987305]], [[3.0480000972747803, 44.15800094604492, 24.445999145507812], [3.9730000495910645, 43.05400085449219, 24.31100082397461], [5.236999988555908, 43.42399978637695, 25.051000595092773]], [[5.953000068664551, 42.41400146484375, 25.520999908447266], [7.195000171661377, 42.619998931884766, 26.243999481201172], [8.23799991607666, 41.75699996948242, 25.541000366210938]], [[9.38700008392334, 42.33300018310547, 25.208999633789062], [10.434000015258789, 41.55500030517578, 24.555999755859375], [11.670000076293945, 41.64799880981445, 25.420000076293945]], [[12.444999694824219, 40.569000244140625, 25.48200035095215], [13.67300033569336, 40.5620002746582, 26.266000747680664], [14.638999938964844, 39.58300018310547, 25.642000198364258]], [[15.918000221252441, 39.94599914550781, 25.641000747680664], [16.944000244140625, 39.10100173950195, 25.072999954223633], [18.290000915527344, 39.61899948120117, 25.53700065612793]], [[19.33099937438965, 38.79999923706055, 25.382999420166016], [20.67300033569336, 39.17100143432617, 25.80500030517578], [21.715999603271484, 38.7869987487793, 24.784000396728516]], [[22.82699966430664, 39.500999450683594, 24.809999465942383], [23.915000915527344, 39.231998443603516, 23.895000457763672], [24.858999252319336, 38.31800079345703, 24.641000747680664]], [[25.884000778198242, 37.84600067138672, 23.945999145507812], [26.86199951171875, 36.96900177001953, 24.56399917602539], [28.275999069213867, 37.54600143432617, 24.447999954223633]], [[29.059999465942383, 37.37099838256836, 25.51300048828125], [30.441999435424805, 37.847999572753906, 25.56399917602539], [31.253999710083008, 37.249000549316406, 24.406999588012695]], [[31.339000701904297, 38.000999450683594, 23.308000564575195], [32.06100082397461, 37.58300018310547, 22.10300064086914], [31.451000213623047, 36.323001861572266, 21.492000579833984]], [[30.530000686645508, 35.70100021362305, 22.22100067138672], [29.84600067138672, 34.50600051879883, 21.73900032043457], [28.7189998626709, 35.01100158691406, 20.841999053955078]], [[28.788999557495117, 34.654998779296875, 19.56399917602539], [27.77400016784668, 35.0890007019043, 18.621999740600586], [26.358999252319336, 34.80400085449219, 19.094999313354492]], [[25.389999389648438, 35.35300064086914, 18.365999221801758], [23.976999282836914, 35.17399978637695, 18.677000045776367], [23.58799934387207, 35.887001037597656, 19.976999282836914]], [[22.4689998626709, 35.446998596191406, 20.541000366210938], [21.94300079345703, 36.03900146484375, 21.75], [20.93400001525879, 35.10100173950195, 22.43600082397461]], [[19.96299934387207, 35.715999603271484, 23.11400032043457], [18.868999481201172, 35.04100036621094, 23.825000762939453], [17.599000930786133, 35.77799987792969, 23.341999053955078]], [[16.429000854492188, 35.314998626708984, 23.767000198364258], [15.196000099182129, 35.97600173950195, 23.368000030517578], [13.913999557495117, 35.321998596191406, 23.90399932861328]], [[12.869000434875488, 36.14099884033203, 24.025999069213867], [11.579999923706055, 35.698001861572266, 24.538999557495117], [10.611000061035156, 36.874000549316406, 24.486000061035156]], [[9.366000175476074, 36.606998443603516, 24.115999221801758], [8.354000091552734, 37.65599822998047, 24.052000045776367], [7.111000061035156, 37.22100067138672, 24.82900047302246]], [[6.318999767303467, 38.20000076293945, 25.273000717163086], [5.086999893188477, 37.95600128173828, 26.020000457763672], [4.002999782562256, 38.9630012512207, 25.618000030517578]], [[2.757999897003174, 38.500999450683594, 25.60300064086914], [1.6670000553131104, 39.382999420166016, 25.24799919128418], [0.6650000214576721, 39.439998626708984, 26.382999420166016]], [[0.27300000190734863, 40.650001525878906, 26.760000228881836], [-0.675000011920929, 40.8390007019043, 27.84600067138672], [-1.840999960899353, 41.67599868774414, 27.356000900268555]], [[-3.0450000762939453, 41.31700134277344, 27.768999099731445], [-4.209000110626221, 42.10499954223633, 27.415000915527344], [-4.860000133514404, 42.34299850463867, 28.763999938964844]], [[-5.065000057220459, 43.608001708984375, 29.108999252319336], [-5.64300012588501, 43.94900131225586, 30.406999588012695], [-4.824999809265137, 43.25, 31.490999221801758]], [[-3.5, 43.34600067138672, 31.3700008392334], [-2.559999942779541, 42.750999450683594, 32.31999969482422], [-2.7039999961853027, 41.257999420166016, 32.520999908447266]], [[-3.318000078201294, 40.5989990234375, 31.548999786376953], [-3.51200008392334, 39.1609992980957, 31.600000381469727], [-2.630000114440918, 38.551998138427734, 30.525999069213867]], [[-1.6610000133514404, 37.73400115966797, 30.933000564575195], [-0.7519999742507935, 37.1150016784668, 29.9689998626709], [-1.5089999437332153, 36.09700012207031, 29.128999710083008]], [[-1.5110000371932983, 36.30099868774414, 27.815000534057617], [-2.2090001106262207, 35.395999908447266, 26.927000045776367], [-1.2970000505447388, 34.68000030517578, 25.951000213623047]], [[-0.0989999994635582, 35.207000732421875, 25.71299934387207], [0.8190000057220459, 34.53499984741211, 24.788000106811523], [2.2960000038146973, 34.58000183105469, 25.183000564575195]], [[2.9830000400543213, 33.46900177001953, 24.930999755859375], [4.3979997634887695, 33.33100128173828, 25.238000869750977], [5.10099983215332, 32.95100021362305, 23.947999954223633]], [[6.364999771118164, 33.334999084472656, 23.812999725341797], [7.114999771118164, 33.02399826049805, 22.610000610351562], [8.5649995803833, 32.83399963378906, 22.976999282836914]], [[9.16100025177002, 31.780000686645508, 22.447999954223633], [10.550000190734863, 31.47100067138672, 22.72800064086914], [11.336000442504883, 31.56999969482422, 21.430999755859375]], [[12.654999732971191, 31.604000091552734, 21.538000106811523], [13.491999626159668, 31.643999099731445, 20.354000091552734], [14.151000022888184, 30.27199935913086, 20.32699966430664]], [[14.442999839782715, 29.75200080871582, 21.524999618530273], [15.0649995803833, 28.434999465942383, 21.70199966430664], [14.166999816894531, 27.35099983215332, 21.106000900268555]], [[12.862000465393066, 27.47100067138672, 21.336000442504883], [11.890000343322754, 26.517000198364258, 20.80500030517578], [11.23799991607666, 27.077999114990234, 19.538000106811523]], [[11.180999755859375, 28.400999069213867, 19.44099998474121], [10.597999572753906, 29.082000732421875, 18.288000106811523], [9.109000205993652, 28.797000885009766, 18.10700035095215]], [[8.472999572753906, 28.329999923706055, 19.17799949645996], [7.047999858856201, 28.003000259399414, 19.17300033569336], [6.214000225067139, 29.009000778198242, 19.975000381469727]], [[5.046000003814697, 29.350000381469727, 19.44700050354004], [4.1529998779296875, 30.295000076293945, 20.101999282836914], [3.127000093460083, 29.583999633789062, 20.979000091552734]], [[3.371999979019165, 29.533000946044922, 22.28499984741211], [2.444999933242798, 28.875999450683594, 23.204999923706055], [1.3489999771118164, 29.812999725341797, 23.731000900268555]], [[0.19900000095367432, 29.243000030517578, 24.135000228881836], [-0.9179999828338623, 30.02899932861328, 24.666000366210938], [-0.7850000262260437, 30.108999252319336, 26.18600082397461]], [[-1.4479999542236328, 31.08099937438965, 26.79599952697754], [-1.3530000448226929, 31.231000900268555, 28.23900032043457], [-2.7070000171661377, 31.35700035095215, 28.92799949645996]], [[-3.7709999084472656, 31.27899932861328, 28.136999130249023], [-5.131999969482422, 31.341999053955078, 28.659000396728516], [-6.015999794006348, 30.410999298095703, 27.832000732421875]], [[-7.132999897003174, 29.95400047302246, 28.413999557495117], [-8.050999641418457, 29.059999465942383, 27.700000762939453], [-8.722999572753906, 29.691999435424805, 26.47100067138672]], [[-9.15999984741211, 30.94499969482422, 26.597999572753906], [-9.847999572753906, 31.65399932861328, 25.51300048828125], [-9.053999900817871, 32.029998779296875, 24.253999710083008]], [[-7.71999979019165, 31.98699951171875, 24.392000198364258], [-6.8420000076293945, 32.31100082397461, 23.270999908447266], [-5.923999786376953, 31.111000061035156, 23.040000915527344]], [[-6.494999885559082, 30.045000076293945, 22.483999252319336], [-5.756999969482422, 28.81399917602539, 22.22100067138672], [-5.26800012588501, 28.684999465942383, 20.774999618530273]], [[-4.314000129699707, 27.775999069213867, 20.569000244140625], [-3.739000082015991, 27.52400016784668, 19.25], [-4.823999881744385, 27.107999801635742, 18.257999420166016]], [[-5.51800012588501, 26.01799964904785, 18.575000762939453], [-6.586999893188477, 25.503999710083008, 17.722000122070312], [-7.927000045776367, 26.216999053955078, 17.968000411987305]], [[-8.197999954223633, 26.586999893188477, 19.224000930786133], [-9.442000389099121, 27.2810001373291, 19.58300018310547], [-9.439000129699707, 28.684999465942383, 18.981000900268555]], [[-10.21500015258789, 28.947999954223633, 17.90399932861328], [-10.25, 30.2450008392334, 17.231000900268555], [-8.831999778747559, 30.55299949645996, 16.738000869750977]], [[-8.29699993133545, 29.635000228881836, 15.930000305175781], [-6.942999839782715, 29.732999801635742, 15.376999855041504], [-6.756999969482422, 30.743000030517578, 14.243000030517578]], [[-7.7729997634887695, 31.562000274658203, 13.982999801635742], [-7.672999858856201, 32.573001861572266, 12.930000305175781], [-6.640999794006348, 33.6150016784668, 13.36299991607666]], [[-6.764999866485596, 34.0620002746582, 14.61400032043457], [-5.867000102996826, 35.055999755859375, 15.196999549865723], [-4.495999813079834, 34.46799850463867, 15.526000022888184]], [[-4.474999904632568, 33.237998962402344, 16.033000946044922], [-3.2160000801086426, 32.59199905395508, 16.3799991607666], [-2.257999897003174, 32.52799987792969, 15.196000099182129]], [[-2.796999931335449, 32.7400016784668, 13.99899959564209], [-2.0, 32.71900177001953, 12.776000022888184], [-1.180999994277954, 34.01100158691406, 12.645999908447266]], [[-1.8609999418258667, 35.15599822998047, 12.5649995803833], [-1.1710000038146973, 36.4370002746582, 12.440999984741211], [-0.4129999876022339, 36.779998779296875, 13.72700023651123]], [[-0.9459999799728394, 36.358001708984375, 14.868000030517578], [-0.29100000858306885, 36.61800003051758, 16.141000747680664], [1.031999945640564, 35.86600112915039, 16.18400001525879]], [[1.0740000009536743, 34.70899963378906, 15.531999588012695], [2.2950000762939453, 33.922000885009766, 15.482999801635742], [3.2279999256134033, 34.55699920654297, 14.446000099182129]], [[2.6570000648498535, 35.30400085449219, 13.50100040435791], [3.453000068664551, 35.97999954223633, 12.472999572753906], [4.0, 37.2599983215332, 13.098999977111816]], [[3.0999999046325684, 38.000999450683594, 13.737000465393066], [3.4189999103546143, 39.25699996948242, 14.404000282287598], [4.669000148773193, 39.143001556396484, 15.293999671936035]], [[4.738999843597412, 38.07099914550781, 16.082000732421875], [5.869999885559082, 37.85300064086914, 16.97800064086914], [7.156000137329102, 37.4630012512207, 16.246000289916992]], [[7.059999942779541, 36.57899856567383, 15.26099967956543], [8.24899959564209, 36.18000030517578, 14.519000053405762], [8.822999954223633, 37.41899871826172, 13.84000015258789]], [[7.934999942779541, 38.323001861572266, 13.427000045776367], [8.371999740600586, 39.54899978637695, 12.777999877929688], [9.116000175476074, 40.43299865722656, 13.760000228881836]], [[8.522000312805176, 40.60900115966797, 14.932999610900879], [9.118000030517578, 41.42499923706055, 15.972000122070312], [10.435999870300293, 40.823001861572266, 16.400999069213867]], [[10.515000343322754, 39.499000549316406, 16.375999450683594], [11.74899959564209, 38.82600021362305, 16.7549991607666], [12.87399959564209, 39.34299850463867, 15.847999572753906]], [[12.63700008392334, 39.33000183105469, 14.532999992370605], [13.625, 39.80099868774414, 13.550000190734863], [14.008999824523926, 41.24800109863281, 13.79699993133545]], [[12.984999656677246, 42.0890007019043, 13.928999900817871], [13.163000106811523, 43.5099983215332, 14.184000015258789], [13.961999893188477, 43.744998931884766, 15.482999801635742]], [[13.58899974822998, 43.0260009765625, 16.52899932861328], [14.279000282287598, 43.12300109863281, 17.80900001525879], [15.765000343322754, 42.78200149536133, 17.584999084472656]], [[16.025999069213867, 41.69300079345703, 16.87700080871582], [17.406999588012695, 41.31399917602539, 16.59000015258789], [18.179000854492188, 42.4379997253418, 15.88599967956543]], [[17.572999954223633, 43.0099983215332, 14.843000411987305], [18.204999923706055, 44.0989990234375, 14.11299991607666], [18.454999923706055, 45.25899887084961, 15.057999610900879]], [[17.43199920654297, 45.60300064086914, 15.84000015258789], [17.533000946044922, 46.702999114990234, 16.792999267578125], [18.62700080871582, 46.44599914550781, 17.839000701904297]], [[18.658000946044922, 45.23099899291992, 18.368999481201172], [19.663000106811523, 44.86600112915039, 19.36199951171875], [21.073999404907227, 45.11899948120117, 18.805999755859375]], [[21.308000564575195, 44.65399932861328, 17.58300018310547], [22.611000061035156, 44.83000183105469, 16.974000930786133], [22.920000076293945, 46.29999923706055, 16.76099967956543]], [[21.92300033569336, 47.04499816894531, 16.29599952697754], [22.11199951171875, 48.4630012512207, 16.034000396728516], [22.520000457763672, 49.196998596191406, 17.30699920654297]], [[21.77400016784668, 48.95800018310547, 18.38599967956543], [22.041000366210938, 49.599998474121094, 19.672000885009766], [23.39699935913086, 49.154998779296875, 20.22100067138672]], [[23.66200065612793, 47.85599899291992, 20.16900062561035], [24.93400001525879, 47.367000579833984, 20.66900062561035], [26.10700035095215, 47.95100021362305, 19.89299964904785]], [[26.042999267578125, 47.90399932861328, 18.570999145507812], [27.136999130249023, 48.44300079345703, 17.77899932861328], [27.329999923706055, 49.948001861572266, 17.975000381469727]], [[26.27199935913086, 50.665000915527344, 18.360000610351562], [26.378000259399414, 52.106998443603516, 18.55299949645996], [26.9060001373291, 52.483001708984375, 19.93600082397461]], [[27.371000289916992, 51.48500061035156, 20.673999786376953], [27.957000732421875, 51.6870002746582, 21.999000549316406], [29.378000259399414, 51.159000396728516, 21.900999069213867]], [[29.700000762939453, 50.59299850463867, 20.742000579833984], [31.003999710083008, 49.99800109863281, 20.51099967956543], [31.09600067138672, 48.8650016784668, 21.531999588012695]], [[30.04199981689453, 48.04499816894531, 21.584999084472656], [29.94700050354004, 46.89699935913086, 22.5], [30.06100082397461, 45.59299850463867, 21.697999954223633]], [[30.981000900268555, 44.7140007019043, 22.093000411987305], [31.16699981689453, 43.4640007019043, 21.356000900268555], [31.525999069213867, 42.25299835205078, 22.211000442504883]], [[32.67900085449219, 42.334999084472656, 22.87299919128418], [33.17300033569336, 41.255001068115234, 23.729000091552734], [32.444000244140625, 41.30400085449219, 25.06399917602539]], [[31.523000717163086, 42.25600051879883, 25.1560001373291], [30.722999572753906, 42.46099853515625, 26.349000930786133], [29.386999130249023, 41.70800018310547, 26.243000030517578]], [[28.75, 41.45100021362305, 27.381000518798828], [27.45199966430664, 40.779998779296875, 27.391000747680664], [26.413000106811523, 41.84700012207031, 27.743000030517578]], [[25.378000259399414, 41.983001708984375, 26.927000045776367], [24.37299919128418, 43.00699996948242, 27.18899917602539], [22.959999084472656, 42.512001037597656, 27.033000946044922]], [[22.020999908447266, 43.314998626708984, 27.509000778198242], [20.621000289916992, 42.95899963378906, 27.39299964904785], [19.81599998474121, 44.071998596191406, 26.72599983215332]], [[18.740999221801758, 43.67900085449219, 26.06599998474121], [17.850000381469727, 44.645999908447266, 25.43899917602539], [16.427000045776367, 44.224998474121094, 25.784000396728516]], [[15.628000259399414, 45.178001403808594, 26.236000061035156], [14.23799991607666, 44.90299987792969, 26.555999755859375], [13.380000114440918, 45.90999984741211, 25.827999114990234]], [[12.234999656677246, 45.45899963378906, 25.336000442504883], [11.322999954223633, 46.347999572753906, 24.649999618530273], [9.937000274658203, 46.1609992980957, 25.231000900268555]], [[9.189000129699707, 47.25299835205078, 25.242000579833984], [7.827000141143799, 47.25, 25.729999542236328], [7.074999809265137, 48.09400177001953, 24.697999954223633]], [[5.9770002365112305, 47.551998138427734, 24.18600082397461], [5.165999889373779, 48.27899932861328, 23.208999633789062], [3.736999988555908, 47.792999267578125, 23.336000442504883]], [[2.802000045776367, 48.69599914550781, 23.073999404907227], [1.4079999923706055, 48.35499954223633, 23.18199920654297], [0.5770000219345093, 49.59400177001953, 23.006000518798828]], [[-0.7200000286102295, 49.45600128173828, 23.260000228881836], [-1.659000039100647, 50.55500030517578, 23.11199951171875], [-2.681999921798706, 50.50899887084961, 24.231000900268555]], [[-3.193000078201294, 51.68199920654297, 24.576000213623047], [-4.171999931335449, 51.82400131225586, 25.636999130249023], [-5.427000045776367, 52.479000091552734, 25.059999465942383]], [[-6.485000133514404, 51.68600082397461, 24.895000457763672], [-7.763999938964844, 52.1609992980957, 24.356000900268555], [-8.696000099182129, 52.64899826049805, 25.4689998626709]], [[-9.309000015258789, 53.814998626708984, 25.274999618530273], [-10.213000297546387, 54.3650016784668, 26.284000396728516], [-11.651000022888184, 53.92300033569336, 26.040000915527344]], [[-12.517000198364258, 54.191001892089844, 27.014999389648438], [-13.927000045776367, 53.819000244140625, 26.927000045776367], [-14.553000450134277, 54.349998474121094, 25.636999130249023]], [[-13.923999786376953, 55.38600158691406, 25.086999893188477], [-14.392000198364258, 56.05400085449219, 23.875], [-13.644000053405762, 55.70399856567383, 22.586000442504883]], [[-12.812999725341797, 54.667999267578125, 22.628000259399414], [-12.055999755859375, 54.24300003051758, 21.450000762939453], [-10.878000259399414, 55.14799880981445, 21.052000045776367]], [[-10.585000038146973, 56.16299819946289, 21.863000869750977], [-9.458000183105469, 57.05400085449219, 21.586000442504883], [-8.208000183105469, 56.321998596191406, 22.086000442504883]], [[-7.247000217437744, 56.084999084472656, 21.19700050354004], [-6.046000003814697, 55.34400177001953, 21.566999435424805], [-4.7729997634887695, 56.1150016784668, 21.88800048828125]], [[-3.984999895095825, 55.525001525878906, 22.78499984741211], [-2.700000047683716, 56.06100082397461, 23.21299934387207], [-1.6959999799728394, 54.909000396728516, 23.091999053955078]], [[-0.609000027179718, 55.12699890136719, 22.363000869750977], [0.367000013589859, 54.064998626708984, 22.20800018310547], [1.7630000114440918, 54.388999938964844, 22.698999404907227]], [[2.5910000801086426, 53.36000061035156, 22.83799934387207], [3.947999954223633, 53.577999114990234, 23.299999237060547], [4.848999977111816, 52.4739990234375, 22.792999267578125]], [[6.142000198364258, 52.78099822998047, 22.775999069213867], [7.171999931335449, 51.87699890136719, 22.304000854492188], [8.4399995803833, 52.35499954223633, 23.020000457763672]], [[8.932000160217285, 51.54600143432617, 23.951000213623047], [10.10099983215332, 51.904998779296875, 24.743000030517578], [11.196999549865723, 50.854000091552734, 24.67099952697754]], [[12.444999694824219, 51.321998596191406, 24.634000778198242], [13.602999687194824, 50.4379997253418, 24.589000701904297], [14.428000450134277, 50.57699966430664, 25.868999481201172]], [[14.763999938964844, 49.43199920654297, 26.472999572753906], [15.586000442504883, 49.39500045776367, 27.673999786376953], [16.885000228881836, 48.65800094604492, 27.368999481201172]], [[18.017000198364258, 49.314998626708984, 27.607999801635742], [19.312000274658203, 48.7130012512207, 27.329999923706055], [20.084999084472656, 48.4739990234375, 28.621999740600586]], [[20.444000244140625, 47.224998474121094, 28.86400032043457], [21.16200065612793, 46.91899871826172, 30.086000442504883], [20.334999084472656, 47.42300033569336, 31.274999618530273]], [[19.03700065612793, 47.11800003051758, 31.243999481201172], [18.141000747680664, 47.494998931884766, 32.32500076293945], [17.84600067138672, 48.95800018310547, 32.5880012512207]], [[18.233999252319336, 49.827999114990234, 31.663000106811523], [17.974000930786133, 51.25699996948242, 31.825000762939453], [17.229000091552734, 51.82699966430664, 30.632999420166016]], [[16.42799949645996, 52.84700012207031, 30.893999099731445], [15.701000213623047, 53.50699996948242, 29.82200050354004], [16.767000198364258, 53.87699890136719, 28.774999618530273]], [[16.424999237060547, 53.77399826049805, 27.493999481201172], [17.37700080871582, 54.08700180053711, 26.44099998474121], [16.75, 54.926998138427734, 25.340999603271484]], [[15.758000373840332, 54.362998962402344, 24.674999237060547], [15.086999893188477, 55.08599853515625, 23.618999481201172], [13.598999977111816, 54.941001892089844, 23.833999633789062]], [[12.864999771118164, 56.005001068115234, 23.55900001525879], [11.425000190734863, 55.96099853515625, 23.711999893188477], [10.758000373840332, 56.63600158691406, 22.545000076293945]], [[9.85099983215332, 55.900001525878906, 21.916000366210938], [9.128000259399414, 56.41699981689453, 20.777999877929688], [8.128999710083008, 57.46900177001953, 21.21299934387207]], [[8.119999885559082, 58.57699966430664, 20.488000869750977], [7.203000068664551, 59.680999755859375, 20.743999481201172], [6.4079999923706055, 59.78300094604492, 19.44099998474121]], [[5.261000156402588, 59.095001220703125, 19.3700008392334], [4.390999794006348, 59.07899856567383, 18.184999465942383], [3.936000108718872, 60.45800018310547, 17.673999786376953]], [[3.5429999828338623, 61.31999969482422, 18.608999252319336], [3.052999973297119, 62.65299987792969, 18.285999298095703], [4.02400016784668, 63.45100021362305, 17.43400001525879]], [[5.304999828338623, 63.36600112915039, 17.766000747680664], [6.329999923706055, 64.11100006103516, 17.048999786376953], [7.091000080108643, 63.24399948120117, 16.034000396728516]], [[6.627999782562256, 62.00600051879883, 15.85200023651123], [7.260000228881836, 61.073001861572266, 14.92199993133545], [8.777999877929688, 61.04199981689453, 15.10099983215332]], [[9.232999801635742, 60.762001037597656, 16.315000534057617], [10.65999984741211, 60.66699981689453, 16.554000854492188], [10.923999786376953, 59.882999420166016, 17.832000732421875]], [[12.168000221252441, 59.869998931884766, 18.297000885009766], [12.491000175476074, 59.130001068115234, 19.509000778198242], [13.220000267028809, 59.99399948120117, 20.500999450683594]], [[12.96399974822998, 59.744998931884766, 21.77899932861328], [13.640999794006348, 60.4640007019043, 22.84600067138672], [14.805000305175781, 59.59199905395508, 23.29800033569336]], [[16.003000259399414, 60.15399932861328, 23.29599952697754], [17.16699981689453, 59.41699981689453, 23.750999450683594], [17.243999481201172, 59.715999603271484, 25.243000030517578]], [[17.121000289916992, 58.68600082397461, 26.062999725341797], [17.152000427246094, 58.86800003051758, 27.50200080871582], [18.551000595092773, 59.15299987792969, 28.045000076293945]], [[19.569000244140625, 58.874000549316406, 27.243999481201172], [20.948999404907227, 59.07099914550781, 27.660999298095703], [21.829999923706055, 59.301998138427734, 26.424999237060547]], [[22.899999618530273, 60.0880012512207, 26.57200050354004], [23.80699920654297, 60.38100051879883, 25.45400047302246], [24.07900047302246, 59.231998443603516, 24.48200035095215]], [[24.54400062561035, 58.10599899291992, 25.0049991607666], [24.878999710083008, 56.965999603271484, 24.158000946044922], [23.73900032043457, 56.40800094604492, 23.290000915527344]], [[22.531999588012695, 56.952999114990234, 23.44099998474121], [21.368000030517578, 56.52299880981445, 22.652999877929688], [21.121000289916992, 57.41899871826172, 21.43000030517578]], [[21.753000259399414, 58.5880012512207, 21.409000396728516], [21.593000411987305, 59.5260009765625, 20.30500030517578], [22.01799964904785, 58.97999954223633, 18.93400001525879]], [[23.14699935913086, 58.25400161743164, 18.858999252319336], [23.57200050354004, 57.720001220703125, 17.552000045776367], [22.479999542236328, 56.88600158691406, 16.913000106811523]], [[21.801000595092773, 56.097999572753906, 17.74799919128418], [20.7189998626709, 55.23699951171875, 17.29599952697754], [19.506999969482422, 56.077999114990234, 16.94700050354004]], [[19.25200080871582, 57.0880012512207, 17.777999877929688], [18.125, 57.981998443603516, 17.55299949645996], [18.292999267578125, 58.630001068115234, 16.18000030517578]], [[19.492000579833984, 59.130001068115234, 15.916000366210938], [19.746999740600586, 59.75899887084961, 14.633999824523926], [19.59600067138672, 58.768001556396484, 13.475000381469727]], [[19.97100067138672, 57.516998291015625, 13.689000129699707], [19.8439998626709, 56.53799819946289, 12.628000259399414], [18.3799991607666, 56.34400177001953, 12.272000312805176]], [[17.562999725341797, 56.06800079345703, 13.282999992370605], [16.13800048828125, 55.821998596191406, 13.079000473022461], [15.401000022888184, 57.02000045776367, 12.559000015258789]], [[15.97700023651123, 58.189998626708984, 12.772000312805176], [15.329000473022461, 59.4119987487793, 12.33899974822998], [15.732000350952148, 59.87900161743164, 10.9399995803833]], [[16.698999404907227, 59.2140007019043, 10.310999870300293], [17.128999710083008, 59.619998931884766, 8.970999717712402], [16.0, 59.60599899291992, 7.940000057220459]], [[15.071000099182129, 58.665000915527344, 8.071999549865723], [13.928999900817871, 58.56700134277344, 7.163000106811523], [12.680000305175781, 58.40299987792969, 8.017999649047852]], [[11.812999725341797, 59.40599822998047, 7.9730000495910645], [10.602999687194824, 59.404998779296875, 8.777000427246094], [9.708999633789062, 58.189998626708984, 8.543000221252441]], [[9.930999755859375, 57.48500061035156, 7.441999912261963], [9.107999801635742, 56.31800079345703, 7.160999774932861], [9.055999755859375, 55.33000183105469, 8.34000015258789]], [[10.1899995803833, 55.13100051879883, 9.017000198364258], [10.194999694824219, 54.2140007019043, 10.135000228881836], [9.23799991607666, 54.67300033569336, 11.220999717712402]], [[9.345999717712402, 55.9379997253418, 11.604000091552734], [8.463000297546387, 56.448001861572266, 12.63599967956543], [7.014999866485596, 56.31999969482422, 12.161999702453613]], [[6.758999824523926, 56.66400146484375, 10.906999588012695], [5.4079999923706055, 56.58000183105469, 10.357000350952148], [4.809999942779541, 55.176998138427734, 10.463000297546387]], [[5.584000110626221, 54.15700149536133, 10.097999572753906], [5.083000183105469, 52.78799819946289, 10.182999610900879], [4.8480000495910645, 52.42499923706055, 11.640999794006348]], [[5.817999839782715, 52.766998291015625, 12.48799991607666], [5.743000030517578, 52.48500061035156, 13.918999671936035], [4.478000164031982, 53.06999969482422, 14.519000053405762]], [[4.229000091552734, 54.349998474121094, 14.256999969482422], [3.0409998893737793, 55.0099983215332, 14.784000396728516], [1.7899999618530273, 54.358001708984375, 14.232000350952148]], [[1.8480000495910645, 53.97999954223633, 12.961999893188477], [0.7120000123977661, 53.3390007019043, 12.32800006866455], [0.3869999945163727, 52.04199981689453, 13.064000129699707]], [[1.4079999923706055, 51.30699920654297, 13.484000205993652], [1.128999948501587, 50.084999084472656, 14.217000007629395], [0.41499999165534973, 50.42900085449219, 15.531999588012695]], [[0.9900000095367432, 51.361000061035156, 16.283000946044922], [0.4350000023841858, 51.74399948120117, 17.57200050354004], [-0.9739999771118164, 52.31999969482422, 17.558000564575195]], [[-1.3200000524520874, 53.058998107910156, 16.51300048828125], [-2.631999969482422, 53.689998626708984, 16.46299934387207], [-3.697999954223633, 52.96200180053711, 15.680000305175781]], [[-3.2769999504089355, 52.0260009765625, 14.843999862670898], [-4.2179999351501465, 51.28200149536133, 14.02299976348877], [-4.159999847412109, 49.784000396728516, 14.28600025177002]], [[-3.0920000076293945, 49.1510009765625, 13.822999954223633], [-2.921999931335449, 47.715999603271484, 13.991000175476074], [-3.135999917984009, 47.215999603271484, 15.425999641418457]], [[-2.433000087738037, 47.81399917602539, 16.38599967956543], [-2.546999931335449, 47.404998779296875, 17.784000396728516], [-3.9790000915527344, 47.457000732421875, 18.327999114990234]], [[-4.650000095367432, 48.61399841308594, 18.233999252319336], [-6.019999980926514, 48.685001373291016, 18.756000518798828], [-6.964000225067139, 47.66600036621094, 18.125]], [[-6.7789998054504395, 47.42599868774414, 16.833999633789062], [-7.61899995803833, 46.4739990234375, 16.139999389648438], [-7.374000072479248, 45.08700180053711, 16.708999633789062]], [[-6.124000072479248, 44.784000396728516, 17.047000885009766], [-5.815999984741211, 43.47800064086914, 17.610000610351562], [-6.355999946594238, 43.37900161743164, 19.040000915527344]], [[-6.455999851226807, 44.520999908447266, 19.7189998626709], [-6.960000038146973, 44.55099868774414, 21.083999633789062], [-8.437999725341797, 44.2140007019043, 21.04599952697754]], [[-9.168000221252441, 44.90399932861328, 20.174999237060547], [-10.600000381469727, 44.683998107910156, 20.011999130249023], [-10.878000259399414, 43.21799850463867, 19.73900032043457]], [[-10.166000366210938, 42.65999984741211, 18.764999389648438], [-10.340999603271484, 41.25899887084961, 18.39900016784668], [-10.07800006866455, 40.327999114990234, 19.58099937438965]], [[-9.152000427246094, 40.71699905395508, 20.452999114990234], [-8.817000389099121, 39.9010009765625, 21.613000869750977], [-9.875, 40.05799865722656, 22.68899917602539]], [[-10.336000442504883, 41.290000915527344, 22.8799991607666], [-11.36299991607666, 41.59199905395508, 23.868999481201172], [-12.619000434875488, 40.816001892089844, 23.516000747680664]], [[-12.666000366210938, 40.354000091552734, 22.270999908447266], [-13.786999702453613, 39.57500076293945, 21.76300048828125], [-13.758999824523926, 38.167999267578125, 22.343000411987305]], [[-12.866000175476074, 37.32500076293945, 21.836000442504883], [-12.784000396728516, 35.95800018310547, 22.334999084472656], [-12.41100025177002, 35.939998626708984, 23.808000564575195]], [[-12.307999610900879, 37.125, 24.4060001373291], [-11.954999923706055, 37.22999954223633, 25.812000274658203], [-13.154000282287598, 37.387001037597656, 26.733999252319336]], [[-14.208999633789062, 38.034000396728516, 26.23699951171875], [-15.430000305175781, 38.249000549316406, 27.016000747680664], [-15.866000175476074, 36.95800018310547, 27.714000701904297]], [[-16.173999786376953, 37.06999969482422, 29.00200080871582], [-16.597000122070312, 35.91299819946289, 29.757999420166016], [-15.5, 35.375999450683594, 30.650999069213867]], [[-14.25, 35.573001861572266, 30.249000549316406], [-13.128000259399414, 35.09000015258789, 31.04199981689453], [-12.246000289916992, 36.255001068115234, 31.483999252319336]], [[-11.991000175476074, 37.1609992980957, 30.551000595092773], [-11.154999732971191, 38.32099914550781, 30.80900001525879], [-11.774999618530273, 39.257999420166016, 31.83300018310547]], [[-13.02299976348877, 39.624000549316406, 31.579999923706055], [-13.777000427246094, 40.51900100708008, 32.44599914550781], [-14.230999946594238, 39.78799819946289, 33.71200180053711]], [[-14.024999618530273, 38.474998474121094, 33.7400016784668], [-14.435999870300293, 37.66600036621094, 34.880001068115234], [-13.635000228881836, 37.97800064086914, 36.13399887084961]], [[-14.319000244140625, 38.39799880981445, 37.1870002746582], [-13.621999740600586, 38.6879997253418, 38.42599868774414], [-14.032999992370605, 37.74599838256836, 39.55500030517578]], [[-13.045999526977539, 37.18899917602539, 40.24800109863281], [-13.319000244140625, 36.30500030517578, 41.36899948120117], [-12.840999603271484, 37.012001037597656, 42.63199996948242]], [[-13.744999885559082, 37.196998596191406, 43.60100173950195], [-13.425999641418457, 37.86899948120117, 44.867000579833984], [-12.463000297546387, 37.07600021362305, 45.72800064086914]], [[-11.793000221252441, 37.7599983215332, 46.66400146484375], [-10.835000038146973, 37.11899948120117, 47.5620002746582], [-11.489999771118164, 36.50400161743164, 48.784000396728516]], [[-10.829000473022461, 35.48400115966797, 49.310001373291016], [-11.241999626159668, 34.8129997253418, 50.5260009765625], [-10.4350004196167, 35.61399841308594, 51.5359992980957]], [[-11.097999572753906, 36.209999084472656, 52.5099983215332], [-10.36299991607666, 36.994998931884766, 53.48400115966797], [-10.225000381469727, 36.29999923706055, 54.832000732421875]], [[-8.986000061035156, 36.202999114990234, 55.31100082397461], [-8.713000297546387, 35.56100082397461, 56.595001220703125], [-7.77400016784668, 36.388999938964844, 57.45399856567383]], [[-8.072999954223633, 36.439998626708984, 58.742000579833984], [-7.256999969482422, 37.194000244140625, 59.66899871826172], [-6.5879998207092285, 36.242000579833984, 60.630001068115234]], [[-5.304999828338623, 35.97200012207031, 60.42499923706055], [-4.567999839782715, 35.073001861572266, 61.30699920654297], [-3.8269999027252197, 35.933998107910156, 62.323001861572266]], [[-3.1549999713897705, 35.29800033569336, 63.277000427246094], [-2.427999973297119, 36.0359992980957, 64.31400299072266], [-1.2450000047683716, 35.257999420166016, 64.9000015258789]], [[-0.3019999861717224, 36.00400161743164, 65.46299743652344], [0.878000020980835, 35.43899917602539, 66.09700012207031], [0.9850000143051147, 36.180999755859375, 67.40699768066406]], [[0.7120000123977661, 35.46699905395508, 68.49500274658203], [0.7279999852180481, 36.051998138427734, 69.8280029296875], [1.9800000190734863, 35.7760009765625, 70.66999816894531]], [[2.4159998893737793, 36.80799865722656, 71.3949966430664], [3.565000057220459, 36.74599838256836, 72.2979965209961], [3.2060000896453857, 37.62799835205078, 73.48699951171875]], [[4.01800012588501, 37.57899856567383, 74.52999877929688], [3.742000102996826, 38.34600067138672, 75.73500061035156], [3.7739999294281006, 39.84700012207031, 75.53600311279297]], [[4.591000080108643, 40.29899978637695, 74.59500122070312], [4.730000019073486, 41.733001708984375, 74.38999938964844], [3.867000102996826, 42.345001220703125, 73.3030014038086]], [[3.5369999408721924, 41.54800033569336, 72.2969970703125], [2.753000020980835, 42.08300018310547, 71.18900299072266], [2.0739998817443848, 40.98099899291992, 70.40399932861328]], [[0.9539999961853027, 41.3129997253418, 69.7750015258789], [0.2540000081062317, 40.345001220703125, 68.94499969482422], [0.30799999833106995, 40.88600158691406, 67.51899719238281]], [[0.6890000104904175, 40.02899932861328, 66.5790023803711], [0.7630000114440918, 40.439998626708984, 65.18499755859375], [-0.453000009059906, 39.94900131225586, 64.41699981689453]], [[-1.2760000228881836, 40.875999450683594, 63.94200134277344], [-2.4489998817443848, 40.50600051879883, 63.16400146484375], [-2.0360000133514404, 40.481998443603516, 61.69200134277344]], [[-2.5810000896453857, 39.5099983215332, 60.972999572753906], [-2.2769999504089355, 39.36399841308594, 59.5620002746582], [-3.5869998931884766, 39.194000244140625, 58.810001373291016]], [[-3.7890000343322754, 40.005001068115234, 57.790000915527344], [-5.013000011444092, 39.895999908447266, 57.0260009765625], [-4.630000114440918, 39.51900100708008, 55.604000091552734]], [[-5.158999919891357, 38.39500045776367, 55.150001525878906], [-4.843999862670898, 37.90399932861328, 53.821998596191406], [-6.057000160217285, 37.84700012207031, 52.91299819946289]], [[-5.848999977111816, 38.20899963378906, 51.654998779296875], [-6.9079999923706055, 38.178001403808594, 50.659000396728516], [-6.380000114440918, 37.11000061035156, 49.72100067138672]], [[-7.144999980926514, 36.0369987487793, 49.53300094604492], [-6.6570000648498535, 34.93199920654297, 48.71200180053711], [-7.379000186920166, 34.569000244140625, 47.441001892089844]], [[-6.598999977111816, 33.98699951171875, 46.5260009765625], [-7.063000202178955, 33.49800109863281, 45.224998474121094], [-8.031999588012695, 34.41999816894531, 44.494998931884766]], [[-7.692999839782715, 35.70800018310547, 44.40800094604492], [-8.572999954223633, 36.67900085449219, 43.762001037597656], [-8.055999755859375, 37.16999816894531, 42.417999267578125]], [[-8.871000289916992, 37.96900177001953, 41.72999954223633], [-8.527000427246094, 38.48500061035156, 40.40599822998047], [-9.637999534606934, 39.391998291015625, 39.882999420166016]], [[-9.28499984741211, 40.51100158691406, 39.24800109863281], [-7.947999954223633, 41.05099868774414, 38.944000244140625], [-7.140999794006348, 41.39500045776367, 40.191001892089844]], [[-5.873000144958496, 41.750999450683594, 39.994998931884766], [-4.97599983215332, 42.04499816894531, 41.1150016784668], [-5.236999988555908, 43.2859992980957, 41.95399856567383]], [[-5.765999794006348, 44.33700180053711, 41.345001220703125], [-6.0229997634887695, 45.54199981689453, 42.11399841308594], [-7.053999900817871, 45.263999938964844, 43.19900131225586]], [[-6.6620001792907715, 45.54600143432617, 44.4370002746582], [-7.49399995803833, 45.305999755859375, 45.60200119018555], [-7.245999813079834, 46.37900161743164, 46.67100143432617]], [[-8.128999710083008, 46.44900131225586, 47.65999984741211], [-7.997000217437744, 47.41899871826172, 48.74100112915039], [-8.079999923706055, 46.71900177001953, 50.09400177001953]], [[-7.056000232696533, 46.92100143432617, 50.90999984741211], [-7.014999866485596, 46.29399871826172, 52.21500015258789], [-6.604000091552734, 47.30400085449219, 53.25899887084961]], [[-7.36299991607666, 47.369998931884766, 54.349998474121094], [-7.0329999923706055, 48.30500030517578, 55.41699981689453], [-7.510000228881836, 47.82400131225586, 56.77799987792969]], [[-6.874000072479248, 48.34600067138672, 57.821998596191406], [-7.236999988555908, 48.00699996948242, 59.196998596191406], [-8.236000061035156, 49.00299835205078, 59.7760009765625]], [[-9.121999740600586, 48.49800109863281, 60.62099838256836], [-10.118000030517578, 49.33300018310547, 61.26300048828125], [-9.95300006866455, 49.165000915527344, 62.7599983215332]], [[-9.892000198364258, 50.290000915527344, 63.45899963378906], [-9.772000312805176, 50.292999267578125, 64.90699768066406], [-11.109000205993652, 50.79800033569336, 65.4489974975586]], [[-11.809000015258789, 49.928001403808594, 66.1719970703125], [-13.121000289916992, 50.25699996948242, 66.72799682617188], [-14.109999656677246, 50.555999755859375, 65.59300231933594]], [[-14.114999771118164, 49.689998626708984, 64.58499908447266], [-14.994999885559082, 49.83599853515625, 63.430999755859375], [-14.720999717712402, 51.111000061035156, 62.630001068115234]], [[-13.689000129699707, 51.861000061035156, 63.00699996948242], [-13.37600040435791, 53.0890007019043, 62.28499984741211], [-12.015000343322754, 53.04100036621094, 61.61899948120117]], [[-11.892999649047852, 53.67399978637695, 60.44300079345703], [-10.652000427246094, 53.722999572753906, 59.659000396728516], [-9.446999549865723, 54.20100021362305, 60.45000076293945]], [[-8.295999526977539, 53.6150016784668, 60.13399887084961], [-7.03000020980835, 53.93899917602539, 60.78200149536133], [-6.210000038146973, 54.70600128173828, 59.770999908447266]], [[-5.535999774932861, 55.757999420166016, 60.222999572753906], [-4.704999923706055, 56.564998626708984, 59.3390007019043], [-3.8299999237060547, 55.61199951171875, 58.52799987792969]], [[-3.8239998817443848, 55.797000885009766, 57.20899963378906], [-3.0510001182556152, 54.95899963378906, 56.29199981689453], [-1.6019999980926514, 54.742000579833984, 56.73099899291992]], [[-1.0329999923706055, 55.74700164794922, 57.38999938964844], [0.3540000021457672, 55.70899963378906, 57.85599899291992], [0.574999988079071, 54.90599822998047, 59.141998291015625]], [[-0.4020000100135803, 54.07600021362305, 59.49399948120117], [-0.335999995470047, 53.25, 60.698001861572266], [0.1120000034570694, 51.8129997253418, 60.39799880981445]], [[-0.0430000014603138, 51.39899826049805, 59.145999908447266], [0.3400000035762787, 50.06100082397461, 58.7130012512207], [0.8759999871253967, 50.178001403808594, 57.29199981689453]], [[1.9989999532699585, 49.52299880981445, 57.020999908447266], [2.5810000896453857, 49.54600143432617, 55.685001373291016], [1.4609999656677246, 49.17599868774414, 54.71799850463867]], [[1.5399999618530273, 49.64500045776367, 53.459999084472656], [0.48500001430511475, 49.32099914550781, 52.48500061035156], [0.4690000116825104, 47.81399917602539, 52.20199966430664]], [[-0.5690000057220459, 47.319000244140625, 51.529998779296875], [-0.6309999823570251, 45.888999938964844, 51.23699951171875], [0.574999988079071, 45.41400146484375, 50.43600082397461]], [[0.6769999861717224, 44.104000091552734, 50.305999755859375], [1.7339999675750732, 43.46500015258789, 49.547000885009766], [0.9160000085830688, 42.5629997253418, 48.64500045776367]], [[1.1299999952316284, 42.65800094604492, 47.33399963378906], [0.38100001215934753, 41.8390007019043, 46.38100051879883], [1.2899999618530273, 40.88199996948242, 45.64500045776367]], [[0.9539999961853027, 39.608001708984375, 45.750999450683594], [1.7610000371932983, 38.577999114990234, 45.14500045776367], [0.9459999799728394, 37.680999755859375, 44.28499984741211]], [[1.5920000076293945, 37.08599853515625, 43.28499984741211], [0.902999997138977, 36.17599868774414, 42.37900161743164], [1.0160000324249268, 34.724998474121094, 42.84400177001953]], [[0.07000000029802322, 33.93199920654297, 42.36600112915039], [0.02199999988079071, 32.49700164794922, 42.61399841308594], [0.28200000524520874, 32.02799987792969, 41.18600082397461]], [[0.8799999952316284, 30.868000030517578, 41.0099983215332], [1.1460000276565552, 30.461000442504883, 39.64099884033203], [-0.0729999989271164, 30.277999877929688, 38.74800109863281]], [[-1.2319999933242798, 30.055999755859375, 39.36600112915039], [-2.4839999675750732, 29.784000396728516, 38.659000396728516], [-3.260999917984009, 30.94099998474121, 38.069000244140625]], [[-2.681999921798706, 32.13100051879883, 38.01900100708008], [-3.427999973297119, 33.237998962402344, 37.44499969482422], [-4.23199987411499, 34.04999923706055, 38.446998596191406]], [[-4.163000106811523, 33.68899917602539, 39.724998474121094], [-4.875, 34.45500183105469, 40.744998931884766], [-3.8469998836517334, 35.18600082397461, 41.590999603271484]], [[-4.322000026702881, 36.04100036621094, 42.481998443603516], [-3.4170000553131104, 36.78900146484375, 43.33300018310547], [-3.8350000381469727, 36.702999114990234, 44.77799987792969]], [[-2.9159998893737793, 37.125999450683594, 45.645999908447266], [-3.1689999103546143, 37.202999114990234, 47.07699966430664], [-2.4630000591278076, 38.45199966430664, 47.608001708984375]], [[-2.9709999561309814, 38.96799850463867, 48.71799850463867], [-2.375999927520752, 40.130001068115234, 49.32699966430664], [-2.4739999771118164, 40.06999969482422, 50.83100128173828]], [[-1.6660000085830688, 40.86399841308594, 51.513999938964844], [-1.7640000581741333, 40.858001708984375, 52.96200180053711], [-1.2890000343322754, 42.13800048828125, 53.59600067138672]], [[-1.8320000171661377, 42.41299819946289, 54.775001525878906], [-1.4079999923706055, 43.56399917602539, 55.53799819946289], [-1.1449999809265137, 43.06999969482422, 56.95500183105469]], [[-0.3009999990463257, 43.79999923706055, 57.67499923706055], [0.06300000101327896, 43.42900085449219, 59.02899932861328], [-0.19699999690055847, 44.54999923706055, 60.020999908447266]], [[-0.4320000112056732, 44.17599868774414, 61.275001525878906], [-0.671999990940094, 45.15800094604492, 62.316001892089844], [-0.20100000500679016, 44.625, 63.64899826049805]], [[0.6959999799728394, 45.35900115966797, 64.29299926757812], [1.190000057220459, 44.94599914550781, 65.6050033569336], [0.3790000081062317, 45.66899871826172, 66.67900085449219]], [[-0.25999999046325684, 44.90700149536133, 67.56099700927734], [-1.0720000267028809, 45.499000549316406, 68.6240005493164], [-0.8040000200271606, 44.8849983215332, 70.01200103759766]], [[-1.1480000019073486, 45.624000549316406, 71.08000183105469], [-0.9269999861717224, 45.090999603271484, 72.43099975585938], [-1.8140000104904175, 43.86600112915039, 72.64800262451172]], [[-1.2640000581741333, 42.797000885009766, 73.24099731445312], [-2.0929999351501465, 41.60300064086914, 73.46900177001953], [-3.3380000591278076, 41.986000061035156, 74.27300262451172]], [[-4.48199987411499, 41.43600082397461, 73.88600158691406], [-5.7179999351501465, 41.77399826049805, 74.56400299072266], [-6.565999984741211, 42.65599822998047, 73.66500091552734]], [[-5.942999839782715, 43.68000030517578, 73.08100128173828], [-6.61299991607666, 44.624000549316406, 72.18499755859375], [-7.0279998779296875, 44.08100128173828, 70.81500244140625]], [[-6.640999794006348, 42.84600067138672, 70.49800109863281], [-6.956999778747559, 42.25699996948242, 69.19300079345703], [-8.36299991607666, 42.58599853515625, 68.69300079345703]], [[-9.357000350952148, 42.24399948120117, 69.50499725341797], [-10.758999824523926, 42.48099899291992, 69.1780014038086], [-11.07800006866455, 43.8849983215332, 68.65699768066406]], [[-10.347000122070312, 44.90399932861328, 69.10299682617188], [-10.619999885559082, 46.263999938964844, 68.6240005493164], [-10.234000205993652, 46.46500015258789, 67.16699981689453]], [[-9.538000106811523, 45.492000579833984, 66.58899688720703], [-9.109000205993652, 45.617000579833984, 65.20099639892578], [-9.770999908447266, 44.619998931884766, 64.26899719238281]], [[-10.149999618530273, 45.13600158691406, 63.10599899291992], [-10.805999755859375, 44.345001220703125, 62.08100128173828], [-10.170999526977539, 44.66299819946289, 60.733001708984375]], [[-10.293000221252441, 43.71699905395508, 59.81100082397461], [-9.729999542236328, 43.909000396728516, 58.50299835205078], [-10.843000411987305, 44.233001708984375, 57.518001556396484]], [[-10.621000289916992, 45.249000549316406, 56.691001892089844], [-11.609999656677246, 45.606998443603516, 55.68600082397461], [-10.980999946594238, 45.46900177001953, 54.321998596191406]], [[-11.619000434875488, 44.67100143432617, 53.47600173950195], [-11.119000434875488, 44.428001403808594, 52.132999420166016], [-12.14799976348877, 44.797000885009766, 51.07400131225586]], [[-11.697999954223633, 45.483001708984375, 50.03200149536133], [-12.586999893188477, 45.887001037597656, 48.95000076293945], [-12.02400016784668, 45.40700149536133, 47.630001068115234]], [[-12.902999877929688, 44.87099838256836, 46.78900146484375], [-12.531000137329102, 44.35900115966797, 45.4739990234375], [-13.776000022888184, 44.375, 44.5989990234375]], [[-13.619999885559082, 44.65999984741211, 43.29999923706055], [-14.781999588012695, 44.6870002746582, 42.40599822998047], [-15.595999717712402, 43.382999420166016, 42.426998138427734]], [[-14.946999549865723, 42.28200149536133, 42.79899978637695], [-15.637999534606934, 41.00899887084961, 42.84299850463867], [-16.59000015258789, 40.95100021362305, 44.02299880981445]], [[-16.371000289916992, 41.821998596191406, 45.000999450683594], [-17.211000442504883, 41.861000061035156, 46.19900131225586], [-18.316999435424805, 42.887001037597656, 46.029998779296875]], [[-19.527999877929688, 42.53300094604492, 46.448001861572266], [-20.663999557495117, 43.44200134277344, 46.314998626708984], [-20.496999740600586, 44.709999084472656, 47.145999908447266]], [[-19.847000122070312, 44.571998596191406, 48.29399871826172], [-19.577999114990234, 45.698001861572266, 49.180999755859375], [-18.378999710083008, 45.3120002746582, 50.034000396728516]], [[-17.759000778198242, 46.29800033569336, 50.707000732421875], [-16.59600067138672, 46.01100158691406, 51.547000885009766], [-16.858999252319336, 44.8489990234375, 52.49100112915039]], [[-15.869000434875488, 43.96200180053711, 52.590999603271484], [-15.934000015258789, 42.777000427246094, 53.43899917602539], [-15.081999778747559, 43.020999908447266, 54.68299865722656]], [[-15.675000190734863, 42.83599853515625, 55.856998443603516], [-14.97700023651123, 43.07699966430664, 57.1150016784668], [-14.819000244140625, 41.790000915527344, 57.90599822998047]], [[-13.567999839782715, 41.435001373291016, 58.20000076293945], [-13.253999710083008, 40.20199966430664, 58.92599868774414], [-12.666999816894531, 40.446998596191406, 60.321998596191406]], [[-13.107000350952148, 39.63800048828125, 61.28799819946289], [-12.668000221252441, 39.71900177001953, 62.6870002746582], [-11.456999778747559, 38.82400131225586, 62.952999114990234]], [[-10.6899995803833, 39.165000915527344, 63.98899841308594], [-9.494999885559082, 38.40399932861328, 64.36399841308594], [-9.784000396728516, 37.0260009765625, 64.97699737548828]]], "seq": "RLLRSHSLHYLFMGASEQDLGLSLFEALGYVDDQLFVFYDHESRRVEPRTPWVSSRISSQMWLQLSQSLKGWDHMFTVDFWTIMENHNHSKESHTLQVILGCEMQEDNSTEGYWKYGYDGQDHLEFCPDTLDWRAAEPRAWPTKLEWERHKIRARQNRAYLERDCPAQLQQLLELGRGVLDQQVPPLVKVTHHVTSSVTTLRCRALNYYPQNITMKWLKDKQPMDAKEFEPKDVLPNGDGTYQGWITLAVPPGEEQRYTCQVEHPGLDQPLIVIW"}, {"coords": [[[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[12.140000343322754, 45.65700149536133, 29.216999053955078], [12.67199993133545, 44.888999938964844, 30.316999435424805], [11.619000434875488, 44.62300109863281, 31.375999450683594]], [[11.89900016784668, 43.68899917602539, 32.242000579833984], [11.043000221252441, 43.31800079345703, 33.37099838256836], [12.010000228881836, 43.19900131225586, 34.560001373291016]], [[11.85099983215332, 44.05099868774414, 35.55799865722656], [12.711999893188477, 44.01100158691406, 36.729000091552734], [12.244999885559082, 42.91400146484375, 37.667999267578125]], [[12.871999740600586, 41.763999938964844, 37.64699935913086], [12.418999671936035, 40.69900131225586, 38.555999755859375], [12.812999725341797, 41.03200149536133, 39.981998443603516]], [[13.897000312805176, 41.80400085449219, 40.17100143432617], [14.274999618530273, 42.215999603271484, 41.51100158691406], [15.055999755859375, 43.520999908447266, 41.465999603271484]], [[15.654999732971191, 43.792999267578125, 42.63399887084961], [16.43000030517578, 45.025001525878906, 42.76300048828125], [17.66699981689453, 44.9900016784668, 41.900001525878906]], [[18.344999313354492, 43.862998962402344, 41.75699996948242], [19.535999298095703, 43.7400016784668, 40.94499969482422], [19.395000457763672, 42.965999603271484, 39.63100051879883]], [[18.187000274658203, 42.51599884033203, 39.28799819946289], [18.100000381469727, 41.68299865722656, 38.060001373291016], [16.930999755859375, 42.10599899291992, 37.20399856567383]], [[17.219999313354492, 42.26300048828125, 35.91299819946289], [16.1299991607666, 42.62099838256836, 34.98699951171875], [16.29599952697754, 41.66299819946289, 33.810001373291016]], [[15.239999771118164, 41.448001861572266, 33.06999969482422], [15.199000358581543, 40.51300048828125, 31.968000411987305], [14.569000244140625, 41.13999938964844, 30.739999771118164]], [[15.074000358581543, 40.847999572753906, 29.56399917602539], [14.505999565124512, 41.3849983215332, 28.322999954223633], [14.211000442504883, 40.250999450683594, 27.378000259399414]], [[13.020000457763672, 40.17499923706055, 26.770999908447266], [12.776000022888184, 39.042999267578125, 25.84000015258789], [13.29699993133545, 39.39799880981445, 24.482999801635742]], [[14.119000434875488, 38.584999084472656, 23.82200050354004], [14.593999862670898, 38.904998779296875, 22.49799919128418], [13.803999900817871, 38.23099899291992, 21.374000549316406]], [[13.307000160217285, 37.0620002746582, 21.64299964904785], [12.600000381469727, 36.18299865722656, 20.731000900268555], [11.652999877929688, 35.37200164794922, 21.586999893188477]], [[10.861000061035156, 34.55699920654297, 20.87700080871582], [9.869999885559082, 33.79499816894531, 21.694000244140625], [10.576000213623047, 32.84299850463867, 22.607999801635742]], [[11.791000366210938, 32.388999938964844, 22.270999908447266], [12.430000305175781, 31.437000274658203, 23.163999557495117], [13.657999992370605, 31.937000274658203, 23.891000747680664]], [[14.024999618530273, 33.194000244140625, 23.683000564575195], [15.246000289916992, 33.750999450683594, 24.200000762939453], [15.069999694824219, 35.01900100708008, 24.97599983215332]], [[15.696999549865723, 35.0, 26.174999237060547], [15.5649995803833, 36.18899917602539, 27.023000717163086], [17.009000778198242, 36.542999267578125, 27.402000427246094]], [[17.243000030517578, 37.821998596191406, 27.635000228881836], [18.60099983215332, 38.143001556396484, 28.077999114990234], [18.454999923706055, 38.78300094604492, 29.433000564575195]], [[19.302000045776367, 38.499000549316406, 30.408000946044922], [19.19300079345703, 39.12300109863281, 31.700000762939453], [20.429000854492188, 39.994998931884766, 32.00299835205078]], [[20.214000701904297, 40.93600082397461, 32.94200134277344], [21.288999557495117, 41.80099868774414, 33.402000427246094], [21.391000747680664, 41.58300018310547, 34.91899871826172]], [[22.527000427246094, 41.244998931884766, 35.48899841308594], [22.56800079345703, 41.112998962402344, 36.96500015258789], [23.604000091552734, 42.1609992980957, 37.44900131225586]], [[23.172000885009766, 42.90299987792969, 38.479000091552734], [24.000999450683594, 43.96699905395508, 39.016998291015625], [24.65399932861328, 43.61199951171875, 40.3390007019043]], [[25.923999786376953, 43.94900131225586, 40.48099899291992], [26.649999618530273, 43.652000427246094, 41.71900177001953], [27.750999450683594, 44.72100067138672, 41.86199951171875]], [[27.888999938964844, 45.16699981689453, 43.11800003051758], [28.94499969482422, 46.15299987792969, 43.4379997253418], [30.219999313354492, 45.34700012207031, 43.715999603271484]], [[31.261999130249023, 45.560001373291016, 42.948001861572266], [32.48099899291992, 44.78799819946289, 43.11000061035156], [33.5880012512207, 45.75699996948242, 43.433998107910156]], [[34.23699951171875, 45.58399963378906, 44.58700180053711], [35.290000915527344, 46.50600051879883, 44.986000061035156], [36.36000061035156, 46.61600112915039, 43.915000915527344]], [[36.71500015258789, 47.89400100708008, 43.61800003051758], [37.71200180053711, 48.21699905395508, 42.62200164794922], [37.152000427246094, 48.49800109863281, 41.23099899291992]], [[35.91600036621094, 48.064998626708984, 40.99300003051758], [35.25899887084961, 48.20399856567383, 39.70399856567383], [33.95500183105469, 48.9640007019043, 39.81700134277344]], [[33.22600173950195, 48.922000885009766, 40.928001403808594], [32.000999450683594, 49.67100143432617, 41.034000396728516], [30.761999130249023, 48.80699920654297, 40.808998107910156]], [[29.690000534057617, 49.4119987487793, 40.32899856567383], [28.457000732421875, 48.71500015258789, 40.090999603271484], [28.55900001525879, 48.04800033569336, 38.7239990234375]], [[28.749000549316406, 46.737998962402344, 38.643001556396484], [28.945999145507812, 46.13800048828125, 37.30099868774414], [27.74799919128418, 45.33599853515625, 36.869998931884766]], [[27.233999252319336, 45.58300018310547, 35.67399978637695], [26.121000289916992, 44.83100128173828, 35.104000091552734], [26.738000869750977, 43.62200164794922, 34.36000061035156]], [[26.31999969482422, 42.417999267578125, 34.66699981689453], [26.849000930786133, 41.20100021362305, 34.01100158691406], [25.641000747680664, 40.57699966430664, 33.266998291015625]], [[25.850000381469727, 40.16299819946289, 32.027000427246094], [24.749000549316406, 39.65800094604492, 31.224000930786133], [24.81599998474121, 38.16299819946289, 31.00200080871582]], [[23.641000747680664, 37.55699920654297, 30.829999923706055], [23.615999221801758, 36.13399887084961, 30.424999237060547], [22.363000869750977, 35.944000244140625, 29.559999465942383]], [[22.072999954223633, 34.691001892089844, 29.243000030517578], [20.874000549316406, 34.404998779296875, 28.490999221801758], [20.11199951171875, 33.2400016784668, 29.107999801635742]], [[18.847999572753906, 33.25, 28.788000106811523], [17.909000396728516, 32.220001220703125, 29.219999313354492], [17.259000778198242, 31.711000442504883, 27.937000274658203]], [[17.399999618530273, 30.447999954223633, 27.618000030517578], [16.856000900268555, 29.898000717163086, 26.356000900268555], [15.845999717712402, 28.829999923706055, 26.768999099731445]], [[14.651000022888184, 28.868000030517578, 26.17099952697754], [13.611000061035156, 27.920000076293945, 26.569000244140625], [12.991999626159668, 27.163999557495117, 25.398000717163086]], [[12.82699966430664, 25.868999481201172, 25.66699981689453], [12.21399974822998, 25.02899932861328, 24.638999938964844], [11.479999542236328, 23.909000396728516, 25.3700008392334]], [[10.168999671936035, 23.854000091552734, 25.055999755859375], [9.315999984741211, 22.802000045776367, 25.62299919128418], [9.46500015258789, 22.64299964904785, 27.117000579833984]], [[9.378999710083008, 23.708999633789062, 27.886999130249023], [9.463000297546387, 23.708999633789062, 29.332000732421875], [10.8149995803833, 23.388999938964844, 29.94499969482422]], [[11.842000007629395, 23.400999069213867, 29.084999084472656], [13.204999923706055, 23.167999267578125, 29.540000915527344], [13.96399974822998, 24.469999313354492, 29.222000122070312]], [[14.906999588012695, 24.791000366210938, 30.111000061035156], [15.654999732971191, 26.020999908447266, 29.889999389648438], [17.1560001373291, 25.781999588012695, 29.966999053955078]], [[17.867000579833984, 26.64699935913086, 29.243999481201172], [19.3439998626709, 26.583999633789062, 29.333999633789062], [19.743000030517578, 27.979999542236328, 29.841999053955078]], [[20.700000762939453, 28.04800033569336, 30.767000198364258], [21.20599937438965, 29.329999923706055, 31.261999130249023], [22.57200050354004, 29.545000076293945, 30.636999130249023]], [[22.851999282836914, 30.670000076293945, 30.0049991607666], [24.202999114990234, 30.915000915527344, 29.472999572753906], [24.791000366210938, 31.97100067138672, 30.42300033569336]], [[25.75200080871582, 31.58799934387207, 31.23900032043457], [26.38800048828125, 32.32500076293945, 32.27000045776367], [25.45800018310547, 32.457000732421875, 33.47700119018555]], [[26.07200050354004, 32.49800109863281, 34.69499969482422], [25.2549991607666, 32.763999938964844, 35.86600112915039], [25.638999938964844, 34.1879997253418, 36.35300064086914]], [[24.79599952697754, 34.790000915527344, 37.165000915527344], [25.08300018310547, 36.02399826049805, 37.869998931884766], [26.332000732421875, 35.66699981689453, 38.744998931884766]], [[26.86199951171875, 36.630001068115234, 39.43000030517578], [28.065000534057617, 36.558998107910156, 40.24100112915039], [28.033000946044922, 35.643001556396484, 41.44900131225586]], [[26.847999572753906, 35.28499984741211, 41.9739990234375], [26.868000030517578, 34.41299819946289, 43.12200164794922], [25.655000686645508, 33.499000549316406, 43.130001068115234]], [[25.69099998474121, 32.54499816894531, 44.09199905395508], [24.58799934387207, 31.611000061035156, 44.15299987792969], [23.253999710083008, 32.27399826049805, 44.457000732421875]], [[23.27899932861328, 33.28799819946289, 45.30099868774414], [22.065000534057617, 33.95199966430664, 45.70899963378906], [21.266000747680664, 34.55500030517578, 44.56399917602539]], [[21.950000762939453, 35.34199905395508, 43.750999450683594], [21.32900047302246, 35.93899917602539, 42.57899856567383], [20.97599983215332, 34.84700012207031, 41.57699966430664]], [[21.77400016784668, 33.76300048828125, 41.50699996948242], [21.461000442504883, 32.67599868774414, 40.5989990234375], [20.152999877929688, 31.972000122070312, 41.000999450683594]], [[20.04599952697754, 31.679000854492188, 42.29600143432617], [18.865999221801758, 31.048999786376953, 42.84299850463867], [17.656999588012695, 31.93899917602539, 42.584999084472656]], [[17.79599952697754, 33.25299835205078, 42.70100021362305], [16.621000289916992, 34.10900115966797, 42.45100021362305], [16.20400047302246, 34.060001373291016, 40.98699951171875]], [[17.190000534057617, 34.012001037597656, 40.10900115966797], [16.94499969482422, 33.974998474121094, 38.64400100708008], [16.325000762939453, 32.652000427246094, 38.2400016784668]], [[16.819000244140625, 31.566999435424805, 38.78099822998047], [16.299999237060547, 30.229999542236328, 38.46099853515625], [14.857999801635742, 30.09600067138672, 38.97100067138672]], [[14.567000389099121, 30.663999557495117, 40.1510009765625], [13.213000297546387, 30.597000122070312, 40.729000091552734], [12.263999938964844, 31.340999603271484, 39.82099914550781]], [[12.684000015258789, 32.42300033569336, 39.17300033569336], [11.842000007629395, 33.21799850463867, 38.29899978637695], [11.54699993133545, 32.4900016784668, 37.00899887084961]], [[12.524999618530273, 31.750999450683594, 36.499000549316406], [12.329999923706055, 31.04199981689453, 35.22800064086914], [11.26200008392334, 29.98699951171875, 35.46799850463867]], [[11.404999732971191, 29.29599952697754, 36.60100173950195], [10.40999984741211, 28.280000686645508, 36.96699905395508], [8.98900032043457, 28.836999893188477, 37.12699890136719]], [[8.829000473022461, 29.780000686645508, 38.0620002746582], [7.573999881744385, 30.413000106811523, 38.362998962402344], [7.000999927520752, 31.316999435424805, 37.28300094604492]], [[7.761000156402588, 32.112998962402344, 36.566001892089844], [7.2170000076293945, 33.000999450683594, 35.59400177001953], [7.385000228881836, 32.52199935913086, 34.18899917602539]], [[8.392999649047852, 31.68899917602539, 33.928001403808594], [8.562999725341797, 31.19700050354004, 32.573001861572266], [8.107999801635742, 29.750999450683594, 32.46099853515625]], [[7.716000080108643, 29.15399932861328, 33.553001403808594], [7.214000225067139, 27.79199981689453, 33.59199905395508], [8.133000373840332, 26.759000778198242, 32.96500015258789]], [[9.446999549865723, 26.94499969482422, 33.159000396728516], [10.420000076293945, 26.009000778198242, 32.604000091552734], [11.404999732971191, 25.67099952697754, 33.72700119018555]], [[12.041000366210938, 24.524999618530273, 33.54100036621094], [13.0, 24.01099967956543, 34.4739990234375], [14.390000343322754, 24.138999938964844, 33.854000091552734]], [[15.3149995803833, 24.69300079345703, 34.625], [16.674999237060547, 24.812999725341797, 34.09000015258789], [17.365999221801758, 23.459999084472656, 34.11899948120117]], [[17.753999710083008, 22.91900062561035, 32.970001220703125], [18.42300033569336, 21.625, 32.91899871826172], [19.834999084472656, 21.70599937438965, 32.36000061035156]], [[20.19700050354004, 22.798999786376953, 31.731000900268555], [21.451000213623047, 22.940000534057617, 31.020999908447266], [22.14299964904785, 24.261999130249023, 31.25200080871582]], [[23.440000534057617, 24.23699951171875, 31.351999282836914], [24.215999603271484, 25.457000732421875, 31.6200008392334], [25.479999542236328, 25.48200035095215, 30.827999114990234]], [[25.823999404907227, 26.660999298095703, 30.295000076293945], [27.093000411987305, 26.860000610351562, 29.600000381469727], [27.69300079345703, 28.187000274658203, 29.999000549316406]], [[28.933000564575195, 28.238000869750977, 30.55299949645996], [29.597000122070312, 29.447999954223633, 30.899999618530273], [30.429000854492188, 29.974000930786133, 29.73200035095215]], [[30.514999389648438, 31.29800033569336, 29.604000091552734], [31.361000061035156, 31.860000610351562, 28.55699920654297], [32.85200119018555, 31.659000396728516, 28.799999237060547]], [[33.33700180053711, 31.743000030517578, 30.042999267578125], [34.73899841308594, 31.57200050354004, 30.364999771118164], [34.84000015258789, 31.374000549316406, 31.88599967956543]], [[35.981998443603516, 31.28499984741211, 32.520999908447266], [36.013999938964844, 30.986000061035156, 33.96200180053711], [35.84199905395508, 32.084999084472656, 34.95600128173828]], [[35.619998931884766, 33.3390007019043, 34.52899932861328], [35.452999114990234, 34.41899871826172, 35.507999420166016], [34.13600158691406, 34.347999572753906, 36.263999938964844]], [[34.09600067138672, 35.117000579833984, 37.395999908447266], [32.95500183105469, 35.0890007019043, 38.279998779296875], [31.672000885009766, 35.534000396728516, 37.652000427246094]], [[31.735000610351562, 36.45000076293945, 36.68899917602539], [30.489999771118164, 36.85100173950195, 36.05099868774414], [30.0049991607666, 35.81399917602539, 35.040000915527344]], [[30.56800079345703, 34.625999450683594, 34.95100021362305], [30.179000854492188, 33.53300094604492, 34.13600158691406], [29.82900047302246, 32.28499984741211, 34.941001892089844]], [[30.57900047302246, 31.94099998474121, 36.00400161743164], [30.354000091552734, 30.743000030517578, 36.79899978637695], [30.222000122070312, 31.014999389648438, 38.29999923706055]], [[29.934999465942383, 32.233001708984375, 38.67599868774414], [29.827999114990234, 32.625, 40.090999603271484], [28.76799964904785, 31.82900047302246, 40.816001892089844]], [[27.724000930786133, 31.40999984741211, 40.14799880981445], [26.634000778198242, 30.655000686645508, 40.74800109863281], [26.6200008392334, 29.211000442504883, 40.2859992980957]], [[27.674999237060547, 28.68199920654297, 39.68899917602539], [27.628999710083008, 27.273000717163086, 39.24700164794922], [27.452999114990234, 26.302000045776367, 40.41999816894531]], [[27.968000411987305, 26.60099983215332, 41.63600158691406], [27.801000595092773, 25.761999130249023, 42.79999923706055], [26.361000061035156, 25.378999710083008, 43.05099868774414]], [[25.424999237060547, 26.35099983215332, 43.05400085449219], [24.016000747680664, 26.167999267578125, 43.22600173950195], [23.410999298095703, 25.30699920654297, 42.111000061035156]], [[23.805999755859375, 25.482999801635742, 40.845001220703125], [23.240999221801758, 24.69099998474121, 39.74800109863281], [23.711000442504883, 23.231000900268555, 39.80799865722656]], [[24.97800064086914, 23.011999130249023, 40.215999603271484], [25.4950008392334, 21.660999298095703, 40.28900146484375], [24.78700065612793, 20.804000854492188, 41.34299850463867]], [[24.472000122070312, 21.386999130249023, 42.474998474121094], [23.766000747680664, 20.610000610351562, 43.49399948120117], [22.382999420166016, 20.222999572753906, 43.00600051879883]], [[21.788999557495117, 21.125999450683594, 42.2239990234375], [20.48900032043457, 20.959999084472656, 41.624000549316406], [20.53499984741211, 19.954999923706055, 40.487998962402344]], [[21.68400001525879, 19.44499969482422, 40.077999114990234], [21.618000030517578, 18.45400047302246, 38.99700164794922], [21.594999313354492, 19.04400062561035, 37.61000061035156]], [[21.83799934387207, 20.347000122070312, 37.465999603271484], [21.927000045776367, 20.941999435424805, 36.106998443603516], [23.18400001525879, 20.39699935913086, 35.40999984741211]], [[23.11199951171875, 20.104999542236328, 34.1150016784668], [24.270000457763672, 19.593000411987305, 33.41400146484375], [24.98699951171875, 20.788000106811523, 32.73400115966797]], [[26.297000885009766, 20.781999588012695, 32.904998779296875], [27.1299991607666, 21.856000900268555, 32.38399887084961], [28.076000213623047, 21.440000534057617, 31.284000396728516]], [[28.22800064086914, 22.322999954223633, 30.281999588012695], [29.114999771118164, 22.08099937438965, 29.180999755859375], [30.00200080871582, 23.305999755859375, 29.0310001373291]], [[31.274999618530273, 23.12299919128418, 28.756999969482422], [32.15399932861328, 24.253999710083008, 28.624000549316406], [33.42399978637695, 23.816999435424805, 27.889999389648438]], [[34.13199996948242, 24.87299919128418, 27.530000686645508], [35.422000885009766, 24.711999893188477, 26.82900047302246], [36.29999923706055, 23.95599937438965, 27.83300018310547]], [[36.946998596191406, 22.905000686645508, 27.277999877929688], [37.888999938964844, 22.149999618530273, 28.097000122070312], [38.862998962402344, 23.124000549316406, 28.743000030517578]], [[39.23899841308594, 24.23200035095215, 28.097000122070312], [40.17399978637695, 25.18000030517578, 28.719999313354492], [39.566001892089844, 25.81599998474121, 29.972999572753906]], [[38.25299835205078, 26.054000854492188, 29.951000213623047], [37.58000183105469, 26.613000869750977, 31.132999420166016], [37.61399841308594, 25.61199951171875, 32.28300094604492]], [[37.31399917602539, 24.37299919128418, 31.95400047302246], [37.332000732421875, 23.283000946044922, 32.96200180053711], [38.750999450683594, 23.17799949645996, 33.57899856567383]], [[39.81800079345703, 23.12299919128418, 32.80400085449219], [41.18000030517578, 23.07200050354004, 33.32400131225586], [41.487998962402344, 24.275999069213867, 34.198001861572266]], [[41.055999755859375, 25.485000610351562, 33.83700180053711], [41.26900100708008, 26.66900062561035, 34.66600036621094], [40.507999420166016, 26.60300064086914, 35.981998443603516]], [[39.237998962402344, 26.184999465942383, 35.96200180053711], [38.44599914550781, 26.076000213623047, 37.16899871826172], [39.099998474121094, 25.070999145507812, 38.09400177001953]], [[39.500999450683594, 23.895000457763672, 37.5880012512207], [40.1349983215332, 22.924999237060547, 38.46699905395508], [41.39400100708008, 23.46299934387207, 39.1609992980957]], [[42.220001220703125, 24.1560001373291, 38.426998138427734], [43.44499969482422, 24.715999603271484, 38.96500015258789], [43.2130012512207, 25.731000900268555, 40.05699920654297]], [[42.21500015258789, 26.55299949645996, 39.91600036621094], [41.90399932861328, 27.639999389648438, 40.80099868774414], [41.01300048828125, 27.368999481201172, 41.94499969482422]], [[40.452999114990234, 26.2189998626709, 42.18199920654297], [39.58599853515625, 25.986000061035156, 43.34199905395508], [38.141998291015625, 26.388999938964844, 43.07600021362305]], [[37.67900085449219, 26.45400047302246, 41.83399963378906], [36.29899978637695, 26.836000442504883, 41.54100036621094], [35.444000244140625, 25.66699981689453, 41.13199996948242]], [[34.125, 25.788000106811523, 41.09400177001953], [33.23099899291992, 24.72599983215332, 40.606998443603516], [33.59600067138672, 24.393999099731445, 39.15999984741211]], [[33.61199951171875, 23.114999771118164, 38.87099838256836], [34.02299880981445, 22.621999740600586, 37.54999923706055], [32.92900085449219, 22.006000518798828, 36.7140007019043]], [[32.641998291015625, 22.577999114990234, 35.57699966430664], [31.65999984741211, 22.033000946044922, 34.6510009765625], [32.051998138427734, 20.5939998626709, 34.301998138427734]], [[31.093000411987305, 19.68899917602539, 34.130001068115234], [31.391000747680664, 18.291000366210938, 33.858001708984375], [31.555999755859375, 17.868000030517578, 32.43199920654297]], [[30.98699951171875, 18.490999221801758, 31.406999588012695], [31.211000442504883, 18.06399917602539, 30.017000198364258], [32.11000061035156, 19.066999435424805, 29.341999053955078]], [[33.2239990234375, 18.672000885009766, 28.71500015258789], [34.09400177001953, 19.64900016784668, 28.073999404907227], [34.10300064086914, 19.44099998474121, 26.54800033569336]], [[34.55799865722656, 20.433000564575195, 25.816999435424805], [34.73400115966797, 20.32200050354004, 24.378999710083008], [35.9739990234375, 21.134000778198242, 23.988000869750977]], [[36.645999908447266, 20.701000213623047, 22.89699935913086], [37.85100173950195, 21.510000228881836, 22.527999877929688], [37.61600112915039, 22.4950008392334, 21.420000076293945]], [[36.8380012512207, 22.21299934387207, 20.391000747680664], [36.652000427246094, 23.172000885009766, 19.30900001525879], [35.215999603271484, 23.62299919128418, 19.17799949645996]], [[34.292999267578125, 22.684999465942383, 19.1299991607666], [32.90299987792969, 23.049999237060547, 19.042999267578125], [32.0359992980957, 21.91699981689453, 19.57900047302246]], [[30.791000366210938, 22.243000030517578, 19.82900047302246], [29.87299919128418, 21.253000259399414, 20.381999969482422], [28.45599937438965, 21.78499984741211, 20.216999053955078]], [[27.569000244140625, 20.922000885009766, 19.72599983215332], [26.163999557495117, 21.30500030517578, 19.666000366210938], [25.481000900268555, 20.6560001373291, 20.836999893188477]], [[24.76300048828125, 21.466999053955078, 21.636999130249023], [24.016000747680664, 20.979999542236328, 22.790000915527344], [22.548999786376953, 21.10300064086914, 22.375]], [[21.81599998474121, 20.02899932861328, 22.631999969482422], [20.427000045776367, 20.034000396728516, 22.267000198364258], [19.5310001373291, 20.333999633789062, 23.43400001525879]], [[18.636999130249023, 21.30500030517578, 23.320999145507812], [17.687000274658203, 21.5939998626709, 24.420000076293945], [16.343000411987305, 21.04800033569336, 23.871000289916992]], [[16.195999145507812, 19.738000869750977, 24.05699920654297], [15.04699993133545, 19.025999069213867, 23.518999099731445], [14.86400032043457, 19.2810001373291, 22.023000717163086]], [[15.895000457763672, 19.256000518798828, 21.243999481201172], [15.866000175476074, 19.452999114990234, 19.816999435424805], [16.35099983215332, 20.809999465942383, 19.34600067138672]], [[16.187000274658203, 21.86400032043457, 20.160999298095703], [16.68000030517578, 23.187999725341797, 19.81100082397461], [18.190000534057617, 23.195999145507812, 20.01300048828125]], [[18.93899917602539, 23.523000717163086, 18.966999053955078], [20.375999450683594, 23.49799919128418, 19.07699966430664], [20.9950008392334, 24.763999938964844, 19.659000396728516]], [[21.990999221801758, 24.589000701904297, 20.530000686645508], [22.773000717163086, 25.73200035095215, 21.045000076293945], [24.177000045776367, 25.378999710083008, 20.479000091552734]], [[24.763999938964844, 26.174999237060547, 19.649999618530273], [26.05900001525879, 25.820999145507812, 19.049999237060547], [27.14900016784668, 26.676000595092773, 19.71500015258789]], [[28.003000259399414, 25.927000045776367, 20.395999908447266], [29.149999618530273, 26.44099998474121, 21.091999053955078], [30.41200065612793, 26.32900047302246, 20.240999221801758]], [[31.128000259399414, 27.444000244140625, 20.139999389648438], [32.36399841308594, 27.450000762939453, 19.42099952697754], [33.51300048828125, 28.086000442504883, 20.208999633789062]], [[34.67300033569336, 27.468000411987305, 20.06100082397461], [35.88999938964844, 28.115999221801758, 20.649999618530273], [36.66899871826172, 28.573999404907227, 19.413000106811523]], [[36.775001525878906, 29.85300064086914, 19.16900062561035], [37.459999084472656, 30.393999099731445, 17.992000579833984], [38.78799819946289, 31.05299949645996, 18.240999221801758]], [[39.27000045776367, 31.0049991607666, 19.485000610351562], [40.58300018310547, 31.540000915527344, 19.85300064086914], [40.430999755859375, 32.47200012207031, 21.062999725341797]], [[41.564998626708984, 32.90999984741211, 21.583999633789062], [41.54600143432617, 33.81100082397461, 22.715999603271484], [41.03300094604492, 35.21500015258789, 22.45400047302246]], [[40.71500015258789, 35.891998291015625, 23.584999084472656], [40.25699996948242, 37.263999938964844, 23.468000411987305], [40.29199981689453, 37.87699890136719, 24.841999053955078]], [[39.125999450683594, 38.305999755859375, 25.336000442504883], [39.04399871826172, 38.8849983215332, 26.67799949645996], [39.61600112915039, 37.941001892089844, 27.716999053955078]], [[39.362998962402344, 36.6609992980957, 27.52400016784668], [39.91999816894531, 35.60100173950195, 28.357999801635742], [40.51599884033203, 34.569000244140625, 27.395000457763672]], [[41.52000045776367, 33.77199935913086, 27.788999557495117], [42.04100036621094, 32.779998779296875, 26.857999801635742], [41.09299850463867, 31.672000885009766, 26.457000732421875]], [[40.055999755859375, 31.400999069213867, 27.23699951171875], [39.13800048828125, 30.30500030517578, 26.983999252319336], [37.744998931884766, 30.750999450683594, 26.55900001525879]], [[37.52399826049805, 32.02799987792969, 26.339000701904297], [36.22200012207031, 32.55699920654297, 26.006999969482422], [35.63800048828125, 31.90399932861328, 24.761999130249023]], [[34.38600158691406, 31.583999633789062, 24.798999786376953], [33.67499923706055, 30.937999725341797, 23.66699981689453], [32.57099914550781, 31.865999221801758, 23.19499969482422]], [[31.798999786376953, 31.43000030517578, 22.202999114990234], [30.599000930786133, 32.143001556396484, 21.777000427246094], [29.503000259399414, 31.072999954223633, 21.673999786376953]], [[28.270000457763672, 31.427000045776367, 21.757999420166016], [27.16200065612793, 30.43400001525879, 21.583999633789062], [26.232999801635742, 31.020999908447266, 20.506999969482422]], [[26.000999450683594, 30.305999755859375, 19.427000045776367], [25.136999130249023, 30.69499969482422, 18.292999267578125], [23.82200050354004, 29.94300079345703, 18.424999237060547]], [[22.690000534057617, 30.577999114990234, 18.25], [21.365999221801758, 29.97100067138672, 18.309999465942383], [20.850000381469727, 30.094999313354492, 16.85099983215332]], [[21.142000198364258, 29.131000518798828, 16.01799964904785], [20.750999450683594, 29.22599983215332, 14.604999542236328], [19.27400016784668, 29.415000915527344, 14.36400032043457]], [[18.371000289916992, 28.867000579833984, 15.142999649047852], [16.961999893188477, 29.045000076293945, 14.852999687194824], [16.451000213623047, 30.42300033569336, 15.213000297546387]], [[17.233999252319336, 31.145999908447266, 16.011999130249023], [16.78499984741211, 32.47200012207031, 16.44300079345703], [17.63599967956543, 33.582000732421875, 15.847000122070312]], [[18.815000534057617, 33.29399871826172, 15.309000015258789], [19.74799919128418, 34.21900177001953, 14.770000457763672], [20.28499984741211, 35.21099853515625, 15.843999862670898]], [[20.535999298095703, 34.619998931884766, 16.989999771118164], [21.05900001525879, 35.3489990234375, 18.152000427246094], [22.41200065612793, 34.71200180053711, 18.510000228881836]], [[23.391000747680664, 35.55500030517578, 18.763999938964844], [24.704999923706055, 35.10599899291992, 19.152999877929688], [24.989999771118164, 35.64099884033203, 20.55900001525879]], [[25.402000427246094, 34.79199981689453, 21.499000549316406], [25.849000930786133, 35.33399963378906, 22.82200050354004], [27.37700080871582, 35.417999267578125, 22.687000274658203]], [[27.94099998474121, 36.577999114990234, 22.472999572753906], [29.341999053955078, 36.75600051879883, 22.29400062561035], [30.065000534057617, 36.77399826049805, 23.621000289916992]], [[29.441999435424805, 37.16699981689453, 24.718000411987305], [30.176000595092773, 37.17599868774414, 25.992000579833984], [31.136999130249023, 38.33399963378906, 26.079999923706055]], [[32.05500030517578, 38.268001556396484, 27.075000762939453], [33.007999420166016, 39.310001373291016, 27.304000854492188], [34.0260009765625, 39.62799835205078, 26.243000030517578]], [[34.25299835205078, 38.770999908447266, 25.257999420166016], [35.178001403808594, 39.11899948120117, 24.176000595092773], [34.59400177001953, 40.27799987792969, 23.364999771118164]], [[33.314998626708984, 40.61800003051758, 23.56999969482422], [32.702999114990234, 41.75199890136719, 22.915000915527344], [32.361000061035156, 42.85900115966797, 23.910999298095703]], [[32.59700012207031, 44.0989990234375, 23.538999557495117], [32.26900100708008, 45.2760009765625, 24.298999786376953], [30.99799919128418, 45.91600036621094, 23.714000701904297]], [[30.198999404907227, 46.518001556396484, 24.579999923706055], [28.95800018310547, 47.14500045776367, 24.183000564575195], [29.26300048828125, 48.49800109863281, 23.5310001373291]], [[28.290000915527344, 49.14799880981445, 22.898000717163086], [28.554000854492188, 50.38800048828125, 22.18199920654297], [28.851999282836914, 51.61899948120117, 23.038999557495117]], [[28.6299991607666, 51.58700180053711, 24.32200050354004], [28.86199951171875, 52.7140007019043, 25.226999282836914], [30.25, 52.597999572753906, 25.868000030517578]], [[30.87700080871582, 51.45100021362305, 25.716999053955078], [32.20500183105469, 51.12699890136719, 26.239999771118164], [33.32600021362305, 51.55699920654297, 25.332000732421875]], [[34.319000244140625, 52.27899932861328, 25.875999450683594], [35.3849983215332, 52.834999084472656, 25.06100082397461], [36.75299835205078, 52.29999923706055, 25.402999877929688]], [[36.88199996948242, 51.54499816894531, 26.472000122070312], [38.13800048828125, 50.98699951171875, 26.966999053955078], [38.209999084472656, 49.46099853515625, 26.843000411987305]], [[39.433998107910156, 48.922000885009766, 26.698999404907227], [39.61000061035156, 47.46900177001953, 26.607999801635742], [39.2599983215332, 46.79100036621094, 27.9060001373291]], [[39.30500030517578, 47.534000396728516, 29.02400016784668], [38.92900085449219, 46.965999603271484, 30.31100082397461], [39.94499969482422, 46.020999908447266, 30.945999145507812]], [[39.382999420166016, 45.04999923706055, 31.69499969482422], [40.189998626708984, 44.07099914550781, 32.40299987792969], [40.88600158691406, 43.090999603271484, 31.472000122070312]], [[42.20800018310547, 43.19499969482422, 31.31999969482422], [42.89799880981445, 42.25, 30.42300033569336], [43.904998779296875, 41.375, 31.142000198364258]], [[43.63199996948242, 41.08100128173828, 32.404998779296875], [44.5260009765625, 40.24700164794922, 33.19599914550781], [44.53200149536133, 38.80699920654297, 32.70899963378906]], [[43.54899978637695, 38.26499938964844, 32.00600051879883], [43.57699966430664, 36.9010009765625, 31.50200080871582], [43.349998474121094, 36.888999938964844, 30.003000259399414]], [[43.500999450683594, 38.0369987487793, 29.347999572753906], [43.266998291015625, 38.19300079345703, 27.93899917602539], [44.44900131225586, 37.78799819946289, 27.076000213623047]], [[44.17399978637695, 37.52399826049805, 25.82200050354004], [45.198001861572266, 37.22999954223633, 24.812000274658203], [44.98400115966797, 38.375999450683594, 23.81999969482422]], [[45.494998931884766, 39.534000396728516, 24.12299919128418], [45.32899856567383, 40.74700164794922, 23.34000015258789], [45.77000045776367, 40.54499816894531, 21.902000427246094]], [[46.83599853515625, 39.792999267578125, 21.649999618530273], [47.28900146484375, 39.63999938964844, 20.260000228881836], [46.41699981689453, 38.715999603271484, 19.46299934387207]], [[45.67100143432617, 37.79800033569336, 20.09000015258789], [44.77899932861328, 36.92599868774414, 19.334999084472656], [43.40800094604492, 37.54399871826172, 19.23699951171875]], [[43.104000091552734, 38.48099899291992, 20.148000717163086], [41.755001068115234, 39.06399917602539, 20.143999099731445], [41.24700164794922, 39.56100082397461, 18.82900047302246]], [[41.957000732421875, 40.457000732421875, 18.115999221801758], [41.47600173950195, 40.96500015258789, 16.833999633789062], [41.22200012207031, 39.832000732421875, 15.833000183105469]], [[42.0260009765625, 38.79100036621094, 15.838000297546387], [41.85900115966797, 37.67599868774414, 14.925999641418457], [40.569000244140625, 36.92399978637695, 15.258000373840332]], [[40.393001556396484, 36.66400146484375, 16.552000045776367], [39.14099884033203, 35.97100067138672, 16.961000442504883], [37.902000427246094, 36.72800064086914, 16.533000946044922]], [[37.9109992980957, 38.05099868774414, 16.613000869750977], [36.750999450683594, 38.874000549316406, 16.22800064086914], [36.49100112915039, 38.75600051879883, 14.725000381469727]], [[37.566001892089844, 38.6879997253418, 13.958000183105469], [37.45399856567383, 38.51900100708008, 12.510000228881836], [36.84199905395508, 37.16899871826172, 12.215999603271484]], [[37.138999938964844, 36.152000427246094, 12.979999542236328], [36.51599884033203, 34.819000244140625, 12.753999710083008], [35.060001373291016, 34.86800003051758, 13.192000389099121]], [[34.755001068115234, 35.56700134277344, 14.28600025177002], [33.39799880981445, 35.74800109863281, 14.758999824523926], [32.62799835205078, 36.525001525878906, 13.699999809265137]], [[33.244998931884766, 37.56100082397461, 13.128000259399414], [32.558998107910156, 38.32500076293945, 12.07699966430664], [32.26100158691406, 37.5, 10.845000267028809]], [[33.207000732421875, 36.6870002746582, 10.380000114440918], [32.959999084472656, 35.862998962402344, 9.187999725341797], [31.972999572753906, 34.722999572753906, 9.402000427246094]], [[31.9060001373291, 34.106998443603516, 10.585000038146973], [31.045000076293945, 33.000999450683594, 10.864999771118164], [29.608999252319336, 33.27399826049805, 11.178000450134277]], [[29.20599937438965, 34.42900085449219, 11.71399974822998], [27.829999923706055, 34.689998626708984, 12.062000274658203], [27.32699966430664, 35.999000549316406, 11.520000457763672]], [[27.40999984741211, 36.183998107910156, 10.1899995803833], [26.975000381469727, 37.43199920654297, 9.565999984741211], [25.4950008392334, 37.68600082397461, 9.711000442504883]], [[24.665000915527344, 36.696998596191406, 9.92300033569336], [23.242000579833984, 36.90999984741211, 10.031999588012695], [22.69499969482422, 37.09700012207031, 11.442999839782715]], [[23.523000717163086, 37.3380012512207, 12.437000274658203], [23.006999969482422, 37.51100158691406, 13.788000106811523], [22.11199951171875, 38.72999954223633, 13.845000267028809]], [[20.915000915527344, 38.715999603271484, 14.385000228881836], [20.099000930786133, 39.909000396728516, 14.45199966430664], [20.44700050354004, 40.645999908447266, 15.751999855041504]], [[20.750999450683594, 39.823001861572266, 16.742000579833984], [21.101999282836914, 40.369998931884766, 18.04199981689453], [22.437000274658203, 39.76599884033203, 18.493000030517578]], [[23.285999298095703, 40.63999938964844, 18.9950008392334], [24.597999572753906, 40.12900161743164, 19.482999801635742], [24.698999404907227, 40.611000061035156, 20.930999755859375]], [[24.822999954223633, 39.6870002746582, 21.875999450683594], [24.804000854492188, 40.108001708984375, 23.31100082397461], [26.219999313354492, 40.03900146484375, 23.87700080871582]], [[26.726999282836914, 41.14799880981445, 24.429000854492188], [28.07699966430664, 41.14400100708008, 25.01300048828125], [27.992000579833984, 40.696998596191406, 26.469999313354492]], [[29.142000198364258, 40.59000015258789, 27.155000686645508], [29.284000396728516, 40.1879997253418, 28.520000457763672], [28.77199935913086, 41.25699996948242, 29.475000381469727]], [[28.889999389648438, 42.516998291015625, 29.106000900268555], [28.44300079345703, 43.63399887084961, 29.93600082397461], [27.884000778198242, 44.77000045776367, 29.06800079345703]], [[26.77400016784668, 45.39500045776367, 29.47100067138672], [26.243000030517578, 46.492000579833984, 28.68600082397461], [25.08799934387207, 46.071998596191406, 27.7810001373291]], [[24.569000244140625, 47.00199890136719, 26.996999740600586], [23.457000732421875, 46.71099853515625, 26.104000091552734], [23.840999603271484, 45.83399963378906, 24.915000915527344]], [[22.840999603271484, 45.00199890136719, 24.590999603271484], [23.007999420166016, 44.13999938964844, 23.413000106811523], [22.829999923706055, 45.05500030517578, 22.19499969482422]], [[23.159000396728516, 44.573001861572266, 21.02400016784668], [23.070999145507812, 45.31100082397461, 19.78499984741211], [23.086999893188477, 44.29399871826172, 18.6299991607666]], [[23.68899917602539, 44.766998291015625, 17.541000366210938], [23.777000427246094, 43.91299819946289, 16.34000015258789], [25.266000747680664, 43.65399932861328, 16.065000534057617]], [[25.562999725341797, 43.20000076293945, 14.866999626159668], [26.917999267578125, 42.89699935913086, 14.51099967956543], [27.836999893188477, 44.082000732421875, 14.550000190734863]], [[27.426000595092773, 45.3390007019043, 14.729999542236328], [28.354000091552734, 46.44300079345703, 14.829000473022461], [29.24799919128418, 46.354000091552734, 16.090999603271484]], [[28.77899932861328, 45.5880012512207, 17.062000274658203], [29.516000747680664, 45.417999267578125, 18.319000244140625], [30.81399917602539, 44.71900177001953, 17.982999801635742]], [[30.82699966430664, 43.882999420166016, 16.95599937438965], [32.071998596191406, 43.21799850463867, 16.590999603271484], [33.08300018310547, 44.231998443603516, 16.06599998474121]], [[32.724998474121094, 45.082000732421875, 15.10200023651123], [33.652000427246094, 46.106998443603516, 14.61299991607666], [34.0260009765625, 47.08700180053711, 15.74899959564209]], [[33.0099983215332, 47.44200134277344, 16.541000366210938], [33.25400161743164, 48.37699890136719, 17.64299964904785], [34.340999603271484, 47.832000732421875, 18.577999114990234]], [[34.25899887084961, 46.551998138427734, 18.923999786376953], [35.26599884033203, 45.95199966430664, 19.79800033569336], [36.60200119018555, 45.895999908447266, 19.118999481201172]], [[36.63600158691406, 45.48699951171875, 17.836999893188477], [37.867000579833984, 45.42399978637695, 17.072999954223633], [38.542999267578125, 46.80699920654297, 17.03499984741211]], [[37.80400085449219, 47.88100051879883, 16.94300079345703], [38.45899963378906, 49.22200012207031, 16.947999954223633], [39.14799880981445, 49.48500061035156, 18.274999618530273]], [[38.47999954223633, 49.1510009765625, 19.35700035095215], [39.04199981689453, 49.34299850463867, 20.731000900268555], [40.30099868774414, 48.52899932861328, 20.90999984741211]], [[40.3129997253418, 47.255001068115234, 20.57699966430664], [41.483001708984375, 46.388999938964844, 20.73200035095215], [42.63399887084961, 46.8650016784668, 19.84000015258789]], [[42.36899948120117, 47.09700012207031, 18.554000854492188], [43.37099838256836, 47.56999969482422, 17.60700035095215], [43.92300033569336, 48.935001373291016, 17.988000869750977]], [[43.060001373291016, 49.915000915527344, 18.361000061035156], [43.650001525878906, 51.20199966430664, 18.756000518798828], [44.573001861572266, 51.04999923706055, 19.954999923706055]], [[44.24599838256836, 50.1619987487793, 20.87299919128418], [45.09400177001953, 49.97999954223633, 22.06100082397461], [46.374000549316406, 49.30099868774414, 21.68199920654297]], [[46.3129997253418, 48.26599884033203, 20.8799991607666], [47.50400161743164, 47.55799865722656, 20.39900016784668], [48.388999938964844, 48.57600021362305, 19.648000717163086]], [[47.797000885009766, 49.391998291015625, 18.77899932861328], [48.57699966430664, 50.37900161743164, 18.062999725341797], [49.2869987487793, 51.35200119018555, 18.993999481201172]], [[48.62200164794922, 51.970001220703125, 19.94700050354004], [49.22800064086914, 52.95000076293945, 20.82200050354004], [50.084999084472656, 52.42399978637695, 21.95400047302246]], [[49.9370002746582, 51.19599914550781, 22.349000930786133], [50.79999923706055, 50.69300079345703, 23.45599937438965], [51.99599838256836, 50.00899887084961, 22.822999954223633]], [[52.025001525878906, 50.03200149536133, 21.490999221801758], [53.069000244140625, 49.387001037597656, 20.708999633789062], [53.87099838256836, 50.375, 19.858999252319336]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]], [[Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity], [Infinity, Infinity, Infinity]]], "seq": "AQKSVKISDDISITQLSDKVYTYVSLAEIEGWGMVPSNGMIVINNHQAALLDTPINDAQTEMLVNWVTDSLHAKVTTFIPNHWHGDCIGGLGYLQRKGVQSYANQMTIDLAKEKGLPVPEHGFTDSLTVSLDGMPLQCYYLGGGHATDNIVVWLPTENILFGGCMLKDNQTTSIGNISDADVTAWPKTLDKVKAKFPSARYVVPGHGNYGGTELIEHTKQIVNQYIESTSKP"}] \ No newline at end of file diff --git a/weight/esm-main/tests/test_alphabet.py b/weight/esm-main/tests/test_alphabet.py new file mode 100644 index 0000000000000000000000000000000000000000..e200becaf3d368361d8b58b2ff4155343ad4b820 --- /dev/null +++ b/weight/esm-main/tests/test_alphabet.py @@ -0,0 +1,87 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +def _test_esm1b(alphabet): + import torch + + batch_converter = alphabet.get_batch_converter() + + data = [ + ("protein1", "MKTVRQG"), + ("protein2 with mask", "KALTAISQP"), + ("protein3", "K A I S Q"), + ] + _, _, batch_tokens = batch_converter(data) + expected_tokens = torch.tensor( + [ + [0, 20, 15, 11, 7, 10, 16, 6, 2, 1, 1, 1], + [0, 15, 5, 4, 11, 5, 32, 12, 8, 16, 14, 2], + [0, 15, 5, 32, 12, 8, 16, 2, 1, 1, 1, 1], + ] + ) + assert torch.allclose(batch_tokens, expected_tokens) + + +def _test_esm1b_truncation(alphabet): + import torch + + batch_converter = alphabet.get_batch_converter(truncation_seq_length=10) + + data = [ + ("protein1", "MKTVRQGMKTVRQG"), + ("protein2 with mask", "KALTAISQPISQP"), + ("protein3", "K A I S Q"), + ] + _, _, batch_tokens = batch_converter(data) + expected_tokens = torch.tensor( + [ + [0, 20, 15, 11, 7, 10, 16, 6, 20, 15, 11, 2], + [0, 15, 5, 4, 11, 5, 32, 12, 8, 16, 14, 2], + [0, 15, 5, 32, 12, 8, 16, 2, 1, 1, 1, 1], + ] + ) + assert torch.allclose(batch_tokens, expected_tokens) + + +def test_esm1b_alphabet(): + import esm + + _, alphabet = esm.pretrained.esm1b_t33_650M_UR50S() + _test_esm1b(alphabet) + _test_esm1b_truncation(alphabet) + + +def test_esm1v_alphabet(): + import esm + + _, alphabet = esm.pretrained.esm1v_t33_650M_UR90S_1() + _test_esm1b(alphabet) + _test_esm1b_truncation(alphabet) + + +def test_esm1_msa1b_alphabet(): + import torch + import esm + + # Load ESM-1b model + _, alphabet = esm.pretrained.esm_msa1b_t12_100M_UR50S() + batch_converter = alphabet.get_batch_converter() + + data = [ + ("protein1", "MKTVRQG"), + ("protein2", "KALTRAI"), + ("protein3", "KAAISQQ"), + ] + _, _, batch_tokens = batch_converter(data) + expected_tokens = torch.tensor( + [ + [ + [0, 20, 15, 11, 7, 10, 16, 6], + [0, 15, 5, 4, 11, 10, 5, 12], + [0, 15, 5, 5, 12, 8, 16, 16], + ] + ] + ) + assert torch.allclose(batch_tokens, expected_tokens)