Instructions to use Synthyra/ESMplusplus_6B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Synthyra/ESMplusplus_6B with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("fill-mask", model="Synthyra/ESMplusplus_6B", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForMaskedLM model = AutoModelForMaskedLM.from_pretrained("Synthyra/ESMplusplus_6B", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Upload modeling_esm_plusplus.py with huggingface_hub
Browse files- modeling_esm_plusplus.py +1354 -0
modeling_esm_plusplus.py
ADDED
|
@@ -0,0 +1,1354 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
"""
|
| 3 |
+
ESM++ model implementation.
|
| 4 |
+
|
| 5 |
+
ESM++ is a faithful implementation of ESMC that allows for batching and standard Huggingface compatibility
|
| 6 |
+
The ESM Python package is not required
|
| 7 |
+
|
| 8 |
+
Modified from https://github.com/Biohub/esm
|
| 9 |
+
License: https://github.com/Biohub/esm/blob/main/LICENSE.md
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import math
|
| 13 |
+
import os
|
| 14 |
+
import json
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn as nn
|
| 17 |
+
import torch.nn.functional as F
|
| 18 |
+
from dataclasses import dataclass
|
| 19 |
+
from functools import cache, partial
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
from typing import Optional, Tuple, Union, List
|
| 22 |
+
from einops import rearrange, repeat
|
| 23 |
+
from huggingface_hub import snapshot_download
|
| 24 |
+
from safetensors.torch import load_file as load_safetensors_file
|
| 25 |
+
from tokenizers import Tokenizer
|
| 26 |
+
from tokenizers.models import BPE
|
| 27 |
+
from tokenizers.processors import TemplateProcessing
|
| 28 |
+
from transformers import PreTrainedModel, PreTrainedTokenizerFast, PretrainedConfig
|
| 29 |
+
from transformers.modeling_outputs import ModelOutput
|
| 30 |
+
|
| 31 |
+
try:
|
| 32 |
+
from fastplms.attention import (
|
| 33 |
+
AttentionBackend, VALID_ATTENTION_BACKENDS,
|
| 34 |
+
resolve_attention_backend, get_attention_mask,
|
| 35 |
+
_get_flex_attention_fn,
|
| 36 |
+
_ensure_flash_kernels_loaded, FLASH_KERNEL, FLASH_KERNEL_VARIANT,
|
| 37 |
+
_kernels_flash_forward, _kernels_flash_varlen_forward,
|
| 38 |
+
kernels_flash_attention_func,
|
| 39 |
+
index_first_axis, index_put_first_axis, pad_input, _unpad_input,
|
| 40 |
+
create_block_mask, flex_attention, BlockMask,
|
| 41 |
+
)
|
| 42 |
+
from fastplms.embedding_mixin import Pooler, EmbeddingMixin, ProteinDataset, parse_fasta, build_collator
|
| 43 |
+
except ImportError:
|
| 44 |
+
pass # Running as HF Hub composite; shared definitions are above
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class ESMplusplusConfig(PretrainedConfig):
|
| 48 |
+
"""Configuration class for ESM++ model.
|
| 49 |
+
|
| 50 |
+
Args:
|
| 51 |
+
vocab_size: Size of the vocabulary
|
| 52 |
+
hidden_size: Dimension of hidden layers
|
| 53 |
+
num_attention_heads: Number of attention heads
|
| 54 |
+
num_hidden_layers: Number of transformer layers
|
| 55 |
+
num_labels: Number of output labels for classification
|
| 56 |
+
problem_type: Type of problem - regression, single/multi label classification
|
| 57 |
+
"""
|
| 58 |
+
model_type = "ESMplusplus"
|
| 59 |
+
def __init__(
|
| 60 |
+
self,
|
| 61 |
+
vocab_size: int = 64,
|
| 62 |
+
hidden_size: int = 960,
|
| 63 |
+
num_attention_heads: int = 15,
|
| 64 |
+
num_hidden_layers: int = 30,
|
| 65 |
+
num_labels: int = 2,
|
| 66 |
+
problem_type: Optional[str] = None,
|
| 67 |
+
dropout: float = 0.0,
|
| 68 |
+
initializer_range: float = 0.02,
|
| 69 |
+
attn_backend: str = "sdpa",
|
| 70 |
+
**kwargs,
|
| 71 |
+
):
|
| 72 |
+
super().__init__(**kwargs)
|
| 73 |
+
self.vocab_size = vocab_size
|
| 74 |
+
self.hidden_size = hidden_size
|
| 75 |
+
self.num_attention_heads = num_attention_heads
|
| 76 |
+
self.num_hidden_layers = num_hidden_layers
|
| 77 |
+
self.num_labels = num_labels
|
| 78 |
+
self.problem_type = problem_type
|
| 79 |
+
self.dropout = dropout
|
| 80 |
+
self.initializer_range = initializer_range
|
| 81 |
+
self.tie_word_embeddings = False
|
| 82 |
+
self.attn_backend = attn_backend
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
### Rotary Embeddings
|
| 86 |
+
def rotate_half(x: torch.Tensor, interleaved: bool = False) -> torch.Tensor:
|
| 87 |
+
"""Rotates half the hidden dims of the input."""
|
| 88 |
+
if not interleaved:
|
| 89 |
+
x1, x2 = x.chunk(2, dim=-1)
|
| 90 |
+
return torch.cat((-x2, x1), dim=-1)
|
| 91 |
+
else:
|
| 92 |
+
x1, x2 = x[..., ::2], x[..., 1::2]
|
| 93 |
+
return rearrange(
|
| 94 |
+
torch.stack((-x2, x1), dim=-1), "... d two -> ... (d two)", two=2
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def apply_rotary_emb_torch(
|
| 99 |
+
x: torch.Tensor,
|
| 100 |
+
cos: torch.Tensor,
|
| 101 |
+
sin: torch.Tensor,
|
| 102 |
+
interleaved: bool = False,
|
| 103 |
+
_inplace: bool = False,
|
| 104 |
+
) -> torch.Tensor:
|
| 105 |
+
"""Apply rotary embeddings to input based on cos and sin."""
|
| 106 |
+
ro_dim = cos.shape[-1] * 2
|
| 107 |
+
assert ro_dim <= x.shape[-1]
|
| 108 |
+
seqlen = x.size(1)
|
| 109 |
+
cos = cos[:seqlen]
|
| 110 |
+
sin = sin[:seqlen]
|
| 111 |
+
cos = repeat(cos, "s d -> s 1 (2 d)")
|
| 112 |
+
sin = repeat(sin, "s d -> s 1 (2 d)")
|
| 113 |
+
return torch.cat(
|
| 114 |
+
[
|
| 115 |
+
x[..., :ro_dim] * cos + rotate_half(x[..., :ro_dim], interleaved) * sin,
|
| 116 |
+
x[..., ro_dim:],
|
| 117 |
+
],
|
| 118 |
+
dim=-1,
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
class RotaryEmbedding(torch.nn.Module):
|
| 123 |
+
"""Rotary position embeddings.
|
| 124 |
+
|
| 125 |
+
Based on the paper "RoFormer: Enhanced Transformer with Rotary Position Embedding"
|
| 126 |
+
|
| 127 |
+
Args:
|
| 128 |
+
dim: Dimension of the embedding
|
| 129 |
+
base: Base for computing angular frequencies
|
| 130 |
+
interleaved: Whether to use interleaved rotations
|
| 131 |
+
scale_base: Base for scaling
|
| 132 |
+
scaling_factor: Factor for scaling positions
|
| 133 |
+
pos_idx_in_fp32: Whether to compute position indices in fp32
|
| 134 |
+
device: Computation device
|
| 135 |
+
"""
|
| 136 |
+
def __init__(
|
| 137 |
+
self,
|
| 138 |
+
dim: int,
|
| 139 |
+
base: float = 10000.0,
|
| 140 |
+
interleaved: bool = False,
|
| 141 |
+
scale_base: Optional[float] = None,
|
| 142 |
+
scaling_factor: float = 1.0,
|
| 143 |
+
pos_idx_in_fp32: bool = True,
|
| 144 |
+
device: Optional[torch.device] = None,
|
| 145 |
+
):
|
| 146 |
+
super().__init__()
|
| 147 |
+
self.dim = dim
|
| 148 |
+
self.base = float(base)
|
| 149 |
+
self.pos_idx_in_fp32 = pos_idx_in_fp32
|
| 150 |
+
self.interleaved = interleaved
|
| 151 |
+
self.scale_base = scale_base
|
| 152 |
+
self.scaling_factor = scaling_factor
|
| 153 |
+
self.device = device
|
| 154 |
+
|
| 155 |
+
self._seq_len_cached = 0
|
| 156 |
+
self._cos_cached = None
|
| 157 |
+
self._sin_cached = None
|
| 158 |
+
self._cos_k_cached = None
|
| 159 |
+
self._sin_k_cached = None
|
| 160 |
+
self.reset_parameters()
|
| 161 |
+
|
| 162 |
+
def reset_parameters(self):
|
| 163 |
+
"""Reset the parameters of the embedding."""
|
| 164 |
+
if "inv_freq" in self._buffers and isinstance(self._buffers["inv_freq"], torch.Tensor):
|
| 165 |
+
buffer_device = self._buffers["inv_freq"].device
|
| 166 |
+
else:
|
| 167 |
+
buffer_device = self.device
|
| 168 |
+
inv_freq = self._compute_inv_freq(buffer_device)
|
| 169 |
+
self._seq_len_cached = 0
|
| 170 |
+
self._cos_cached = None
|
| 171 |
+
self._sin_cached = None
|
| 172 |
+
self._cos_k_cached = None
|
| 173 |
+
self._sin_k_cached = None
|
| 174 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
| 175 |
+
arange = torch.arange(0, self.dim, 2, device=buffer_device, dtype=torch.float32)
|
| 176 |
+
scale = (
|
| 177 |
+
(arange + 0.4 * self.dim) / (1.4 * self.dim)
|
| 178 |
+
if self.scale_base is not None
|
| 179 |
+
else None
|
| 180 |
+
)
|
| 181 |
+
self.register_buffer("scale", scale)
|
| 182 |
+
|
| 183 |
+
def _compute_inv_freq(self, device: Optional[torch.device] = None) -> torch.Tensor:
|
| 184 |
+
"""Compute inverse frequency bands.
|
| 185 |
+
|
| 186 |
+
Always computes on CPU then moves to the requested device. This matches
|
| 187 |
+
native Biohub ESMC, which computes inv_freq on CPU at
|
| 188 |
+
`__init__` and migrates via `.to(device)`. Computing directly on GPU
|
| 189 |
+
gives a ~3.7e-9 bit-level difference in inv_freq (fp32 transcendental
|
| 190 |
+
precision differs between CPU and GPU), which compounds through the 30
|
| 191 |
+
attention layers to ~1e-3 mse divergence from native at
|
| 192 |
+
`hidden_states[-2]`. See testing/parity_debug_rotary.py.
|
| 193 |
+
"""
|
| 194 |
+
cpu_inv_freq = 1 / (
|
| 195 |
+
self.base
|
| 196 |
+
** (
|
| 197 |
+
torch.arange(0, self.dim, 2, device="cpu", dtype=torch.float32)
|
| 198 |
+
/ self.dim
|
| 199 |
+
)
|
| 200 |
+
)
|
| 201 |
+
if device is not None and torch.device(device).type != "cpu":
|
| 202 |
+
return cpu_inv_freq.to(device)
|
| 203 |
+
return cpu_inv_freq
|
| 204 |
+
|
| 205 |
+
def _update_cos_sin_cache(self, seqlen: int, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None):
|
| 206 |
+
"""Update the cached cosine and sine values."""
|
| 207 |
+
if (
|
| 208 |
+
seqlen > self._seq_len_cached
|
| 209 |
+
or self._cos_cached is None
|
| 210 |
+
or self._cos_cached.device != device
|
| 211 |
+
or self._cos_cached.dtype != dtype
|
| 212 |
+
or (self.training and self._cos_cached.is_inference())
|
| 213 |
+
):
|
| 214 |
+
self._seq_len_cached = seqlen
|
| 215 |
+
if self.pos_idx_in_fp32:
|
| 216 |
+
t = torch.arange(seqlen, device=device, dtype=torch.float32)
|
| 217 |
+
t /= self.scaling_factor
|
| 218 |
+
if self.inv_freq.dtype != torch.float32:
|
| 219 |
+
inv_freq = self.inv_freq.to(torch.float32)
|
| 220 |
+
else:
|
| 221 |
+
inv_freq = self.inv_freq
|
| 222 |
+
else:
|
| 223 |
+
t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype)
|
| 224 |
+
t /= self.scaling_factor
|
| 225 |
+
inv_freq = self.inv_freq
|
| 226 |
+
freqs = torch.outer(t, inv_freq)
|
| 227 |
+
|
| 228 |
+
if self.scale is None:
|
| 229 |
+
self._cos_cached = torch.cos(freqs).to(dtype)
|
| 230 |
+
self._sin_cached = torch.sin(freqs).to(dtype)
|
| 231 |
+
else:
|
| 232 |
+
power = (
|
| 233 |
+
torch.arange(
|
| 234 |
+
seqlen, dtype=self.scale.dtype, device=self.scale.device
|
| 235 |
+
)
|
| 236 |
+
- seqlen // 2
|
| 237 |
+
) / self.scale_base
|
| 238 |
+
scale = self.scale.to(device=power.device) ** power.unsqueeze(-1)
|
| 239 |
+
self._cos_cached = (torch.cos(freqs) * scale).to(dtype)
|
| 240 |
+
self._sin_cached = (torch.sin(freqs) * scale).to(dtype)
|
| 241 |
+
self._cos_k_cached = (torch.cos(freqs) / scale).to(dtype)
|
| 242 |
+
self._sin_k_cached = (torch.sin(freqs) / scale).to(dtype)
|
| 243 |
+
|
| 244 |
+
def forward(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 245 |
+
"""Apply rotary embeddings to queries and keys.
|
| 246 |
+
|
| 247 |
+
Args:
|
| 248 |
+
q: Query tensor of shape (batch, seqlen, nheads, headdim)
|
| 249 |
+
k: Key tensor of shape (batch, seqlen, nheads, headdim)
|
| 250 |
+
|
| 251 |
+
Returns:
|
| 252 |
+
Tuple of rotated query and key tensors
|
| 253 |
+
"""
|
| 254 |
+
# NOTE: do NOT recompute inv_freq here if device has changed. The native
|
| 255 |
+
# ESMC implementation computes inv_freq once on CPU at __init__ and
|
| 256 |
+
# relies on PyTorch's `.to(device)` to migrate the buffer. Recomputing
|
| 257 |
+
# the values directly on GPU gives a ~3.7e-9 bit-level difference vs the
|
| 258 |
+
# CPU-computed-then-moved values due to fp32 transcendental precision,
|
| 259 |
+
# which compounds through 30 attention layers to ~1e-3 mse divergence
|
| 260 |
+
# from native at `hidden_states[-2]`. See testing/parity_debug_rotary.py.
|
| 261 |
+
self._update_cos_sin_cache(q.shape[1], device=q.device, dtype=q.dtype)
|
| 262 |
+
assert self._cos_cached is not None
|
| 263 |
+
assert self._sin_cached is not None
|
| 264 |
+
if self.scale is None:
|
| 265 |
+
return (
|
| 266 |
+
apply_rotary_emb_torch(
|
| 267 |
+
q,
|
| 268 |
+
self._cos_cached,
|
| 269 |
+
self._sin_cached,
|
| 270 |
+
self.interleaved,
|
| 271 |
+
True, # inplace=True
|
| 272 |
+
),
|
| 273 |
+
apply_rotary_emb_torch(
|
| 274 |
+
k,
|
| 275 |
+
self._cos_cached,
|
| 276 |
+
self._sin_cached,
|
| 277 |
+
self.interleaved,
|
| 278 |
+
True, # inplace=True
|
| 279 |
+
),
|
| 280 |
+
) # type: ignore
|
| 281 |
+
else:
|
| 282 |
+
assert False
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
### Feedforward Network Components
|
| 286 |
+
def swiglu_correction_fn(expansion_ratio: float, d_model: int) -> int:
|
| 287 |
+
"""Compute corrected dimension for SwiGLU."""
|
| 288 |
+
return int(((expansion_ratio * d_model) + 255) // 256 * 256)
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
class SwiGLU(nn.Module):
|
| 292 |
+
"""SwiGLU activation function."""
|
| 293 |
+
def __init__(self):
|
| 294 |
+
super(SwiGLU, self).__init__()
|
| 295 |
+
|
| 296 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 297 |
+
x1, x2 = x.chunk(2, dim=-1)
|
| 298 |
+
return F.silu(x1) * x2
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
def swiglu_ln_ffn(d_model: int, expansion_ratio: float) -> nn.Sequential:
|
| 302 |
+
"""Create SwiGLU feedforward network with layer normalization."""
|
| 303 |
+
return nn.Sequential(
|
| 304 |
+
nn.LayerNorm(d_model),
|
| 305 |
+
nn.Linear(
|
| 306 |
+
d_model, swiglu_correction_fn(expansion_ratio, d_model) * 2, bias=False
|
| 307 |
+
),
|
| 308 |
+
SwiGLU(),
|
| 309 |
+
nn.Linear(swiglu_correction_fn(expansion_ratio, d_model), d_model, bias=False),
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
### Attention
|
| 314 |
+
class MultiHeadAttention(nn.Module):
|
| 315 |
+
"""Multi-head attention with rotary embeddings and configurable backend.
|
| 316 |
+
|
| 317 |
+
Args:
|
| 318 |
+
d_model: Model dimension
|
| 319 |
+
n_heads: Number of attention heads
|
| 320 |
+
attn_backend: One of "auto", "kernels_flash", "flex", "sdpa"
|
| 321 |
+
"""
|
| 322 |
+
def __init__(
|
| 323 |
+
self,
|
| 324 |
+
d_model: int,
|
| 325 |
+
n_heads: int,
|
| 326 |
+
attn_backend: str = "sdpa",
|
| 327 |
+
):
|
| 328 |
+
super().__init__()
|
| 329 |
+
self.d_model = d_model
|
| 330 |
+
self.n_heads = n_heads
|
| 331 |
+
self.d_head = self.d_model // self.n_heads
|
| 332 |
+
self.scale = 1.0 / math.sqrt(self.d_head)
|
| 333 |
+
self.attn_backend = resolve_attention_backend(attn_backend)
|
| 334 |
+
self.layernorm_qkv = nn.Sequential(
|
| 335 |
+
nn.LayerNorm(d_model), nn.Linear(d_model, d_model * 3, bias=False)
|
| 336 |
+
)
|
| 337 |
+
self.out_proj = nn.Linear(d_model, d_model, bias=False)
|
| 338 |
+
self.q_ln = nn.LayerNorm(d_model, bias=False)
|
| 339 |
+
self.k_ln = nn.LayerNorm(d_model, bias=False)
|
| 340 |
+
self.reshaper = partial(rearrange, pattern="b s (h d) -> b h s d", h=n_heads)
|
| 341 |
+
self.rotary = RotaryEmbedding(d_model // n_heads)
|
| 342 |
+
|
| 343 |
+
def _apply_rotary(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 344 |
+
q = q.unflatten(-1, (self.n_heads, self.d_head))
|
| 345 |
+
k = k.unflatten(-1, (self.n_heads, self.d_head))
|
| 346 |
+
q, k = self.rotary(q, k)
|
| 347 |
+
q = q.flatten(-2, -1)
|
| 348 |
+
k = k.flatten(-2, -1)
|
| 349 |
+
return q, k
|
| 350 |
+
|
| 351 |
+
def forward(
|
| 352 |
+
self,
|
| 353 |
+
x: torch.Tensor,
|
| 354 |
+
attention_mask_2d: Optional[torch.Tensor] = None,
|
| 355 |
+
attention_mask_4d: Optional[torch.Tensor] = None,
|
| 356 |
+
flex_block_mask: Optional[BlockMask] = None,
|
| 357 |
+
output_attentions: bool = False,
|
| 358 |
+
output_s_max: bool = False,
|
| 359 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[List[torch.Tensor]]]:
|
| 360 |
+
qkv_BLD3 = self.layernorm_qkv(x)
|
| 361 |
+
query_BLD, key_BLD, value_BLD = torch.chunk(qkv_BLD3, 3, dim=-1)
|
| 362 |
+
query_BLD, key_BLD = (
|
| 363 |
+
self.q_ln(query_BLD).to(query_BLD.dtype),
|
| 364 |
+
self.k_ln(key_BLD).to(query_BLD.dtype),
|
| 365 |
+
)
|
| 366 |
+
query_BLD, key_BLD = self._apply_rotary(query_BLD, key_BLD)
|
| 367 |
+
query_BHLD, key_BHLD, value_BHLD = map(self.reshaper, (query_BLD, key_BLD, value_BLD))
|
| 368 |
+
|
| 369 |
+
attn_output, attn_weights, s_max = self._attn(
|
| 370 |
+
query_BHLD, key_BHLD, value_BHLD,
|
| 371 |
+
attention_mask_2d=attention_mask_2d,
|
| 372 |
+
attention_mask_4d=attention_mask_4d,
|
| 373 |
+
flex_block_mask=flex_block_mask,
|
| 374 |
+
output_attentions=output_attentions,
|
| 375 |
+
output_s_max=output_s_max,
|
| 376 |
+
)
|
| 377 |
+
|
| 378 |
+
output = self.out_proj(attn_output)
|
| 379 |
+
return output, attn_weights, s_max
|
| 380 |
+
|
| 381 |
+
def _attn(
|
| 382 |
+
self,
|
| 383 |
+
query_BHLD: torch.Tensor,
|
| 384 |
+
key_BHLD: torch.Tensor,
|
| 385 |
+
value_BHLD: torch.Tensor,
|
| 386 |
+
attention_mask_2d: Optional[torch.Tensor] = None,
|
| 387 |
+
attention_mask_4d: Optional[torch.Tensor] = None,
|
| 388 |
+
flex_block_mask: Optional[BlockMask] = None,
|
| 389 |
+
output_attentions: bool = False,
|
| 390 |
+
output_s_max: bool = False,
|
| 391 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[List[torch.Tensor]]]:
|
| 392 |
+
if output_attentions:
|
| 393 |
+
return self._manual_attn(query_BHLD, key_BHLD, value_BHLD, attention_mask_4d, output_s_max)
|
| 394 |
+
|
| 395 |
+
if self.attn_backend == AttentionBackend.KERNELS_FLASH:
|
| 396 |
+
attn_output, attn_weights = self._kernels_flash_attn(query_BHLD, key_BHLD, value_BHLD, attention_mask_2d)
|
| 397 |
+
elif self.attn_backend == AttentionBackend.FLEX:
|
| 398 |
+
attn_output, attn_weights = self._flex_attn(query_BHLD, key_BHLD, value_BHLD, flex_block_mask)
|
| 399 |
+
elif self.attn_backend == AttentionBackend.SDPA:
|
| 400 |
+
attn_output, attn_weights = self._sdpa_attn(query_BHLD, key_BHLD, value_BHLD, attention_mask_4d)
|
| 401 |
+
else:
|
| 402 |
+
raise AssertionError(f"Unsupported resolved backend: {self.attn_backend}")
|
| 403 |
+
|
| 404 |
+
s_max = self._compute_s_max(query_BHLD, key_BHLD) if output_s_max else None
|
| 405 |
+
return attn_output, attn_weights, s_max
|
| 406 |
+
|
| 407 |
+
@torch.no_grad()
|
| 408 |
+
def _compute_s_max(self, query_BHLD: torch.Tensor, key_BHLD: torch.Tensor) -> List[torch.Tensor]:
|
| 409 |
+
q_norm = torch.linalg.vector_norm(query_BHLD, dim=-1)
|
| 410 |
+
k_norm = torch.linalg.vector_norm(key_BHLD, dim=-1)
|
| 411 |
+
s_max_bound = (q_norm.max(dim=-1).values * k_norm.max(dim=-1).values).max(dim=0).values * self.scale
|
| 412 |
+
return [s_max_bound[h] for h in range(self.n_heads)]
|
| 413 |
+
|
| 414 |
+
def _manual_attn(
|
| 415 |
+
self,
|
| 416 |
+
query_BHLD: torch.Tensor,
|
| 417 |
+
key_BHLD: torch.Tensor,
|
| 418 |
+
value_BHLD: torch.Tensor,
|
| 419 |
+
attention_mask_4d: Optional[torch.Tensor] = None,
|
| 420 |
+
output_s_max: bool = False,
|
| 421 |
+
) -> Tuple[torch.Tensor, torch.Tensor, Optional[List[torch.Tensor]]]:
|
| 422 |
+
attn_weights = torch.matmul(query_BHLD, key_BHLD.transpose(-2, -1)) * self.scale
|
| 423 |
+
if attention_mask_4d is not None:
|
| 424 |
+
attn_weights = attn_weights.masked_fill(attention_mask_4d.logical_not(), float("-inf"))
|
| 425 |
+
attn_weights = F.softmax(attn_weights, dim=-1)
|
| 426 |
+
context_BHLD = torch.matmul(attn_weights, value_BHLD)
|
| 427 |
+
attn_output = rearrange(context_BHLD, "b h s d -> b s (h d)")
|
| 428 |
+
s_max = self._compute_s_max(query_BHLD, key_BHLD) if output_s_max else None
|
| 429 |
+
return attn_output, attn_weights, s_max
|
| 430 |
+
|
| 431 |
+
def _kernels_flash_attn(
|
| 432 |
+
self,
|
| 433 |
+
query_BHLD: torch.Tensor,
|
| 434 |
+
key_BHLD: torch.Tensor,
|
| 435 |
+
value_BHLD: torch.Tensor,
|
| 436 |
+
attention_mask_2d: Optional[torch.Tensor] = None,
|
| 437 |
+
) -> Tuple[torch.Tensor, None]:
|
| 438 |
+
query_BLHD = query_BHLD.transpose(1, 2).contiguous()
|
| 439 |
+
key_BLHD = key_BHLD.transpose(1, 2).contiguous()
|
| 440 |
+
value_BLHD = value_BHLD.transpose(1, 2).contiguous()
|
| 441 |
+
attn_output = kernels_flash_attention_func(
|
| 442 |
+
query_states=query_BLHD, key_states=key_BLHD, value_states=value_BLHD,
|
| 443 |
+
attention_mask_2d=attention_mask_2d, causal=False,
|
| 444 |
+
)
|
| 445 |
+
return rearrange(attn_output, "b s h d -> b s (h d)"), None
|
| 446 |
+
|
| 447 |
+
def _flex_attn(
|
| 448 |
+
self,
|
| 449 |
+
query_BHLD: torch.Tensor,
|
| 450 |
+
key_BHLD: torch.Tensor,
|
| 451 |
+
value_BHLD: torch.Tensor,
|
| 452 |
+
flex_block_mask: Optional[BlockMask] = None,
|
| 453 |
+
) -> Tuple[torch.Tensor, None]:
|
| 454 |
+
assert flex_attention is not None, "Flex attention is not available in this environment."
|
| 455 |
+
fn = _get_flex_attention_fn()
|
| 456 |
+
context_BHLD = fn(query_BHLD, key_BHLD, value_BHLD, block_mask=flex_block_mask, scale=self.scale)
|
| 457 |
+
return rearrange(context_BHLD, "b h s d -> b s (h d)"), None
|
| 458 |
+
|
| 459 |
+
def _sdpa_attn(
|
| 460 |
+
self,
|
| 461 |
+
query_BHLD: torch.Tensor,
|
| 462 |
+
key_BHLD: torch.Tensor,
|
| 463 |
+
value_BHLD: torch.Tensor,
|
| 464 |
+
attention_mask_4d: Optional[torch.Tensor] = None,
|
| 465 |
+
) -> Tuple[torch.Tensor, None]:
|
| 466 |
+
context_BHLD = F.scaled_dot_product_attention(
|
| 467 |
+
query_BHLD, key_BHLD, value_BHLD, attn_mask=attention_mask_4d, scale=self.scale,
|
| 468 |
+
)
|
| 469 |
+
return rearrange(context_BHLD, "b h s d -> b s (h d)"), None
|
| 470 |
+
|
| 471 |
+
|
| 472 |
+
### Regression Head
|
| 473 |
+
def RegressionHead(d_model: int, output_dim: int, hidden_dim: Optional[int] = None) -> nn.Module:
|
| 474 |
+
"""Create a regression head with optional hidden dimension.
|
| 475 |
+
|
| 476 |
+
Args:
|
| 477 |
+
d_model: Input dimension
|
| 478 |
+
output_dim: Output dimension
|
| 479 |
+
hidden_dim: Optional hidden dimension (defaults to d_model)
|
| 480 |
+
"""
|
| 481 |
+
hidden_dim = hidden_dim if hidden_dim is not None else d_model
|
| 482 |
+
return nn.Sequential(
|
| 483 |
+
nn.Linear(d_model, hidden_dim),
|
| 484 |
+
nn.GELU(),
|
| 485 |
+
nn.LayerNorm(hidden_dim),
|
| 486 |
+
nn.Linear(hidden_dim, output_dim),
|
| 487 |
+
)
|
| 488 |
+
|
| 489 |
+
|
| 490 |
+
### Transformer Block
|
| 491 |
+
class UnifiedTransformerBlock(nn.Module):
|
| 492 |
+
"""Transformer block with attention and feedforward layers."""
|
| 493 |
+
def __init__(
|
| 494 |
+
self,
|
| 495 |
+
d_model: int,
|
| 496 |
+
n_heads: int,
|
| 497 |
+
residue_scaling_factor: float = 1,
|
| 498 |
+
expansion_ratio: float = 8 / 3,
|
| 499 |
+
dropout: float = 0.0,
|
| 500 |
+
attn_backend: str = "sdpa",
|
| 501 |
+
):
|
| 502 |
+
super().__init__()
|
| 503 |
+
self.attn = MultiHeadAttention(d_model=d_model, n_heads=n_heads, attn_backend=attn_backend)
|
| 504 |
+
self.ffn = swiglu_ln_ffn(d_model, expansion_ratio)
|
| 505 |
+
self.scaling_factor = residue_scaling_factor
|
| 506 |
+
self.dropout = nn.Dropout(dropout)
|
| 507 |
+
|
| 508 |
+
def forward(
|
| 509 |
+
self,
|
| 510 |
+
x: torch.Tensor,
|
| 511 |
+
attention_mask_2d: Optional[torch.Tensor] = None,
|
| 512 |
+
attention_mask_4d: Optional[torch.Tensor] = None,
|
| 513 |
+
flex_block_mask: Optional[BlockMask] = None,
|
| 514 |
+
output_attentions: bool = False,
|
| 515 |
+
output_s_max: bool = False,
|
| 516 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[List[torch.Tensor]]]:
|
| 517 |
+
attn_output, attn_weights, s_max = self.attn(
|
| 518 |
+
x,
|
| 519 |
+
attention_mask_2d=attention_mask_2d,
|
| 520 |
+
attention_mask_4d=attention_mask_4d,
|
| 521 |
+
flex_block_mask=flex_block_mask,
|
| 522 |
+
output_attentions=output_attentions,
|
| 523 |
+
output_s_max=output_s_max,
|
| 524 |
+
)
|
| 525 |
+
x = x + self.dropout(attn_output) / self.scaling_factor
|
| 526 |
+
x = x + self.dropout(self.ffn(x)) / self.scaling_factor
|
| 527 |
+
return x, attn_weights, s_max
|
| 528 |
+
|
| 529 |
+
|
| 530 |
+
### Model Outputs
|
| 531 |
+
@dataclass
|
| 532 |
+
class TransformerOutput(ModelOutput):
|
| 533 |
+
"""Output type for transformer encoder."""
|
| 534 |
+
last_hidden_state: Optional[torch.Tensor] = None
|
| 535 |
+
hidden_states: Optional[Tuple[torch.Tensor]] = None
|
| 536 |
+
attentions: Optional[Tuple[torch.Tensor]] = None
|
| 537 |
+
s_max: Optional[Tuple[List[torch.Tensor], ...]] = None
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
@dataclass
|
| 541 |
+
class ESMplusplusOutput(ModelOutput):
|
| 542 |
+
"""Output type for ESM++ models."""
|
| 543 |
+
loss: Optional[torch.Tensor] = None
|
| 544 |
+
logits: Optional[torch.Tensor] = None
|
| 545 |
+
last_hidden_state: Optional[torch.Tensor] = None
|
| 546 |
+
hidden_states: Optional[Tuple[torch.Tensor]] = None
|
| 547 |
+
attentions: Optional[Tuple[torch.Tensor]] = None
|
| 548 |
+
s_max: Optional[Tuple[List[torch.Tensor], ...]] = None
|
| 549 |
+
|
| 550 |
+
|
| 551 |
+
### Transformer Stack
|
| 552 |
+
class TransformerStack(nn.Module):
|
| 553 |
+
"""Stack of transformer blocks."""
|
| 554 |
+
def __init__(
|
| 555 |
+
self,
|
| 556 |
+
d_model: int,
|
| 557 |
+
n_heads: int,
|
| 558 |
+
n_layers: int,
|
| 559 |
+
dropout: float = 0.0,
|
| 560 |
+
attn_backend: str = "sdpa",
|
| 561 |
+
):
|
| 562 |
+
super().__init__()
|
| 563 |
+
self.attention_backend = resolve_attention_backend(attn_backend)
|
| 564 |
+
self.blocks = nn.ModuleList(
|
| 565 |
+
[
|
| 566 |
+
UnifiedTransformerBlock(
|
| 567 |
+
d_model,
|
| 568 |
+
n_heads,
|
| 569 |
+
residue_scaling_factor=math.sqrt(n_layers / 36),
|
| 570 |
+
dropout=dropout,
|
| 571 |
+
attn_backend=attn_backend,
|
| 572 |
+
)
|
| 573 |
+
for i in range(n_layers)
|
| 574 |
+
]
|
| 575 |
+
)
|
| 576 |
+
self.norm = nn.LayerNorm(d_model, bias=False)
|
| 577 |
+
self.gradient_checkpointing = False
|
| 578 |
+
|
| 579 |
+
@property
|
| 580 |
+
def attn_backend(self) -> AttentionBackend:
|
| 581 |
+
return self.attention_backend
|
| 582 |
+
|
| 583 |
+
@attn_backend.setter
|
| 584 |
+
def attn_backend(self, backend: str) -> None:
|
| 585 |
+
resolved = resolve_attention_backend(backend)
|
| 586 |
+
self.attention_backend = resolved
|
| 587 |
+
for block in self.blocks:
|
| 588 |
+
block.attn.attn_backend = resolved
|
| 589 |
+
|
| 590 |
+
def forward(
|
| 591 |
+
self,
|
| 592 |
+
x: torch.Tensor,
|
| 593 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 594 |
+
output_hidden_states: Optional[bool] = False,
|
| 595 |
+
output_attentions: Optional[bool] = False,
|
| 596 |
+
output_s_max: Optional[bool] = False,
|
| 597 |
+
) -> TransformerOutput:
|
| 598 |
+
hidden_states = () if output_hidden_states else None
|
| 599 |
+
attentions = () if output_attentions else None
|
| 600 |
+
full_s_max = () if output_s_max else None
|
| 601 |
+
|
| 602 |
+
attention_mask_2d, attention_mask_4d, flex_block_mask = get_attention_mask(
|
| 603 |
+
effective_backend=self.attention_backend,
|
| 604 |
+
batch_size=x.shape[0],
|
| 605 |
+
seq_len=x.shape[1],
|
| 606 |
+
device=x.device,
|
| 607 |
+
attention_mask=attention_mask,
|
| 608 |
+
)
|
| 609 |
+
|
| 610 |
+
for block in self.blocks:
|
| 611 |
+
if self.gradient_checkpointing and self.training:
|
| 612 |
+
x, attn_weights, s_max = self._gradient_checkpointing_func(
|
| 613 |
+
block.__call__,
|
| 614 |
+
x=x,
|
| 615 |
+
attention_mask_2d=attention_mask_2d,
|
| 616 |
+
attention_mask_4d=attention_mask_4d,
|
| 617 |
+
flex_block_mask=flex_block_mask,
|
| 618 |
+
output_attentions=output_attentions,
|
| 619 |
+
output_s_max=output_s_max,
|
| 620 |
+
)
|
| 621 |
+
else:
|
| 622 |
+
x, attn_weights, s_max = block(
|
| 623 |
+
x=x,
|
| 624 |
+
attention_mask_2d=attention_mask_2d,
|
| 625 |
+
attention_mask_4d=attention_mask_4d,
|
| 626 |
+
flex_block_mask=flex_block_mask,
|
| 627 |
+
output_attentions=output_attentions,
|
| 628 |
+
output_s_max=output_s_max,
|
| 629 |
+
)
|
| 630 |
+
|
| 631 |
+
if attentions is not None:
|
| 632 |
+
attentions += (attn_weights,)
|
| 633 |
+
if output_hidden_states:
|
| 634 |
+
assert hidden_states is not None
|
| 635 |
+
hidden_states += (x,)
|
| 636 |
+
if full_s_max is not None:
|
| 637 |
+
full_s_max += (s_max,)
|
| 638 |
+
|
| 639 |
+
last_hidden_state = self.norm(x)
|
| 640 |
+
if output_hidden_states:
|
| 641 |
+
hidden_states += (last_hidden_state,)
|
| 642 |
+
|
| 643 |
+
return TransformerOutput(
|
| 644 |
+
last_hidden_state=last_hidden_state,
|
| 645 |
+
hidden_states=hidden_states,
|
| 646 |
+
attentions=attentions,
|
| 647 |
+
s_max=full_s_max,
|
| 648 |
+
)
|
| 649 |
+
|
| 650 |
+
|
| 651 |
+
class PreTrainedESMplusplusModel(PreTrainedModel):
|
| 652 |
+
"""
|
| 653 |
+
init weights for ESM++ models
|
| 654 |
+
"""
|
| 655 |
+
config_class = ESMplusplusConfig
|
| 656 |
+
base_model_prefix = "esm++"
|
| 657 |
+
supports_gradient_checkpointing = True
|
| 658 |
+
all_tied_weights_keys = {}
|
| 659 |
+
|
| 660 |
+
@classmethod
|
| 661 |
+
def is_remote_code(cls) -> bool:
|
| 662 |
+
# Prevent post-load reinitialization of tensors already loaded from checkpoints.
|
| 663 |
+
return True
|
| 664 |
+
|
| 665 |
+
def _init_weights(self, module):
|
| 666 |
+
"""Initialize the weights"""
|
| 667 |
+
# HF from_pretrained marks loaded parameters with `_is_hf_initialized`.
|
| 668 |
+
# Skip this module if any local parameter is already marked as loaded.
|
| 669 |
+
for parameter in module.parameters(recurse=False):
|
| 670 |
+
if "_is_hf_initialized" in parameter.__dict__ and parameter.__dict__["_is_hf_initialized"]:
|
| 671 |
+
return
|
| 672 |
+
|
| 673 |
+
if isinstance(module, nn.Linear):
|
| 674 |
+
nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
|
| 675 |
+
if module.bias is not None:
|
| 676 |
+
nn.init.zeros_(module.bias)
|
| 677 |
+
elif isinstance(module, nn.Embedding):
|
| 678 |
+
nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
|
| 679 |
+
if module.padding_idx is not None:
|
| 680 |
+
with torch.no_grad():
|
| 681 |
+
module.weight[module.padding_idx].zero_()
|
| 682 |
+
elif isinstance(module, nn.LayerNorm):
|
| 683 |
+
if module.bias is not None:
|
| 684 |
+
nn.init.zeros_(module.bias)
|
| 685 |
+
nn.init.ones_(module.weight)
|
| 686 |
+
|
| 687 |
+
@property
|
| 688 |
+
def attn_backend(self) -> str:
|
| 689 |
+
return self.config.attn_backend
|
| 690 |
+
|
| 691 |
+
@attn_backend.setter
|
| 692 |
+
def attn_backend(self, backend: str) -> None:
|
| 693 |
+
assert backend in VALID_ATTENTION_BACKENDS, f"Unsupported attn_backend: {backend}. Expected one of {VALID_ATTENTION_BACKENDS}."
|
| 694 |
+
self.config.attn_backend = backend
|
| 695 |
+
for module in self.modules():
|
| 696 |
+
if isinstance(module, TransformerStack):
|
| 697 |
+
module.attn_backend = backend
|
| 698 |
+
|
| 699 |
+
def _reset_rotary_embeddings(self):
|
| 700 |
+
"""Refresh non-persistent rotary buffers after checkpoint loading."""
|
| 701 |
+
for module in self.modules():
|
| 702 |
+
if isinstance(module, RotaryEmbedding):
|
| 703 |
+
module.reset_parameters()
|
| 704 |
+
|
| 705 |
+
@classmethod
|
| 706 |
+
def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
|
| 707 |
+
output_loading_info = bool(kwargs["output_loading_info"]) if "output_loading_info" in kwargs else False
|
| 708 |
+
loaded = super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
|
| 709 |
+
if output_loading_info:
|
| 710 |
+
model, loading_info = loaded
|
| 711 |
+
model._reset_rotary_embeddings()
|
| 712 |
+
return model, loading_info
|
| 713 |
+
loaded._reset_rotary_embeddings()
|
| 714 |
+
return loaded
|
| 715 |
+
|
| 716 |
+
@classmethod
|
| 717 |
+
def from_pretrained_esm(
|
| 718 |
+
cls,
|
| 719 |
+
model_name: str,
|
| 720 |
+
device: Union[torch.device, str] = "cpu",
|
| 721 |
+
):
|
| 722 |
+
"""Load a pretrained ESM++ model."""
|
| 723 |
+
key = _resolve_esmc_checkpoint_key(model_name)
|
| 724 |
+
if key == "esmc-300":
|
| 725 |
+
return ESMplusplus_300M(device=device)
|
| 726 |
+
if key == "esmc-600":
|
| 727 |
+
return ESMplusplus_600M(device=device)
|
| 728 |
+
if key == "esmc-6b":
|
| 729 |
+
return ESMplusplus_6B(device=device)
|
| 730 |
+
raise ValueError(f"Invalid model name: {model_name}")
|
| 731 |
+
|
| 732 |
+
|
| 733 |
+
### ESM++ Models
|
| 734 |
+
class ESMplusplusModel(PreTrainedESMplusplusModel, EmbeddingMixin):
|
| 735 |
+
"""
|
| 736 |
+
ESM++ model. transformer model with no heads
|
| 737 |
+
"""
|
| 738 |
+
config_class = ESMplusplusConfig
|
| 739 |
+
def __init__(self, config: ESMplusplusConfig, **kwargs):
|
| 740 |
+
PreTrainedESMplusplusModel.__init__(self, config, **kwargs)
|
| 741 |
+
self.config = config
|
| 742 |
+
self.vocab_size = config.vocab_size
|
| 743 |
+
self.embed = nn.Embedding(self.vocab_size, config.hidden_size)
|
| 744 |
+
self.transformer = TransformerStack(
|
| 745 |
+
d_model=config.hidden_size,
|
| 746 |
+
n_heads=config.num_attention_heads,
|
| 747 |
+
n_layers=config.num_hidden_layers,
|
| 748 |
+
dropout=config.dropout,
|
| 749 |
+
attn_backend=config.attn_backend,
|
| 750 |
+
)
|
| 751 |
+
self.tokenizer = EsmSequenceTokenizer()
|
| 752 |
+
self.init_weights()
|
| 753 |
+
|
| 754 |
+
def get_input_embeddings(self):
|
| 755 |
+
return self.embed
|
| 756 |
+
|
| 757 |
+
def set_input_embeddings(self, value):
|
| 758 |
+
self.embed = value
|
| 759 |
+
|
| 760 |
+
def _embed(self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 761 |
+
x = self.embed(input_ids)
|
| 762 |
+
return self.transformer(
|
| 763 |
+
x=x,
|
| 764 |
+
attention_mask=attention_mask,
|
| 765 |
+
output_hidden_states=False,
|
| 766 |
+
output_attentions=False,
|
| 767 |
+
).last_hidden_state
|
| 768 |
+
|
| 769 |
+
def forward(
|
| 770 |
+
self,
|
| 771 |
+
input_ids: Optional[torch.Tensor] = None,
|
| 772 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 773 |
+
inputs_embeds: Optional[torch.Tensor] = None,
|
| 774 |
+
output_attentions: Optional[bool] = None,
|
| 775 |
+
output_hidden_states: Optional[bool] = None,
|
| 776 |
+
output_s_max: Optional[bool] = False,
|
| 777 |
+
return_dict: Optional[bool] = None,
|
| 778 |
+
**kwargs,
|
| 779 |
+
) -> ESMplusplusOutput:
|
| 780 |
+
assert input_ids is not None or inputs_embeds is not None, "You have to specify either input_ids or inputs_embeds"
|
| 781 |
+
assert not (input_ids is not None and inputs_embeds is not None), "You cannot specify both input_ids and inputs_embeds at the same time"
|
| 782 |
+
|
| 783 |
+
if inputs_embeds is None:
|
| 784 |
+
x = self.embed(input_ids)
|
| 785 |
+
else:
|
| 786 |
+
x = inputs_embeds
|
| 787 |
+
|
| 788 |
+
transformer_output = self.transformer(
|
| 789 |
+
x=x,
|
| 790 |
+
attention_mask=attention_mask,
|
| 791 |
+
output_hidden_states=output_hidden_states,
|
| 792 |
+
output_attentions=output_attentions,
|
| 793 |
+
output_s_max=output_s_max,
|
| 794 |
+
)
|
| 795 |
+
return ESMplusplusOutput(
|
| 796 |
+
last_hidden_state=transformer_output.last_hidden_state,
|
| 797 |
+
hidden_states=transformer_output.hidden_states,
|
| 798 |
+
attentions=transformer_output.attentions,
|
| 799 |
+
s_max=transformer_output.s_max,
|
| 800 |
+
)
|
| 801 |
+
|
| 802 |
+
class ESMplusplusForMaskedLM(PreTrainedESMplusplusModel, EmbeddingMixin):
|
| 803 |
+
"""
|
| 804 |
+
ESM++ model for masked language modeling.
|
| 805 |
+
Implements the base ESM++ architecture with a masked language modeling head.
|
| 806 |
+
"""
|
| 807 |
+
config_class = ESMplusplusConfig
|
| 808 |
+
def __init__(self, config: ESMplusplusConfig, **kwargs):
|
| 809 |
+
PreTrainedESMplusplusModel.__init__(self, config, **kwargs)
|
| 810 |
+
self.config = config
|
| 811 |
+
self.vocab_size = config.vocab_size
|
| 812 |
+
self.embed = nn.Embedding(self.vocab_size, config.hidden_size)
|
| 813 |
+
self.transformer = TransformerStack(
|
| 814 |
+
d_model=config.hidden_size,
|
| 815 |
+
n_heads=config.num_attention_heads,
|
| 816 |
+
n_layers=config.num_hidden_layers,
|
| 817 |
+
dropout=config.dropout,
|
| 818 |
+
attn_backend=config.attn_backend,
|
| 819 |
+
)
|
| 820 |
+
self.sequence_head = RegressionHead(config.hidden_size, self.vocab_size)
|
| 821 |
+
self.ce_loss = nn.CrossEntropyLoss()
|
| 822 |
+
self.tokenizer = EsmSequenceTokenizer()
|
| 823 |
+
self.init_weights()
|
| 824 |
+
|
| 825 |
+
def get_input_embeddings(self):
|
| 826 |
+
return self.embed
|
| 827 |
+
|
| 828 |
+
def set_input_embeddings(self, value):
|
| 829 |
+
self.embed = value
|
| 830 |
+
|
| 831 |
+
def get_output_embeddings(self):
|
| 832 |
+
return self.sequence_head[-1]
|
| 833 |
+
|
| 834 |
+
def set_output_embeddings(self, new_embeddings):
|
| 835 |
+
self.sequence_head[-1] = new_embeddings
|
| 836 |
+
|
| 837 |
+
def _embed(self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 838 |
+
x = self.embed(input_ids)
|
| 839 |
+
return self.transformer(
|
| 840 |
+
x=x,
|
| 841 |
+
attention_mask=attention_mask,
|
| 842 |
+
output_hidden_states=False,
|
| 843 |
+
output_attentions=False,
|
| 844 |
+
).last_hidden_state
|
| 845 |
+
|
| 846 |
+
def forward(
|
| 847 |
+
self,
|
| 848 |
+
input_ids: Optional[torch.Tensor] = None,
|
| 849 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 850 |
+
inputs_embeds: Optional[torch.Tensor] = None,
|
| 851 |
+
labels: Optional[torch.Tensor] = None,
|
| 852 |
+
output_attentions: Optional[bool] = None,
|
| 853 |
+
output_hidden_states: Optional[bool] = None,
|
| 854 |
+
output_s_max: Optional[bool] = False,
|
| 855 |
+
return_dict: Optional[bool] = None,
|
| 856 |
+
**kwargs,
|
| 857 |
+
) -> ESMplusplusOutput:
|
| 858 |
+
if inputs_embeds is None:
|
| 859 |
+
x = self.embed(input_ids)
|
| 860 |
+
else:
|
| 861 |
+
x = inputs_embeds
|
| 862 |
+
|
| 863 |
+
output = self.transformer(
|
| 864 |
+
x=x,
|
| 865 |
+
attention_mask=attention_mask,
|
| 866 |
+
output_hidden_states=output_hidden_states,
|
| 867 |
+
output_attentions=output_attentions,
|
| 868 |
+
output_s_max=output_s_max,
|
| 869 |
+
)
|
| 870 |
+
|
| 871 |
+
last_hidden_state = output.last_hidden_state
|
| 872 |
+
logits = self.sequence_head(last_hidden_state)
|
| 873 |
+
loss = None
|
| 874 |
+
if labels is not None:
|
| 875 |
+
loss = self.ce_loss(logits.view(-1, self.vocab_size), labels.view(-1))
|
| 876 |
+
|
| 877 |
+
return ESMplusplusOutput(
|
| 878 |
+
loss=loss,
|
| 879 |
+
logits=logits,
|
| 880 |
+
last_hidden_state=last_hidden_state,
|
| 881 |
+
hidden_states=output.hidden_states,
|
| 882 |
+
attentions=output.attentions,
|
| 883 |
+
s_max=output.s_max,
|
| 884 |
+
)
|
| 885 |
+
|
| 886 |
+
|
| 887 |
+
class ESMplusplusForSequenceClassification(ESMplusplusForMaskedLM, EmbeddingMixin):
|
| 888 |
+
"""
|
| 889 |
+
ESM++ model for sequence classification.
|
| 890 |
+
Extends the base ESM++ model with a classification head.
|
| 891 |
+
"""
|
| 892 |
+
def __init__(self, config: ESMplusplusConfig, **kwargs):
|
| 893 |
+
ESMplusplusForMaskedLM.__init__(self, config, **kwargs)
|
| 894 |
+
self.config = config
|
| 895 |
+
self.num_labels = config.num_labels
|
| 896 |
+
self.classifier = RegressionHead(config.hidden_size * 2, config.num_labels, config.hidden_size * 4)
|
| 897 |
+
# Large intermediate projections help with sequence classification tasks (*4)
|
| 898 |
+
self.mse = nn.MSELoss()
|
| 899 |
+
self.ce = nn.CrossEntropyLoss()
|
| 900 |
+
self.bce = nn.BCEWithLogitsLoss()
|
| 901 |
+
# if kwargs has pooling_types, use them, otherwise use ['cls', 'mean']
|
| 902 |
+
if 'pooling_types' in kwargs and isinstance(kwargs['pooling_types'], List[str]) and len(kwargs['pooling_types']) > 0:
|
| 903 |
+
pooling_types = kwargs['pooling_types']
|
| 904 |
+
else:
|
| 905 |
+
pooling_types = ['mean', 'var']
|
| 906 |
+
self.pooler = Pooler(pooling_types)
|
| 907 |
+
self.init_weights()
|
| 908 |
+
|
| 909 |
+
def _embed(self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 910 |
+
x = self.embed(input_ids)
|
| 911 |
+
return self.transformer(
|
| 912 |
+
x=x,
|
| 913 |
+
attention_mask=attention_mask,
|
| 914 |
+
output_hidden_states=False,
|
| 915 |
+
output_attentions=False,
|
| 916 |
+
).last_hidden_state
|
| 917 |
+
|
| 918 |
+
def forward(
|
| 919 |
+
self,
|
| 920 |
+
input_ids: Optional[torch.Tensor] = None,
|
| 921 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 922 |
+
inputs_embeds: Optional[torch.Tensor] = None,
|
| 923 |
+
labels: Optional[torch.Tensor] = None,
|
| 924 |
+
output_attentions: Optional[bool] = None,
|
| 925 |
+
output_hidden_states: Optional[bool] = None,
|
| 926 |
+
output_s_max: Optional[bool] = False,
|
| 927 |
+
return_dict: Optional[bool] = None,
|
| 928 |
+
**kwargs,
|
| 929 |
+
) -> ESMplusplusOutput:
|
| 930 |
+
output = super().forward(
|
| 931 |
+
input_ids=input_ids,
|
| 932 |
+
attention_mask=attention_mask,
|
| 933 |
+
inputs_embeds=inputs_embeds,
|
| 934 |
+
labels=None,
|
| 935 |
+
output_attentions=output_attentions,
|
| 936 |
+
output_hidden_states=output_hidden_states,
|
| 937 |
+
output_s_max=output_s_max,
|
| 938 |
+
)
|
| 939 |
+
|
| 940 |
+
last_hidden_state = output.last_hidden_state
|
| 941 |
+
features = self.pooler(last_hidden_state, attention_mask)
|
| 942 |
+
logits = self.classifier(features)
|
| 943 |
+
|
| 944 |
+
loss = None
|
| 945 |
+
if labels is not None:
|
| 946 |
+
labels = labels.to(logits.device)
|
| 947 |
+
if self.config.problem_type is None:
|
| 948 |
+
if self.num_labels == 1:
|
| 949 |
+
self.config.problem_type = "regression"
|
| 950 |
+
elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
|
| 951 |
+
self.config.problem_type = "single_label_classification"
|
| 952 |
+
else:
|
| 953 |
+
self.config.problem_type = "multi_label_classification"
|
| 954 |
+
|
| 955 |
+
if self.config.problem_type == "regression":
|
| 956 |
+
if self.num_labels == 1:
|
| 957 |
+
loss = self.mse(logits.flatten(), labels.flatten())
|
| 958 |
+
else:
|
| 959 |
+
loss = self.mse(logits, labels)
|
| 960 |
+
elif self.config.problem_type == "single_label_classification":
|
| 961 |
+
loss = self.ce(logits.view(-1, self.num_labels), labels.view(-1))
|
| 962 |
+
elif self.config.problem_type == "multi_label_classification":
|
| 963 |
+
loss = self.bce(logits, labels)
|
| 964 |
+
|
| 965 |
+
return ESMplusplusOutput(
|
| 966 |
+
loss=loss,
|
| 967 |
+
logits=logits,
|
| 968 |
+
last_hidden_state=last_hidden_state,
|
| 969 |
+
hidden_states=output.hidden_states,
|
| 970 |
+
attentions=output.attentions,
|
| 971 |
+
s_max=output.s_max,
|
| 972 |
+
)
|
| 973 |
+
|
| 974 |
+
|
| 975 |
+
class ESMplusplusForTokenClassification(ESMplusplusForMaskedLM, EmbeddingMixin):
|
| 976 |
+
"""
|
| 977 |
+
ESM++ model for token classification.
|
| 978 |
+
Extends the base ESM++ model with a token classification head.
|
| 979 |
+
"""
|
| 980 |
+
def __init__(self, config: ESMplusplusConfig, **kwargs):
|
| 981 |
+
ESMplusplusForMaskedLM.__init__(self, config, **kwargs)
|
| 982 |
+
self.config = config
|
| 983 |
+
self.num_labels = config.num_labels
|
| 984 |
+
self.classifier = RegressionHead(config.hidden_size, config.num_labels, config.hidden_size * 4)
|
| 985 |
+
# Large intermediate projections help with sequence classification tasks (*4)
|
| 986 |
+
self.loss_fct = nn.CrossEntropyLoss()
|
| 987 |
+
self.init_weights()
|
| 988 |
+
|
| 989 |
+
def _embed(self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 990 |
+
x = self.embed(input_ids)
|
| 991 |
+
return self.transformer(x, attention_mask, output_hidden_states=False, output_attentions=False).last_hidden_state
|
| 992 |
+
|
| 993 |
+
def forward(
|
| 994 |
+
self,
|
| 995 |
+
input_ids: Optional[torch.Tensor] = None,
|
| 996 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 997 |
+
inputs_embeds: Optional[torch.Tensor] = None,
|
| 998 |
+
labels: Optional[torch.Tensor] = None,
|
| 999 |
+
output_attentions: Optional[bool] = None,
|
| 1000 |
+
output_hidden_states: Optional[bool] = None,
|
| 1001 |
+
output_s_max: Optional[bool] = False,
|
| 1002 |
+
return_dict: Optional[bool] = None,
|
| 1003 |
+
**kwargs,
|
| 1004 |
+
) -> ESMplusplusOutput:
|
| 1005 |
+
output = super().forward(
|
| 1006 |
+
input_ids=input_ids,
|
| 1007 |
+
attention_mask=attention_mask,
|
| 1008 |
+
inputs_embeds=inputs_embeds,
|
| 1009 |
+
labels=None,
|
| 1010 |
+
output_attentions=output_attentions,
|
| 1011 |
+
output_hidden_states=output_hidden_states,
|
| 1012 |
+
output_s_max=output_s_max,
|
| 1013 |
+
)
|
| 1014 |
+
|
| 1015 |
+
last_hidden_state = output.last_hidden_state
|
| 1016 |
+
logits = self.classifier(last_hidden_state)
|
| 1017 |
+
loss = None
|
| 1018 |
+
if labels is not None:
|
| 1019 |
+
loss = self.loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
|
| 1020 |
+
|
| 1021 |
+
return ESMplusplusOutput(
|
| 1022 |
+
loss=loss,
|
| 1023 |
+
logits=logits,
|
| 1024 |
+
last_hidden_state=last_hidden_state,
|
| 1025 |
+
hidden_states=output.hidden_states,
|
| 1026 |
+
attentions=output.attentions,
|
| 1027 |
+
s_max=output.s_max,
|
| 1028 |
+
)
|
| 1029 |
+
|
| 1030 |
+
|
| 1031 |
+
### Loading from Biohub
|
| 1032 |
+
_ESMC_CHECKPOINT_SPECS = {
|
| 1033 |
+
"esmc-300": {
|
| 1034 |
+
"repo_id": "biohub/ESMC-300M",
|
| 1035 |
+
"hidden_size": 960,
|
| 1036 |
+
"num_attention_heads": 15,
|
| 1037 |
+
"num_hidden_layers": 30,
|
| 1038 |
+
},
|
| 1039 |
+
"esmc-600": {
|
| 1040 |
+
"repo_id": "biohub/ESMC-600M",
|
| 1041 |
+
"hidden_size": 1152,
|
| 1042 |
+
"num_attention_heads": 18,
|
| 1043 |
+
"num_hidden_layers": 36,
|
| 1044 |
+
},
|
| 1045 |
+
"esmc-6b": {
|
| 1046 |
+
"repo_id": "biohub/ESMC-6B",
|
| 1047 |
+
"hidden_size": 2560,
|
| 1048 |
+
"num_attention_heads": 40,
|
| 1049 |
+
"num_hidden_layers": 80,
|
| 1050 |
+
},
|
| 1051 |
+
}
|
| 1052 |
+
|
| 1053 |
+
|
| 1054 |
+
def _resolve_esmc_checkpoint_key(model: str) -> str:
|
| 1055 |
+
normalized = model.lower().replace("_", "-")
|
| 1056 |
+
if "300" in normalized:
|
| 1057 |
+
return "esmc-300"
|
| 1058 |
+
if "600" in normalized:
|
| 1059 |
+
return "esmc-600"
|
| 1060 |
+
if "6b" in normalized:
|
| 1061 |
+
return "esmc-6b"
|
| 1062 |
+
raise ValueError(f"{model=} is an invalid ESMC model name.")
|
| 1063 |
+
|
| 1064 |
+
|
| 1065 |
+
@staticmethod
|
| 1066 |
+
@cache
|
| 1067 |
+
def data_root(model: str):
|
| 1068 |
+
if "INFRA_PROVIDER" in os.environ:
|
| 1069 |
+
return Path("")
|
| 1070 |
+
key = _resolve_esmc_checkpoint_key(model)
|
| 1071 |
+
return Path(snapshot_download(repo_id=_ESMC_CHECKPOINT_SPECS[key]["repo_id"]))
|
| 1072 |
+
|
| 1073 |
+
|
| 1074 |
+
def get_esmc_checkpoint_path(model: str) -> Path:
|
| 1075 |
+
key = _resolve_esmc_checkpoint_key(model)
|
| 1076 |
+
spec = _ESMC_CHECKPOINT_SPECS[key]
|
| 1077 |
+
if "weights_relpath" in spec:
|
| 1078 |
+
return data_root(key) / spec["weights_relpath"]
|
| 1079 |
+
checkpoint_dir = data_root(key)
|
| 1080 |
+
if (checkpoint_dir / "model.safetensors").exists():
|
| 1081 |
+
return checkpoint_dir / "model.safetensors"
|
| 1082 |
+
if (checkpoint_dir / "model.safetensors.index.json").exists():
|
| 1083 |
+
return checkpoint_dir / "model.safetensors.index.json"
|
| 1084 |
+
raise FileNotFoundError(f"No ESMC checkpoint found under {checkpoint_dir}.")
|
| 1085 |
+
|
| 1086 |
+
|
| 1087 |
+
def _normalize_esmc_state_key(key: str) -> Optional[str]:
|
| 1088 |
+
if key.endswith("._extra_state"):
|
| 1089 |
+
return None
|
| 1090 |
+
if key.startswith("esmc."):
|
| 1091 |
+
key = key[len("esmc."):]
|
| 1092 |
+
if key.startswith("lm_head."):
|
| 1093 |
+
key = f"sequence_head.{key[len('lm_head.'):]}"
|
| 1094 |
+
replacements = (
|
| 1095 |
+
(".attn.layernorm_qkv.layer_norm_bias", ".attn.layernorm_qkv.0.bias"),
|
| 1096 |
+
(".attn.layernorm_qkv.layer_norm_weight", ".attn.layernorm_qkv.0.weight"),
|
| 1097 |
+
(".attn.layernorm_qkv.weight", ".attn.layernorm_qkv.1.weight"),
|
| 1098 |
+
(".ffn.layer_norm_bias", ".ffn.0.bias"),
|
| 1099 |
+
(".ffn.layer_norm_weight", ".ffn.0.weight"),
|
| 1100 |
+
(".ffn.fc1_weight", ".ffn.1.weight"),
|
| 1101 |
+
(".ffn.fc2_weight", ".ffn.3.weight"),
|
| 1102 |
+
)
|
| 1103 |
+
for old, new in replacements:
|
| 1104 |
+
key = key.replace(old, new)
|
| 1105 |
+
return key
|
| 1106 |
+
|
| 1107 |
+
|
| 1108 |
+
def _normalize_esmc_state_dict(state_dict: dict) -> dict:
|
| 1109 |
+
normalized = {}
|
| 1110 |
+
for key, tensor in state_dict.items():
|
| 1111 |
+
normalized_key = _normalize_esmc_state_key(key)
|
| 1112 |
+
if normalized_key is None:
|
| 1113 |
+
continue
|
| 1114 |
+
normalized[normalized_key] = tensor
|
| 1115 |
+
return normalized
|
| 1116 |
+
|
| 1117 |
+
|
| 1118 |
+
def _safetensors_checkpoint_files(checkpoint_path: Path) -> List[Path]:
|
| 1119 |
+
if checkpoint_path.name == "model.safetensors":
|
| 1120 |
+
return [checkpoint_path]
|
| 1121 |
+
with checkpoint_path.open("r", encoding="utf-8") as f:
|
| 1122 |
+
index = json.load(f)
|
| 1123 |
+
return [
|
| 1124 |
+
checkpoint_path.parent / filename
|
| 1125 |
+
for filename in sorted(set(index["weight_map"].values()))
|
| 1126 |
+
]
|
| 1127 |
+
|
| 1128 |
+
|
| 1129 |
+
def _load_safetensors_state_dict(
|
| 1130 |
+
model_obj: ESMplusplusForMaskedLM,
|
| 1131 |
+
checkpoint_path: Path,
|
| 1132 |
+
device: Union[torch.device, str],
|
| 1133 |
+
) -> None:
|
| 1134 |
+
expected_keys = set(model_obj.state_dict().keys())
|
| 1135 |
+
loaded_keys = set()
|
| 1136 |
+
device_string = str(torch.device(device))
|
| 1137 |
+
for shard_path in _safetensors_checkpoint_files(checkpoint_path):
|
| 1138 |
+
shard_state_dict = load_safetensors_file(shard_path, device=device_string)
|
| 1139 |
+
normalized = _normalize_esmc_state_dict(shard_state_dict)
|
| 1140 |
+
unexpected = set(normalized.keys()) - expected_keys
|
| 1141 |
+
assert len(unexpected) == 0, (
|
| 1142 |
+
f"Unexpected ESMC checkpoint keys in {shard_path.name}: "
|
| 1143 |
+
f"{sorted(unexpected)[:10]}"
|
| 1144 |
+
)
|
| 1145 |
+
model_obj.load_state_dict(normalized, strict=False)
|
| 1146 |
+
loaded_keys.update(normalized.keys())
|
| 1147 |
+
|
| 1148 |
+
missing = expected_keys - loaded_keys
|
| 1149 |
+
assert len(missing) == 0, (
|
| 1150 |
+
f"ESMC checkpoint did not provide all expected keys: {sorted(missing)[:10]}"
|
| 1151 |
+
)
|
| 1152 |
+
|
| 1153 |
+
|
| 1154 |
+
def _load_esmc_checkpoint_model(
|
| 1155 |
+
config: ESMplusplusConfig,
|
| 1156 |
+
model: str,
|
| 1157 |
+
device: Union[torch.device, str] = "cpu",
|
| 1158 |
+
) -> ESMplusplusForMaskedLM:
|
| 1159 |
+
key = _resolve_esmc_checkpoint_key(model)
|
| 1160 |
+
spec = _ESMC_CHECKPOINT_SPECS[key]
|
| 1161 |
+
assert config.hidden_size == spec["hidden_size"], (
|
| 1162 |
+
f"ESMC loader expected hidden_size={spec['hidden_size']} for {key}, "
|
| 1163 |
+
f"but got {config.hidden_size}."
|
| 1164 |
+
)
|
| 1165 |
+
assert config.num_attention_heads == spec["num_attention_heads"], (
|
| 1166 |
+
f"ESMC loader expected num_attention_heads={spec['num_attention_heads']} for {key}, "
|
| 1167 |
+
f"but got {config.num_attention_heads}."
|
| 1168 |
+
)
|
| 1169 |
+
assert config.num_hidden_layers == spec["num_hidden_layers"], (
|
| 1170 |
+
f"ESMC loader expected num_hidden_layers={spec['num_hidden_layers']} for {key}, "
|
| 1171 |
+
f"but got {config.num_hidden_layers}."
|
| 1172 |
+
)
|
| 1173 |
+
with torch.device(device):
|
| 1174 |
+
model_obj = ESMplusplusForMaskedLM(config)
|
| 1175 |
+
checkpoint_path = get_esmc_checkpoint_path(key)
|
| 1176 |
+
if checkpoint_path.suffix == ".safetensors" or checkpoint_path.name == "model.safetensors.index.json":
|
| 1177 |
+
_load_safetensors_state_dict(
|
| 1178 |
+
model_obj=model_obj,
|
| 1179 |
+
checkpoint_path=checkpoint_path,
|
| 1180 |
+
device=device,
|
| 1181 |
+
)
|
| 1182 |
+
else:
|
| 1183 |
+
state_dict = torch.load(checkpoint_path, map_location=device)
|
| 1184 |
+
model_obj.load_state_dict(_normalize_esmc_state_dict(state_dict))
|
| 1185 |
+
return model_obj
|
| 1186 |
+
|
| 1187 |
+
|
| 1188 |
+
def ESMplusplus_300M(device: Union[torch.device, str] = "cpu"):
|
| 1189 |
+
config = ESMplusplusConfig(
|
| 1190 |
+
hidden_size=960,
|
| 1191 |
+
num_attention_heads=15,
|
| 1192 |
+
num_hidden_layers=30,
|
| 1193 |
+
)
|
| 1194 |
+
return _load_esmc_checkpoint_model(config=config, model="esmc-300", device=device)
|
| 1195 |
+
|
| 1196 |
+
|
| 1197 |
+
def ESMplusplus_600M(device: Union[torch.device, str] = "cpu"):
|
| 1198 |
+
config = ESMplusplusConfig(
|
| 1199 |
+
hidden_size=1152,
|
| 1200 |
+
num_attention_heads=18,
|
| 1201 |
+
num_hidden_layers=36,
|
| 1202 |
+
)
|
| 1203 |
+
return _load_esmc_checkpoint_model(config=config, model="esmc-600", device=device)
|
| 1204 |
+
|
| 1205 |
+
|
| 1206 |
+
def ESMplusplus_6B(device: Union[torch.device, str] = "cpu"):
|
| 1207 |
+
config = ESMplusplusConfig(
|
| 1208 |
+
hidden_size=2560,
|
| 1209 |
+
num_attention_heads=40,
|
| 1210 |
+
num_hidden_layers=80,
|
| 1211 |
+
)
|
| 1212 |
+
return _load_esmc_checkpoint_model(config=config, model="esmc-6b", device=device)
|
| 1213 |
+
|
| 1214 |
+
|
| 1215 |
+
### Tokenization
|
| 1216 |
+
SEQUENCE_VOCAB = [
|
| 1217 |
+
"<cls>", "<pad>", "<eos>", "<unk>",
|
| 1218 |
+
"L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K",
|
| 1219 |
+
"Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z",
|
| 1220 |
+
"O", ".", "-", "|",
|
| 1221 |
+
"<mask>",
|
| 1222 |
+
]
|
| 1223 |
+
|
| 1224 |
+
class EsmSequenceTokenizer(PreTrainedTokenizerFast):
|
| 1225 |
+
model_input_names = ["input_ids", "attention_mask"]
|
| 1226 |
+
|
| 1227 |
+
def __init__(
|
| 1228 |
+
self,
|
| 1229 |
+
unk_token="<unk>",
|
| 1230 |
+
cls_token="<cls>",
|
| 1231 |
+
pad_token="<pad>",
|
| 1232 |
+
mask_token="<mask>",
|
| 1233 |
+
eos_token="<eos>",
|
| 1234 |
+
chain_break_token="|",
|
| 1235 |
+
**kwargs,
|
| 1236 |
+
):
|
| 1237 |
+
all_tokens = SEQUENCE_VOCAB
|
| 1238 |
+
token_to_id = {tok: ind for ind, tok in enumerate(all_tokens)}
|
| 1239 |
+
|
| 1240 |
+
# a character-level tokenizer is the same as BPE with no token merges
|
| 1241 |
+
bpe = BPE(token_to_id, merges=[], unk_token=unk_token)
|
| 1242 |
+
tokenizer = Tokenizer(bpe)
|
| 1243 |
+
special_tokens = [
|
| 1244 |
+
cls_token,
|
| 1245 |
+
pad_token,
|
| 1246 |
+
mask_token,
|
| 1247 |
+
eos_token,
|
| 1248 |
+
chain_break_token,
|
| 1249 |
+
]
|
| 1250 |
+
self.cb_token = chain_break_token
|
| 1251 |
+
additional_special_tokens = [chain_break_token]
|
| 1252 |
+
|
| 1253 |
+
tokenizer.add_special_tokens(special_tokens)
|
| 1254 |
+
|
| 1255 |
+
# This is where we configure the automatic addition of special tokens when we call
|
| 1256 |
+
# tokenizer(text, add_special_tokens=True). Note that you can also configure how two
|
| 1257 |
+
# sequences are merged if you want.
|
| 1258 |
+
tokenizer.post_processor = TemplateProcessing( # type: ignore
|
| 1259 |
+
single="<cls> $A <eos>",
|
| 1260 |
+
pair="<cls>:0 $A:0 <eos>:0 $B:1 <eos>:1",
|
| 1261 |
+
special_tokens=[
|
| 1262 |
+
("<cls>", tokenizer.token_to_id("<cls>")),
|
| 1263 |
+
("<eos>", tokenizer.token_to_id("<eos>")),
|
| 1264 |
+
],
|
| 1265 |
+
)
|
| 1266 |
+
super().__init__(
|
| 1267 |
+
tokenizer_object=tokenizer,
|
| 1268 |
+
unk_token=unk_token,
|
| 1269 |
+
cls_token=cls_token,
|
| 1270 |
+
pad_token=pad_token,
|
| 1271 |
+
mask_token=mask_token,
|
| 1272 |
+
eos_token=eos_token,
|
| 1273 |
+
additional_special_tokens=additional_special_tokens,
|
| 1274 |
+
**kwargs,
|
| 1275 |
+
)
|
| 1276 |
+
|
| 1277 |
+
# These are a footgun, we never use the `bos` token anywhere so we're just overriding it here.
|
| 1278 |
+
@property
|
| 1279 |
+
def bos_token(self):
|
| 1280 |
+
return self.cls_token
|
| 1281 |
+
|
| 1282 |
+
@property
|
| 1283 |
+
def bos_token_id(self):
|
| 1284 |
+
return self.cls_token_id
|
| 1285 |
+
|
| 1286 |
+
@property
|
| 1287 |
+
def chain_break_token(self):
|
| 1288 |
+
return self.cb_token
|
| 1289 |
+
|
| 1290 |
+
@property
|
| 1291 |
+
def chain_break_token_id(self):
|
| 1292 |
+
return self.convert_tokens_to_ids(self.chain_break_token)
|
| 1293 |
+
|
| 1294 |
+
@property
|
| 1295 |
+
def all_token_ids(self):
|
| 1296 |
+
return list(range(self.vocab_size))
|
| 1297 |
+
|
| 1298 |
+
@property
|
| 1299 |
+
def special_token_ids(self):
|
| 1300 |
+
return self.all_special_ids
|
| 1301 |
+
|
| 1302 |
+
|
| 1303 |
+
if __name__ == "__main__":
|
| 1304 |
+
import random
|
| 1305 |
+
|
| 1306 |
+
import torch
|
| 1307 |
+
|
| 1308 |
+
from torch import Tensor
|
| 1309 |
+
|
| 1310 |
+
def print_tensor_shapes(prefix: str, obj):
|
| 1311 |
+
if isinstance(obj, Tensor):
|
| 1312 |
+
print(f"{prefix}{obj.shape}")
|
| 1313 |
+
elif isinstance(obj, dict):
|
| 1314 |
+
for name, value in obj.items():
|
| 1315 |
+
print_tensor_shapes(f"{prefix}{name}.", value)
|
| 1316 |
+
elif isinstance(obj, list):
|
| 1317 |
+
for idx, value in enumerate(obj):
|
| 1318 |
+
print_tensor_shapes(f"{prefix}[{idx}].", value)
|
| 1319 |
+
elif isinstance(obj, tuple):
|
| 1320 |
+
for idx, value in enumerate(obj):
|
| 1321 |
+
print_tensor_shapes(f"{prefix}[{idx}].", value)
|
| 1322 |
+
elif hasattr(obj, "__dict__"):
|
| 1323 |
+
for name, value in vars(obj).items():
|
| 1324 |
+
if name.startswith("_"):
|
| 1325 |
+
continue
|
| 1326 |
+
print_tensor_shapes(f"{prefix}{name}.", value)
|
| 1327 |
+
else:
|
| 1328 |
+
print(f"{prefix}{type(obj)}")
|
| 1329 |
+
|
| 1330 |
+
random.seed(0)
|
| 1331 |
+
torch.manual_seed(0)
|
| 1332 |
+
|
| 1333 |
+
tokenizer = EsmSequenceTokenizer()
|
| 1334 |
+
num_attention_heads = random.choice([2, 4])
|
| 1335 |
+
config = ESMplusplusConfig(
|
| 1336 |
+
vocab_size=tokenizer.vocab_size,
|
| 1337 |
+
hidden_size=16 * num_attention_heads,
|
| 1338 |
+
num_attention_heads=num_attention_heads,
|
| 1339 |
+
num_hidden_layers=random.choice([1, 2]),
|
| 1340 |
+
num_labels=2,
|
| 1341 |
+
dropout=0.0,
|
| 1342 |
+
)
|
| 1343 |
+
|
| 1344 |
+
batch = tokenizer(["ACDEFG", "MKTW"], return_tensors="pt", padding=True)
|
| 1345 |
+
batch["labels"] = batch["input_ids"].clone()
|
| 1346 |
+
model = ESMplusplusForMaskedLM(config=config).eval()
|
| 1347 |
+
|
| 1348 |
+
with torch.no_grad():
|
| 1349 |
+
output = model(**batch, return_dict=True)
|
| 1350 |
+
|
| 1351 |
+
print("Batch shape:")
|
| 1352 |
+
print_tensor_shapes("", batch)
|
| 1353 |
+
print("Output shape:")
|
| 1354 |
+
print_tensor_shapes("", output)
|