super-dainiu commited on
Commit
b3354d2
·
verified ·
1 Parent(s): a97a7ec

Add completed LogiCA v2 150M pretraining checkpoint

Browse files
checkpoints/v2_20261110/150m/README.md ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # LogiCA v2 — BindingDB pretraining
2
+
3
+ Frozen ESM-2 150M + SELFormer; trainable native-head-preserving cross-attention adapter.
4
+ This release uses Aurel's Stage 0 / Stage 1 kit from commit
5
+ `31526bb4a91806da79cddaeb4ccd8b3bd065bee3` and the existing LogiCA checkpoint runtime.
6
+
7
+ Architecture: width 384; 2 bidirectional blocks; 4 heads; tanh-zero gates;
8
+ orthogonal projections; sigmoid drug-to-protein attention; intra-modal attention;
9
+ learned last-four-layer protein mixture. Training uses masked contextual-gain
10
+ InfoNCE, with alpha 0.5 and frozen native language-model heads.
11
+
12
+ Data processing and split counts are in the accompanying preparation manifest.
13
+ The split groups protein families by the kit's k-mer containment rule.
14
+
15
+ `best.pt` contains the trainable-parameter overlay plus registered buffers in
16
+ `model_state_dict`, with a complete `trainable_parameter_names` manifest.
17
+ It is selected by minimum held-out validation loss and may come from an early step.
18
+ `final.pt`, when present, contains the final full-budget adapter without optimizer state.
19
+ The two checkpoints' steps and validation metrics are recorded in `release_summary.json`.
20
+ Instantiate `ImprovedLogiCAModel` from the
21
+ bundled `modeling_logica_v2.py` with `arch_version="v2"` and the architecture above,
22
+ loading the backbone revisions in `config.json`. Freeze both backbones and
23
+ `pair_alpha_logit`; load the adapter overlay after checking the parameter names.
24
+ The v1 checkpoint loader is not the loader for this architecture.
25
+
26
+ This artifact is a completed pretraining run. Downstream benchmark results are
27
+ not implied by publication of the checkpoint.
checkpoints/v2_20261110/150m/SHA256SUMS.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "execution_profile.json": "a340820757f9d00c9f6f74facc93554c4eb8e559282dea1ca3cefe69a594108f",
3
+ "TRAINING_COMPLETE.json": "4aac041e8b422e27e1a2176f76902ac5bbdd86827bd65d694e590a9adb52d6f3",
4
+ "modeling_logica_v2.py": "23279a6e4f6e3af417eedd77c7734772a44e297a8ad57303ed1fb6df451b614b",
5
+ "config.json": "84bf32561b1ab3e1fc913757980fe7b2293715aea12bbfdc1a47d64a97703f21",
6
+ "curation_summary.json": "eff9be86b93df2a4d4506d44a2c1f294c450d05f0fab20e76877d06a65cc88ef",
7
+ "final.pt": "6322c7e595003cced50232bd437b92846bef04f6a9669c7d97f08711b86c1ea4",
8
+ "preparation_manifest.json": "45f61c248a4451dabf5228c9074cd507ca0aa807bc71a962c5a287c87efdbc03",
9
+ "release_summary.json": "fa4b344ffb2d93f593f6fa803d4efe81d483e8358dc30df47612868fd17e6f03",
10
+ "execution_benchmark.json": "34b2bb84985674f82061a14ea118fe14eab647af63b7e6f5a4c6f176abf2371a",
11
+ "README.md": "9ca33b3a79376b3ef77db825c2220d2c3bd9509a418b682531f203f294744a14",
12
+ "best.pt": "604a78e1572b9234ecc9d9a9209fa9621a119b545e1ab1cd0ba883f5daedb384"
13
+ }
checkpoints/v2_20261110/150m/TRAINING_COMPLETE.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "step": 62520,
3
+ "best_loss": 2.659006729722023
4
+ }
checkpoints/v2_20261110/150m/best.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:604a78e1572b9234ecc9d9a9209fa9621a119b545e1ab1cd0ba883f5daedb384
3
+ size 32878986
checkpoints/v2_20261110/150m/config.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "esm_model": "facebook/esm2_t30_150M_UR50D",
3
+ "drug_model": "HUBioDataLab/SELFormer",
4
+ "epochs": 40,
5
+ "anchors_per_epoch": 100000,
6
+ "batch_size": 16,
7
+ "grad_accum": 1,
8
+ "neg_samples": 15,
9
+ "adapter_chunk_size": 128,
10
+ "lr": 0.0001,
11
+ "weight_decay": 0.01,
12
+ "temperature": 0.07,
13
+ "mask_fraction": 0.15,
14
+ "eval_every": 1000,
15
+ "eval_batches": 16,
16
+ "log_every": 10,
17
+ "seed": 42,
18
+ "max_steps": null,
19
+ "resume": true,
20
+ "ddp": true,
21
+ "checkpoint_adapter": false,
22
+ "performance_profile_sha256": "03289afeab74863ede26d920bb6d077d0ebd3b5221804d7b2e5c167558e7bf06",
23
+ "arch_version": "v2",
24
+ "gate_mode": "tanh_zero",
25
+ "width": 384,
26
+ "heads": 4,
27
+ "layers": 2,
28
+ "num_protein_fusion_layers": 4,
29
+ "d2p_attn_mode": "sigmoid",
30
+ "use_intra_self_attn": true,
31
+ "protein_revision": "a695f6045e2e32885fa60af20c13cb35398ce30c",
32
+ "drug_revision": "177d98b158e999a6cb7fc9743dbfe1e8a17c57e5",
33
+ "world_size": 4,
34
+ "steps_per_epoch": 1563,
35
+ "train_mutant_cliffs": 1353,
36
+ "validation_mutant_cliffs": 19,
37
+ "train_protein_families": 2578,
38
+ "protein_mask_token_id": 32,
39
+ "drug_mask_token_id": 4,
40
+ "trainable_parameters": 8189966
41
+ }
checkpoints/v2_20261110/150m/curation_summary.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "num_unique_proteins": 7256,
3
+ "num_protein_families": 5476,
4
+ "num_unique_drugs": 595365,
5
+ "num_total_pairs": 806165,
6
+ "num_positive_pairs": 329741,
7
+ "num_verified_negative_pairs": 186326,
8
+ "num_gray_zone_pairs": 290098,
9
+ "num_train_pairs": 700580,
10
+ "num_val_pairs": 105585
11
+ }
checkpoints/v2_20261110/150m/execution_benchmark.json ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "esm_model": "facebook/esm2_t30_150M_UR50D",
3
+ "arch_version": "v2",
4
+ "checkpoint_step": 8546,
5
+ "gpu": "NVIDIA H100 80GB HBM3",
6
+ "checkpoint_sha256": "a6def128e59f91603d2e61c7e9e225fa6a7ec6d0df5ada33bb2f1599a58e247b",
7
+ "batch_size": 16,
8
+ "negative_samples": 15,
9
+ "protein_tokens": 1024,
10
+ "drug_tokens": 138,
11
+ "data_preparation_seconds": [
12
+ 0.004722184035927057,
13
+ 0.0026600889395922422,
14
+ 0.003557940013706684,
15
+ 0.0034004130866378546
16
+ ],
17
+ "correctness": [
18
+ {
19
+ "chunk": 2,
20
+ "checkpoint_adapter": true,
21
+ "loss": 1.1956030130386353,
22
+ "loss_abs_error": 0.0,
23
+ "gradient_relative_l2": 0.0
24
+ },
25
+ {
26
+ "chunk": 2,
27
+ "checkpoint_adapter": false,
28
+ "loss": 1.1956030130386353,
29
+ "loss_abs_error": 0.0,
30
+ "gradient_relative_l2": 1.0836556896265392e-07
31
+ },
32
+ {
33
+ "chunk": 64,
34
+ "checkpoint_adapter": true,
35
+ "loss": 1.1956026554107666,
36
+ "loss_abs_error": 3.5762786865234375e-07,
37
+ "gradient_relative_l2": 1.4153346228340524e-06
38
+ },
39
+ {
40
+ "chunk": 64,
41
+ "checkpoint_adapter": false,
42
+ "loss": 1.1956026554107666,
43
+ "loss_abs_error": 3.5762786865234375e-07,
44
+ "gradient_relative_l2": 1.4153182519294205e-06
45
+ },
46
+ {
47
+ "chunk": 128,
48
+ "checkpoint_adapter": true,
49
+ "loss": 1.1956026554107666,
50
+ "loss_abs_error": 3.5762786865234375e-07,
51
+ "gradient_relative_l2": 1.4138399819785263e-06
52
+ },
53
+ {
54
+ "chunk": 128,
55
+ "checkpoint_adapter": false,
56
+ "loss": 1.1956026554107666,
57
+ "loss_abs_error": 3.5762786865234375e-07,
58
+ "gradient_relative_l2": 1.4158223393678782e-06
59
+ }
60
+ ],
61
+ "timings": [
62
+ {
63
+ "chunk": 2,
64
+ "checkpoint_adapter": true,
65
+ "median_step_seconds": 5.648621527012438,
66
+ "samples_seconds": [
67
+ 5.692857360001653,
68
+ 5.517517200903967,
69
+ 5.645145141985267,
70
+ 5.648621527012438,
71
+ 5.699445354985073
72
+ ],
73
+ "peak_allocated_gib": 4.269045352935791,
74
+ "peak_reserved_gib": 5.037109375,
75
+ "stress_protein_tokens": 1024,
76
+ "stress_drug_tokens": 256,
77
+ "stress_peak_reserved_gib": 5.123046875
78
+ },
79
+ {
80
+ "chunk": 2,
81
+ "checkpoint_adapter": false,
82
+ "median_step_seconds": 2.9281322590541095,
83
+ "samples_seconds": [
84
+ 2.9363738850224763,
85
+ 2.9281322590541095,
86
+ 3.0685506730806082,
87
+ 2.9186167139559984,
88
+ 2.922416811110452
89
+ ],
90
+ "peak_allocated_gib": 42.45229482650757,
91
+ "peak_reserved_gib": 42.919921875,
92
+ "stress_protein_tokens": 1024,
93
+ "stress_drug_tokens": 256,
94
+ "stress_peak_reserved_gib": 52.603515625
95
+ },
96
+ {
97
+ "chunk": 64,
98
+ "checkpoint_adapter": true,
99
+ "median_step_seconds": 0.7713121310807765,
100
+ "samples_seconds": [
101
+ 0.7730167899280787,
102
+ 0.7714591689873487,
103
+ 0.7710773290600628,
104
+ 0.7713121310807765,
105
+ 0.7710377559997141
106
+ ],
107
+ "peak_allocated_gib": 7.269124984741211,
108
+ "peak_reserved_gib": 7.7265625,
109
+ "stress_protein_tokens": 1024,
110
+ "stress_drug_tokens": 256,
111
+ "stress_peak_reserved_gib": 9.18359375
112
+ },
113
+ {
114
+ "chunk": 64,
115
+ "checkpoint_adapter": false,
116
+ "median_step_seconds": 0.616375891957432,
117
+ "samples_seconds": [
118
+ 0.6166188640054315,
119
+ 0.6161552940029651,
120
+ 0.6163672010879964,
121
+ 0.616375891957432,
122
+ 0.6165828828234226
123
+ ],
124
+ "peak_allocated_gib": 40.01216220855713,
125
+ "peak_reserved_gib": 40.720703125,
126
+ "stress_protein_tokens": 1024,
127
+ "stress_drug_tokens": 256,
128
+ "stress_peak_reserved_gib": 50.04296875
129
+ },
130
+ {
131
+ "chunk": 128,
132
+ "checkpoint_adapter": true,
133
+ "median_step_seconds": 0.7444617659784853,
134
+ "samples_seconds": [
135
+ 0.7444617659784853,
136
+ 0.7442538880277425,
137
+ 0.7448720769025385,
138
+ 0.7443247509654611,
139
+ 0.7447491327766329
140
+ ],
141
+ "peak_allocated_gib": 12.79415512084961,
142
+ "peak_reserved_gib": 14.376953125,
143
+ "stress_protein_tokens": 1024,
144
+ "stress_drug_tokens": 256,
145
+ "stress_peak_reserved_gib": 16.986328125
146
+ },
147
+ {
148
+ "chunk": 128,
149
+ "checkpoint_adapter": false,
150
+ "median_step_seconds": 0.5958139861468226,
151
+ "samples_seconds": [
152
+ 0.5960364181082696,
153
+ 0.5959548719692975,
154
+ 0.5958139130380005,
155
+ 0.5958139861468226,
156
+ 0.5956389489583671
157
+ ],
158
+ "peak_allocated_gib": 40.69248104095459,
159
+ "peak_reserved_gib": 41.669921875,
160
+ "stress_protein_tokens": 1024,
161
+ "stress_drug_tokens": 256,
162
+ "stress_peak_reserved_gib": 51.462890625
163
+ }
164
+ ],
165
+ "top_cpu_events": [],
166
+ "recommended_chunk": 128,
167
+ "recommended_checkpoint_adapter": false,
168
+ "measured_single_gpu_speedup": 9.48051180124611
169
+ }
checkpoints/v2_20261110/150m/execution_profile.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "esm_model": "facebook/esm2_t30_150M_UR50D",
3
+ "arch_version": "v2",
4
+ "batch_size": 16,
5
+ "negative_samples": 15,
6
+ "adapter_chunk_size": 128,
7
+ "checkpoint_adapter": false,
8
+ "validation_report_sha256": "34b2bb84985674f82061a14ea118fe14eab647af63b7e6f5a4c6f176abf2371a",
9
+ "validation_report": "execution_benchmark.json",
10
+ "source_profile_sha256": "03289afeab74863ede26d920bb6d077d0ebd3b5221804d7b2e5c167558e7bf06"
11
+ }
checkpoints/v2_20261110/150m/final.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6322c7e595003cced50232bd437b92846bef04f6a9669c7d97f08711b86c1ea4
3
+ size 32798078
checkpoints/v2_20261110/150m/modeling_logica_v2.py ADDED
@@ -0,0 +1,906 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Upgraded LogiCA Architecture & Scoring Interface (Stage 1 v2 + Stage 2 v1-Checkpoint-Compatible).
2
+
3
+ Built directly on top of the original released `LogiCAModel` architecture, adding:
4
+ - [Stage 1 Fix A1] `tanh(0)` / per-channel LayerScale gating (unit gradient at step 0 instead of
5
+ `sigmoid(-6.0)`'s 400x gradient attenuation) + full-rank orthogonal `proj_in` & zero-init
6
+ `proj_out` (preventing truncation of channels `384:1280` in ESM-2 650M).
7
+ - [Stage 1 Fix A2] Strict masked-token log-likelihood gain (`forward_masked_gain`), eliminating
8
+ unmasked `pair_scores_from_tokens` self-token leakage (`H_ligand = 0.068` nats).
9
+ - [Stage 1 Fix A3] Length-invariant `sigmoid` or `topk_softmax` attention on `drug <- protein`
10
+ cross-attention (preventing ~1,000 non-pocket residues from diluting ligand representations)
11
+ + optional intra-modality pocket self-attention (`SelfAttn -> CrossAttn -> FFN`) for
12
+ second-shell / allosteric coupling.
13
+ - [Stage 1 Fix A4] Learned multi-layer ESM-2 hidden-state aggregation (`num_protein_layers=4`)
14
+ for cross-attention `Q/K/V`, while injecting the contextual residual directly before the
15
+ frozen native `lm_head`.
16
+ - [Stage 2 Fixes] Seamless `arch_version="v1_compat"` loading of existing BindingDB checkpoints,
17
+ late-layer-only `LoRALinear` injection (`top_k_esm=4`, `top_k_drug=2`), differential optimizer
18
+ parameter groups (`5x` LR on gates + cross-attention), and single-pass site-masked 20-AA
19
+ holo + apo scoring (`score_masked_sites_20aa`) with `alpha = 1.0`.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ from dataclasses import dataclass
25
+ import hashlib
26
+ import math
27
+ from pathlib import Path
28
+ from typing import Iterable, Literal, Sequence
29
+
30
+ import numpy as np
31
+ import torch
32
+ from torch import nn
33
+ import torch.nn.functional as F
34
+
35
+ try:
36
+ import selfies as sf
37
+ except ImportError: # pragma: no cover - optional for pure tensor unit tests
38
+ sf = None # type: ignore[assignment]
39
+
40
+ try:
41
+ from transformers import AutoModelForMaskedLM, AutoTokenizer
42
+ except ImportError: # pragma: no cover
43
+ AutoModelForMaskedLM = None # type: ignore[assignment]
44
+ AutoTokenizer = None # type: ignore[assignment]
45
+
46
+
47
+ CANONICAL_AA_ORDER: tuple[str, ...] = tuple("ACDEFGHIKLMNPQRSTVWY")
48
+
49
+
50
+ # =============================================================================
51
+ # 1. Lightweight Built-In LoRA Wrapper for Stage 2 Late-Layer Adaptation
52
+ # =============================================================================
53
+
54
+
55
+ class LoRALinear(nn.Module):
56
+ """Drop-in low-rank adapter around a frozen `nn.Linear` layer (Hu et al., 2022)."""
57
+
58
+ def __init__(
59
+ self,
60
+ base_linear: nn.Linear,
61
+ rank: int = 8,
62
+ alpha: float = 16.0,
63
+ dropout: float = 0.1,
64
+ ) -> None:
65
+ super().__init__()
66
+ if rank <= 0:
67
+ raise ValueError(f"LoRA rank must be positive; got {rank}")
68
+ self.base_linear = base_linear
69
+ for param in self.base_linear.parameters():
70
+ param.requires_grad_(False)
71
+
72
+ in_features = base_linear.in_features
73
+ out_features = base_linear.out_features
74
+ self.rank = rank
75
+ self.scaling = float(alpha) / float(rank)
76
+ self.lora_dropout = nn.Dropout(dropout) if dropout > 0.0 else nn.Identity()
77
+ self.lora_a = nn.Parameter(torch.empty(rank, in_features))
78
+ self.lora_b = nn.Parameter(torch.zeros(out_features, rank))
79
+ nn.init.kaiming_uniform_(self.lora_a, a=math.sqrt(5))
80
+
81
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
82
+ base_out = self.base_linear(x)
83
+ lora_out = F.linear(F.linear(self.lora_dropout(x), self.lora_a), self.lora_b)
84
+ return base_out + self.scaling * lora_out
85
+
86
+
87
+ def inject_late_layer_lora(
88
+ encoder_module: nn.Module,
89
+ top_k_layers: int = 4,
90
+ rank: int = 8,
91
+ alpha: float = 16.0,
92
+ dropout: float = 0.1,
93
+ target_Linear_suffixes: tuple[str, ...] = ("query", "key", "value", "dense"),
94
+ ) -> list[nn.Parameter]:
95
+ """Injects LoRA strictly into the top `top_k_layers` transformer blocks of an encoder.
96
+
97
+ Why only late layers (Stage 2 Fix):
98
+ Applying LoRA across all ESM-2 layers with `lr=3e-4` overwrites early evolutionary
99
+ conservation features. Restricting LoRA to the top 4-6 ESM-2 blocks and top 2 SELFormer
100
+ blocks adapts high-level pocket/warhead representations while preserving core folding priors.
101
+ """
102
+ transformer_layers: Sequence[nn.Module] | None = None
103
+ for attr_path in (
104
+ ("esm", "encoder", "layer"),
105
+ ("roberta", "encoder", "layer"),
106
+ ("encoder", "layer"),
107
+ ("layers",),
108
+ ):
109
+ curr: object = encoder_module
110
+ for attr in attr_path:
111
+ curr = getattr(curr, attr, None)
112
+ if curr is None:
113
+ break
114
+ if isinstance(curr, (nn.ModuleList, list, tuple)) and len(curr) > 0:
115
+ transformer_layers = curr # type: ignore[assignment]
116
+ break
117
+
118
+ if transformer_layers is None or top_k_layers <= 0:
119
+ return []
120
+
121
+ selected_blocks = list(transformer_layers)[-top_k_layers:]
122
+ lora_params: list[nn.Parameter] = []
123
+ for block in selected_blocks:
124
+ for parent_name, parent_mod in list(block.named_modules()):
125
+ for child_name, child_mod in list(parent_mod.named_children()):
126
+ if isinstance(child_mod, nn.Linear) and child_name.endswith(target_Linear_suffixes):
127
+ wrapped = LoRALinear(child_mod, rank=rank, alpha=alpha, dropout=dropout)
128
+ wrapped.to(
129
+ device=child_mod.weight.device,
130
+ dtype=child_mod.weight.dtype,
131
+ )
132
+ setattr(parent_mod, child_name, wrapped)
133
+ lora_params.extend([wrapped.lora_a, wrapped.lora_b])
134
+ return lora_params
135
+
136
+
137
+ # =============================================================================
138
+ # 2. Upgraded Cross-Attention Block (Stage 1 Fixes A3: Sigmoid/Top-K + Self-Attn)
139
+ # =============================================================================
140
+
141
+
142
+ AttentionMode = Literal["softmax", "sigmoid", "topk_softmax"]
143
+ GateMode = Literal["v1_sigmoid", "tanh_zero", "layerscale_zero"]
144
+ ArchVersion = Literal["v1_compat", "v2"]
145
+
146
+
147
+ class CrossAttentionBlock(nn.Module):
148
+ """Cross-modal attention block supporting both v1 state-dict compatibility and v2 upgrades.
149
+
150
+ Upgrades over original v1 `CrossAttentionBlock`:
151
+ 1. `attn_mode="sigmoid"` or `"topk_softmax"` (used on `drug <- protein` `d2p` blocks in v2):
152
+ Eliminates sequence-length softmax dilution when a ligand attends over L=512..1024 protein
153
+ residues of which ~95-98% are non-pocket background residues.
154
+ 2. `use_intra_self_attn=True` (v2):
155
+ Adds a pre-norm intra-modality self-attention sublayer (`SelfAttn -> CrossAttn -> FFN`) so
156
+ second-shell / allosteric residues can exchange ligand-conditioned messages with direct
157
+ pocket-contacting residues between cross-attention hops.
158
+ """
159
+
160
+ def __init__(
161
+ self,
162
+ width: int,
163
+ heads: int = 4,
164
+ dropout: float = 0.1,
165
+ attn_mode: AttentionMode = "softmax",
166
+ topk_keys: int = 32,
167
+ use_intra_self_attn: bool = False,
168
+ ) -> None:
169
+ super().__init__()
170
+ if width % heads != 0:
171
+ raise ValueError(f"width ({width}) must be divisible by heads ({heads})")
172
+ self.width = width
173
+ self.heads = heads
174
+ self.head_dim = width // heads
175
+ self.attn_mode = attn_mode
176
+ self.topk_keys = topk_keys
177
+ self.use_intra_self_attn = use_intra_self_attn
178
+
179
+ if use_intra_self_attn:
180
+ self.norm_self = nn.LayerNorm(width)
181
+ self.self_attn = nn.MultiheadAttention(
182
+ width, heads, dropout=dropout, batch_first=True
183
+ )
184
+ # Zero-initialized LayerScale on the added self-attention path so it starts smoothly
185
+ self.self_scale = nn.Parameter(torch.zeros(width))
186
+
187
+ # Exact attribute names `norm_q`, `norm_kv`, `attn`, `norm_ff`, `ff` match v1 checkpoints!
188
+ self.norm_q = nn.LayerNorm(width)
189
+ self.norm_kv = nn.LayerNorm(width)
190
+ self.attn = nn.MultiheadAttention(width, heads, dropout=dropout, batch_first=True)
191
+ if attn_mode == "sigmoid":
192
+ # Initialize bias to -log(64) so average summed sigmoid mass starts ~ O(1) independent of L
193
+ self.sigmoid_attn_bias = nn.Parameter(torch.full((1, heads, 1, 1), -math.log(64.0)))
194
+ self.norm_ff = nn.LayerNorm(width)
195
+ self.ff = nn.Sequential(
196
+ nn.Linear(width, width * 2),
197
+ nn.GELU(),
198
+ nn.Dropout(dropout),
199
+ nn.Linear(width * 2, width),
200
+ )
201
+ self.dropout = nn.Dropout(dropout)
202
+
203
+ def _custom_cross_attention(
204
+ self,
205
+ q_in: torch.Tensor,
206
+ kv_in: torch.Tensor,
207
+ context_padding_mask: torch.Tensor,
208
+ ) -> torch.Tensor:
209
+ """Executes `sigmoid` or `topk_softmax` multi-head attention using `self.attn` weights."""
210
+ bsz, q_len, _ = q_in.shape
211
+ kv_len = kv_in.shape[1]
212
+ q_proj, k_proj, v_proj = F._in_projection_packed(
213
+ q_in, kv_in, kv_in, self.attn.in_proj_weight, self.attn.in_proj_bias
214
+ )
215
+ q = q_proj.view(bsz, q_len, self.heads, self.head_dim).transpose(1, 2)
216
+ k = k_proj.view(bsz, kv_len, self.heads, self.head_dim).transpose(1, 2)
217
+ v = v_proj.view(bsz, kv_len, self.heads, self.head_dim).transpose(1, 2)
218
+
219
+ scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.head_dim)
220
+ pad_mask = context_padding_mask[:, None, None, :].bool()
221
+
222
+ if self.attn_mode == "sigmoid":
223
+ scores = scores + self.sigmoid_attn_bias
224
+ weights = torch.sigmoid(scores).masked_fill(pad_mask, 0.0)
225
+ # Normalize by active mass + 1.0 stabilizer so completely non-contacting queries stay ~0
226
+ denom = weights.sum(dim=-1, keepdim=True) + 1.0
227
+ weights = weights / denom
228
+ elif self.attn_mode == "topk_softmax":
229
+ scores = scores.masked_fill(pad_mask, -torch.inf)
230
+ k_eff = min(self.topk_keys, kv_len)
231
+ if k_eff < kv_len:
232
+ topk_vals, topk_idx = torch.topk(scores, k=k_eff, dim=-1)
233
+ sparse_scores = torch.full_like(scores, -torch.inf)
234
+ sparse_scores.scatter_(-1, topk_idx, topk_vals)
235
+ scores = sparse_scores
236
+ weights = torch.softmax(scores, dim=-1)
237
+ weights = torch.nan_to_num(weights, nan=0.0)
238
+ else:
239
+ raise ValueError(f"Unsupported custom attn_mode: {self.attn_mode}")
240
+
241
+ weights = F.dropout(weights, p=self.attn.dropout, training=self.training)
242
+ out = torch.matmul(weights, v)
243
+ out = out.transpose(1, 2).contiguous().view(bsz, q_len, self.width)
244
+ return F.linear(out, self.attn.out_proj.weight, self.attn.out_proj.bias)
245
+
246
+ def forward(
247
+ self,
248
+ query: torch.Tensor,
249
+ context: torch.Tensor,
250
+ context_padding_mask: torch.Tensor,
251
+ query_padding_mask: torch.Tensor | None = None,
252
+ ) -> torch.Tensor:
253
+ if self.use_intra_self_attn:
254
+ q_norm = self.norm_self(query)
255
+ self_out, _ = self.self_attn(
256
+ q_norm,
257
+ q_norm,
258
+ q_norm,
259
+ key_padding_mask=query_padding_mask,
260
+ need_weights=False,
261
+ )
262
+ query = query + self.dropout(self_out) * torch.tanh(self.self_scale)
263
+
264
+ q_pre = self.norm_q(query)
265
+ kv_pre = self.norm_kv(context)
266
+ if self.attn_mode == "softmax":
267
+ attended, _ = self.attn(
268
+ q_pre,
269
+ kv_pre,
270
+ kv_pre,
271
+ key_padding_mask=context_padding_mask,
272
+ need_weights=False,
273
+ )
274
+ else:
275
+ attended = self._custom_cross_attention(q_pre, kv_pre, context_padding_mask)
276
+
277
+ hidden = query + self.dropout(attended)
278
+ return hidden + self.dropout(self.ff(self.norm_ff(hidden)))
279
+
280
+
281
+ class BidirectionalCrossAttention(nn.Module):
282
+ """Coupled bidirectional (`p2d` and `d2p`) cross-modal interaction tower."""
283
+
284
+ def __init__(
285
+ self,
286
+ width: int = 384,
287
+ heads: int = 4,
288
+ layers: int = 2,
289
+ dropout: float = 0.1,
290
+ p2d_attn_mode: AttentionMode = "softmax",
291
+ d2p_attn_mode: AttentionMode = "softmax",
292
+ d2p_topk_keys: int = 32,
293
+ use_intra_self_attn: bool = False,
294
+ ) -> None:
295
+ super().__init__()
296
+ self.p2d = nn.ModuleList([
297
+ CrossAttentionBlock(
298
+ width,
299
+ heads,
300
+ dropout=dropout,
301
+ attn_mode=p2d_attn_mode,
302
+ use_intra_self_attn=use_intra_self_attn,
303
+ )
304
+ for _ in range(layers)
305
+ ])
306
+ self.d2p = nn.ModuleList([
307
+ CrossAttentionBlock(
308
+ width,
309
+ heads,
310
+ dropout=dropout,
311
+ attn_mode=d2p_attn_mode,
312
+ topk_keys=d2p_topk_keys,
313
+ use_intra_self_attn=use_intra_self_attn,
314
+ )
315
+ for _ in range(layers)
316
+ ])
317
+
318
+ def forward(
319
+ self,
320
+ protein: torch.Tensor,
321
+ drug: torch.Tensor,
322
+ protein_padding_mask: torch.Tensor,
323
+ drug_padding_mask: torch.Tensor,
324
+ protein_interface_mask: torch.Tensor | None = None,
325
+ ) -> tuple[torch.Tensor, torch.Tensor]:
326
+ if protein_interface_mask is not None:
327
+ allowed = protein_interface_mask.bool() & ~protein_padding_mask
328
+ if not bool(allowed.any(dim=1).all()):
329
+ raise ValueError("Every pair must have at least one attended protein residue")
330
+ protein_key_mask = ~allowed
331
+ else:
332
+ allowed = None
333
+ protein_key_mask = protein_padding_mask
334
+
335
+ for protein_block, drug_block in zip(self.p2d, self.d2p, strict=True):
336
+ next_protein = protein_block(
337
+ protein,
338
+ drug,
339
+ drug_padding_mask,
340
+ query_padding_mask=protein_padding_mask,
341
+ )
342
+ if allowed is not None:
343
+ next_protein = torch.where(allowed[:, :, None], next_protein, protein)
344
+ next_drug = drug_block(
345
+ drug,
346
+ protein,
347
+ protein_key_mask,
348
+ query_padding_mask=drug_padding_mask,
349
+ )
350
+ protein, drug = next_protein, next_drug
351
+ return protein, drug
352
+
353
+
354
+ # =============================================================================
355
+ # 3. Multi-Layer Hidden Mixer (Stage 1 Fix A4)
356
+ # =============================================================================
357
+
358
+
359
+ class MultiLayerHiddenMixer(nn.Module):
360
+ """Softmax-weighted mixture over the last `num_layers` backbone hidden states.
361
+
362
+ Why (Stage 1 Fix A4):
363
+ ESM-2's final layer (`last_hidden_state`) is specialized for unconditional amino-acid MLM
364
+ token reconstruction, whereas geometric/contact/physicochemical representations peak in the
365
+ late-intermediate layers (65%-85% depth). Mixing the last K=4 layers for `prot_proj_in`
366
+ gives cross-attention `Q/K/V` rich structural features while keeping the residual output
367
+ aligned with `last_hidden_state` right before the native `lm_head`.
368
+ """
369
+
370
+ def __init__(self, num_layers: int = 4) -> None:
371
+ super().__init__()
372
+ if num_layers < 1:
373
+ raise ValueError("num_layers must be >= 1")
374
+ self.num_layers = num_layers
375
+ self.layer_logits = nn.Parameter(torch.zeros(num_layers))
376
+
377
+ def forward(self, hidden_states: Sequence[torch.Tensor]) -> torch.Tensor:
378
+ selected = list(hidden_states)[-self.num_layers :]
379
+ if len(selected) == 1:
380
+ return selected[0]
381
+ weights = torch.softmax(self.layer_logits[-len(selected) :], dim=0)
382
+ stacked = torch.stack(selected, dim=0) # (K, B, L, D)
383
+ return (weights[:, None, None, None] * stacked).sum(dim=0)
384
+
385
+
386
+ # =============================================================================
387
+ # 4. Improved LogiCA Core Model (Supports `v1_compat` AND `v2` Pretraining)
388
+ # =============================================================================
389
+
390
+
391
+ class ImprovedLogiCAModel(nn.Module):
392
+ """Upgraded native-head-preserving LogiCA model.
393
+
394
+ Modes:
395
+ - `arch_version="v1_compat"`:
396
+ Matches the exact parameter names and shapes of released v1 BindingDB checkpoints
397
+ (`prot_gate`, `drug_gate`, `prot_proj_*`, `drug_proj_*`, `cross_attn.*`) so you can
398
+ load `best.pt` directly for Stage 2 Drug-Resistance fine-tuning today!
399
+ - `arch_version="v2"`:
400
+ Activates all Stage 1 BindingDB pretraining architectural upgrades:
401
+ * `gate_mode="tanh_zero"` (unit gradient at step 0, exact zero residual at init)
402
+ * Full-rank orthogonal `proj_in` + zero-initialized `proj_out` (all 1280 ESM-2 channels active)
403
+ * `d2p_attn_mode="sigmoid"` (no 1/L background dilution over 1024 protein residues)
404
+ * `use_intra_self_attn=True` (pocket residue-residue propagation between cross-attn hops)
405
+ * `num_protein_fusion_layers=4` (intermediate+final ESM-2 hidden mixture for `Q/K/V`)
406
+ """
407
+
408
+ def __init__(
409
+ self,
410
+ protein_encoder: nn.Module,
411
+ drug_encoder: nn.Module,
412
+ width: int = 384,
413
+ heads: int = 4,
414
+ layers: int = 2,
415
+ dropout: float = 0.1,
416
+ arch_version: ArchVersion = "v2",
417
+ gate_mode: GateMode | None = None,
418
+ d2p_attn_mode: AttentionMode | None = None,
419
+ d2p_topk_keys: int = 32,
420
+ use_intra_self_attn: bool | None = None,
421
+ num_protein_fusion_layers: int | None = None,
422
+ ) -> None:
423
+ super().__init__()
424
+ self.esm = protein_encoder
425
+ self.drug_encoder = drug_encoder
426
+ self.arch_version = arch_version
427
+
428
+ protein_width = int(self.esm.config.hidden_size)
429
+ drug_width = int(self.drug_encoder.config.hidden_size)
430
+ self.protein_width = protein_width
431
+ self.drug_width = drug_width
432
+ self.interaction_width = width
433
+
434
+ if gate_mode is None:
435
+ gate_mode = "v1_sigmoid" if arch_version == "v1_compat" else "tanh_zero"
436
+ if d2p_attn_mode is None:
437
+ d2p_attn_mode = "softmax" if arch_version == "v1_compat" else "sigmoid"
438
+ if use_intra_self_attn is None:
439
+ use_intra_self_attn = arch_version == "v2"
440
+ if num_protein_fusion_layers is None:
441
+ num_protein_fusion_layers = 1 if arch_version == "v1_compat" else 4
442
+
443
+ self.gate_mode = gate_mode
444
+ self.num_protein_fusion_layers = num_protein_fusion_layers
445
+
446
+ if num_protein_fusion_layers > 1:
447
+ self.prot_layer_mixer = MultiLayerHiddenMixer(num_protein_fusion_layers)
448
+ else:
449
+ self.prot_layer_mixer = None
450
+
451
+ # Projections between native encoder widths and shared interaction width `d_s`
452
+ self.prot_proj_in = nn.Linear(protein_width, width, bias=False)
453
+ self.prot_proj_out = nn.Linear(width, protein_width, bias=False)
454
+ self.drug_proj_in = nn.Linear(drug_width, width, bias=False)
455
+ self.drug_proj_out = nn.Linear(width, drug_width, bias=False)
456
+
457
+ if gate_mode == "v1_sigmoid":
458
+ self.prot_gate = nn.Parameter(torch.tensor([-6.0]))
459
+ self.drug_gate = nn.Parameter(torch.tensor([-6.0]))
460
+ for proj in (
461
+ self.prot_proj_in,
462
+ self.prot_proj_out,
463
+ self.drug_proj_in,
464
+ self.drug_proj_out,
465
+ ):
466
+ self._init_v1_partial_identity(proj)
467
+ elif gate_mode == "tanh_zero":
468
+ # Fix A1: tanh(0) == 0.0 (exact identity at step 0), derivative sech^2(0) == 1.0 (400x larger!)
469
+ self.prot_gate = nn.Parameter(torch.zeros(1))
470
+ self.drug_gate = nn.Parameter(torch.zeros(1))
471
+ nn.init.orthogonal_(self.prot_proj_in.weight)
472
+ nn.init.orthogonal_(self.drug_proj_in.weight)
473
+ nn.init.orthogonal_(self.prot_proj_out.weight)
474
+ nn.init.orthogonal_(self.drug_proj_out.weight)
475
+ elif gate_mode == "layerscale_zero":
476
+ # Per-channel gating + orthogonal proj_in & small orthogonal proj_out
477
+ self.prot_gate = nn.Parameter(torch.zeros(protein_width))
478
+ self.drug_gate = nn.Parameter(torch.zeros(drug_width))
479
+ nn.init.orthogonal_(self.prot_proj_in.weight)
480
+ nn.init.orthogonal_(self.drug_proj_in.weight)
481
+ nn.init.orthogonal_(self.prot_proj_out.weight)
482
+ nn.init.orthogonal_(self.drug_proj_out.weight)
483
+ else:
484
+ raise ValueError(f"Unknown gate_mode: {gate_mode}")
485
+
486
+ self.cross_attn = BidirectionalCrossAttention(
487
+ width=width,
488
+ heads=heads,
489
+ layers=layers,
490
+ dropout=dropout,
491
+ p2d_attn_mode="softmax",
492
+ d2p_attn_mode=d2p_attn_mode,
493
+ d2p_topk_keys=d2p_topk_keys,
494
+ use_intra_self_attn=use_intra_self_attn,
495
+ )
496
+ self.pair_alpha_logit = nn.Parameter(torch.zeros(1))
497
+
498
+ @classmethod
499
+ def from_pretrained_backbones(
500
+ cls,
501
+ protein_model: str = "facebook/esm2_t30_150M_UR50D",
502
+ drug_model: str = "HUBioDataLab/SELFormer",
503
+ width: int = 384,
504
+ cache_dir: str | None = None,
505
+ arch_version: ArchVersion = "v2",
506
+ **kwargs: object,
507
+ ) -> "ImprovedLogiCAModel":
508
+ if AutoModelForMaskedLM is None:
509
+ raise RuntimeError("`transformers` is required to load HuggingFace backbones")
510
+ esm = AutoModelForMaskedLM.from_pretrained(protein_model, cache_dir=cache_dir)
511
+ drug = AutoModelForMaskedLM.from_pretrained(drug_model, cache_dir=cache_dir)
512
+ return cls(
513
+ esm,
514
+ drug,
515
+ width=width,
516
+ arch_version=arch_version,
517
+ **kwargs, # type: ignore[arg-type]
518
+ )
519
+
520
+ @staticmethod
521
+ def _init_v1_partial_identity(layer: nn.Linear) -> None:
522
+ with torch.no_grad():
523
+ layer.weight.zero_()
524
+ diag = min(layer.out_features, layer.in_features)
525
+ layer.weight[:diag, :diag] = torch.eye(diag)
526
+
527
+ def _gate_scale(self, gate_param: torch.Tensor) -> torch.Tensor:
528
+ if self.gate_mode == "v1_sigmoid":
529
+ return torch.sigmoid(gate_param)
530
+ if self.gate_mode in ("tanh_zero", "layerscale_zero"):
531
+ return torch.tanh(gate_param)
532
+ raise ValueError(f"Unsupported gate_mode: {self.gate_mode}")
533
+
534
+ def pair_alpha(self) -> torch.Tensor:
535
+ return torch.sigmoid(self.pair_alpha_logit)
536
+
537
+ def load_v1_checkpoint(self, checkpoint_path: Path | str) -> dict[str, object]:
538
+ """Loads an official v1 BindingDB checkpoint into `arch_version='v1_compat'` cleanly."""
539
+ payload = torch.load(checkpoint_path, map_location="cpu", weights_only=False)
540
+ state = payload.get("model_state_dict", payload)
541
+ incompatible = self.load_state_dict(state, strict=False)
542
+ critical_prefixes = (
543
+ "prot_gate",
544
+ "drug_gate",
545
+ "prot_proj_",
546
+ "drug_proj_",
547
+ "cross_attn.",
548
+ )
549
+ critical_missing = [
550
+ key for key in incompatible.missing_keys if key.startswith(critical_prefixes)
551
+ ]
552
+ if critical_missing or incompatible.unexpected_keys:
553
+ raise RuntimeError(
554
+ f"Checkpoint mismatch: critical_missing={critical_missing}, "
555
+ f"unexpected={incompatible.unexpected_keys}"
556
+ )
557
+ return {
558
+ "epoch": payload.get("epoch"),
559
+ "metrics": payload.get("metrics", {}),
560
+ }
561
+
562
+ def encode_protein_hidden(
563
+ self,
564
+ protein_input_ids: torch.Tensor,
565
+ protein_attention_mask: torch.Tensor,
566
+ ) -> tuple[torch.Tensor, torch.Tensor]:
567
+ """Returns `(protein_last_hidden, protein_adapter_query_hidden)`."""
568
+ need_multi = self.prot_layer_mixer is not None
569
+ out = self.esm.base_model(
570
+ input_ids=protein_input_ids,
571
+ attention_mask=protein_attention_mask,
572
+ output_hidden_states=need_multi,
573
+ return_dict=True,
574
+ )
575
+ last_hidden = out.last_hidden_state
576
+ if self.prot_layer_mixer is not None and out.hidden_states is not None:
577
+ adapter_hidden = self.prot_layer_mixer(out.hidden_states)
578
+ else:
579
+ adapter_hidden = last_hidden
580
+ return last_hidden, adapter_hidden
581
+
582
+ def encode_drug_hidden(
583
+ self,
584
+ drug_input_ids: torch.Tensor,
585
+ drug_attention_mask: torch.Tensor,
586
+ ) -> torch.Tensor:
587
+ out = self.drug_encoder.base_model(
588
+ input_ids=drug_input_ids,
589
+ attention_mask=drug_attention_mask,
590
+ return_dict=True,
591
+ )
592
+ return out.last_hidden_state
593
+
594
+ def encode_hidden(
595
+ self,
596
+ protein_input_ids: torch.Tensor,
597
+ protein_attention_mask: torch.Tensor,
598
+ drug_input_ids: torch.Tensor,
599
+ drug_attention_mask: torch.Tensor,
600
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
601
+ p_last, p_qkv = self.encode_protein_hidden(protein_input_ids, protein_attention_mask)
602
+ d_last = self.encode_drug_hidden(drug_input_ids, drug_attention_mask)
603
+ return p_last, p_qkv, d_last
604
+
605
+ def forward_from_hidden(
606
+ self,
607
+ protein_last_hidden: torch.Tensor,
608
+ protein_attention_mask: torch.Tensor,
609
+ drug_hidden: torch.Tensor,
610
+ drug_attention_mask: torch.Tensor,
611
+ protein_adapter_input: torch.Tensor | None = None,
612
+ protein_interface_mask: torch.Tensor | None = None,
613
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
614
+ """Runs cross-attention adapter + native heads, returning holo and apo logits."""
615
+ p_in = protein_last_hidden if protein_adapter_input is None else protein_adapter_input
616
+ protein_interaction_0 = self.prot_proj_in(p_in)
617
+ drug_interaction_0 = self.drug_proj_in(drug_hidden)
618
+ protein_interaction, drug_interaction = self.cross_attn(
619
+ protein_interaction_0,
620
+ drug_interaction_0,
621
+ protein_attention_mask == 0,
622
+ drug_attention_mask == 0,
623
+ protein_interface_mask=protein_interface_mask,
624
+ )
625
+ p_scale = self._gate_scale(self.prot_gate)
626
+ d_scale = self._gate_scale(self.drug_gate)
627
+
628
+ protein_contextual = protein_last_hidden + p_scale * self.prot_proj_out(
629
+ protein_interaction - protein_interaction_0
630
+ )
631
+ drug_contextual = drug_hidden + d_scale * self.drug_proj_out(
632
+ drug_interaction - drug_interaction_0
633
+ )
634
+ protein_logits = self.esm.lm_head(protein_contextual)
635
+ drug_logits = self.drug_encoder.lm_head(drug_contextual)
636
+ protein_apo_logits = self.esm.lm_head(protein_last_hidden)
637
+ drug_apo_logits = self.drug_encoder.lm_head(drug_hidden)
638
+ return protein_logits, drug_logits, protein_apo_logits, drug_apo_logits
639
+
640
+ def forward(
641
+ self,
642
+ protein_input_ids: torch.Tensor,
643
+ protein_attention_mask: torch.Tensor,
644
+ drug_input_ids: torch.Tensor,
645
+ drug_attention_mask: torch.Tensor,
646
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
647
+ p_last, p_qkv, d_last = self.encode_hidden(
648
+ protein_input_ids,
649
+ protein_attention_mask,
650
+ drug_input_ids,
651
+ drug_attention_mask,
652
+ )
653
+ return self.forward_from_hidden(
654
+ p_last,
655
+ protein_attention_mask,
656
+ d_last,
657
+ drug_attention_mask,
658
+ protein_adapter_input=p_qkv,
659
+ )
660
+
661
+ def score_masked_sites_20aa(
662
+ self,
663
+ wt_input_ids: torch.Tensor,
664
+ protein_attention_mask: torch.Tensor,
665
+ residue_zero_based_indices: Sequence[int],
666
+ mask_token_id: int,
667
+ aa_vocab_token_ids: Sequence[int],
668
+ drug_input_ids: torch.Tensor,
669
+ drug_attention_mask: torch.Tensor,
670
+ ) -> dict[str, torch.Tensor]:
671
+ """Stage 2 Single-Pass Masked-Site 20-AA Scorer (`alpha = 1.0`, zero self-token leakage).
672
+
673
+ Given a wild-type sequence (`1 x L_tok`), masks the queried site(s) `i in M` (`token = i + 1`
674
+ after `<cls>`), encodes the masked protein ONCE, and evaluates contextualized holo and apo
675
+ 20-AA log-probability distributions across `B_drugs` ligands simultaneously!
676
+
677
+ Returns dict with tensors of shape `(B_drugs, |M|, 20)`:
678
+ - `holo_log_probs_20aa`: log pi_theta(a | x_{\\M}, y) normalized over the 20 canonical AAs
679
+ - `apo_log_probs_20aa`: log pi_theta(a | x_{\\M}, empty) normalized over the 20 canonical AAs
680
+ - `dd_ll_drug_20aa`: holo_log_probs_20aa - apo_log_probs_20aa (pure drug shift per AA!)
681
+ """
682
+ if wt_input_ids.shape[0] != 1:
683
+ raise ValueError("wt_input_ids must have shape (1, L_tok) for a single target gene")
684
+ if len(aa_vocab_token_ids) != 20:
685
+ raise ValueError("aa_vocab_token_ids must list the 20 canonical amino-acid token IDs")
686
+
687
+ token_positions = torch.tensor(
688
+ [int(idx) + 1 for idx in residue_zero_based_indices],
689
+ dtype=torch.long,
690
+ device=wt_input_ids.device,
691
+ )
692
+ masked_ids = wt_input_ids.clone()
693
+ masked_ids[0, token_positions] = int(mask_token_id)
694
+
695
+ # Encode masked protein ONCE (1 x L_tok x D), then broadcast across B_drugs ligands!
696
+ p_last, p_qkv = self.encode_protein_hidden(masked_ids, protein_attention_mask)
697
+ d_last = self.encode_drug_hidden(drug_input_ids, drug_attention_mask)
698
+ b_drugs = d_last.shape[0]
699
+
700
+ p_last_b = p_last.expand(b_drugs, -1, -1)
701
+ p_qkv_b = p_qkv.expand(b_drugs, -1, -1)
702
+ p_mask_b = protein_attention_mask.expand(b_drugs, -1)
703
+
704
+ p_holo_logits, _, p_apo_logits, _ = self.forward_from_hidden(
705
+ p_last_b,
706
+ p_mask_b,
707
+ d_last,
708
+ drug_attention_mask,
709
+ protein_adapter_input=p_qkv_b,
710
+ )
711
+
712
+ aa_ids = torch.tensor(aa_vocab_token_ids, dtype=torch.long, device=wt_input_ids.device)
713
+ holo_site_20 = p_holo_logits[:, token_positions, :][:, :, aa_ids]
714
+ apo_site_20 = p_apo_logits[:, token_positions, :][:, :, aa_ids]
715
+
716
+ holo_log_probs = F.log_softmax(holo_site_20, dim=-1)
717
+ apo_log_probs = F.log_softmax(apo_site_20, dim=-1)
718
+ return {
719
+ "holo_log_probs_20aa": holo_log_probs,
720
+ "apo_log_probs_20aa": apo_log_probs,
721
+ "dd_ll_drug_20aa": holo_log_probs - apo_log_probs,
722
+ }
723
+
724
+
725
+ # Drop-in alias matching the original module's class name
726
+ LogiCAModel = ImprovedLogiCAModel
727
+
728
+
729
+ # =============================================================================
730
+ # 5. Masking, Log-Probability & Stage 2 Differential Optimizer Helpers
731
+ # =============================================================================
732
+
733
+
734
+ def _stable_seed(text: str, seed: int) -> int:
735
+ digest = hashlib.blake2b(f"{seed}:{text}".encode(), digest_size=8).digest()
736
+ return int.from_bytes(digest, "little")
737
+
738
+
739
+ def make_masked_inputs(
740
+ input_ids: torch.Tensor,
741
+ attention_mask: torch.Tensor,
742
+ special_tokens_mask: torch.Tensor,
743
+ mask_token_id: int,
744
+ mask_fraction: float = 0.15,
745
+ generator: torch.Generator | None = None,
746
+ forced_mask_positions: torch.Tensor | None = None,
747
+ ) -> tuple[torch.Tensor, torch.Tensor]:
748
+ """Masks `mask_fraction` of valid non-special tokens in each row (Stage 1 Fix A2).
749
+
750
+ If `forced_mask_positions` (`(B, L)` bool) is provided (e.g. exact differing residue
751
+ indices of a real BindingDB clinical mutant cliff), all valid forced positions are
752
+ guaranteed to be masked and included in `selected`, supplemented by random MLM tokens
753
+ up to `max(count, n_forced)`.
754
+ """
755
+ if not 0.0 < mask_fraction <= 1.0:
756
+ raise ValueError(f"mask_fraction must lie in (0, 1]; got {mask_fraction}")
757
+ masked = input_ids.clone()
758
+ selected = torch.zeros_like(input_ids, dtype=torch.bool)
759
+ for row in range(input_ids.shape[0]):
760
+ valid_mask = (attention_mask[row] == 1) & (special_tokens_mask[row] == 0)
761
+ candidates = torch.nonzero(valid_mask, as_tuple=False).flatten()
762
+ if candidates.numel() == 0:
763
+ raise ValueError(f"Row {row} has zero scoreable non-special tokens")
764
+ target_count = max(1, int(round(candidates.numel() * mask_fraction)))
765
+
766
+ if forced_mask_positions is not None:
767
+ forced_valid = torch.nonzero(
768
+ valid_mask & forced_mask_positions[row].bool(), as_tuple=False
769
+ ).flatten()
770
+ else:
771
+ forced_valid = candidates.new_empty((0,))
772
+
773
+ if forced_valid.numel() > 0:
774
+ selected[row, forced_valid] = True
775
+ masked[row, forced_valid] = int(mask_token_id)
776
+
777
+ remaining_needed = max(0, target_count - int(forced_valid.numel()))
778
+ if remaining_needed > 0:
779
+ rem_candidates = torch.nonzero(
780
+ valid_mask & (~selected[row]), as_tuple=False
781
+ ).flatten()
782
+ if rem_candidates.numel() > 0:
783
+ perm = torch.randperm(
784
+ rem_candidates.numel(), generator=generator, device=rem_candidates.device
785
+ )
786
+ chosen = rem_candidates[perm[:remaining_needed]]
787
+ selected[row, chosen] = True
788
+ masked[row, chosen] = int(mask_token_id)
789
+ return masked, selected
790
+
791
+
792
+ def mean_selected_log_probability(
793
+ logits: torch.Tensor,
794
+ original_ids: torch.Tensor,
795
+ selected_mask: torch.Tensor,
796
+ ) -> torch.Tensor:
797
+ """Computes `(1 / |M|) * sum_{i in M} log p(original_ids[i])` per row."""
798
+ token_log_probs = (
799
+ F.log_softmax(logits.float(), dim=-1)
800
+ .gather(-1, original_ids.unsqueeze(-1))
801
+ .squeeze(-1)
802
+ )
803
+ sel_f = selected_mask.to(token_log_probs.dtype)
804
+ return (token_log_probs * sel_f).sum(dim=1) / sel_f.sum(dim=1).clamp_min(1.0)
805
+
806
+
807
+ def build_stage2_optimizer_groups(
808
+ model: ImprovedLogiCAModel,
809
+ thermo_scorer: nn.Module | None = None,
810
+ base_lora_lr: float = 1e-4,
811
+ adapter_lr_multiplier: float = 5.0,
812
+ weight_decay: float = 1e-2,
813
+ top_k_esm_lora: int = 4,
814
+ top_k_drug_lora: int = 2,
815
+ lora_rank: int = 8,
816
+ lora_alpha: float = 16.0,
817
+ lora_dropout: float = 0.1,
818
+ ) -> list[dict[str, object]]:
819
+ """Configures Stage 2 fine-tuning with late-layer LoRA + 5x higher adapter/gate LR.
820
+
821
+ Why (Stage 2 Fix):
822
+ 1. Freezes early/mid ESM-2 and SELFormer layers completely (prevents catastrophic
823
+ destruction of evolutionary conservation priors under `lr=3e-4`).
824
+ 2. Injects `LoRALinear` (`r=8`) strictly into the top `top_k_esm_lora` ESM-2 blocks and
825
+ top `top_k_drug_lora` SELFormer blocks (so warhead differences like covalent Osimertinib
826
+ vs. reversible Gefitinib can also adapt!).
827
+ 3. Assigns `5x` higher learning rate to `prot_gate`, `cross_attn`, and projections so
828
+ cross-modal conditioning adapts rapidly without large backbone drift.
829
+ """
830
+ for param in model.parameters():
831
+ param.requires_grad_(False)
832
+
833
+ lora_params: list[nn.Parameter] = []
834
+ lora_params.extend(
835
+ inject_late_layer_lora(
836
+ model.esm,
837
+ top_k_layers=top_k_esm_lora,
838
+ rank=lora_rank,
839
+ alpha=lora_alpha,
840
+ dropout=lora_dropout,
841
+ )
842
+ )
843
+ lora_params.extend(
844
+ inject_late_layer_lora(
845
+ model.drug_encoder,
846
+ top_k_layers=top_k_drug_lora,
847
+ rank=lora_rank,
848
+ alpha=lora_alpha,
849
+ dropout=lora_dropout,
850
+ )
851
+ )
852
+
853
+ adapter_modules = [
854
+ model.prot_proj_in,
855
+ model.prot_proj_out,
856
+ model.drug_proj_in,
857
+ model.drug_proj_out,
858
+ model.cross_attn,
859
+ ]
860
+ if model.prot_layer_mixer is not None:
861
+ adapter_modules.append(model.prot_layer_mixer)
862
+
863
+ adapter_decay: list[nn.Parameter] = []
864
+ adapter_no_decay: list[nn.Parameter] = [model.prot_gate, model.drug_gate]
865
+ model.prot_gate.requires_grad_(True)
866
+ model.drug_gate.requires_grad_(True)
867
+
868
+ for mod in adapter_modules:
869
+ for name, param in mod.named_parameters():
870
+ param.requires_grad_(True)
871
+ if param.ndim < 2 or "norm" in name or "bias" in name or "scale" in name:
872
+ adapter_no_decay.append(param)
873
+ else:
874
+ adapter_decay.append(param)
875
+
876
+ groups: list[dict[str, object]] = [
877
+ {
878
+ "params": adapter_decay,
879
+ "lr": base_lora_lr * adapter_lr_multiplier,
880
+ "weight_decay": weight_decay,
881
+ "name": "adapter_decay",
882
+ },
883
+ {
884
+ "params": adapter_no_decay,
885
+ "lr": base_lora_lr * adapter_lr_multiplier,
886
+ "weight_decay": 0.0,
887
+ "name": "adapter_gates_and_norms",
888
+ },
889
+ ]
890
+ if lora_params:
891
+ groups.append({
892
+ "params": lora_params,
893
+ "lr": base_lora_lr,
894
+ "weight_decay": weight_decay,
895
+ "name": "late_layer_lora",
896
+ })
897
+ if thermo_scorer is not None:
898
+ thermo_params = [p for p in thermo_scorer.parameters() if p.requires_grad]
899
+ if thermo_params:
900
+ groups.append({
901
+ "params": thermo_params,
902
+ "lr": base_lora_lr * adapter_lr_multiplier,
903
+ "weight_decay": 0.0,
904
+ "name": "thermodynamic_scalars",
905
+ })
906
+ return groups
checkpoints/v2_20261110/150m/preparation_manifest.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "source_url": "https://bindingdb.org/rwd/bind/downloads/BindingDB_All_202611_tsv.zip",
3
+ "raw_md5": "932346d34d4a3ab823b58ae6ff63b7e8",
4
+ "upstream_commit": "31526bb4a91806da79cddaeb4ccd8b3bd065bee3",
5
+ "framework_commit": "77d29888ef5e7aaa698fdad1478262a94a63c57a",
6
+ "max_rows": null,
7
+ "heldout_target_sequences": 3071,
8
+ "decontamination_removed": 1306324,
9
+ "protein_family_overlap": 0,
10
+ "seconds": 663.6932878610096,
11
+ "heldout_construct_min_length": 30,
12
+ "residual_heldout_protein_matches": 0,
13
+ "max_protein_tokens": 1024,
14
+ "max_drug_tokens": 256,
15
+ "protein_mask_token_id": 32,
16
+ "drug_mask_token_id": 4,
17
+ "protein_unknown_tokens": 0,
18
+ "num_unique_proteins": 7256,
19
+ "num_protein_families": 5476,
20
+ "num_unique_drugs": 595365,
21
+ "num_total_pairs": 806165,
22
+ "num_positive_pairs": 329741,
23
+ "num_verified_negative_pairs": 186326,
24
+ "num_gray_zone_pairs": 290098,
25
+ "num_train_pairs": 700580,
26
+ "num_val_pairs": 105585
27
+ }
checkpoints/v2_20261110/150m/release_summary.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "scale": "150M",
3
+ "architecture": "v2",
4
+ "completed_run_step": 62520,
5
+ "selected_checkpoint_step": 1000,
6
+ "selected_checkpoint_epoch": 1,
7
+ "selected_checkpoint_metrics": {
8
+ "acc_drug_anchor": 0.1142578125,
9
+ "acc_protein_anchor": 0.1328125,
10
+ "loss": 2.659006729722023,
11
+ "loss_drug_anchor": 2.6828183494508266,
12
+ "loss_protein_anchor": 2.6351951584219933
13
+ },
14
+ "final_checkpoint_step": 62520,
15
+ "final_checkpoint_metrics": {
16
+ "acc_drug_anchor": 0.2734375,
17
+ "acc_protein_anchor": 0.287109375,
18
+ "loss": 4.379277817904949,
19
+ "loss_drug_anchor": 4.298316143453121,
20
+ "loss_protein_anchor": 4.460239566862583
21
+ },
22
+ "checkpoint_selection": "minimum held-out validation loss"
23
+ }