File size: 16,797 Bytes
ee82dfc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
# deployment_common.py — shared code for the liver deployment refits.
# The tox_common block below is copied VERBATIM from the arms' own notebooks
# (kaggle/notebooks/liver_arch3_lazypredict.py, shared programme conventions
# cell), and the graph block (ELEMS / mol_to_graph / GATNet) is copied
# VERBATIM from kaggle/notebooks/liver_arch2_gat.py. Deployment scripts reuse
# the arms' exact identity, featurization and model code — no re-implementation.
import os, json, time
import numpy as np
import pandas as pd

# ---- shared programme conventions (tox_common, inlined verbatim) ----
"""Shared conventions for the organ-toxicity programme (liver + kidney).

Binding rules (PROJECT_SPEC.md + READING_NOTES harmonization protocols):
- classification ONLY; identity = InChIKey connectivity block (first 14 chars)
  of the RDKit-standardized parent structure (largest fragment).
- seed-42 scaffold-disjoint (Bemis-Murcko) train/val/test split.
- preprocessing (imputation / scaling) fit on the TRAIN fold only.
- regulatory benchmark compounds are NEVER in training; lockout verified = 0
  with the dual key (deposited connectivity UNION standardized-parent
  connectivity) against EACH benchmark file separately.
- benchmark scoring is reported PER SET (withdrawn / clinical failures /
  pass), never pooled; metrics at full coverage.
"""
import json
import numpy as np
import pandas as pd

SEED = 42

# ---------------------------------------------------------------- identity
def _inchi_key(mol):
    """InChIKey via whichever RDKit API the installed build provides.
    DEPLOYMENT COMPATIBILITY FIX (2026-11-15): the arms' verbatim code calls
    rdMolDescriptors.CalcInchiKey, which does not exist in RDKit 2024.03.6 or
    2026.09.1 (it silently made parent_connectivity return None there).
    rdkit.Chem.inchi.MolToInchiKey is the same InChI computation; verified
    byte-identical keys across both installed RDKit builds on a sample set
    (salts, stereo, charged species)."""
    try:
        from rdkit.Chem import inchi
        return inchi.MolToInchiKey(mol)
    except Exception:
        from rdkit.Chem import rdMolDescriptors
        return rdMolDescriptors.CalcInchiKey(mol)


def parent_connectivity(smiles):
    """Connectivity block of the standardized parent (largest fragment)."""
    try:
        from rdkit import Chem
        from rdkit.Chem.MolStandardize import rdMolStandardize
        if smiles is None or (isinstance(smiles, float) and np.isnan(smiles)):
            return None
        mol = Chem.MolFromSmiles(str(smiles))
        if mol is None:
            return None
        parent = rdMolStandardize.LargestFragmentChooser().choose(mol)
        ik = _inchi_key(parent)
        return ik[:14] if ik else None
    except Exception:
        return None


def deposited_connectivity(inchikey):
    if inchikey is None or (isinstance(inchikey, float) and np.isnan(inchikey)):
        return None
    s = str(inchikey).strip()
    return s[:14] if len(s) >= 14 else None


def dual_keys(df, smiles_col="canonical_smiles", inchikey_col="inchikey"):
    """Set of connectivity keys: deposited UNION standardized-parent."""
    keys = set()
    dep = df[inchikey_col].map(deposited_connectivity) if inchikey_col in df else pd.Series([None] * len(df))
    par = df[smiles_col].map(parent_connectivity)
    for a, b in zip(dep, par):
        if a:
            keys.add(a)
        if b:
            keys.add(b)
    return keys


def verify_lockout(train_df, bench_dfs, train_smiles="canonical_smiles"):
    """bench_dfs: dict name -> df (with 'smiles' + 'inchikey' columns).
    Returns dict of overlap counts; every count must be 0."""
    train_keys = dual_keys(train_df, smiles_col=train_smiles)
    out = {}
    for name, bdf in bench_dfs.items():
        bk = dual_keys(bdf, smiles_col="smiles")
        out[name] = len(train_keys & bk)
    return out


# ---------------------------------------------------------------- loading
def load_train(path):
    df = pd.read_csv(path)
    df = df.rename(columns={"canonical_smiles": "smiles"})
    df["label"] = df["final_label"].astype(int)
    df["name"] = df["drug_name"] if "drug_name" in df else ""
    df["conn"] = df["inchikey_connectivity"]
    keep = ["name", "smiles", "inchikey", "conn", "label"]
    for c in ["sources", "sources_present", "final_label_source"]:
        if c in df.columns:
            keep.append(c)
    return df[keep].reset_index(drop=True)


def load_benchmark(path, expected_label):
    df = pd.read_csv(path)
    name_col = "drug_name" if "drug_name" in df.columns else "compound_name"
    out = pd.DataFrame({
        "name": df[name_col],
        "smiles": df["smiles"] if "smiles" in df else None,
        "inchikey": df["inchikey"] if "inchikey" in df else None,
    })
    if "evidence_grade" in df.columns:
        out["evidence_grade"] = df["evidence_grade"]
    out["label"] = expected_label
    return out


def dedup_benchmark(df):
    """Dedup benchmark rows to unique compounds (deposited connectivity,
    falling back to standardized-parent connectivity). Rows with no usable
    structure are counted as unscored."""
    keys, keep_idx, unscored = {}, [], 0
    for i, row in df.iterrows():
        k = deposited_connectivity(row.get("inchikey")) or parent_connectivity(row.get("smiles"))
        if k is None:
            unscored += 1
            continue
        if k in keys:
            continue
        keys[k] = True
        keep_idx.append(i)
    out = df.loc[keep_idx].copy()
    out["conn"] = [deposited_connectivity(r.get("inchikey")) or parent_connectivity(r.get("smiles"))
                    for _, r in out.iterrows()]
    return out.reset_index(drop=True), unscored


# ---------------------------------------------------------------- split
def scaffold_split(df, seed=SEED, fracs=(0.80, 0.10, 0.10)):
    """Seed-42 Bemis-Murcko scaffold-disjoint train/val/test split."""
    from rdkit import Chem
    from rdkit.Chem.Scaffolds import MurckoScaffold
    rng = np.random.RandomState(seed)
    scaffolds = []
    for smi in df["smiles"]:
        mol = Chem.MolFromSmiles(str(smi))
        try:
            scaf = MurckoScaffold.MurckoScaffoldSmiles(mol=mol) if mol is not None else ""
        except Exception:
            scaf = ""
        scaffolds.append(scaf if scaf else f"ACYCLIC::{smi}")
    groups = {}
    for i, s in enumerate(scaffolds):
        groups.setdefault(s, []).append(i)
    group_list = list(groups.values())
    rng.shuffle(group_list)
    group_list.sort(key=len, reverse=True)
    n = len(df)
    targets = [f * n for f in fracs]
    buckets = [[], [], []]
    counts = [0, 0, 0]
    for g in group_list:
        # assign each scaffold group to the bucket furthest below its target
        j = int(np.argmax([targets[k] - counts[k] for k in range(3)]))
        buckets[j].extend(g)
        counts[j] += len(g)
    names = ["train", "val", "test"]
    split = pd.Series(index=df.index, dtype=object)
    for j, idxs in enumerate(buckets):
        split.iloc[idxs] = names[j]
    # scaffold disjointness assertion
    seen = {}
    for j, idxs in enumerate(buckets):
        for i in idxs:
            s = scaffolds[i]
            assert seen.get(s, j) == j, "scaffold leaked across splits"
            seen[s] = j
    return split


# ---------------------------------------------------------------- features
PHYSCHEM_NAMES = ["MolWt", "MolLogP", "TPSA", "NumHDonors", "NumHAcceptors",
                  "NumRotatableBonds", "RingCount", "NumAromaticRings",
                  "NumAliphaticRings", "FractionCSP3", "HeavyAtomCount",
                  "NumValenceElectrons", "BertzCT"]


def featurize(smiles_list):
    """Returns dict of float32 blocks + validity mask. Blocks:
    morgan_bits (1024), morgan_counts (1024), maccs (167),
    desc (full RDKit descriptor list), physchem (PHYSCHEM_NAMES subset)."""
    from rdkit import Chem
    from rdkit.Chem import AllChem, MACCSkeys, Descriptors
    from rdkit import RDLogger
    RDLogger.DisableLog("rdApp.*")
    desc_names = [d[0] for d in Descriptors._descList]
    phys_idx = [desc_names.index(n) for n in PHYSCHEM_NAMES]
    n = len(smiles_list)
    blocks = {
        "morgan_bits": np.zeros((n, 1024), dtype=np.float32),
        "morgan_counts": np.zeros((n, 1024), dtype=np.float32),
        "maccs": np.zeros((n, 167), dtype=np.float32),
        "desc": np.full((n, len(desc_names)), np.nan, dtype=np.float32),
    }
    valid = np.zeros(n, dtype=bool)
    from rdkit import DataStructs
    gen_bits = AllChem.GetMorganGenerator(radius=2, fpSize=1024)
    gen_counts = AllChem.GetMorganGenerator(radius=2, fpSize=1024, countSimulation=True)
    for i, smi in enumerate(smiles_list):
        mol = Chem.MolFromSmiles(str(smi)) if smi is not None and str(smi) != "nan" else None
        if mol is None:
            continue
        valid[i] = True
        DataStructs.ConvertToNumpyArray(gen_bits.GetFingerprint(mol), blocks["morgan_bits"][i])
        DataStructs.ConvertToNumpyArray(gen_counts.GetCountFingerprint(mol), blocks["morgan_counts"][i])
        blocks["maccs"][i] = np.asarray(MACCSkeys.GenMACCSKeys(mol), dtype=np.float32)
        # iterate _descList directly: identical values to CalcMolDescriptors
        # at ~1/3 the cost (measured on this VM, RDKit 2026.09)
        blocks["desc"][i] = np.array([fn(mol) for _, fn in Descriptors._descList], dtype=np.float32)
    blocks["physchem"] = blocks["desc"][:, phys_idx].copy()
    blocks["desc_names"] = desc_names
    return blocks, valid


class DescPrep:
    """Train-fold-fitted descriptor preprocessing: drop columns >50% missing
    or near-constant ON TRAIN, median-impute, clip to train 0.1/99.9 pct,
    standardize with train mean/std."""

    def fit(self, X):
        X = np.asarray(X, dtype=np.float64)
        miss = np.mean(~np.isfinite(X), axis=0)
        std = np.nanstd(np.where(np.isfinite(X), X, np.nan), axis=0)
        self.keep_ = (miss <= 0.5) & (std > 1e-8)
        Xk = X[:, self.keep_]
        self.med_ = np.nanmedian(np.where(np.isfinite(Xk), Xk, np.nan), axis=0)
        Xi = np.where(np.isfinite(Xk), Xk, self.med_)
        self.lo_ = np.percentile(Xi, 0.1, axis=0)
        self.hi_ = np.percentile(Xi, 99.9, axis=0)
        Xc = np.clip(Xi, self.lo_, self.hi_)
        self.mu_ = Xc.mean(axis=0)
        self.sd_ = Xc.std(axis=0) + 1e-9
        return self

    def transform(self, X):
        X = np.asarray(X, dtype=np.float64)[:, self.keep_]
        Xi = np.where(np.isfinite(X), X, self.med_)
        return ((np.clip(Xi, self.lo_, self.hi_) - self.mu_) / self.sd_).astype(np.float32)


def prep_matrix(blocks, prep, kind):
    """kind: 'fp' (morgan bits) / 'fp_desc' (bits+maccs+desc) /
    'desc_maccs_counts' (arch-5 block b) / 'physchem' raw physchem."""
    if kind == "fp":
        return blocks["morgan_bits"]
    if kind == "fp_desc":
        return np.hstack([blocks["morgan_bits"], blocks["maccs"], prep.transform(blocks["desc"])]).astype(np.float32)
    if kind == "desc_maccs_counts":
        return np.hstack([prep.transform(blocks["desc"]), blocks["maccs"], blocks["morgan_counts"]]).astype(np.float32)
    if kind == "physchem":
        return blocks["physchem"]
    raise ValueError(kind)


# ---------------------------------------------------------------- metrics
def metrics_dict(y_true, prob, threshold=0.5):
    from sklearn.metrics import (accuracy_score, balanced_accuracy_score,
                                 matthews_corrcoef, roc_auc_score, confusion_matrix)
    y_true = np.asarray(y_true, dtype=int)
    prob = np.asarray(prob, dtype=float)
    pred = (prob >= threshold).astype(int)
    out = {"n": int(len(y_true)), "threshold": threshold,
           "accuracy": float(accuracy_score(y_true, pred)),
           "balanced_accuracy": float(balanced_accuracy_score(y_true, pred))}
    tn, fp, fn, tp = confusion_matrix(y_true, pred, labels=[0, 1]).ravel()
    out.update({"tn": int(tn), "fp": int(fp), "fn": int(fn), "tp": int(tp),
                "sensitivity_recall_pos": float(tp / (tp + fn)) if (tp + fn) else None,
                "specificity_recall_neg": float(tn / (tn + fp)) if (tn + fp) else None})
    if len(np.unique(y_true)) == 2 and len(np.unique(pred)) == 2:
        out["mcc"] = float(matthews_corrcoef(y_true, pred))
    else:
        out["mcc"] = None
        out["mcc_note"] = "single-class ground truth or single-class predictions"
    if len(np.unique(y_true)) == 2:
        out["auroc"] = float(roc_auc_score(y_true, prob))
    else:
        out["auroc"] = None
    return out


def score_benchmark_sets(predict_proba_fn, bench_sets):
    """bench_sets: dict name -> deduped df (with 'smiles','label','name').
    predict_proba_fn(smiles_list) -> P(label=1) array (NaN where unparseable).
    Returns (metrics dict per set, predictions dataframe)."""
    all_metrics, pred_frames = {}, []
    for set_name, bdf in bench_sets.items():
        prob = predict_proba_fn(list(bdf["smiles"]))
        scored = np.isfinite(prob)
        m = metrics_dict(bdf.loc[scored, "label"], prob[scored]) if scored.any() else {"n": 0}
        m["n_rows_unique"] = int(len(bdf))
        m["n_scored"] = int(scored.sum())
        m["n_unscored_invalid_structure"] = int((~scored).sum())
        all_metrics[set_name] = m
        pf = bdf[["name", "smiles", "label"]].copy()
        if "evidence_grade" in bdf.columns:
            pf["evidence_grade"] = bdf["evidence_grade"]
        pf["set"] = set_name
        pf["prob_toxic"] = prob
        pf["pred_label"] = np.where(scored, (prob >= 0.5).astype(float), np.nan)
        pf["correct"] = np.where(scored, (pf["pred_label"] == pf["label"]).astype(float), np.nan)
        pred_frames.append(pf)
    return all_metrics, pd.concat(pred_frames, ignore_index=True)

# ---- GAT graph code (verbatim from liver_arch2_gat.py) ----
# Import guard (deployment plumbing only, model code below unchanged): the
# sklearn-side deployment scripts import this module in an env without
# torch_geometric; the graph functions are only called in the pyg env.
try:
    import torch
    import torch.nn.functional as F
    from torch_geometric.data import Data
    from torch_geometric.nn import GATConv, global_mean_pool
    _TORCH_OK = True
except ImportError:
    torch = None
    _TORCH_OK = False

ELEMS = [6, 7, 8, 16, 9, 17, 35, 53, 15, 14]
def mol_to_graph(smi):
    from rdkit import Chem
    HYB = [Chem.rdchem.HybridizationType.SP, Chem.rdchem.HybridizationType.SP2,
           Chem.rdchem.HybridizationType.SP3, Chem.rdchem.HybridizationType.SP3D,
           Chem.rdchem.HybridizationType.SP3D2]
    BOND = {Chem.rdchem.BondType.SINGLE: 0, Chem.rdchem.BondType.DOUBLE: 1,
            Chem.rdchem.BondType.TRIPLE: 2, Chem.rdchem.BondType.AROMATIC: 3}
    def oh(v, n):
        o = [0.0] * n; o[min(max(v, 0), n - 1)] = 1.0; return o
    mol = Chem.MolFromSmiles(str(smi)) if smi is not None and str(smi) != "nan" else None
    if mol is None or mol.GetNumAtoms() == 0:
        return None
    xs = []
    for a in mol.GetAtoms():
        z = a.GetAtomicNum()
        f = oh(ELEMS.index(z) if z in ELEMS else len(ELEMS), len(ELEMS) + 1)
        f += oh(a.GetDegree(), 6)
        f += oh(a.GetFormalCharge() + 2, 5)
        f += oh(a.GetTotalNumHs(), 5)
        f += oh(HYB.index(a.GetHybridization()) if a.GetHybridization() in HYB else 5, 6)
        f += [1.0 if a.GetIsAromatic() else 0.0, 1.0 if a.IsInRing() else 0.0]
        xs.append(f)
    ei, ea = [], []
    for b in mol.GetBonds():
        i, j = b.GetBeginAtomIdx(), b.GetEndAtomIdx()
        t = BOND.get(b.GetBondType(), 0)
        ei += [[i, j], [j, i]]; ea += [oh(t, 4), oh(t, 4)]
    return Data(x=torch.tensor(xs, dtype=torch.float),
                edge_index=torch.tensor(ei, dtype=torch.long).t().contiguous() if ei else torch.empty((2, 0), dtype=torch.long),
                edge_attr=torch.tensor(ea, dtype=torch.float) if ea else torch.empty((0, 4)))

if _TORCH_OK:
    class GATNet(torch.nn.Module):
        def __init__(self, in_dim, hidden, dropout):
            super().__init__()
            self.c1 = GATConv(in_dim, hidden, heads=4, dropout=dropout, edge_dim=4)
            self.c2 = GATConv(hidden * 4, hidden, heads=1, dropout=dropout, edge_dim=4)
            self.head = torch.nn.Sequential(torch.nn.Linear(hidden, hidden // 2), torch.nn.ReLU(),
                                            torch.nn.Dropout(dropout), torch.nn.Linear(hidden // 2, 1))
            self.dropout = dropout
        def forward(self, data):
            x = F.elu(self.c1(data.x, data.edge_index, data.edge_attr))
            x = F.dropout(x, p=self.dropout, training=self.training)
            x = F.elu(self.c2(x, data.edge_index, data.edge_attr))
            x = global_mean_pool(x, data.batch)
            return self.head(x).ravel()