# deployment_common.py — shared code for the liver deployment refits. # The tox_common block below is copied VERBATIM from the arms' own notebooks # (kaggle/notebooks/liver_arch3_lazypredict.py, shared programme conventions # cell), and the graph block (ELEMS / mol_to_graph / GATNet) is copied # VERBATIM from kaggle/notebooks/liver_arch2_gat.py. Deployment scripts reuse # the arms' exact identity, featurization and model code — no re-implementation. import os, json, time import numpy as np import pandas as pd # ---- shared programme conventions (tox_common, inlined verbatim) ---- """Shared conventions for the organ-toxicity programme (liver + kidney). Binding rules (PROJECT_SPEC.md + READING_NOTES harmonization protocols): - classification ONLY; identity = InChIKey connectivity block (first 14 chars) of the RDKit-standardized parent structure (largest fragment). - seed-42 scaffold-disjoint (Bemis-Murcko) train/val/test split. - preprocessing (imputation / scaling) fit on the TRAIN fold only. - regulatory benchmark compounds are NEVER in training; lockout verified = 0 with the dual key (deposited connectivity UNION standardized-parent connectivity) against EACH benchmark file separately. - benchmark scoring is reported PER SET (withdrawn / clinical failures / pass), never pooled; metrics at full coverage. """ import json import numpy as np import pandas as pd SEED = 42 # ---------------------------------------------------------------- identity def _inchi_key(mol): """InChIKey via whichever RDKit API the installed build provides. DEPLOYMENT COMPATIBILITY FIX (2026-11-15): the arms' verbatim code calls rdMolDescriptors.CalcInchiKey, which does not exist in RDKit 2024.03.6 or 2026.09.1 (it silently made parent_connectivity return None there). rdkit.Chem.inchi.MolToInchiKey is the same InChI computation; verified byte-identical keys across both installed RDKit builds on a sample set (salts, stereo, charged species).""" try: from rdkit.Chem import inchi return inchi.MolToInchiKey(mol) except Exception: from rdkit.Chem import rdMolDescriptors return rdMolDescriptors.CalcInchiKey(mol) def parent_connectivity(smiles): """Connectivity block of the standardized parent (largest fragment).""" try: from rdkit import Chem from rdkit.Chem.MolStandardize import rdMolStandardize if smiles is None or (isinstance(smiles, float) and np.isnan(smiles)): return None mol = Chem.MolFromSmiles(str(smiles)) if mol is None: return None parent = rdMolStandardize.LargestFragmentChooser().choose(mol) ik = _inchi_key(parent) return ik[:14] if ik else None except Exception: return None def deposited_connectivity(inchikey): if inchikey is None or (isinstance(inchikey, float) and np.isnan(inchikey)): return None s = str(inchikey).strip() return s[:14] if len(s) >= 14 else None def dual_keys(df, smiles_col="canonical_smiles", inchikey_col="inchikey"): """Set of connectivity keys: deposited UNION standardized-parent.""" keys = set() dep = df[inchikey_col].map(deposited_connectivity) if inchikey_col in df else pd.Series([None] * len(df)) par = df[smiles_col].map(parent_connectivity) for a, b in zip(dep, par): if a: keys.add(a) if b: keys.add(b) return keys def verify_lockout(train_df, bench_dfs, train_smiles="canonical_smiles"): """bench_dfs: dict name -> df (with 'smiles' + 'inchikey' columns). Returns dict of overlap counts; every count must be 0.""" train_keys = dual_keys(train_df, smiles_col=train_smiles) out = {} for name, bdf in bench_dfs.items(): bk = dual_keys(bdf, smiles_col="smiles") out[name] = len(train_keys & bk) return out # ---------------------------------------------------------------- loading def load_train(path): df = pd.read_csv(path) df = df.rename(columns={"canonical_smiles": "smiles"}) df["label"] = df["final_label"].astype(int) df["name"] = df["drug_name"] if "drug_name" in df else "" df["conn"] = df["inchikey_connectivity"] keep = ["name", "smiles", "inchikey", "conn", "label"] for c in ["sources", "sources_present", "final_label_source"]: if c in df.columns: keep.append(c) return df[keep].reset_index(drop=True) def load_benchmark(path, expected_label): df = pd.read_csv(path) name_col = "drug_name" if "drug_name" in df.columns else "compound_name" out = pd.DataFrame({ "name": df[name_col], "smiles": df["smiles"] if "smiles" in df else None, "inchikey": df["inchikey"] if "inchikey" in df else None, }) if "evidence_grade" in df.columns: out["evidence_grade"] = df["evidence_grade"] out["label"] = expected_label return out def dedup_benchmark(df): """Dedup benchmark rows to unique compounds (deposited connectivity, falling back to standardized-parent connectivity). Rows with no usable structure are counted as unscored.""" keys, keep_idx, unscored = {}, [], 0 for i, row in df.iterrows(): k = deposited_connectivity(row.get("inchikey")) or parent_connectivity(row.get("smiles")) if k is None: unscored += 1 continue if k in keys: continue keys[k] = True keep_idx.append(i) out = df.loc[keep_idx].copy() out["conn"] = [deposited_connectivity(r.get("inchikey")) or parent_connectivity(r.get("smiles")) for _, r in out.iterrows()] return out.reset_index(drop=True), unscored # ---------------------------------------------------------------- split def scaffold_split(df, seed=SEED, fracs=(0.80, 0.10, 0.10)): """Seed-42 Bemis-Murcko scaffold-disjoint train/val/test split.""" from rdkit import Chem from rdkit.Chem.Scaffolds import MurckoScaffold rng = np.random.RandomState(seed) scaffolds = [] for smi in df["smiles"]: mol = Chem.MolFromSmiles(str(smi)) try: scaf = MurckoScaffold.MurckoScaffoldSmiles(mol=mol) if mol is not None else "" except Exception: scaf = "" scaffolds.append(scaf if scaf else f"ACYCLIC::{smi}") groups = {} for i, s in enumerate(scaffolds): groups.setdefault(s, []).append(i) group_list = list(groups.values()) rng.shuffle(group_list) group_list.sort(key=len, reverse=True) n = len(df) targets = [f * n for f in fracs] buckets = [[], [], []] counts = [0, 0, 0] for g in group_list: # assign each scaffold group to the bucket furthest below its target j = int(np.argmax([targets[k] - counts[k] for k in range(3)])) buckets[j].extend(g) counts[j] += len(g) names = ["train", "val", "test"] split = pd.Series(index=df.index, dtype=object) for j, idxs in enumerate(buckets): split.iloc[idxs] = names[j] # scaffold disjointness assertion seen = {} for j, idxs in enumerate(buckets): for i in idxs: s = scaffolds[i] assert seen.get(s, j) == j, "scaffold leaked across splits" seen[s] = j return split # ---------------------------------------------------------------- features PHYSCHEM_NAMES = ["MolWt", "MolLogP", "TPSA", "NumHDonors", "NumHAcceptors", "NumRotatableBonds", "RingCount", "NumAromaticRings", "NumAliphaticRings", "FractionCSP3", "HeavyAtomCount", "NumValenceElectrons", "BertzCT"] def featurize(smiles_list): """Returns dict of float32 blocks + validity mask. Blocks: morgan_bits (1024), morgan_counts (1024), maccs (167), desc (full RDKit descriptor list), physchem (PHYSCHEM_NAMES subset).""" from rdkit import Chem from rdkit.Chem import AllChem, MACCSkeys, Descriptors from rdkit import RDLogger RDLogger.DisableLog("rdApp.*") desc_names = [d[0] for d in Descriptors._descList] phys_idx = [desc_names.index(n) for n in PHYSCHEM_NAMES] n = len(smiles_list) blocks = { "morgan_bits": np.zeros((n, 1024), dtype=np.float32), "morgan_counts": np.zeros((n, 1024), dtype=np.float32), "maccs": np.zeros((n, 167), dtype=np.float32), "desc": np.full((n, len(desc_names)), np.nan, dtype=np.float32), } valid = np.zeros(n, dtype=bool) from rdkit import DataStructs gen_bits = AllChem.GetMorganGenerator(radius=2, fpSize=1024) gen_counts = AllChem.GetMorganGenerator(radius=2, fpSize=1024, countSimulation=True) for i, smi in enumerate(smiles_list): mol = Chem.MolFromSmiles(str(smi)) if smi is not None and str(smi) != "nan" else None if mol is None: continue valid[i] = True DataStructs.ConvertToNumpyArray(gen_bits.GetFingerprint(mol), blocks["morgan_bits"][i]) DataStructs.ConvertToNumpyArray(gen_counts.GetCountFingerprint(mol), blocks["morgan_counts"][i]) blocks["maccs"][i] = np.asarray(MACCSkeys.GenMACCSKeys(mol), dtype=np.float32) # iterate _descList directly: identical values to CalcMolDescriptors # at ~1/3 the cost (measured on this VM, RDKit 2026.09) blocks["desc"][i] = np.array([fn(mol) for _, fn in Descriptors._descList], dtype=np.float32) blocks["physchem"] = blocks["desc"][:, phys_idx].copy() blocks["desc_names"] = desc_names return blocks, valid class DescPrep: """Train-fold-fitted descriptor preprocessing: drop columns >50% missing or near-constant ON TRAIN, median-impute, clip to train 0.1/99.9 pct, standardize with train mean/std.""" def fit(self, X): X = np.asarray(X, dtype=np.float64) miss = np.mean(~np.isfinite(X), axis=0) std = np.nanstd(np.where(np.isfinite(X), X, np.nan), axis=0) self.keep_ = (miss <= 0.5) & (std > 1e-8) Xk = X[:, self.keep_] self.med_ = np.nanmedian(np.where(np.isfinite(Xk), Xk, np.nan), axis=0) Xi = np.where(np.isfinite(Xk), Xk, self.med_) self.lo_ = np.percentile(Xi, 0.1, axis=0) self.hi_ = np.percentile(Xi, 99.9, axis=0) Xc = np.clip(Xi, self.lo_, self.hi_) self.mu_ = Xc.mean(axis=0) self.sd_ = Xc.std(axis=0) + 1e-9 return self def transform(self, X): X = np.asarray(X, dtype=np.float64)[:, self.keep_] Xi = np.where(np.isfinite(X), X, self.med_) return ((np.clip(Xi, self.lo_, self.hi_) - self.mu_) / self.sd_).astype(np.float32) def prep_matrix(blocks, prep, kind): """kind: 'fp' (morgan bits) / 'fp_desc' (bits+maccs+desc) / 'desc_maccs_counts' (arch-5 block b) / 'physchem' raw physchem.""" if kind == "fp": return blocks["morgan_bits"] if kind == "fp_desc": return np.hstack([blocks["morgan_bits"], blocks["maccs"], prep.transform(blocks["desc"])]).astype(np.float32) if kind == "desc_maccs_counts": return np.hstack([prep.transform(blocks["desc"]), blocks["maccs"], blocks["morgan_counts"]]).astype(np.float32) if kind == "physchem": return blocks["physchem"] raise ValueError(kind) # ---------------------------------------------------------------- metrics def metrics_dict(y_true, prob, threshold=0.5): from sklearn.metrics import (accuracy_score, balanced_accuracy_score, matthews_corrcoef, roc_auc_score, confusion_matrix) y_true = np.asarray(y_true, dtype=int) prob = np.asarray(prob, dtype=float) pred = (prob >= threshold).astype(int) out = {"n": int(len(y_true)), "threshold": threshold, "accuracy": float(accuracy_score(y_true, pred)), "balanced_accuracy": float(balanced_accuracy_score(y_true, pred))} tn, fp, fn, tp = confusion_matrix(y_true, pred, labels=[0, 1]).ravel() out.update({"tn": int(tn), "fp": int(fp), "fn": int(fn), "tp": int(tp), "sensitivity_recall_pos": float(tp / (tp + fn)) if (tp + fn) else None, "specificity_recall_neg": float(tn / (tn + fp)) if (tn + fp) else None}) if len(np.unique(y_true)) == 2 and len(np.unique(pred)) == 2: out["mcc"] = float(matthews_corrcoef(y_true, pred)) else: out["mcc"] = None out["mcc_note"] = "single-class ground truth or single-class predictions" if len(np.unique(y_true)) == 2: out["auroc"] = float(roc_auc_score(y_true, prob)) else: out["auroc"] = None return out def score_benchmark_sets(predict_proba_fn, bench_sets): """bench_sets: dict name -> deduped df (with 'smiles','label','name'). predict_proba_fn(smiles_list) -> P(label=1) array (NaN where unparseable). Returns (metrics dict per set, predictions dataframe).""" all_metrics, pred_frames = {}, [] for set_name, bdf in bench_sets.items(): prob = predict_proba_fn(list(bdf["smiles"])) scored = np.isfinite(prob) m = metrics_dict(bdf.loc[scored, "label"], prob[scored]) if scored.any() else {"n": 0} m["n_rows_unique"] = int(len(bdf)) m["n_scored"] = int(scored.sum()) m["n_unscored_invalid_structure"] = int((~scored).sum()) all_metrics[set_name] = m pf = bdf[["name", "smiles", "label"]].copy() if "evidence_grade" in bdf.columns: pf["evidence_grade"] = bdf["evidence_grade"] pf["set"] = set_name pf["prob_toxic"] = prob pf["pred_label"] = np.where(scored, (prob >= 0.5).astype(float), np.nan) pf["correct"] = np.where(scored, (pf["pred_label"] == pf["label"]).astype(float), np.nan) pred_frames.append(pf) return all_metrics, pd.concat(pred_frames, ignore_index=True) # ---- GAT graph code (verbatim from liver_arch2_gat.py) ---- # Import guard (deployment plumbing only, model code below unchanged): the # sklearn-side deployment scripts import this module in an env without # torch_geometric; the graph functions are only called in the pyg env. try: import torch import torch.nn.functional as F from torch_geometric.data import Data from torch_geometric.nn import GATConv, global_mean_pool _TORCH_OK = True except ImportError: torch = None _TORCH_OK = False ELEMS = [6, 7, 8, 16, 9, 17, 35, 53, 15, 14] def mol_to_graph(smi): from rdkit import Chem HYB = [Chem.rdchem.HybridizationType.SP, Chem.rdchem.HybridizationType.SP2, Chem.rdchem.HybridizationType.SP3, Chem.rdchem.HybridizationType.SP3D, Chem.rdchem.HybridizationType.SP3D2] BOND = {Chem.rdchem.BondType.SINGLE: 0, Chem.rdchem.BondType.DOUBLE: 1, Chem.rdchem.BondType.TRIPLE: 2, Chem.rdchem.BondType.AROMATIC: 3} def oh(v, n): o = [0.0] * n; o[min(max(v, 0), n - 1)] = 1.0; return o mol = Chem.MolFromSmiles(str(smi)) if smi is not None and str(smi) != "nan" else None if mol is None or mol.GetNumAtoms() == 0: return None xs = [] for a in mol.GetAtoms(): z = a.GetAtomicNum() f = oh(ELEMS.index(z) if z in ELEMS else len(ELEMS), len(ELEMS) + 1) f += oh(a.GetDegree(), 6) f += oh(a.GetFormalCharge() + 2, 5) f += oh(a.GetTotalNumHs(), 5) f += oh(HYB.index(a.GetHybridization()) if a.GetHybridization() in HYB else 5, 6) f += [1.0 if a.GetIsAromatic() else 0.0, 1.0 if a.IsInRing() else 0.0] xs.append(f) ei, ea = [], [] for b in mol.GetBonds(): i, j = b.GetBeginAtomIdx(), b.GetEndAtomIdx() t = BOND.get(b.GetBondType(), 0) ei += [[i, j], [j, i]]; ea += [oh(t, 4), oh(t, 4)] return Data(x=torch.tensor(xs, dtype=torch.float), edge_index=torch.tensor(ei, dtype=torch.long).t().contiguous() if ei else torch.empty((2, 0), dtype=torch.long), edge_attr=torch.tensor(ea, dtype=torch.float) if ea else torch.empty((0, 4))) if _TORCH_OK: class GATNet(torch.nn.Module): def __init__(self, in_dim, hidden, dropout): super().__init__() self.c1 = GATConv(in_dim, hidden, heads=4, dropout=dropout, edge_dim=4) self.c2 = GATConv(hidden * 4, hidden, heads=1, dropout=dropout, edge_dim=4) self.head = torch.nn.Sequential(torch.nn.Linear(hidden, hidden // 2), torch.nn.ReLU(), torch.nn.Dropout(dropout), torch.nn.Linear(hidden // 2, 1)) self.dropout = dropout def forward(self, data): x = F.elu(self.c1(data.x, data.edge_index, data.edge_attr)) x = F.dropout(x, p=self.dropout, training=self.training) x = F.elu(self.c2(x, data.edge_index, data.edge_attr)) x = global_mean_pool(x, data.batch) return self.head(x).ravel()