Instructions to use Delower/liver-extratrees-deployment with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Scikit-learn
How to use Delower/liver-extratrees-deployment with Scikit-learn:
from huggingface_hub import hf_hub_download import joblib model = joblib.load( hf_hub_download("Delower/liver-extratrees-deployment", "sklearn_model.joblib") ) # only load pickle files from sources you trust # read more about it here https://skops.readthedocs.io/en/stable/persistence.html - Notebooks
- Google Colab
- Kaggle
File size: 16,797 Bytes
ee82dfc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 | # deployment_common.py — shared code for the liver deployment refits.
# The tox_common block below is copied VERBATIM from the arms' own notebooks
# (kaggle/notebooks/liver_arch3_lazypredict.py, shared programme conventions
# cell), and the graph block (ELEMS / mol_to_graph / GATNet) is copied
# VERBATIM from kaggle/notebooks/liver_arch2_gat.py. Deployment scripts reuse
# the arms' exact identity, featurization and model code — no re-implementation.
import os, json, time
import numpy as np
import pandas as pd
# ---- shared programme conventions (tox_common, inlined verbatim) ----
"""Shared conventions for the organ-toxicity programme (liver + kidney).
Binding rules (PROJECT_SPEC.md + READING_NOTES harmonization protocols):
- classification ONLY; identity = InChIKey connectivity block (first 14 chars)
of the RDKit-standardized parent structure (largest fragment).
- seed-42 scaffold-disjoint (Bemis-Murcko) train/val/test split.
- preprocessing (imputation / scaling) fit on the TRAIN fold only.
- regulatory benchmark compounds are NEVER in training; lockout verified = 0
with the dual key (deposited connectivity UNION standardized-parent
connectivity) against EACH benchmark file separately.
- benchmark scoring is reported PER SET (withdrawn / clinical failures /
pass), never pooled; metrics at full coverage.
"""
import json
import numpy as np
import pandas as pd
SEED = 42
# ---------------------------------------------------------------- identity
def _inchi_key(mol):
"""InChIKey via whichever RDKit API the installed build provides.
DEPLOYMENT COMPATIBILITY FIX (2026-11-15): the arms' verbatim code calls
rdMolDescriptors.CalcInchiKey, which does not exist in RDKit 2024.03.6 or
2026.09.1 (it silently made parent_connectivity return None there).
rdkit.Chem.inchi.MolToInchiKey is the same InChI computation; verified
byte-identical keys across both installed RDKit builds on a sample set
(salts, stereo, charged species)."""
try:
from rdkit.Chem import inchi
return inchi.MolToInchiKey(mol)
except Exception:
from rdkit.Chem import rdMolDescriptors
return rdMolDescriptors.CalcInchiKey(mol)
def parent_connectivity(smiles):
"""Connectivity block of the standardized parent (largest fragment)."""
try:
from rdkit import Chem
from rdkit.Chem.MolStandardize import rdMolStandardize
if smiles is None or (isinstance(smiles, float) and np.isnan(smiles)):
return None
mol = Chem.MolFromSmiles(str(smiles))
if mol is None:
return None
parent = rdMolStandardize.LargestFragmentChooser().choose(mol)
ik = _inchi_key(parent)
return ik[:14] if ik else None
except Exception:
return None
def deposited_connectivity(inchikey):
if inchikey is None or (isinstance(inchikey, float) and np.isnan(inchikey)):
return None
s = str(inchikey).strip()
return s[:14] if len(s) >= 14 else None
def dual_keys(df, smiles_col="canonical_smiles", inchikey_col="inchikey"):
"""Set of connectivity keys: deposited UNION standardized-parent."""
keys = set()
dep = df[inchikey_col].map(deposited_connectivity) if inchikey_col in df else pd.Series([None] * len(df))
par = df[smiles_col].map(parent_connectivity)
for a, b in zip(dep, par):
if a:
keys.add(a)
if b:
keys.add(b)
return keys
def verify_lockout(train_df, bench_dfs, train_smiles="canonical_smiles"):
"""bench_dfs: dict name -> df (with 'smiles' + 'inchikey' columns).
Returns dict of overlap counts; every count must be 0."""
train_keys = dual_keys(train_df, smiles_col=train_smiles)
out = {}
for name, bdf in bench_dfs.items():
bk = dual_keys(bdf, smiles_col="smiles")
out[name] = len(train_keys & bk)
return out
# ---------------------------------------------------------------- loading
def load_train(path):
df = pd.read_csv(path)
df = df.rename(columns={"canonical_smiles": "smiles"})
df["label"] = df["final_label"].astype(int)
df["name"] = df["drug_name"] if "drug_name" in df else ""
df["conn"] = df["inchikey_connectivity"]
keep = ["name", "smiles", "inchikey", "conn", "label"]
for c in ["sources", "sources_present", "final_label_source"]:
if c in df.columns:
keep.append(c)
return df[keep].reset_index(drop=True)
def load_benchmark(path, expected_label):
df = pd.read_csv(path)
name_col = "drug_name" if "drug_name" in df.columns else "compound_name"
out = pd.DataFrame({
"name": df[name_col],
"smiles": df["smiles"] if "smiles" in df else None,
"inchikey": df["inchikey"] if "inchikey" in df else None,
})
if "evidence_grade" in df.columns:
out["evidence_grade"] = df["evidence_grade"]
out["label"] = expected_label
return out
def dedup_benchmark(df):
"""Dedup benchmark rows to unique compounds (deposited connectivity,
falling back to standardized-parent connectivity). Rows with no usable
structure are counted as unscored."""
keys, keep_idx, unscored = {}, [], 0
for i, row in df.iterrows():
k = deposited_connectivity(row.get("inchikey")) or parent_connectivity(row.get("smiles"))
if k is None:
unscored += 1
continue
if k in keys:
continue
keys[k] = True
keep_idx.append(i)
out = df.loc[keep_idx].copy()
out["conn"] = [deposited_connectivity(r.get("inchikey")) or parent_connectivity(r.get("smiles"))
for _, r in out.iterrows()]
return out.reset_index(drop=True), unscored
# ---------------------------------------------------------------- split
def scaffold_split(df, seed=SEED, fracs=(0.80, 0.10, 0.10)):
"""Seed-42 Bemis-Murcko scaffold-disjoint train/val/test split."""
from rdkit import Chem
from rdkit.Chem.Scaffolds import MurckoScaffold
rng = np.random.RandomState(seed)
scaffolds = []
for smi in df["smiles"]:
mol = Chem.MolFromSmiles(str(smi))
try:
scaf = MurckoScaffold.MurckoScaffoldSmiles(mol=mol) if mol is not None else ""
except Exception:
scaf = ""
scaffolds.append(scaf if scaf else f"ACYCLIC::{smi}")
groups = {}
for i, s in enumerate(scaffolds):
groups.setdefault(s, []).append(i)
group_list = list(groups.values())
rng.shuffle(group_list)
group_list.sort(key=len, reverse=True)
n = len(df)
targets = [f * n for f in fracs]
buckets = [[], [], []]
counts = [0, 0, 0]
for g in group_list:
# assign each scaffold group to the bucket furthest below its target
j = int(np.argmax([targets[k] - counts[k] for k in range(3)]))
buckets[j].extend(g)
counts[j] += len(g)
names = ["train", "val", "test"]
split = pd.Series(index=df.index, dtype=object)
for j, idxs in enumerate(buckets):
split.iloc[idxs] = names[j]
# scaffold disjointness assertion
seen = {}
for j, idxs in enumerate(buckets):
for i in idxs:
s = scaffolds[i]
assert seen.get(s, j) == j, "scaffold leaked across splits"
seen[s] = j
return split
# ---------------------------------------------------------------- features
PHYSCHEM_NAMES = ["MolWt", "MolLogP", "TPSA", "NumHDonors", "NumHAcceptors",
"NumRotatableBonds", "RingCount", "NumAromaticRings",
"NumAliphaticRings", "FractionCSP3", "HeavyAtomCount",
"NumValenceElectrons", "BertzCT"]
def featurize(smiles_list):
"""Returns dict of float32 blocks + validity mask. Blocks:
morgan_bits (1024), morgan_counts (1024), maccs (167),
desc (full RDKit descriptor list), physchem (PHYSCHEM_NAMES subset)."""
from rdkit import Chem
from rdkit.Chem import AllChem, MACCSkeys, Descriptors
from rdkit import RDLogger
RDLogger.DisableLog("rdApp.*")
desc_names = [d[0] for d in Descriptors._descList]
phys_idx = [desc_names.index(n) for n in PHYSCHEM_NAMES]
n = len(smiles_list)
blocks = {
"morgan_bits": np.zeros((n, 1024), dtype=np.float32),
"morgan_counts": np.zeros((n, 1024), dtype=np.float32),
"maccs": np.zeros((n, 167), dtype=np.float32),
"desc": np.full((n, len(desc_names)), np.nan, dtype=np.float32),
}
valid = np.zeros(n, dtype=bool)
from rdkit import DataStructs
gen_bits = AllChem.GetMorganGenerator(radius=2, fpSize=1024)
gen_counts = AllChem.GetMorganGenerator(radius=2, fpSize=1024, countSimulation=True)
for i, smi in enumerate(smiles_list):
mol = Chem.MolFromSmiles(str(smi)) if smi is not None and str(smi) != "nan" else None
if mol is None:
continue
valid[i] = True
DataStructs.ConvertToNumpyArray(gen_bits.GetFingerprint(mol), blocks["morgan_bits"][i])
DataStructs.ConvertToNumpyArray(gen_counts.GetCountFingerprint(mol), blocks["morgan_counts"][i])
blocks["maccs"][i] = np.asarray(MACCSkeys.GenMACCSKeys(mol), dtype=np.float32)
# iterate _descList directly: identical values to CalcMolDescriptors
# at ~1/3 the cost (measured on this VM, RDKit 2026.09)
blocks["desc"][i] = np.array([fn(mol) for _, fn in Descriptors._descList], dtype=np.float32)
blocks["physchem"] = blocks["desc"][:, phys_idx].copy()
blocks["desc_names"] = desc_names
return blocks, valid
class DescPrep:
"""Train-fold-fitted descriptor preprocessing: drop columns >50% missing
or near-constant ON TRAIN, median-impute, clip to train 0.1/99.9 pct,
standardize with train mean/std."""
def fit(self, X):
X = np.asarray(X, dtype=np.float64)
miss = np.mean(~np.isfinite(X), axis=0)
std = np.nanstd(np.where(np.isfinite(X), X, np.nan), axis=0)
self.keep_ = (miss <= 0.5) & (std > 1e-8)
Xk = X[:, self.keep_]
self.med_ = np.nanmedian(np.where(np.isfinite(Xk), Xk, np.nan), axis=0)
Xi = np.where(np.isfinite(Xk), Xk, self.med_)
self.lo_ = np.percentile(Xi, 0.1, axis=0)
self.hi_ = np.percentile(Xi, 99.9, axis=0)
Xc = np.clip(Xi, self.lo_, self.hi_)
self.mu_ = Xc.mean(axis=0)
self.sd_ = Xc.std(axis=0) + 1e-9
return self
def transform(self, X):
X = np.asarray(X, dtype=np.float64)[:, self.keep_]
Xi = np.where(np.isfinite(X), X, self.med_)
return ((np.clip(Xi, self.lo_, self.hi_) - self.mu_) / self.sd_).astype(np.float32)
def prep_matrix(blocks, prep, kind):
"""kind: 'fp' (morgan bits) / 'fp_desc' (bits+maccs+desc) /
'desc_maccs_counts' (arch-5 block b) / 'physchem' raw physchem."""
if kind == "fp":
return blocks["morgan_bits"]
if kind == "fp_desc":
return np.hstack([blocks["morgan_bits"], blocks["maccs"], prep.transform(blocks["desc"])]).astype(np.float32)
if kind == "desc_maccs_counts":
return np.hstack([prep.transform(blocks["desc"]), blocks["maccs"], blocks["morgan_counts"]]).astype(np.float32)
if kind == "physchem":
return blocks["physchem"]
raise ValueError(kind)
# ---------------------------------------------------------------- metrics
def metrics_dict(y_true, prob, threshold=0.5):
from sklearn.metrics import (accuracy_score, balanced_accuracy_score,
matthews_corrcoef, roc_auc_score, confusion_matrix)
y_true = np.asarray(y_true, dtype=int)
prob = np.asarray(prob, dtype=float)
pred = (prob >= threshold).astype(int)
out = {"n": int(len(y_true)), "threshold": threshold,
"accuracy": float(accuracy_score(y_true, pred)),
"balanced_accuracy": float(balanced_accuracy_score(y_true, pred))}
tn, fp, fn, tp = confusion_matrix(y_true, pred, labels=[0, 1]).ravel()
out.update({"tn": int(tn), "fp": int(fp), "fn": int(fn), "tp": int(tp),
"sensitivity_recall_pos": float(tp / (tp + fn)) if (tp + fn) else None,
"specificity_recall_neg": float(tn / (tn + fp)) if (tn + fp) else None})
if len(np.unique(y_true)) == 2 and len(np.unique(pred)) == 2:
out["mcc"] = float(matthews_corrcoef(y_true, pred))
else:
out["mcc"] = None
out["mcc_note"] = "single-class ground truth or single-class predictions"
if len(np.unique(y_true)) == 2:
out["auroc"] = float(roc_auc_score(y_true, prob))
else:
out["auroc"] = None
return out
def score_benchmark_sets(predict_proba_fn, bench_sets):
"""bench_sets: dict name -> deduped df (with 'smiles','label','name').
predict_proba_fn(smiles_list) -> P(label=1) array (NaN where unparseable).
Returns (metrics dict per set, predictions dataframe)."""
all_metrics, pred_frames = {}, []
for set_name, bdf in bench_sets.items():
prob = predict_proba_fn(list(bdf["smiles"]))
scored = np.isfinite(prob)
m = metrics_dict(bdf.loc[scored, "label"], prob[scored]) if scored.any() else {"n": 0}
m["n_rows_unique"] = int(len(bdf))
m["n_scored"] = int(scored.sum())
m["n_unscored_invalid_structure"] = int((~scored).sum())
all_metrics[set_name] = m
pf = bdf[["name", "smiles", "label"]].copy()
if "evidence_grade" in bdf.columns:
pf["evidence_grade"] = bdf["evidence_grade"]
pf["set"] = set_name
pf["prob_toxic"] = prob
pf["pred_label"] = np.where(scored, (prob >= 0.5).astype(float), np.nan)
pf["correct"] = np.where(scored, (pf["pred_label"] == pf["label"]).astype(float), np.nan)
pred_frames.append(pf)
return all_metrics, pd.concat(pred_frames, ignore_index=True)
# ---- GAT graph code (verbatim from liver_arch2_gat.py) ----
# Import guard (deployment plumbing only, model code below unchanged): the
# sklearn-side deployment scripts import this module in an env without
# torch_geometric; the graph functions are only called in the pyg env.
try:
import torch
import torch.nn.functional as F
from torch_geometric.data import Data
from torch_geometric.nn import GATConv, global_mean_pool
_TORCH_OK = True
except ImportError:
torch = None
_TORCH_OK = False
ELEMS = [6, 7, 8, 16, 9, 17, 35, 53, 15, 14]
def mol_to_graph(smi):
from rdkit import Chem
HYB = [Chem.rdchem.HybridizationType.SP, Chem.rdchem.HybridizationType.SP2,
Chem.rdchem.HybridizationType.SP3, Chem.rdchem.HybridizationType.SP3D,
Chem.rdchem.HybridizationType.SP3D2]
BOND = {Chem.rdchem.BondType.SINGLE: 0, Chem.rdchem.BondType.DOUBLE: 1,
Chem.rdchem.BondType.TRIPLE: 2, Chem.rdchem.BondType.AROMATIC: 3}
def oh(v, n):
o = [0.0] * n; o[min(max(v, 0), n - 1)] = 1.0; return o
mol = Chem.MolFromSmiles(str(smi)) if smi is not None and str(smi) != "nan" else None
if mol is None or mol.GetNumAtoms() == 0:
return None
xs = []
for a in mol.GetAtoms():
z = a.GetAtomicNum()
f = oh(ELEMS.index(z) if z in ELEMS else len(ELEMS), len(ELEMS) + 1)
f += oh(a.GetDegree(), 6)
f += oh(a.GetFormalCharge() + 2, 5)
f += oh(a.GetTotalNumHs(), 5)
f += oh(HYB.index(a.GetHybridization()) if a.GetHybridization() in HYB else 5, 6)
f += [1.0 if a.GetIsAromatic() else 0.0, 1.0 if a.IsInRing() else 0.0]
xs.append(f)
ei, ea = [], []
for b in mol.GetBonds():
i, j = b.GetBeginAtomIdx(), b.GetEndAtomIdx()
t = BOND.get(b.GetBondType(), 0)
ei += [[i, j], [j, i]]; ea += [oh(t, 4), oh(t, 4)]
return Data(x=torch.tensor(xs, dtype=torch.float),
edge_index=torch.tensor(ei, dtype=torch.long).t().contiguous() if ei else torch.empty((2, 0), dtype=torch.long),
edge_attr=torch.tensor(ea, dtype=torch.float) if ea else torch.empty((0, 4)))
if _TORCH_OK:
class GATNet(torch.nn.Module):
def __init__(self, in_dim, hidden, dropout):
super().__init__()
self.c1 = GATConv(in_dim, hidden, heads=4, dropout=dropout, edge_dim=4)
self.c2 = GATConv(hidden * 4, hidden, heads=1, dropout=dropout, edge_dim=4)
self.head = torch.nn.Sequential(torch.nn.Linear(hidden, hidden // 2), torch.nn.ReLU(),
torch.nn.Dropout(dropout), torch.nn.Linear(hidden // 2, 1))
self.dropout = dropout
def forward(self, data):
x = F.elu(self.c1(data.x, data.edge_index, data.edge_attr))
x = F.dropout(x, p=self.dropout, training=self.training)
x = F.elu(self.c2(x, data.edge_index, data.edge_attr))
x = global_mean_pool(x, data.batch)
return self.head(x).ravel()
|