Instructions to use Delower/liver-extratrees-deployment with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Scikit-learn
How to use Delower/liver-extratrees-deployment with Scikit-learn:
from huggingface_hub import hf_hub_download import joblib model = joblib.load( hf_hub_download("Delower/liver-extratrees-deployment", "sklearn_model.joblib") ) # only load pickle files from sources you trust # read more about it here https://skops.readthedocs.io/en/stable/persistence.html - Notebooks
- Google Colab
- Kaggle
Download deployment_common.py from Delower/liver-extratrees-deployment: direct link, hf CLI and curl.
- Browser
- Download file 16.8 kB
-
https://huggingface.co/Delower/liver-extratrees-deployment/resolve/main/deployment_common.py
- Command line
-
hf download hf://Delower/liver-extratrees-deployment/deployment_common.py
-
curl -L -o deployment_common.py https://huggingface.co/Delower/liver-extratrees-deployment/resolve/main/deployment_common.py
16.8 kB
| # deployment_common.py — shared code for the liver deployment refits. | |
| # The tox_common block below is copied VERBATIM from the arms' own notebooks | |
| # (kaggle/notebooks/liver_arch3_lazypredict.py, shared programme conventions | |
| # cell), and the graph block (ELEMS / mol_to_graph / GATNet) is copied | |
| # VERBATIM from kaggle/notebooks/liver_arch2_gat.py. Deployment scripts reuse | |
| # the arms' exact identity, featurization and model code — no re-implementation. | |
| import os, json, time | |
| import numpy as np | |
| import pandas as pd | |
| # ---- shared programme conventions (tox_common, inlined verbatim) ---- | |
| """Shared conventions for the organ-toxicity programme (liver + kidney). | |
| Binding rules (PROJECT_SPEC.md + READING_NOTES harmonization protocols): | |
| - classification ONLY; identity = InChIKey connectivity block (first 14 chars) | |
| of the RDKit-standardized parent structure (largest fragment). | |
| - seed-42 scaffold-disjoint (Bemis-Murcko) train/val/test split. | |
| - preprocessing (imputation / scaling) fit on the TRAIN fold only. | |
| - regulatory benchmark compounds are NEVER in training; lockout verified = 0 | |
| with the dual key (deposited connectivity UNION standardized-parent | |
| connectivity) against EACH benchmark file separately. | |
| - benchmark scoring is reported PER SET (withdrawn / clinical failures / | |
| pass), never pooled; metrics at full coverage. | |
| """ | |
| import json | |
| import numpy as np | |
| import pandas as pd | |
| SEED = 42 | |
| # ---------------------------------------------------------------- identity | |
| def _inchi_key(mol): | |
| """InChIKey via whichever RDKit API the installed build provides. | |
| DEPLOYMENT COMPATIBILITY FIX (2026-11-15): the arms' verbatim code calls | |
| rdMolDescriptors.CalcInchiKey, which does not exist in RDKit 2024.03.6 or | |
| 2026.09.1 (it silently made parent_connectivity return None there). | |
| rdkit.Chem.inchi.MolToInchiKey is the same InChI computation; verified | |
| byte-identical keys across both installed RDKit builds on a sample set | |
| (salts, stereo, charged species).""" | |
| try: | |
| from rdkit.Chem import inchi | |
| return inchi.MolToInchiKey(mol) | |
| except Exception: | |
| from rdkit.Chem import rdMolDescriptors | |
| return rdMolDescriptors.CalcInchiKey(mol) | |
| def parent_connectivity(smiles): | |
| """Connectivity block of the standardized parent (largest fragment).""" | |
| try: | |
| from rdkit import Chem | |
| from rdkit.Chem.MolStandardize import rdMolStandardize | |
| if smiles is None or (isinstance(smiles, float) and np.isnan(smiles)): | |
| return None | |
| mol = Chem.MolFromSmiles(str(smiles)) | |
| if mol is None: | |
| return None | |
| parent = rdMolStandardize.LargestFragmentChooser().choose(mol) | |
| ik = _inchi_key(parent) | |
| return ik[:14] if ik else None | |
| except Exception: | |
| return None | |
| def deposited_connectivity(inchikey): | |
| if inchikey is None or (isinstance(inchikey, float) and np.isnan(inchikey)): | |
| return None | |
| s = str(inchikey).strip() | |
| return s[:14] if len(s) >= 14 else None | |
| def dual_keys(df, smiles_col="canonical_smiles", inchikey_col="inchikey"): | |
| """Set of connectivity keys: deposited UNION standardized-parent.""" | |
| keys = set() | |
| dep = df[inchikey_col].map(deposited_connectivity) if inchikey_col in df else pd.Series([None] * len(df)) | |
| par = df[smiles_col].map(parent_connectivity) | |
| for a, b in zip(dep, par): | |
| if a: | |
| keys.add(a) | |
| if b: | |
| keys.add(b) | |
| return keys | |
| def verify_lockout(train_df, bench_dfs, train_smiles="canonical_smiles"): | |
| """bench_dfs: dict name -> df (with 'smiles' + 'inchikey' columns). | |
| Returns dict of overlap counts; every count must be 0.""" | |
| train_keys = dual_keys(train_df, smiles_col=train_smiles) | |
| out = {} | |
| for name, bdf in bench_dfs.items(): | |
| bk = dual_keys(bdf, smiles_col="smiles") | |
| out[name] = len(train_keys & bk) | |
| return out | |
| # ---------------------------------------------------------------- loading | |
| def load_train(path): | |
| df = pd.read_csv(path) | |
| df = df.rename(columns={"canonical_smiles": "smiles"}) | |
| df["label"] = df["final_label"].astype(int) | |
| df["name"] = df["drug_name"] if "drug_name" in df else "" | |
| df["conn"] = df["inchikey_connectivity"] | |
| keep = ["name", "smiles", "inchikey", "conn", "label"] | |
| for c in ["sources", "sources_present", "final_label_source"]: | |
| if c in df.columns: | |
| keep.append(c) | |
| return df[keep].reset_index(drop=True) | |
| def load_benchmark(path, expected_label): | |
| df = pd.read_csv(path) | |
| name_col = "drug_name" if "drug_name" in df.columns else "compound_name" | |
| out = pd.DataFrame({ | |
| "name": df[name_col], | |
| "smiles": df["smiles"] if "smiles" in df else None, | |
| "inchikey": df["inchikey"] if "inchikey" in df else None, | |
| }) | |
| if "evidence_grade" in df.columns: | |
| out["evidence_grade"] = df["evidence_grade"] | |
| out["label"] = expected_label | |
| return out | |
| def dedup_benchmark(df): | |
| """Dedup benchmark rows to unique compounds (deposited connectivity, | |
| falling back to standardized-parent connectivity). Rows with no usable | |
| structure are counted as unscored.""" | |
| keys, keep_idx, unscored = {}, [], 0 | |
| for i, row in df.iterrows(): | |
| k = deposited_connectivity(row.get("inchikey")) or parent_connectivity(row.get("smiles")) | |
| if k is None: | |
| unscored += 1 | |
| continue | |
| if k in keys: | |
| continue | |
| keys[k] = True | |
| keep_idx.append(i) | |
| out = df.loc[keep_idx].copy() | |
| out["conn"] = [deposited_connectivity(r.get("inchikey")) or parent_connectivity(r.get("smiles")) | |
| for _, r in out.iterrows()] | |
| return out.reset_index(drop=True), unscored | |
| # ---------------------------------------------------------------- split | |
| def scaffold_split(df, seed=SEED, fracs=(0.80, 0.10, 0.10)): | |
| """Seed-42 Bemis-Murcko scaffold-disjoint train/val/test split.""" | |
| from rdkit import Chem | |
| from rdkit.Chem.Scaffolds import MurckoScaffold | |
| rng = np.random.RandomState(seed) | |
| scaffolds = [] | |
| for smi in df["smiles"]: | |
| mol = Chem.MolFromSmiles(str(smi)) | |
| try: | |
| scaf = MurckoScaffold.MurckoScaffoldSmiles(mol=mol) if mol is not None else "" | |
| except Exception: | |
| scaf = "" | |
| scaffolds.append(scaf if scaf else f"ACYCLIC::{smi}") | |
| groups = {} | |
| for i, s in enumerate(scaffolds): | |
| groups.setdefault(s, []).append(i) | |
| group_list = list(groups.values()) | |
| rng.shuffle(group_list) | |
| group_list.sort(key=len, reverse=True) | |
| n = len(df) | |
| targets = [f * n for f in fracs] | |
| buckets = [[], [], []] | |
| counts = [0, 0, 0] | |
| for g in group_list: | |
| # assign each scaffold group to the bucket furthest below its target | |
| j = int(np.argmax([targets[k] - counts[k] for k in range(3)])) | |
| buckets[j].extend(g) | |
| counts[j] += len(g) | |
| names = ["train", "val", "test"] | |
| split = pd.Series(index=df.index, dtype=object) | |
| for j, idxs in enumerate(buckets): | |
| split.iloc[idxs] = names[j] | |
| # scaffold disjointness assertion | |
| seen = {} | |
| for j, idxs in enumerate(buckets): | |
| for i in idxs: | |
| s = scaffolds[i] | |
| assert seen.get(s, j) == j, "scaffold leaked across splits" | |
| seen[s] = j | |
| return split | |
| # ---------------------------------------------------------------- features | |
| PHYSCHEM_NAMES = ["MolWt", "MolLogP", "TPSA", "NumHDonors", "NumHAcceptors", | |
| "NumRotatableBonds", "RingCount", "NumAromaticRings", | |
| "NumAliphaticRings", "FractionCSP3", "HeavyAtomCount", | |
| "NumValenceElectrons", "BertzCT"] | |
| def featurize(smiles_list): | |
| """Returns dict of float32 blocks + validity mask. Blocks: | |
| morgan_bits (1024), morgan_counts (1024), maccs (167), | |
| desc (full RDKit descriptor list), physchem (PHYSCHEM_NAMES subset).""" | |
| from rdkit import Chem | |
| from rdkit.Chem import AllChem, MACCSkeys, Descriptors | |
| from rdkit import RDLogger | |
| RDLogger.DisableLog("rdApp.*") | |
| desc_names = [d[0] for d in Descriptors._descList] | |
| phys_idx = [desc_names.index(n) for n in PHYSCHEM_NAMES] | |
| n = len(smiles_list) | |
| blocks = { | |
| "morgan_bits": np.zeros((n, 1024), dtype=np.float32), | |
| "morgan_counts": np.zeros((n, 1024), dtype=np.float32), | |
| "maccs": np.zeros((n, 167), dtype=np.float32), | |
| "desc": np.full((n, len(desc_names)), np.nan, dtype=np.float32), | |
| } | |
| valid = np.zeros(n, dtype=bool) | |
| from rdkit import DataStructs | |
| gen_bits = AllChem.GetMorganGenerator(radius=2, fpSize=1024) | |
| gen_counts = AllChem.GetMorganGenerator(radius=2, fpSize=1024, countSimulation=True) | |
| for i, smi in enumerate(smiles_list): | |
| mol = Chem.MolFromSmiles(str(smi)) if smi is not None and str(smi) != "nan" else None | |
| if mol is None: | |
| continue | |
| valid[i] = True | |
| DataStructs.ConvertToNumpyArray(gen_bits.GetFingerprint(mol), blocks["morgan_bits"][i]) | |
| DataStructs.ConvertToNumpyArray(gen_counts.GetCountFingerprint(mol), blocks["morgan_counts"][i]) | |
| blocks["maccs"][i] = np.asarray(MACCSkeys.GenMACCSKeys(mol), dtype=np.float32) | |
| # iterate _descList directly: identical values to CalcMolDescriptors | |
| # at ~1/3 the cost (measured on this VM, RDKit 2026.09) | |
| blocks["desc"][i] = np.array([fn(mol) for _, fn in Descriptors._descList], dtype=np.float32) | |
| blocks["physchem"] = blocks["desc"][:, phys_idx].copy() | |
| blocks["desc_names"] = desc_names | |
| return blocks, valid | |
| class DescPrep: | |
| """Train-fold-fitted descriptor preprocessing: drop columns >50% missing | |
| or near-constant ON TRAIN, median-impute, clip to train 0.1/99.9 pct, | |
| standardize with train mean/std.""" | |
| def fit(self, X): | |
| X = np.asarray(X, dtype=np.float64) | |
| miss = np.mean(~np.isfinite(X), axis=0) | |
| std = np.nanstd(np.where(np.isfinite(X), X, np.nan), axis=0) | |
| self.keep_ = (miss <= 0.5) & (std > 1e-8) | |
| Xk = X[:, self.keep_] | |
| self.med_ = np.nanmedian(np.where(np.isfinite(Xk), Xk, np.nan), axis=0) | |
| Xi = np.where(np.isfinite(Xk), Xk, self.med_) | |
| self.lo_ = np.percentile(Xi, 0.1, axis=0) | |
| self.hi_ = np.percentile(Xi, 99.9, axis=0) | |
| Xc = np.clip(Xi, self.lo_, self.hi_) | |
| self.mu_ = Xc.mean(axis=0) | |
| self.sd_ = Xc.std(axis=0) + 1e-9 | |
| return self | |
| def transform(self, X): | |
| X = np.asarray(X, dtype=np.float64)[:, self.keep_] | |
| Xi = np.where(np.isfinite(X), X, self.med_) | |
| return ((np.clip(Xi, self.lo_, self.hi_) - self.mu_) / self.sd_).astype(np.float32) | |
| def prep_matrix(blocks, prep, kind): | |
| """kind: 'fp' (morgan bits) / 'fp_desc' (bits+maccs+desc) / | |
| 'desc_maccs_counts' (arch-5 block b) / 'physchem' raw physchem.""" | |
| if kind == "fp": | |
| return blocks["morgan_bits"] | |
| if kind == "fp_desc": | |
| return np.hstack([blocks["morgan_bits"], blocks["maccs"], prep.transform(blocks["desc"])]).astype(np.float32) | |
| if kind == "desc_maccs_counts": | |
| return np.hstack([prep.transform(blocks["desc"]), blocks["maccs"], blocks["morgan_counts"]]).astype(np.float32) | |
| if kind == "physchem": | |
| return blocks["physchem"] | |
| raise ValueError(kind) | |
| # ---------------------------------------------------------------- metrics | |
| def metrics_dict(y_true, prob, threshold=0.5): | |
| from sklearn.metrics import (accuracy_score, balanced_accuracy_score, | |
| matthews_corrcoef, roc_auc_score, confusion_matrix) | |
| y_true = np.asarray(y_true, dtype=int) | |
| prob = np.asarray(prob, dtype=float) | |
| pred = (prob >= threshold).astype(int) | |
| out = {"n": int(len(y_true)), "threshold": threshold, | |
| "accuracy": float(accuracy_score(y_true, pred)), | |
| "balanced_accuracy": float(balanced_accuracy_score(y_true, pred))} | |
| tn, fp, fn, tp = confusion_matrix(y_true, pred, labels=[0, 1]).ravel() | |
| out.update({"tn": int(tn), "fp": int(fp), "fn": int(fn), "tp": int(tp), | |
| "sensitivity_recall_pos": float(tp / (tp + fn)) if (tp + fn) else None, | |
| "specificity_recall_neg": float(tn / (tn + fp)) if (tn + fp) else None}) | |
| if len(np.unique(y_true)) == 2 and len(np.unique(pred)) == 2: | |
| out["mcc"] = float(matthews_corrcoef(y_true, pred)) | |
| else: | |
| out["mcc"] = None | |
| out["mcc_note"] = "single-class ground truth or single-class predictions" | |
| if len(np.unique(y_true)) == 2: | |
| out["auroc"] = float(roc_auc_score(y_true, prob)) | |
| else: | |
| out["auroc"] = None | |
| return out | |
| def score_benchmark_sets(predict_proba_fn, bench_sets): | |
| """bench_sets: dict name -> deduped df (with 'smiles','label','name'). | |
| predict_proba_fn(smiles_list) -> P(label=1) array (NaN where unparseable). | |
| Returns (metrics dict per set, predictions dataframe).""" | |
| all_metrics, pred_frames = {}, [] | |
| for set_name, bdf in bench_sets.items(): | |
| prob = predict_proba_fn(list(bdf["smiles"])) | |
| scored = np.isfinite(prob) | |
| m = metrics_dict(bdf.loc[scored, "label"], prob[scored]) if scored.any() else {"n": 0} | |
| m["n_rows_unique"] = int(len(bdf)) | |
| m["n_scored"] = int(scored.sum()) | |
| m["n_unscored_invalid_structure"] = int((~scored).sum()) | |
| all_metrics[set_name] = m | |
| pf = bdf[["name", "smiles", "label"]].copy() | |
| if "evidence_grade" in bdf.columns: | |
| pf["evidence_grade"] = bdf["evidence_grade"] | |
| pf["set"] = set_name | |
| pf["prob_toxic"] = prob | |
| pf["pred_label"] = np.where(scored, (prob >= 0.5).astype(float), np.nan) | |
| pf["correct"] = np.where(scored, (pf["pred_label"] == pf["label"]).astype(float), np.nan) | |
| pred_frames.append(pf) | |
| return all_metrics, pd.concat(pred_frames, ignore_index=True) | |
| # ---- GAT graph code (verbatim from liver_arch2_gat.py) ---- | |
| # Import guard (deployment plumbing only, model code below unchanged): the | |
| # sklearn-side deployment scripts import this module in an env without | |
| # torch_geometric; the graph functions are only called in the pyg env. | |
| try: | |
| import torch | |
| import torch.nn.functional as F | |
| from torch_geometric.data import Data | |
| from torch_geometric.nn import GATConv, global_mean_pool | |
| _TORCH_OK = True | |
| except ImportError: | |
| torch = None | |
| _TORCH_OK = False | |
| ELEMS = [6, 7, 8, 16, 9, 17, 35, 53, 15, 14] | |
| def mol_to_graph(smi): | |
| from rdkit import Chem | |
| HYB = [Chem.rdchem.HybridizationType.SP, Chem.rdchem.HybridizationType.SP2, | |
| Chem.rdchem.HybridizationType.SP3, Chem.rdchem.HybridizationType.SP3D, | |
| Chem.rdchem.HybridizationType.SP3D2] | |
| BOND = {Chem.rdchem.BondType.SINGLE: 0, Chem.rdchem.BondType.DOUBLE: 1, | |
| Chem.rdchem.BondType.TRIPLE: 2, Chem.rdchem.BondType.AROMATIC: 3} | |
| def oh(v, n): | |
| o = [0.0] * n; o[min(max(v, 0), n - 1)] = 1.0; return o | |
| mol = Chem.MolFromSmiles(str(smi)) if smi is not None and str(smi) != "nan" else None | |
| if mol is None or mol.GetNumAtoms() == 0: | |
| return None | |
| xs = [] | |
| for a in mol.GetAtoms(): | |
| z = a.GetAtomicNum() | |
| f = oh(ELEMS.index(z) if z in ELEMS else len(ELEMS), len(ELEMS) + 1) | |
| f += oh(a.GetDegree(), 6) | |
| f += oh(a.GetFormalCharge() + 2, 5) | |
| f += oh(a.GetTotalNumHs(), 5) | |
| f += oh(HYB.index(a.GetHybridization()) if a.GetHybridization() in HYB else 5, 6) | |
| f += [1.0 if a.GetIsAromatic() else 0.0, 1.0 if a.IsInRing() else 0.0] | |
| xs.append(f) | |
| ei, ea = [], [] | |
| for b in mol.GetBonds(): | |
| i, j = b.GetBeginAtomIdx(), b.GetEndAtomIdx() | |
| t = BOND.get(b.GetBondType(), 0) | |
| ei += [[i, j], [j, i]]; ea += [oh(t, 4), oh(t, 4)] | |
| return Data(x=torch.tensor(xs, dtype=torch.float), | |
| edge_index=torch.tensor(ei, dtype=torch.long).t().contiguous() if ei else torch.empty((2, 0), dtype=torch.long), | |
| edge_attr=torch.tensor(ea, dtype=torch.float) if ea else torch.empty((0, 4))) | |
| if _TORCH_OK: | |
| class GATNet(torch.nn.Module): | |
| def __init__(self, in_dim, hidden, dropout): | |
| super().__init__() | |
| self.c1 = GATConv(in_dim, hidden, heads=4, dropout=dropout, edge_dim=4) | |
| self.c2 = GATConv(hidden * 4, hidden, heads=1, dropout=dropout, edge_dim=4) | |
| self.head = torch.nn.Sequential(torch.nn.Linear(hidden, hidden // 2), torch.nn.ReLU(), | |
| torch.nn.Dropout(dropout), torch.nn.Linear(hidden // 2, 1)) | |
| self.dropout = dropout | |
| def forward(self, data): | |
| x = F.elu(self.c1(data.x, data.edge_index, data.edge_attr)) | |
| x = F.dropout(x, p=self.dropout, training=self.training) | |
| x = F.elu(self.c2(x, data.edge_index, data.edge_attr)) | |
| x = global_mean_pool(x, data.batch) | |
| return self.head(x).ravel() | |