#!/usr/bin/env python3 """Validate a MolMistral bundle end to end. Checks, in order: 1. model.keras SHA-256 matches MANIFEST.json (when present). 2. The model loads and reports its parameter count. 3. Tokenizer round-trip: encode->decode reproduces sample SMILES exactly, and the regex round-trip guard passes (no silent character drops). 4. Loss / perplexity of the model on a small SMILES set (built-in drug examples, or --csv with a `smiles` column). Reference point: the foundation's recorded best validation loss is 0.2239 (perplexity 1.251) on its 20,000-row mixed holdout -- a 6-molecule smoke set will differ; this is a wiring check, not a benchmark. 5. Sampling smoke test: generates --n_sample sequences at T=0.8 and reports the RDKit-valid fraction. Usage: python validate_bundle.py --bundle . python validate_bundle.py --bundle . --csv my_smiles.csv --n_sample 64 """ import argparse import hashlib import json import os import sys import numpy as np SMOKE_SMILES = [ "CC(=O)Oc1ccccc1C(=O)O", # aspirin "CN1C=NC2=C1C(=O)N(C(=O)N2C)C", # caffeine "CC(C)Cc1ccc(C(C)C(=O)O)cc1", # ibuprofen "CC(=O)Nc1ccc(O)cc1", # paracetamol "CN(C)C(=N)NC(=N)N", # metformin "OC[C@H]1OC(O)[C@H](O)[C@@H](O)[C@@H]1O", # glucose ] def sha256(path): h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def main(): ap = argparse.ArgumentParser() ap.add_argument("--bundle", default=".") ap.add_argument("--csv", default="") ap.add_argument("--n_sample", type=int, default=32) ap.add_argument("--max_len", type=int, default=96) args = ap.parse_args() b = args.bundle ok = True # 1. hash check man_path = os.path.join(b, "MANIFEST.json") if os.path.exists(man_path): man = json.load(open(man_path)) expect = man["files"]["model.keras"]["sha256"] got = sha256(os.path.join(b, "model.keras")) status = "PASS" if got == expect else "FAIL" ok &= got == expect print(f"[1] model.keras sha256 {status}: {got[:16]}...") else: print("[1] MANIFEST.json absent - hash check skipped") # 2. load model + tokenizer sys.path.insert(0, b) import train_llm_foundation as T import keras import tensorflow as tf tok = T.SmilesTokenizer(T.load_vocab(os.path.join(b, "vocab.txt"))) tok.add_property_tokens() tok.add_fix_tokens() tok.property_token_ids |= {tok.vocab[t] for t in T.PROPERTY_TOKENS if t in tok.vocab} model = keras.models.load_model(os.path.join(b, "model.keras")) print(f"[2] model loaded: {model.count_params():,} parameters, " f"vocab {tok.vocab_size}") # 3. tokenizer round-trip smiles = SMOKE_SMILES if args.csv: import pandas as pd smiles = pd.read_csv(args.csv)["smiles"].astype(str).tolist() rt_ok = all(tok.roundtrip_ok(s) for s in smiles) dec_ok = True for s in smiles: ids, _ = tok.encode(s, max_len=args.max_len) if tok.decode(ids) != s: dec_ok = False ok &= rt_ok and dec_ok print(f"[3] tokenizer round-trip {'PASS' if rt_ok and dec_ok else 'FAIL'} " f"on {len(smiles)} SMILES") # 4. loss / perplexity on the set model.compile( loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]) ds = T.build_tf_dataset(smiles, tok, max_len=args.max_len, batch_size=64, shuffle=False) loss, acc = model.evaluate(ds, verbose=0) print(f"[4] smoke loss {loss:.4f} | perplexity {np.exp(loss):.3f} | " f"accuracy {acc:.3f} (n={len(smiles)})") # 5. sampling smoke test np.random.seed(0) gen, valid = [], 0 for _ in range(args.n_sample): ids = T.sample_sequence(model, tok, [tok.bos], steps=128, temperature=0.8, top_k=50, top_p=0.95) smi = tok.decode(ids) gen.append(smi) try: from rdkit import Chem valid += bool(Chem.MolFromSmiles(smi)) if smi else 0 except Exception: pass print(f"[5] sampled {args.n_sample} @ T=0.8: RDKit-valid " f"{valid}/{args.n_sample}") for s in gen[:5]: print(" ", s) print("VALIDATION", "PASS" if ok else "FAIL") return 0 if ok else 1 if __name__ == "__main__": sys.exit(main())