molmistral-foundation-v61 / validate_bundle.py
Delower's picture
Upload validate_bundle.py with huggingface_hub
7f0be31 verified
Raw History Blame Contribute Delete
4.64 kB
#!/usr/bin/env python3
"""Validate a MolMistral bundle end to end.
Checks, in order:
1. model.keras SHA-256 matches MANIFEST.json (when present).
2. The model loads and reports its parameter count.
3. Tokenizer round-trip: encode->decode reproduces sample SMILES exactly,
and the regex round-trip guard passes (no silent character drops).
4. Loss / perplexity of the model on a small SMILES set (built-in drug
examples, or --csv with a `smiles` column). Reference point: the
foundation's recorded best validation loss is 0.2239 (perplexity 1.251)
on its 20,000-row mixed holdout -- a 6-molecule smoke set will differ;
this is a wiring check, not a benchmark.
5. Sampling smoke test: generates --n_sample sequences at T=0.8 and
reports the RDKit-valid fraction.
Usage:
python validate_bundle.py --bundle .
python validate_bundle.py --bundle . --csv my_smiles.csv --n_sample 64
"""
import argparse
import hashlib
import json
import os
import sys
import numpy as np
SMOKE_SMILES = [
"CC(=O)Oc1ccccc1C(=O)O", # aspirin
"CN1C=NC2=C1C(=O)N(C(=O)N2C)C", # caffeine
"CC(C)Cc1ccc(C(C)C(=O)O)cc1", # ibuprofen
"CC(=O)Nc1ccc(O)cc1", # paracetamol
"CN(C)C(=N)NC(=N)N", # metformin
"OC[C@H]1OC(O)[C@H](O)[C@@H](O)[C@@H]1O", # glucose
]
def sha256(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--bundle", default=".")
ap.add_argument("--csv", default="")
ap.add_argument("--n_sample", type=int, default=32)
ap.add_argument("--max_len", type=int, default=96)
args = ap.parse_args()
b = args.bundle
ok = True
# 1. hash check
man_path = os.path.join(b, "MANIFEST.json")
if os.path.exists(man_path):
man = json.load(open(man_path))
expect = man["files"]["model.keras"]["sha256"]
got = sha256(os.path.join(b, "model.keras"))
status = "PASS" if got == expect else "FAIL"
ok &= got == expect
print(f"[1] model.keras sha256 {status}: {got[:16]}...")
else:
print("[1] MANIFEST.json absent - hash check skipped")
# 2. load model + tokenizer
sys.path.insert(0, b)
import train_llm_foundation as T
import keras
import tensorflow as tf
tok = T.SmilesTokenizer(T.load_vocab(os.path.join(b, "vocab.txt")))
tok.add_property_tokens()
tok.add_fix_tokens()
tok.property_token_ids |= {tok.vocab[t] for t in T.PROPERTY_TOKENS
if t in tok.vocab}
model = keras.models.load_model(os.path.join(b, "model.keras"))
print(f"[2] model loaded: {model.count_params():,} parameters, "
f"vocab {tok.vocab_size}")
# 3. tokenizer round-trip
smiles = SMOKE_SMILES
if args.csv:
import pandas as pd
smiles = pd.read_csv(args.csv)["smiles"].astype(str).tolist()
rt_ok = all(tok.roundtrip_ok(s) for s in smiles)
dec_ok = True
for s in smiles:
ids, _ = tok.encode(s, max_len=args.max_len)
if tok.decode(ids) != s:
dec_ok = False
ok &= rt_ok and dec_ok
print(f"[3] tokenizer round-trip {'PASS' if rt_ok and dec_ok else 'FAIL'} "
f"on {len(smiles)} SMILES")
# 4. loss / perplexity on the set
model.compile(
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
ds = T.build_tf_dataset(smiles, tok, max_len=args.max_len,
batch_size=64, shuffle=False)
loss, acc = model.evaluate(ds, verbose=0)
print(f"[4] smoke loss {loss:.4f} | perplexity {np.exp(loss):.3f} | "
f"accuracy {acc:.3f} (n={len(smiles)})")
# 5. sampling smoke test
np.random.seed(0)
gen, valid = [], 0
for _ in range(args.n_sample):
ids = T.sample_sequence(model, tok, [tok.bos], steps=128,
temperature=0.8, top_k=50, top_p=0.95)
smi = tok.decode(ids)
gen.append(smi)
try:
from rdkit import Chem
valid += bool(Chem.MolFromSmiles(smi)) if smi else 0
except Exception:
pass
print(f"[5] sampled {args.n_sample} @ T=0.8: RDKit-valid "
f"{valid}/{args.n_sample}")
for s in gen[:5]:
print(" ", s)
print("VALIDATION", "PASS" if ok else "FAIL")
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())