scripts/verify_minimal_training.py

#!/usr/bin/env python3
"""Run a few in-memory training steps on DCU without modifying official weights."""

import argparse
import pickle
from pathlib import Path

from _bootstrap import DATA_DIR, WEIGHT_DIR

import numpy as np
import tensorflow as tf
import tf_keras


DEFAULT_DATA_DIR = DATA_DIR / "features" / "cdr_kmer3_ag_kmer1"
DEFAULT_WEIGHT_PATH = WEIGHT_DIR / "cdr_kmer3_ag_kmer1" / "Model99.h5"


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--data-dir", type=Path, default=DEFAULT_DATA_DIR)
    parser.add_argument("--weights", type=Path, default=DEFAULT_WEIGHT_PATH)
    parser.add_argument("--batch-size", type=int, default=8)
    parser.add_argument("--steps", type=int, default=1)
    return parser.parse_args()


def load_training_batch(data_dir, batch_size):
    # Register features.InputFeatures before unpickling the official artifacts.
    import model  # noqa: F401

    with (data_dir / "cdr_features_tr.pickle").open("rb") as reader:
        cdr_features = pickle.load(reader)
    with (data_dir / "ag_features_tr.pickle").open("rb") as reader:
        ag_features = pickle.load(reader)

    if len(cdr_features) != len(ag_features):
        raise ValueError(
            f"Unpaired training features: CDR={len(cdr_features)}, AG={len(ag_features)}"
        )
    if batch_size < 2 or batch_size > len(cdr_features):
        raise ValueError(
            f"--batch-size must be between 2 and {len(cdr_features)}, got {batch_size}"
        )

    selected_cdr = cdr_features[:batch_size]
    selected_ag = ag_features[:batch_size]
    inputs = [
        np.asarray([item.input_ids for item in selected_cdr], dtype=np.int32),
        np.asarray([item.cdr_number_ids for item in selected_cdr], dtype=np.int32),
        np.asarray([item.input_ids for item in selected_ag], dtype=np.int32),
    ]
    labels = np.asarray(
        [[item.label_id] for item in selected_cdr], dtype=np.float32
    )

    expected_shapes = ((batch_size, 24), (batch_size, 24), (batch_size, 2371))
    actual_shapes = tuple(array.shape for array in inputs)
    if actual_shapes != expected_shapes:
        raise ValueError(f"Unexpected input shapes: {actual_shapes}, expected {expected_shapes}")
    if labels.shape != (batch_size, 1):
        raise ValueError(f"Unexpected label shape: {labels.shape}")

    return inputs, labels


def inference_loss(model, inputs, labels):
    predictions = model(inputs, training=False)
    losses = tf_keras.losses.binary_crossentropy(labels, predictions)
    loss = float(tf.reduce_mean(losses).numpy())
    predictions = predictions.numpy()
    if not np.isfinite(loss) or not np.all(np.isfinite(predictions)):
        raise FloatingPointError("Inference loss or predictions contain NaN/Inf")
    return loss


def main():
    args = parse_args()
    if args.steps < 1:
        raise ValueError(f"--steps must be at least 1, got {args.steps}")
    if not args.data_dir.is_dir():
        raise FileNotFoundError(f"Data directory not found: {args.data_dir}")
    if not args.weights.is_file():
        raise FileNotFoundError(f"Weight file not found: {args.weights}")

    gpus = tf.config.list_physical_devices("GPU")
    if not gpus:
        raise RuntimeError(
            "TensorFlow cannot see a DCU. Set HIP_VISIBLE_DEVICES and "
            "CUDA_VISIBLE_DEVICES before starting Python."
        )
    for gpu in gpus:
        tf.config.experimental.set_memory_growth(gpu, True)

    inputs, labels = load_training_batch(args.data_dir, args.batch_size)

    # Import after configuring the accelerator to avoid initializing it early.
    from model import get_model

    with tf.device("/GPU:0"):
        model = get_model()
        model.load_weights(str(args.weights))
        loss_before = inference_loss(model, inputs, labels)
        weights_before = [variable.numpy().copy() for variable in model.trainable_variables]

        training_losses = []
        for _ in range(args.steps):
            loss = model.train_on_batch(inputs, labels)
            training_losses.append(float(np.asarray(loss).reshape(-1)[0]))

        loss_after = inference_loss(model, inputs, labels)

    if not np.all(np.isfinite(training_losses)):
        raise FloatingPointError(f"Training loss contains NaN/Inf: {training_losses}")

    changed_count = 0
    max_weight_change = 0.0
    for before, variable in zip(weights_before, model.trainable_variables):
        after = variable.numpy()
        if not np.all(np.isfinite(after)):
            raise FloatingPointError(f"Trainable variable contains NaN/Inf: {variable.name}")
        change = float(np.max(np.abs(after - before)))
        max_weight_change = max(max_weight_change, change)
        if change > 0.0:
            changed_count += 1

    dcu_variable_count = sum("GPU:0" in variable.device for variable in model.variables)

    print(f"TensorFlow: {tf.__version__}")
    print(f"tf_keras: {tf_keras.__version__}")
    print(f"DCU: {gpus[0]}")
    print(f"Batch size: {args.batch_size}")
    print(f"Steps: {args.steps}")
    print(f"Labels: {labels.reshape(-1).tolist()}")
    print(f"Inference loss before: {loss_before:.10g}")
    print(f"Training losses: {training_losses}")
    print(f"Inference loss after: {loss_after:.10g}")
    print(f"Trainable tensors changed: {changed_count}/{len(weights_before)}")
    print(f"Max absolute weight change: {max_weight_change:.10g}")
    print(f"Variables placed on GPU:0: {dcu_variable_count}/{len(model.variables)}")

    if changed_count == 0 or max_weight_change == 0.0:
        raise AssertionError("No trainable weights changed after train_on_batch")
    if dcu_variable_count == 0:
        raise RuntimeError("No model variables were placed on GPU:0")

    print("Minimal DCU training: PASS")
    print("Official Model99.h5 was loaded read-only and was not overwritten.")


if __name__ == "__main__":
    main()