scripts/smoke_structure.py

"""Random tiny-model CPU test of custom generate/prefill. NOT a 4B accuracy test."""
import argparse
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from grounding_demo.native_runtime import inspect_contract, user_messages


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("model_path", type=Path, help="Reviewed native code/config/tokenizer directory")
    args = parser.parse_args()
    root = str(args.model_path.resolve())
    inspect_contract(Path(root), require_weights=False)
    import torch
    from PIL import Image
    from transformers import AutoConfig, AutoProcessor
    from transformers.dynamic_module_utils import get_class_from_dynamic_module

    c = AutoConfig.from_pretrained(root, trust_remote_code=True)
    for key, value in dict(hidden_size=32, intermediate_size=64, num_hidden_layers=1,
                           num_attention_heads=4, num_key_value_heads=2, head_dim=8).items():
        setattr(c.text_config, key, value)
    for key, value in dict(hidden_size=32, qkv_hidden_size=32, intermediate_size=64,
                           num_hidden_layers=1, num_attention_heads=4, out_hidden_size=32,
                           projector_hidden_size=64, init_pos_emb_height=2,
                           init_pos_emb_width=2).items():
        setattr(c.vision_config, key, value)
    for config in [c, c.text_config, c.vision_config]:
        config._attn_implementation = "sdpa"
    model_class = get_class_from_dynamic_module(c.auto_map["AutoModelForImageTextToText"], root)
    # Full float32 only for this CPU/random-weight structural test; production is BF16.
    model = model_class(c).eval().float()
    processor = AutoProcessor.from_pretrained(root, trust_remote_code=True)
    rendered = processor.apply_chat_template(user_messages("Find the red square."),
                                             tokenize=False, add_generation_prompt=True)
    inputs = processor(text=[rendered], images=[Image.new("RGB", (28, 28), "red")],
                       return_tensors="pt")
    calls = []
    hook = model.model.visual.register_forward_pre_hook(lambda *args: calls.append("vision"))
    try:
        with torch.inference_mode():
            output = model.generate(**inputs, max_new_tokens=3, do_sample=False,
                                    use_cache=True, eos_token_id=[151645, 151643])
    finally:
        hook.remove()
    generated = int(output.shape[-1] - inputs["input_ids"].shape[-1])
    if len(calls) != 1 or generated < 1:
        raise RuntimeError("Vision must execute exactly once during cached generate prefill.")
    print(json.dumps({"scope": "RANDOM_TINY_MODEL_CPU_STRUCTURE_ONLY_NOT_4B_ACCURACY",
                      "vision_calls": len(calls), "new_tokens": generated,
                      "parameters": sum(p.numel() for p in model.parameters())}, indent=2))


if __name__ == "__main__":
    main()