# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: OpenMDW-1.1 import json from pathlib import Path from types import SimpleNamespace from typing import Any from unittest.mock import Mock import pytest def _make_v2v_sample_args(**overrides: Any) -> SimpleNamespace: """v2v ``OmniSampleArgs`` stand-in for ``get_sample_data`` tests.""" from cosmos_framework.inference.args import ModelMode, NegativeMetadataMode defaults = dict( action_path=None, aspect_ratio="16,9", autoregressive=False, camera_trajectory=None, condition_frame_indexes_vision=[0, 1], condition_video_keep=None, condition_vision_mode="video", duration_template=None, enable_sound=False, fps=24, inverse_duration_template=None, inverse_resolution_template=None, model_mode=ModelMode.VIDEO2VIDEO, native_prompt_upsampling=False, negative_metadata_mode=NegativeMetadataMode.NONE, negative_prompt=None, num_frames=125, num_outputs=1, prompt="prompt", resolution_template=None, transfer_hints={}, vision_path="conditioning.mp4", vision_size=(32, 16), ) defaults.update(overrides) return SimpleNamespace(**defaults) @pytest.mark.parametrize( ("condition_video_keep", "expected_loader_keep"), [ ("last", "last"), ("first", "first"), (None, "first"), # default ], ) def test_video_conditioning_plumbs_keep_and_pixel_frame_count( monkeypatch: pytest.MonkeyPatch, condition_video_keep: str | None, expected_loader_keep: str, ) -> None: """v2v: tokenizer derives pixel-frame count from latent count, ``keep`` passes through to the loader.""" torch = pytest.importorskip("torch") from cosmos_framework.inference import inference class Tokenizer: calls: list[int] def __init__(self) -> None: self.calls = [] def get_pixel_num_frames(self, num_latent_frames: int) -> int: self.calls.append(num_latent_frames) return 5 tokenizer = Tokenizer() model = SimpleNamespace( input_image_key="image", input_video_key="video", input_caption_key="caption", tokenizer_vision_gen=tokenizer, ) sample_args = _make_v2v_sample_args(condition_video_keep=condition_video_keep) conditioning_frames = torch.zeros(3, 5, 16, 32) sequence_plan = ["sequence-plan"] load_conditioning_video_mock = Mock(return_value=conditioning_frames) build_conditioned_video_batch_mock = Mock( return_value={ "video": [torch.zeros(1, 3, 125, 16, 32)], "sequence_plan": sequence_plan, } ) monkeypatch.setattr(inference, "load_conditioning_video", load_conditioning_video_mock) monkeypatch.setattr(inference, "build_conditioned_video_batch", build_conditioned_video_batch_mock) out = inference.get_sample_data(sample_args, model, device="cpu") assert tokenizer.calls == [2] # max([0, 1]) + 1 load_conditioning_video_mock.assert_called_once_with( Path("conditioning.mp4"), target_h=16, target_w=32, max_frames=5, keep=expected_loader_keep, ) build_conditioned_video_batch_mock.assert_called_once() build_args, build_kwargs = build_conditioned_video_batch_mock.call_args assert build_args == (conditioning_frames,) assert build_kwargs == { "condition_frames_vision": [0, 1], "w": 32, "h": 16, "num_frames": 125, "fps": 24, "batch_size": 1, } assert out["sequence_plan"] is sequence_plan def test_json_prompt_metadata_for_single_frame_omits_temporal_fields() -> None: from cosmos_framework.inference.inference import _format_json_prompt_with_template prompt = _format_json_prompt_with_template( {"subjects": [], "duration": "8s", "fps": 24.0}, fps=24, num_frames=1, aspect_ratio="1,1", h=1024, w=1024, include_temporal_metadata=False, ) assert prompt == '{"subjects": [], "resolution": {"H": 1024, "W": 1024}, "aspect_ratio": "1,1"}' parsed = json.loads(prompt) assert parsed["resolution"] == {"H": 1024, "W": 1024} assert parsed["aspect_ratio"] == "1,1" assert "duration" not in parsed assert "fps" not in parsed def test_json_prompt_metadata_for_video_keeps_temporal_fields() -> None: from cosmos_framework.inference.inference import _format_json_prompt_with_template prompt = _format_json_prompt_with_template( {"subjects": []}, fps=24, num_frames=189, aspect_ratio="16,9", h=720, w=1280, include_temporal_metadata=True, ) assert prompt == ( '{"subjects": [], "duration": "7s", "fps": 24.0, "resolution": {"H": 720, "W": 1280}, "aspect_ratio": "16,9"}' ) assert json.loads(prompt) == { "subjects": [], "duration": "7s", "fps": 24.0, "resolution": {"H": 720, "W": 1280}, "aspect_ratio": "16,9", } def _make_reasoner_sample_args(**overrides: Any) -> SimpleNamespace: """Reasoner ``OmniSampleArgs`` stand-in for ``get_sample_data`` tests.""" from cosmos_framework.inference.args import ModelMode defaults = dict( model_mode=ModelMode.REASONER, prompt="Describe a robotic arm.", vision_path=None, max_new_tokens=8, do_sample=False, temperature=1.0, top_k=None, top_p=None, num_outputs=1, ) defaults.update(overrides) return SimpleNamespace(**defaults) @pytest.mark.L0 def test_get_sample_data_reasoner_text_only() -> None: from cosmos_framework.inference import inference model = SimpleNamespace(input_caption_key="caption") sample_args = _make_reasoner_sample_args() out = inference.get_sample_data(sample_args, model, device="cpu") assert out == {"caption": ["Describe a robotic arm."], "reasoner_images": [None]} @pytest.mark.L0 def test_get_sample_data_reasoner_with_image(tmp_path: Path) -> None: PIL = pytest.importorskip("PIL.Image") from cosmos_framework.inference import inference img_path = tmp_path / "arm.png" PIL.new("RGB", (8, 8), color="red").save(img_path) model = SimpleNamespace(input_caption_key="caption") sample_args = _make_reasoner_sample_args(vision_path=str(img_path)) out = inference.get_sample_data(sample_args, model, device="cpu") assert list(out) == ["caption", "reasoner_images"] assert out["caption"] == ["Describe a robotic arm."] assert len(out["reasoner_images"]) == 1 assert out["reasoner_images"][0].size == (8, 8) assert out["reasoner_images"][0].mode == "RGB" @pytest.mark.L0 def test_reasoner_defaults_json_round_trip() -> None: import json as _json from cosmos_framework.inference.args import PACKAGE_DIR, _load_modality_defaults defaults = _load_modality_defaults("reasoner") assert defaults["model_mode"] == "reasoner" assert defaults["max_new_tokens"] == 64 on_disk = _json.loads((PACKAGE_DIR / "defaults/reasoner/sample_args.json").read_text()) assert defaults == on_disk @pytest.mark.L0 def test_reasoner_overrides_round_trip() -> None: import pydantic from cosmos_framework.inference.args import ModelMode, ReasonerDataOverrides overrides = ReasonerDataOverrides(max_new_tokens=128, temperature=0.7, top_p=0.9) assert overrides.max_new_tokens == 128 assert overrides.temperature == 0.7 assert overrides.top_p == 0.9 with pytest.raises(pydantic.ValidationError): ReasonerDataOverrides(top_p=1.5) with pytest.raises(pydantic.ValidationError): ReasonerDataOverrides(temperature=0) assert ModelMode.REASONER.is_reasoner assert not ModelMode.TEXT2VIDEO.is_reasoner assert not ModelMode.REASONER.is_action @pytest.mark.L0 def test_generate_reasoner_batch_writes_outputs(tmp_path: Path) -> None: pytest.importorskip("torch") from cosmos_framework.inference import inference from cosmos_framework.inference.args import ModelMode out_dir = tmp_path / "hello" class _SA(SimpleNamespace): def model_dump(self, **_): return {"name": self.name, "model_mode": self.model_mode} def model_dump_json(self, **_): import json as _json return _json.dumps(self.model_dump()) sample_args = _SA( name="hello", model_mode=ModelMode.REASONER, output_dir=out_dir, prompt="Describe a robotic arm.", max_new_tokens=8, do_sample=False, temperature=1.0, top_k=None, top_p=None, repetition_penalty=1.0, presence_penalty=0.0, seed=None, ) def _fake_generate_reasoner_text(prompts, *, images=None, **kwargs): assert prompts == ["Describe a robotic arm."] assert images is None return ["A six-axis arm with a parallel-jaw gripper."] model = SimpleNamespace( input_caption_key="caption", generate_reasoner_text=_fake_generate_reasoner_text, ) pipe = inference.OmniInference.__new__(inference.OmniInference) pipe.model = model pipe.should_process_sample = lambda sa: True # type: ignore[attr-defined] pipe._run_text_guardrail = lambda *_a, **_kw: None # type: ignore[attr-defined] pipe._handle_sample_exception = lambda sa, e: (_ for _ in ()).throw(e) # type: ignore[attr-defined] from contextlib import nullcontext pipe._get_timer = lambda *_a, **_kw: nullcontext() # type: ignore[attr-defined] data_batch = {"caption": ["Describe a robotic arm."], "reasoner_images": [None]} results = pipe._generate_reasoner_batch([sample_args], data_batch, warmup=False) assert len(results) == 1 so = results[0] assert so.outputs[0].content == {"reasoner_text": "A six-axis arm with a parallel-jaw gripper."} txt_file = out_dir / "reasoner_text.txt" assert txt_file.read_text() == "A six-axis arm with a parallel-jaw gripper." assert (out_dir / "sample_args.json").is_file() assert (out_dir / "sample_outputs.json").is_file() @pytest.mark.L0 def test_generate_reasoner_batch_rejects_mixed_image_text_only(tmp_path: Path) -> None: PIL = pytest.importorskip("PIL.Image") from cosmos_framework.inference import inference from cosmos_framework.inference.args import ModelMode pipe = inference.OmniInference.__new__(inference.OmniInference) pipe.model = SimpleNamespace(input_caption_key="caption") pipe.should_process_sample = lambda sa: False # type: ignore[attr-defined] sa1 = SimpleNamespace(model_mode=ModelMode.REASONER, output_dir=tmp_path / "a") sa2 = SimpleNamespace(model_mode=ModelMode.REASONER, output_dir=tmp_path / "b") data_batch = { "caption": ["p1", "p2"], "reasoner_images": [PIL.new("RGB", (8, 8)), None], } with pytest.raises(ValueError, match="mixes image-conditioned and text-only"): pipe._generate_reasoner_batch([sa1, sa2], data_batch, warmup=False) @pytest.mark.L0 def test_reasoner_build_rejects_empty_prompt() -> None: from cosmos_framework.inference.args import ModelMode, OmniSampleOverrides, SampleMeta, VisionMode overrides = OmniSampleOverrides(prompt=" ") meta = SampleMeta(model_mode=ModelMode.REASONER, vision_mode=VisionMode.IMAGE, condition_vision_mode=None) with pytest.raises(ValueError, match="non-empty 'prompt'"): overrides._build_reasoner_data(model_config=None, sample_meta=meta) @pytest.mark.L0 def test_reasoner_defaults_validate_against_overrides() -> None: """The defaults JSON must validate against ``OmniSampleOverrides`` so ``build_sample`` cannot silently drop a field after an upstream rename.""" from cosmos_framework.inference.args import OmniSampleOverrides, _load_modality_defaults defaults = _load_modality_defaults("reasoner") filtered = {k: v for k, v in defaults.items() if k in OmniSampleOverrides.model_fields} assert set(defaults) - set(filtered) == set(), f"defaults has unknown fields: {set(defaults) - set(filtered)}" OmniSampleOverrides.model_validate(filtered)