File size: 3,096 Bytes
a8ef2f8 f470c42 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | from pathlib import Path
import pytest
import torch
from transformers import AutoTokenizer
from processing_salmonn import SalmonnProcessor
ROOT = Path(__file__).resolve().parents[1]
@pytest.fixture(scope="module")
def processor():
tokenizer = AutoTokenizer.from_pretrained(ROOT, fix_mistral_regex=False)
return SalmonnProcessor(tokenizer)
def test_basic_audio_request(processor):
inputs = processor(
audios=torch.zeros(1600),
sampling_rate=16000,
instruction="Describe the audio.",
)
assert inputs["input_ids"].shape[0] == 1
assert inputs["audio_features"].shape == (1, 10, 128)
assert inputs["audio_lengths"].tolist() == [10]
assert inputs["audio_counts"].tolist() == [1]
def test_text_only_context_does_not_add_audio(processor):
prompt = processor.build_prompt(
"Transcribe the speech.",
context=["howes", "wszelaki"],
)
assert prompt.count("<audio>") == 1
assert "[howes, wszelaki]" in prompt
def test_audio_text_context_preserves_placeholder_order(processor):
waveform = torch.zeros(1600)
context = [
{"text": "howes", "audio": (waveform, 16000)},
{"text": "wszelaki", "audio": (waveform, 16000)},
]
prompt = processor.build_prompt("Transcribe the speech.", context=context)
inputs = processor(
audios=(waveform, 16000),
instruction="Transcribe the speech.",
context=context,
)
assert prompt.count("<audio>") == 3
assert "<audio>howes\n<audio>wszelaki" in prompt
assert inputs["audio_features"].shape == (3, 10, 128)
assert inputs["audio_lengths"].tolist() == [10, 10, 10]
assert inputs["audio_counts"].tolist() == [3]
def test_mixed_context_modalities_are_rejected(processor):
with pytest.raises(ValueError, match="all include audio or all be text-only"):
processor.build_prompt(
"Transcribe the speech.",
context=[
"howes",
{"text": "wszelaki", "audio": (torch.zeros(1600), 16000)},
],
)
def test_formatted_prompt_preserves_custom_audio_placement(processor):
waveform = torch.zeros(1600)
inputs = processor(
audios=[(waveform, 16000), (waveform, 16000)],
formatted_prompt="<audio>Compare this recording with <audio>this example.",
)
assert (inputs["input_ids"] == 151652).sum().item() == 2
assert inputs["audio_counts"].tolist() == [2]
def test_formatted_prompt_validates_placeholder_count(processor):
with pytest.raises(ValueError, match="1 audio placeholders but received 2"):
processor(
audios=[(torch.zeros(1600), 16000), (torch.zeros(1600), 16000)],
formatted_prompt="<audio>Describe both recordings.",
)
def test_instruction_and_formatted_prompt_are_mutually_exclusive(processor):
with pytest.raises(ValueError, match="exactly one"):
processor(
audios=(torch.zeros(1600), 16000),
instruction="Describe the audio.",
formatted_prompt="<audio>Describe the audio.",
)
|