File size: 280 Bytes
688e1f3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 | import numpy as np
from openpi_value.models import tokenizer as _tokenizer
def test_tokenize():
tokenizer = _tokenizer.PaligemmaTokenizer(max_len=10)
tokens, masks = tokenizer.tokenize("Hello, world!")
assert tokens.shape == (10,)
assert masks.shape == (10,)
|