// Copyright 2025 The ODML Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "runtime/components/token_id_util.h" #include #include #include #include "absl/status/status.h" // from @com_google_absl #include "runtime/util/test_utils.h" // NOLINT namespace litert::lm { namespace { using ::testing::ElementsAre; using ::testing::status::IsOkAndHolds; using ::testing::status::StatusIs; TEST(TokenIdUtilTest, PreprocessTokenIds) { std::vector token_ids = {1, 2, 3, 4, 5}; EXPECT_OK(PreprocessTokenIds(token_ids, /*start_token_id=*/0, /*max_num_tokens=*/10, /*context_length_ratio_threhold=*/0.9f)); EXPECT_THAT(token_ids, ElementsAre(0, 1, 2, 3, 4, 5)); } TEST(TokenIdUtilTest, PreprocessTokenIdsExceeedThreshold) { std::vector token_ids = {1, 2, 3, 4, 5, 6, 7, 8, 9}; EXPECT_THAT(PreprocessTokenIds(token_ids, /*start_token_id=*/0, /*max_num_tokens=*/10, /*context_length_ratio_threhold=*/0.9f), StatusIs(absl::StatusCode::kInvalidArgument)); } TEST(TokenIdUtilTest, StopTokenFoundTrue) { std::vector decoded_token_ids = {0, 2, 0, 4, 5}; std::vector stop_token_found = {false, true, false, true, true}; EXPECT_THAT(StopTokenFound(decoded_token_ids, /*stop_token_ids=*/{0}, stop_token_found), IsOkAndHolds(true)); } TEST(TokenIdUtilTest, MultiStopTokenFoundTrue) { std::vector decoded_token_ids = {0, 2, 1, 4, 5}; std::vector stop_token_found = {false, true, false, true, true}; EXPECT_THAT(StopTokenFound(decoded_token_ids, /*stop_token_ids=*/{0, 1}, stop_token_found), IsOkAndHolds(true)); } TEST(TokenIdUtilTest, StopTokenFoundFalse) { std::vector decoded_token_ids = {0, 2, 0, 4, 5}; std::vector stop_token_found = {false, false, false, false, false}; EXPECT_THAT(StopTokenFound(decoded_token_ids, /*stop_token_ids=*/{0}, stop_token_found), IsOkAndHolds(false)); EXPECT_THAT(stop_token_found, ElementsAre(true, false, true, false, false)); } TEST(TokenIdUtilTest, StopTokenFoundInvalidInput) { std::vector decoded_token_ids = {1, 2, 3, 4, 5}; std::vector stop_token_found = {false, false, false, false}; EXPECT_THAT(StopTokenFound(decoded_token_ids, /*stop_token_ids=*/{4}, stop_token_found), StatusIs(absl::StatusCode::kInvalidArgument)); } } // namespace } // namespace litert::lm