runtime/conversation/model_data_processor/BUILD

# Copyright 2025 The ODML Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# [Google-internal load of `cc_library`]
# [Google-internal load of `cc_test`]

package(
    default_hdrs_check = "strict",
    default_visibility = [
        "//visibility:public",
    ],
)

cc_library(
    name = "config_registry",
    hdrs = [
        "config_registry.h",
    ],
    deps = [
        ":function_gemma_data_processor_config",
        ":gemma3_data_processor_config",
        ":gemma4_data_processor_config",
        ":generic_data_processor_config",
        ":qwen3_data_processor_config",
    ],
)

cc_library(
    name = "data_utils",
    srcs = ["data_utils.cc"],
    hdrs = ["data_utils.h"],
    deps = [
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
        "@nlohmann_json//:json",
        "//runtime/util:memory_mapped_file",
    ],
)

cc_test(
    name = "data_utils_test",
    srcs = ["data_utils_test.cc"],
    deps = [
        ":data_utils",
        "@com_google_googletest//:gtest_main",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/strings:string_view",
        "@nlohmann_json//:json",
        "//runtime/util:memory_mapped_file",
        "//runtime/util:test_utils",
    ],
)

cc_library(
    name = "gemma3_data_processor_config",
    hdrs = [
        "gemma3_data_processor_config.h",
    ],
)

cc_library(
    name = "function_gemma_data_processor_config",
    hdrs = ["function_gemma_data_processor_config.h"],
)

cc_library(
    name = "generic_data_processor_config",
    hdrs = [
        "generic_data_processor_config.h",
    ],
)

cc_library(
    name = "qwen3_data_processor_config",
    hdrs = [
        "qwen3_data_processor_config.h",
    ],
)

cc_library(
    name = "model_data_processor",
    hdrs = ["model_data_processor.h"],
    deps = [
        ":config_registry",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings:string_view",
        "@nlohmann_json//:json",
        "//runtime/components:prompt_template",
        "//runtime/components/constrained_decoding:constraint",
        "//runtime/conversation:io_types",
        "//runtime/engine:io_types",
    ],
)

cc_library(
    name = "model_data_processor_factory",
    srcs = ["model_data_processor_factory.cc"],
    hdrs = ["model_data_processor_factory.h"],
    deps = [
        ":config_registry",
        ":function_gemma_data_processor",
        ":function_gemma_data_processor_config",
        ":gemma3_data_processor",
        ":gemma3_data_processor_config",
        ":gemma4_data_processor",
        ":gemma4_data_processor_config",
        ":generic_data_processor",
        ":generic_data_processor_config",
        ":model_data_processor",
        ":qwen3_data_processor",
        ":qwen3_data_processor_config",
        "@com_google_absl//absl/log:absl_log",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "//runtime/components:prompt_template",
        "//runtime/components:tokenizer",
        "//runtime/conversation:io_types",
        "//runtime/proto:llm_model_type_cc_proto",
        "//runtime/proto:token_cc_proto",
        "//runtime/util:litert_status_util",
    ],
)

cc_test(
    name = "model_data_processor_factory_test",
    srcs = ["model_data_processor_factory_test.cc"],
    data = ["//runtime/components/testdata"],
    deps = [
        ":config_registry",
        ":function_gemma_data_processor_config",
        ":gemma3_data_processor_config",
        ":gemma4_data_processor_config",
        ":generic_data_processor_config",
        ":model_data_processor",
        ":model_data_processor_factory",
        ":qwen3_data_processor_config",
        "@com_google_googletest//:gtest_main",
        "@com_google_absl//absl/status",
        "//runtime/components:sentencepiece_tokenizer",
        "//runtime/components:tokenizer",
        "//runtime/conversation:io_types",
        "//runtime/engine:io_types",
        "//runtime/proto:llm_model_type_cc_proto",
        "//runtime/util:litert_status_util",
        "//runtime/util:test_utils",
    ],
)

cc_library(
    name = "generic_data_processor",
    srcs = ["generic_data_processor.cc"],
    hdrs = ["generic_data_processor.h"],
    deps = [
        ":generic_data_processor_config",
        ":model_data_processor",
        "@com_google_absl//absl/log:absl_log",
        "@com_google_absl//absl/memory",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings:string_view",
        "@nlohmann_json//:json",
        "//runtime/components:prompt_template",
        "//runtime/conversation:io_types",
        "//runtime/engine:io_types",
        "//runtime/util:litert_status_util",
    ],
)

cc_test(
    name = "generic_data_processor_test",
    srcs = ["generic_data_processor_test.cc"],
    deps = [
        ":generic_data_processor",
        ":generic_data_processor_config",
        "@com_google_googletest//:gtest_main",
        "@nlohmann_json//:json",
        "//runtime/components:prompt_template",
        "//runtime/conversation:io_types",
        "//runtime/engine:io_types",
        "//runtime/util:test_utils",
    ],
)

cc_library(
    name = "qwen3_data_processor",
    srcs = ["qwen3_data_processor.cc"],
    hdrs = ["qwen3_data_processor.h"],
    deps = [
        ":model_data_processor",
        ":qwen3_data_processor_config",
        "@com_google_absl//absl/log:absl_log",
        "@com_google_absl//absl/memory",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings:string_view",
        "@nlohmann_json//:json",
        "//runtime/components/tool_use:parser_utils",
        "//runtime/conversation:io_types",
        "//runtime/engine:io_types",
        "//runtime/util:litert_status_util",
    ],
)

cc_test(
    name = "qwen3_data_processor_test",
    srcs = ["qwen3_data_processor_test.cc"],
    deps = [
        ":qwen3_data_processor",
        ":qwen3_data_processor_config",
        "@com_google_googletest//:gtest_main",
        "@nlohmann_json//:json",
        "//runtime/conversation:io_types",
        "//runtime/engine:io_types",
        "//runtime/util:test_utils",
    ],
)

cc_library(
    name = "gemma3_data_processor",
    srcs = ["gemma3_data_processor.cc"],
    hdrs = ["gemma3_data_processor.h"],
    deps = [
        ":data_utils",
        ":gemma3_data_processor_config",
        ":model_data_processor",
        "@com_google_absl//absl/memory",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/strings:string_view",
        "@nlohmann_json//:json",
        "@litert//litert/cc:litert_layout",
        "//runtime/components:prompt_template",
        "//runtime/components:sentencepiece_tokenizer",
        "//runtime/components:tokenizer",
        "//runtime/components/constrained_decoding:constraint",
        "//runtime/components/constrained_decoding:gemma_model_constraint_provider_lib",
        "//runtime/components/preprocessor:audio_preprocessor",
        "//runtime/components/preprocessor:audio_preprocessor_miniaudio",
        "//runtime/components/preprocessor:image_preprocessor",
        "//runtime/components/preprocessor:stb_image_preprocessor",
        "//runtime/components/tool_use:parser_utils",
        "//runtime/components/tool_use:python_tool_format_utils",
        "//runtime/conversation:io_types",
        "//runtime/conversation:prompt_utils",
        "//runtime/engine:io_types",
        "//runtime/util:litert_status_util",
        "//runtime/util:memory_mapped_file",
        "@com_googlesource_code_re2//:re2",
        "@sentencepiece//:sentencepiece_model_cc_proto",
    ],
)

cc_test(
    name = "gemma3_data_processor_test",
    srcs = ["gemma3_data_processor_test.cc"],
    data = [
        "//runtime/components/preprocessor/testdata",
        "//runtime/components/testdata",
    ],
    deps = [
        ":gemma3_data_processor",
        ":gemma3_data_processor_config",
        "@com_google_googletest//:gtest_main",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
        "@nlohmann_json//:json",
        "@litert//litert/cc:litert_layout",
        "//runtime/components:prompt_template",
        "//runtime/components:sentencepiece_tokenizer",
        "//runtime/components:tokenizer",
        "//runtime/components/preprocessor:audio_preprocessor",
        "//runtime/components/preprocessor:audio_preprocessor_miniaudio",
        "//runtime/components/preprocessor:image_preprocessor",
        "//runtime/components/preprocessor:stb_image_preprocessor",
        "//runtime/conversation:io_types",
        "//runtime/engine:io_types",
        "//runtime/util:convert_tensor_buffer",
        "//runtime/util:test_utils",
    ],
)

cc_library(
    name = "function_gemma_data_processor",
    srcs = ["function_gemma_data_processor.cc"],
    hdrs = ["function_gemma_data_processor.h"],
    deps = [
        ":function_gemma_data_processor_config",
        ":model_data_processor",
        "@com_google_absl//absl/log:absl_log",
        "@com_google_absl//absl/memory",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/strings:string_view",
        "@nlohmann_json//:json",
        "//runtime/components:sentencepiece_tokenizer",
        "//runtime/components:tokenizer",
        "//runtime/components/constrained_decoding:constraint",
        "//runtime/components/constrained_decoding:gemma_model_constraint_provider_lib",
        "//runtime/components/tool_use:fc_tool_format_utils",
        "//runtime/components/tool_use:parser_utils",
        "//runtime/conversation:io_types",
        "//runtime/engine:io_types",
        "//runtime/util:litert_status_util",
        "@sentencepiece//:sentencepiece_model_cc_proto",
    ],
)

cc_test(
    name = "function_gemma_data_processor_test",
    srcs = ["function_gemma_data_processor_test.cc"],
    data = [
        "//runtime/components/testdata",
    ],
    shard_count = 32,
    deps = [
        ":function_gemma_data_processor",
        ":function_gemma_data_processor_config",
        "@com_google_googletest//:gtest_main",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
        "@nlohmann_json//:json",
        "//runtime/components:prompt_template",
        "//runtime/components:sentencepiece_tokenizer",
        "//runtime/components:tokenizer",
        "//runtime/conversation:io_types",
        "//runtime/engine:io_types",
        "//runtime/util:test_utils",
    ],
)

cc_library(
    name = "gemma4_data_processor_config",
    hdrs = ["gemma4_data_processor_config.h"],
)

cc_library(
    name = "gemma4_data_processor",
    srcs = ["gemma4_data_processor.cc"],
    hdrs = ["gemma4_data_processor.h"],
    deps = [
        ":data_utils",
        ":gemma4_data_processor_config",
        ":model_data_processor",
        "@com_google_absl//absl/log:absl_log",
        "@com_google_absl//absl/memory",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/strings:string_view",
        "@nlohmann_json//:json",
        "//runtime/components:sentencepiece_tokenizer",
        "//runtime/components:tokenizer",
        "//runtime/components/constrained_decoding:constraint",
        "//runtime/components/constrained_decoding:gemma_model_constraint_provider_lib",
        "//runtime/components/preprocessor:audio_preprocessor",
        "//runtime/components/preprocessor:audio_preprocessor_miniaudio",
        "//runtime/components/preprocessor:image_preprocessor",
        "//runtime/components/preprocessor:stb_image_preprocessor",
        "//runtime/components/tool_use:fc_tool_format_utils",
        "//runtime/components/tool_use:parser_utils",
        "//runtime/conversation:io_types",
        "//runtime/engine:io_types",
        "//runtime/util:litert_status_util",
        "//runtime/util:memory_mapped_file",
        "@com_googlesource_code_re2//:re2",
        "@sentencepiece//:sentencepiece_model_cc_proto",
    ],
)