runtime/proto/llm_metadata.proto

// Copyright 2025 The ODML Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


syntax = "proto3";

package litert.lm.proto;

import "runtime/proto/llm_model_type.proto";
import "runtime/proto/sampler_params.proto";
import "runtime/proto/token.proto";

message PromptAffixes {
  // Text prepended to the input prompt on the first chunck of the prefill call.
  // This is useful for adding start of user's turn markers.
  string prefix = 1;

  // Text appended to the input prompt upon transition to decode. This is useful
  // for adding start of model of model's turn markers.
  string suffix = 2;
}

// Definition of a channel for responses, e.g. thinking channel.
message Channel {
  // The channel name which will be used the key for the channel content.
  string channel_name = 1;

  // A string that marks the start of the channel, e.g. "<|channel>thought".
  string start = 2;

  // A string that marks the end of the channel, e.g. "<channel|>".
  string end = 3;
}

// Prompt template used by the session to format the input prompt.
message PromptTemplates {
  // The template for user role.
  optional PromptAffixes user = 1;

  // The template for model role.
  optional PromptAffixes model = 2;

  // The template for system role.
  optional PromptAffixes system = 3;
}

// Parameters for Large Language Models (LLM).
message LlmMetadata {
  // Start token prepended to the beginning of input sequence.
  // If in the future we support multiple start tokens, we can add a new field
  // that is a repeated TokenUnion.
  TokenUnion start_token = 1;

  // Stop tokens to determine the end of output stream.
  repeated TokenUnion stop_tokens = 2;

  // Prompt templates for different roles.
  // Note: Conversation APIs support both Jinja and non-Jinja prompt
  // templates with Jinja templates being preferred. However, the session
  // layer only supports non-Jinja prompt templates defined in this field.
  optional PromptTemplates prompt_templates = 3;

  // Default sampler parameters for the LLM model.
  optional SamplerParameters sampler_params = 4;

  // Maximum number of tokens that can be processed by the LLM.
  int32 max_num_tokens = 5;

  // LLM model type.
  LlmModelType llm_model_type = 6;

  // Default Jinja prompt template for the LLM model. If the field is not set,
  // LiteRT-LM will use the prompt_templates field to generate the
  // Jinja prompt template for backward compatibility. If the prompt_templates
  // field is also not set, the input will not be formatted with jinja template.
  optional string jinja_prompt_template = 7;

  // The channels defined for this model.
  repeated Channel channels = 8;
}