mahynski commited on
Commit
4cb9000
·
verified ·
1 Parent(s): 4e9119f

updated google tokenizer

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -93,7 +93,7 @@ def main():
93
  temperature=temperature,
94
  max_tokens=MAX_OUTPUT_TOKENS
95
  )
96
- Settings.tokenizer = tokenization.get_tokenizer_for_model(llm_name) #.count_tokens
97
  Settings.num_output = MAX_OUTPUT_TOKENS
98
  Settings.embed_model = GeminiEmbedding(
99
  model_name="models/text-embedding-004", api_key=os.environ.get("GOOGLE_API_KEY") #, title="this is a document"
 
93
  temperature=temperature,
94
  max_tokens=MAX_OUTPUT_TOKENS
95
  )
96
+ Settings.tokenizer = tokenization.get_tokenizer_for_model(llm_name).compute_tokens
97
  Settings.num_output = MAX_OUTPUT_TOKENS
98
  Settings.embed_model = GeminiEmbedding(
99
  model_name="models/text-embedding-004", api_key=os.environ.get("GOOGLE_API_KEY") #, title="this is a document"