updated google tokenizer
Browse files
app.py
CHANGED
|
@@ -93,7 +93,7 @@ def main():
|
|
| 93 |
temperature=temperature,
|
| 94 |
max_tokens=MAX_OUTPUT_TOKENS
|
| 95 |
)
|
| 96 |
-
Settings.tokenizer = tokenization.get_tokenizer_for_model(llm_name)
|
| 97 |
Settings.num_output = MAX_OUTPUT_TOKENS
|
| 98 |
Settings.embed_model = GeminiEmbedding(
|
| 99 |
model_name="models/text-embedding-004", api_key=os.environ.get("GOOGLE_API_KEY") #, title="this is a document"
|
|
|
|
| 93 |
temperature=temperature,
|
| 94 |
max_tokens=MAX_OUTPUT_TOKENS
|
| 95 |
)
|
| 96 |
+
Settings.tokenizer = tokenization.get_tokenizer_for_model(llm_name).compute_tokens
|
| 97 |
Settings.num_output = MAX_OUTPUT_TOKENS
|
| 98 |
Settings.embed_model = GeminiEmbedding(
|
| 99 |
model_name="models/text-embedding-004", api_key=os.environ.get("GOOGLE_API_KEY") #, title="this is a document"
|