Update project_model.py
Browse files- project_model.py +8 -3
project_model.py
CHANGED
|
@@ -198,8 +198,13 @@ def process_inputs(
|
|
| 198 |
|
| 199 |
vqa_prompt = "You are a helpful visual assistant designed for visually impaired users that assists users by answering their questions. Answer the following question with the help of the shared visual context: " + question + "Shared visual context: " + visual_context
|
| 200 |
|
| 201 |
-
# Gemma
|
| 202 |
-
messages = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
"role": "user",
|
| 204 |
"content": [
|
| 205 |
{"type": "image", "image": session.current_image},
|
|
@@ -212,7 +217,7 @@ def process_inputs(
|
|
| 212 |
|
| 213 |
# Handle the output from Gemma model safely
|
| 214 |
if isinstance(gemma_output, list) and len(gemma_output) > 0:
|
| 215 |
-
gemma_text = gemma_output[0]["generated_text"]
|
| 216 |
if isinstance(gemma_text, str):
|
| 217 |
answer = gemma_text
|
| 218 |
else:
|
|
|
|
| 198 |
|
| 199 |
vqa_prompt = "You are a helpful visual assistant designed for visually impaired users that assists users by answering their questions. Answer the following question with the help of the shared visual context: " + question + "Shared visual context: " + visual_context
|
| 200 |
|
| 201 |
+
# Gemma prompt input
|
| 202 |
+
messages = [
|
| 203 |
+
{
|
| 204 |
+
"role": "assistant",
|
| 205 |
+
"content": "You are a helpful visual assistant designed for visually impaired users."
|
| 206 |
+
},
|
| 207 |
+
{
|
| 208 |
"role": "user",
|
| 209 |
"content": [
|
| 210 |
{"type": "image", "image": session.current_image},
|
|
|
|
| 217 |
|
| 218 |
# Handle the output from Gemma model safely
|
| 219 |
if isinstance(gemma_output, list) and len(gemma_output) > 0:
|
| 220 |
+
gemma_text = gemma_output[0]["generated_text"][-1]["content"]
|
| 221 |
if isinstance(gemma_text, str):
|
| 222 |
answer = gemma_text
|
| 223 |
else:
|