Spaces:
Sleeping
Sleeping
Niki Zhang
commited on
Update app.py
Browse files
app.py
CHANGED
|
@@ -187,12 +187,13 @@ def inference_click(image_input, point_prompt, click_mode, enable_wiki, language
|
|
| 187 |
|
| 188 |
click_state[0] = input_points
|
| 189 |
click_state[1] = input_labels
|
|
|
|
| 190 |
|
| 191 |
return state, click_state
|
| 192 |
|
| 193 |
def submit_caption(image_input, enable_wiki, language, sentiment, factuality, length, image_embedding, state, click_state, original_size, input_size, text_refiner, visual_chatgpt):
|
| 194 |
-
|
| 195 |
-
prompt = get_click_prompt(
|
| 196 |
input_points = prompt['input_point']
|
| 197 |
input_labels = prompt['input_label']
|
| 198 |
|
|
@@ -216,15 +217,16 @@ def submit_caption(image_input, enable_wiki, language, sentiment, factuality, le
|
|
| 216 |
enable_wiki = True if enable_wiki in ['True', 'TRUE', 'true', True, 'Yes', 'YES', 'yes'] else False
|
| 217 |
out = model.inference(image_input, prompt, controls, disable_gpt=True, enable_wiki=enable_wiki, verbose=True, args={'clip_filter': False})[0]
|
| 218 |
|
| 219 |
-
state = state + [("Image point: {}, Input label: {}".format(prompt["input_point"], prompt["input_label"]), None)]
|
| 220 |
state = state + [(None, "raw_caption: {}".format(out['generated_captions']['raw_caption']))]
|
| 221 |
-
|
|
|
|
| 222 |
text = out['generated_captions']['raw_caption']
|
| 223 |
input_mask = np.array(out['mask'].convert('P'))
|
| 224 |
image_input = mask_painter(np.array(image_input), input_mask)
|
| 225 |
origin_image_input = image_input
|
| 226 |
image_input = create_bubble_frame(image_input, text, (input_points[-1][0], input_points[-1][1]), input_mask,
|
| 227 |
input_points=input_points, input_labels=input_labels)
|
|
|
|
| 228 |
if visual_chatgpt is not None:
|
| 229 |
new_image_path = get_new_image_name('chat_image', func_name='upload')
|
| 230 |
image_input.save(new_image_path)
|
|
@@ -246,9 +248,10 @@ def submit_caption(image_input, enable_wiki, language, sentiment, factuality, le
|
|
| 246 |
input_mask,
|
| 247 |
input_points=input_points, input_labels=input_labels)
|
| 248 |
return state, state, click_state, refined_image_input
|
| 249 |
-
|
| 250 |
return state, state, click_state, image_input
|
| 251 |
|
|
|
|
| 252 |
def get_sketch_prompt(mask: Image.Image):
|
| 253 |
"""
|
| 254 |
Get the prompt for the sketcher.
|
|
@@ -556,7 +559,7 @@ def create_ui():
|
|
| 556 |
submit_button_click.click(submit_caption,
|
| 557 |
[origin_image, enable_wiki, language, sentiment, factuality, length, image_embedding, state, click_state, original_size, input_size, text_refiner, visual_chatgpt],
|
| 558 |
[chatbot, state, click_state, image_input])
|
| 559 |
-
|
| 560 |
image_input.clear(
|
| 561 |
lambda: (None, [], [], [[], [], []], "", "", ""),
|
| 562 |
[],
|
|
|
|
| 187 |
|
| 188 |
click_state[0] = input_points
|
| 189 |
click_state[1] = input_labels
|
| 190 |
+
state = state + [("Image point: {}, Input label: {}".format(click_state[0], click_state[1]), None)]
|
| 191 |
|
| 192 |
return state, click_state
|
| 193 |
|
| 194 |
def submit_caption(image_input, enable_wiki, language, sentiment, factuality, length, image_embedding, state, click_state, original_size, input_size, text_refiner, visual_chatgpt):
|
| 195 |
+
coordinates = json.dumps(list(zip(click_state[0], click_state[1])))
|
| 196 |
+
prompt = get_click_prompt(coordinates, click_state, 'Single')
|
| 197 |
input_points = prompt['input_point']
|
| 198 |
input_labels = prompt['input_label']
|
| 199 |
|
|
|
|
| 217 |
enable_wiki = True if enable_wiki in ['True', 'TRUE', 'true', True, 'Yes', 'YES', 'yes'] else False
|
| 218 |
out = model.inference(image_input, prompt, controls, disable_gpt=True, enable_wiki=enable_wiki, verbose=True, args={'clip_filter': False})[0]
|
| 219 |
|
|
|
|
| 220 |
state = state + [(None, "raw_caption: {}".format(out['generated_captions']['raw_caption']))]
|
| 221 |
+
|
| 222 |
+
update_click_state(click_state, out['generated_captions']['raw_caption'], 'Single')
|
| 223 |
text = out['generated_captions']['raw_caption']
|
| 224 |
input_mask = np.array(out['mask'].convert('P'))
|
| 225 |
image_input = mask_painter(np.array(image_input), input_mask)
|
| 226 |
origin_image_input = image_input
|
| 227 |
image_input = create_bubble_frame(image_input, text, (input_points[-1][0], input_points[-1][1]), input_mask,
|
| 228 |
input_points=input_points, input_labels=input_labels)
|
| 229 |
+
|
| 230 |
if visual_chatgpt is not None:
|
| 231 |
new_image_path = get_new_image_name('chat_image', func_name='upload')
|
| 232 |
image_input.save(new_image_path)
|
|
|
|
| 248 |
input_mask,
|
| 249 |
input_points=input_points, input_labels=input_labels)
|
| 250 |
return state, state, click_state, refined_image_input
|
| 251 |
+
|
| 252 |
return state, state, click_state, image_input
|
| 253 |
|
| 254 |
+
|
| 255 |
def get_sketch_prompt(mask: Image.Image):
|
| 256 |
"""
|
| 257 |
Get the prompt for the sketcher.
|
|
|
|
| 559 |
submit_button_click.click(submit_caption,
|
| 560 |
[origin_image, enable_wiki, language, sentiment, factuality, length, image_embedding, state, click_state, original_size, input_size, text_refiner, visual_chatgpt],
|
| 561 |
[chatbot, state, click_state, image_input])
|
| 562 |
+
|
| 563 |
image_input.clear(
|
| 564 |
lambda: (None, [], [], [[], [], []], "", "", ""),
|
| 565 |
[],
|