{%- set image_placeholder = '<|vision_start|><|image_pad|><|vision_end|>' -%} {%- for message in messages -%} {%- if message['role'] == 'system' -%} {{- '<|im_start|>system ' -}} {%- if message['content'] is string -%} {{- message['content'] | trim -}} {%- endif -%} {{- '<|im_end|> ' -}} {%- elif message['role'] == 'user' -%} {%- if loop.first -%} {{- '<|im_start|>system ' -}} {%- if template -%} {#--- If template, extraction task ---#} {{- 'You are NuExtract, an information extraction tool created by NuMind.' -}} {%- else -%} {#--- Else, template generation task ---#} {{- 'You are a helpful assistant.' -}} {%- endif -%} {{ '<|im_end|> ' }} {%- endif -%} {{- '<|im_start|>' + message['role'] + ' ' -}} {%- if template -%} {#--- Template Section ---#} {{- '# Template: ' -}} {{- template -}} {{- ' ' -}} {%- if examples -%} {#--- Examples can only exist in the extraction task ---#} {{- '# Examples: ' -}} {%- for example in examples -%} {{- '## Input: ' -}} {%- if example['input'] is mapping and (example['input']['type'] == 'image' or example['input']['type'] == 'image_url') -%} {{- image_placeholder | trim -}} {%- elif example['input'] == '' -%} {#--- Keep compatibility with for now ---#} {{- image_placeholder | trim -}} {%- else -%} {#--- Text input example ---#} {{- example['input'] -}} {%- endif -%} {{- ' ' -}} {{- '## Output: ' -}} {{- example['output'] -}} {{- ' ' -}} {%- endfor -%} {%- endif -%} {{- '# Context: ' -}} {%- endif -%} {%- if message['content'] is string -%} {#--- Simple string content ---#} {{- message['content'] | trim -}} {%- elif message['content'] is mapping and (message['content']['type'] == 'image' or message['content']['type'] == 'image_url') -%} {{- image_placeholder | trim -}} {%- else -%} {#--- List of content items (mixed text/images) ---#} {#--- First, determine what the actual input content is (not ICL images) ---#} {%- set ns = namespace(has_text_input=false, text_content='') -%} {#--- Count content types and identify actual input document ---#} {%- for content in message['content'] -%} {%- if content is mapping and content.get('type') == 'text' -%} {%- if content.get('text') != '' -%} {#--- Keep compatibility with for now ---#} {%- set ns.has_text_input = true -%} {%- set ns.text_content = content['text'] -%} {%- endif -%} {%- elif content is string -%} {%- if content != '' -%} {#--- Keep compatibility with for now ---#} {%- set ns.has_text_input = true -%} {%- set ns.text_content = content -%} {%- endif -%} {%- endif -%} {%- endfor -%} {#--- Determine what to output based on actual input type ---#} {%- if ns.has_text_input -%} {#--- Main input is text, so output the text content ---#} {{- ns.text_content | trim -}} {%- else -%} {#--- Main input is image or placeholder ---#} {%- set ns2 = namespace(found_image=false) -%} {%- for content in message['content'] -%} {%- if content is mapping and (content.get('type') == 'image' or content.get('type') == 'image_url') and not ns2.found_image -%} {{- image_placeholder | trim -}} {%- set ns2.found_image = true -%} {%- elif content is mapping and content.get('type') == 'text' and content.get('text') == '' and not ns2.found_image -%} {#--- Keep compatibility with for now ---#} {{- image_placeholder | trim -}} {%- set ns2.found_image = true -%} {%- elif content is string and content == '' and not ns2.found_image -%} {#--- Keep compatibility with for now ---#} {{- image_placeholder | trim -}} {%- set ns2.found_image = true -%} {%- endif -%} {%- endfor -%} {%- endif -%} {%- endif -%} {{- '<|im_end|> '}} {%- elif message['role'] == 'assistant' -%} {{- '<|im_start|>assistant ' -}} {%- if message['content'] is string -%} {{- message['content'] | trim -}} {%- elif message['content'] is iterable and message['content'] is not string -%} {%- for content in message['content'] -%} {%- if content is mapping and content.get('type') == 'text' -%} {{- content['text'] | trim -}} {%- elif content is string -%} {{- content | trim -}} {%- endif -%} {%- endfor -%} {%- endif -%} {{- '<|im_end|> ' -}} {%- endif -%} {%- endfor -%} {%- if add_generation_prompt -%} {{- '<|im_start|>assistant ' -}} {%- endif -%}