ValueError: Model architectures ['Idefics3ForConditionalGeneration'] failed to be inspected. Please check the logs for more details.
How can I fix this issue?
How can I replicate this error any tips? Are you sure you are on the latest transformers version?
Below is the code to reproduce it. I am actually on the 4.50.0 dev version
import time
import os
from vllm import LLM, SamplingParams
from PIL import Image
from docling_core.types.doc import DoclingDocument
from docling_core.types.doc.document import DocTagsDocument
MODEL_PATH = "ds4sd/SmolDocling-256M-preview"
IMAGE_DIR = "img/" # Place your page images here
OUTPUT_DIR = "out_docling/"
PROMPT_TEXT = "Convert page to Docling."
os.makedirs(OUTPUT_DIR, exist_ok=True)
llm = LLM(model=MODEL_PATH, limit_mm_per_prompt = {'image': 1})
sampling_params = SamplingParams(
temperature = 0.0,
max_tokens = 8192
)
chat_template = f"<|im_start|>User:{PROMPT_TEXT}Assistant:"
image_files= ["example_data/Images/0194b1c3-a881-7062-a861-e1bb66bb75d6_4.png"]
start_time = time.time()
total_tokens = 0
images = [Image.open(image_file) for image_file in image_files]
llm_input = [{'prompt': chat_template, 'multi_modal_data': {'image': image}} for image in images]
outputs = llm.generate([llm_input], sampling_params=sampling_params)
print(f"Total time: {time.time() - start_time:.2f} sec")
for output, img_file in zip(outputs,image_files):
doctags = output.outputs[0].text
image = Image.open(img_file)
img_fn = os.path.splittext(img_file)[0]
output_filename =img_fn + ".dt"
output_path = os.path.join(OUTPUT_DIR, output_filename)
with open(output_path, "w", encoding="utf-8") as f:
f.write(doctags)
doctags_doc = DocTagsDocument.from_doctags_and_image_pairs([doctags], [image])
doc = DoclingDocument(name='Document')
doc.load_from_doctags(doctags_doc)
output_filename_md = img_fn + '.md'
output_path_md = os.path.join(OUTPUT_DIR, output_filename_md)
doc.save_as_markdown(output_path_md)
output_filename_md = img_fn + '.xml'
output_path_md = os.path.join(OUTPUT_DIR, output_filename_md)
doc.save_as_xml(output_path_md)