init

app.py

@@ -0,0 +1,63 @@
+# Imports
+import re
+import gradio as gr
+import torch
+from transformers import DonutProcessor, VisionEncoderDecoderModel
+from dotenv import load_dotenv
+import os
+from torchvision import transforms
+import torch
+from PIL import Image
+
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+model_name = 'naver-clova-ix/donut-base-finetuned-docvqa'
+
+# Importante esta app esta pensada para que el modelo corra en CPU
+processor = DonutProcessor.from_pretrained(model_name)
+model = VisionEncoderDecoderModel.from_pretrained(model_name)
+
+# Defino la funcion principal que ejecuta el modelo y obtiene los resultados
+def process_image(image, question):
+    # Paso por el procesador la imagen y especifico que los outputs sean tensores de pytorch
+    pixel_values = processor(image, return_tensors='pt').pixel_values
+    # Seteo el prompt
+    prompt = f"<s_docvqa><s_question>{question}</s_question><s_answer>"
+    # Generamos la sequencia de tokens de salida esto es un vector largo con los ids
+    # Esta parte encodea la pregunta y se la pasa al decoder junto con la representación de la imagen post encoder
+    decoder_input_ids = processor.tokenizer(prompt, add_special_tokens=False, return_tensors='pt').input_ids
+    # Defino los outputs
+    outputs = model.generate(
+        pixel_values.to(device).half(),
+        decoder_input_ids=decoder_input_ids.to(device),
+        max_length=model.decoder.config.max_position_embeddings,
+        early_stopping=True,
+        pad_token_id=processor.tokenizer.pad_token_id,
+        eos_token_id=processor.tokenizer.eos_token_id,
+        use_cache=True,
+        num_beams=1, # Probar cambiando este parametro
+        bad_words_ids=[[processor.tokenizer.unk_token_id]],
+        return_dict_in_generate=True
+    )
+    # Realizo el Post-processing de la salida del modelo
+    sequence = processor.batch_decode(outputs.sequences)[0]
+    sequence = sequence.replace(processor.tokenizer.eos_token, "").replace(processor.tokenizer.pad_token, "")
+    processor.tokenizer.eos_token
+    sequence = re.sub(r"<.*?>", "", sequence, count=1).strip()
+    
+    return processor.token2json(sequence)
+
+description = "Esta es una aplicacción realizada con el modelo Donut fine tuned en DocVQA"
+
+demo = gr.Interface(
+    fn=process_image,
+    inputs=['image', 'text'],
+    outputs='json',
+    title='Demo: Document Question Answering',
+    description=description,
+    enable_queue=True,
+    examples=[
+        ['examples/dni_25.jpg', 'cual es el documento / document number?'],
+        ['examples/extracto.jpg', 'cual es el telefono de centros servicios de banco galicia?'],
+        ['examples/factura_5.jpg', 'cual es el total de la factura?'],
+    ]
+)

requirements.txt

@@ -0,0 +1,152 @@
+accelerate @ git+https://github.com/huggingface/accelerate.git@109f3272f542cbd0e34022f5455078a4ab99d7eb
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.0.1
+anyio==3.7.0
+arxiv==1.4.7
+asttokens==2.2.1
+async-timeout==4.0.2
+attrs==23.1.0
+backcall==0.2.0
+backoff==2.2.1
+bitsandbytes @ https://github.com/acpopescu/bitsandbytes/releases/download/v0.38.0-win0/bitsandbytes-0.38.1-py3-none-any.whl
+certifi==2023.5.7
+charset-normalizer==3.1.0
+chromadb==0.3.25
+click==8.1.3
+clickhouse-connect==0.5.25
+colorama==0.4.6
+coloredlogs==15.0.1
+comm==0.1.3
+contourpy==1.0.7
+cycler==0.11.0
+dataclasses-json==0.5.7
+debugpy==1.6.7
+decorator==5.1.1
+diffusers==0.16.1
+duckdb==0.8.0
+einops==0.6.1
+exceptiongroup==1.1.1
+executing==1.2.0
+fastapi==0.96.0
+feedparser==6.0.10
+ffmpy==0.3.0
+filelock==3.12.0
+flatbuffers==23.5.26
+fonttools==4.39.4
+frozenlist==1.3.3
+fsspec==2023.5.0
+gradio==3.34.0
+gradio_client==0.2.6
+greenlet==2.0.2
+h11==0.14.0
+hnswlib==0.7.0
+httpcore==0.17.2
+httptools==0.5.0
+httpx==0.24.1
+huggingface-hub==0.15.1
+humanfriendly==10.0
+idna==3.4
+importlib-metadata==6.6.0
+ipykernel==6.23.1
+ipython==8.14.0
+ipywidgets==8.0.6
+jedi==0.18.2
+Jinja2==3.1.2
+joblib==1.2.0
+jsonschema==4.17.3
+jupyter_client==8.2.0
+jupyter_core==5.3.0
+jupyterlab-widgets==3.0.7
+kiwisolver==1.4.4
+langchain==0.0.189
+linkify-it-py==2.0.2
+lz4==4.3.2
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+matplotlib==3.7.1
+matplotlib-inline==0.1.6
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+nest-asyncio==1.5.6
+networkx==3.1
+nltk==3.8.1
+numexpr==2.8.4
+numpy==1.24.3
+onnxruntime==1.15.0
+openai==0.27.7
+openapi-schema-pydantic==1.2.4
+orjson==3.9.0
+overrides==7.3.1
+packaging==23.1
+pandas==2.0.2
+parso==0.8.3
+peft @ git+https://github.com/huggingface/peft.git@fcff23f005fc7bfb816ad1f55360442c170cd5f5
+pickleshare==0.7.5
+Pillow==9.3.0
+platformdirs==3.5.1
+posthog==3.0.1
+prompt-toolkit==3.0.38
+protobuf==4.23.2
+psutil==5.9.5
+pure-eval==0.2.2
+pydantic==1.10.8
+pydub==0.25.1
+Pygments==2.15.1
+pyodbc==4.0.39
+pyparsing==3.0.9
+pypdf==3.9.1
+pyreadline3==3.4.1
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-multipart==0.0.6
+pytz==2023.3
+PyYAML==6.0
+pyzmq==25.1.0
+regex==2023.5.5
+requests==2.31.0
+safetensors==0.3.1
+scikit-learn==1.2.2
+scipy==1.10.1
+semantic-version==2.10.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.99
+sgmllib3k==1.0.0
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==2.0.15
+stack-data==0.6.2
+starlette==0.27.0
+sympy==1.12
+tenacity==8.2.2
+text-generation==0.6.0
+threadpoolctl==3.1.0
+tiktoken==0.4.0
+tokenizers==0.13.3
+toolz==0.12.0
+torch
+tornado==6.3.2
+tqdm==4.65.0
+traitlets==5.9.0
+transformers @ git+https://github.com/huggingface/transformers.git@bacaab1629972b85664fe61ec3caa4da7b55b041
+typing-inspect==0.9.0
+typing_extensions==4.6.3
+tzdata==2023.3
+uc-micro-py==1.0.2
+urllib3==2.0.2
+uvicorn==0.22.0
+watchfiles==0.19.0
+wcwidth==0.2.6
+websockets==11.0.3
+widgetsnbextension==4.0.7
+yarl==1.9.2
+zipp==3.15.0
+zstandard==0.21.0