build version:1.9

app.py

@@ -2,40 +2,42 @@ import streamlit as st
 import altair as alt
 from transformers import pipeline
 from transformers import AutoModelForSeq2SeqLM , AutoTokenizer, TranslationPipeline
+from PIL import Image
 
 
+st.title("Image-Based Question Answering 🕵️‍♂️")
 
-st.title('Question Answering 🕵️‍♂️')
-st.write('Ask a question about NLP, and I will answer based on the provided context! 🔄')
-context_input = st.text_area("please provice some context", "Many NLP tasks are now benchmarked using datasets like GLUE and SuperGLUE.  Multilingual NLP models like mBERT support multiple languages in a single framework.")
+st.subheader("Ask questions directly from images!")
+
+st.write("""
+Upload an image (e.g., receipts, documents), type your question, and get precise answers in real-time. 
+Powered by the advanced `naver-clova-ix/donut-base-finetuned-docvqa` model.
+""")
+
+# context_input = st.text_area("please provice some context", "Many NLP tasks are now benchmarked using datasets like GLUE and SuperGLUE.  Multilingual NLP models like mBERT support multiple languages in a single framework.")
 
 # question_input = st.text_area("enter question about NLP", "what model support multilingual nlp?")
 @st.cache_resource
 def load_model():
     print("Loading model...")
-    return pipeline("question-answering", model="deepset/roberta-base-squad2")
+    return pipeline("document-question-answering", model="naver-clova-ix/donut-base-finetuned-docvqa")
 
 
 dunno_answerer  = load_model()
 # with open('NLP_History_and_Facts.txt', 'r') as file:
 #     context = file.read()
+uploaded_image = st.file_uploader("Upload an image", type=["png", "jpg", "jpeg"])
+
+if uploaded_image is not None:
+    image = Image.open(uploaded_image)
+    st.image(image, caption="Uploaded Image", use_column_width=True)
 
+    question_input = st.text_area("Enter your question", "Any questions ?")
 
-if context_input.strip():
-    question_input = st.text_area("enter question about NLP", "what model support multilingual nlp?")
     if st.button("Answer!"):
         if question_input.strip():
-            # Generate an answer using the model
-            result = dunno_answerer(question=question_input, context=context_input)
-
-            # Display the answer and additional information
-            st.write(f"**Answer:** {result['answer']}")
-            st.write(f"**Confidence Score:** {round(result['score'], 4)}")
-            st.write(f"**Answer Start Position:** {result['start']}")
-            st.write(f"**Answer End Position:** {result['end']}")
+            result = dunno_answerer(image=image, question=question_input)
+
+            st.write(f"**Answer:** {result[0]['answer']}")
         else:
             st.write("Please enter a valid question!")
-
-# x = st.slider('Select a value')
-# st.write(x, 'squared is', x * x)
-# print(x)

requirements.txt

@@ -1,3 +1,4 @@
 streamlit==1.41.1
 transformers
-torch
+torch
+sentencepiece