adding ocr

app.py

@@ -9,7 +9,7 @@ from recording_analysis import ui_recording_analysis
 from prompts import ui_prompt_builder
 from chat_bot import ui_simple_chat
 from slack_sentiment_analysis import ui_sentiment_analysis
-
+from image_to_text import image_to_text_translator as ui_image_to_text
 
 openai.api_key = api_keys.APIKeys().get_key('OPENAI_API_KEY')
 
@@ -19,9 +19,10 @@ image_generator=ui_image_generation.ui
 video_analysis=ui_recording_analysis.ui
 slack_sentiment=ui_sentiment_analysis.ui
 usaa_advisor=ui_simple_chat.ui
+image_to_text=ui_image_to_text.ui
 
-ui = gr.TabbedInterface([prompt_builder, summarize, image_generator, video_analysis,slack_sentiment, usaa_advisor], 
-                        (constants.UI_1, constants.UI_2, constants.UI_3,constants.UI_4, constants.UI_5,constants.UI_6), 
+ui = gr.TabbedInterface([prompt_builder, summarize, image_generator, video_analysis,slack_sentiment, image_to_text, usaa_advisor], 
+                        (constants.UI_1, constants.UI_2, constants.UI_3,constants.UI_4, constants.UI_5,constants.UI_6,constants.UI_7), 
                         theme=SoftBlue())
 
 ui.launch()    

chat_bot/ui_simple_chat.py

@@ -11,25 +11,25 @@ def InitDF():
     commDF=pd.DataFrame({"role": [""], "content": [""] })
     simple_chat.CreateEmbeddings(constants.PDF_INPUT_PATH, constants.TXT_OUTPUT_PATH)
 
-def respond(message, chat_history):
+def Respond(message, chat_history):
     bot_message,df=simple_chat.QueryEmbeddingsSimple(message)
     chat_history.append((message, bot_message))
     return "", chat_history,df
 
 with gr.Blocks() as ui:
-    label = gr.Label(show_label=False, value=constants.UI_6, container=False)
+    label = gr.Label(show_label=False, value=constants.UI_7, container=False)
     with gr.Row():
         with gr.Column():
             gr.Markdown(constants.PURPOSE_MD)
-            gr.Markdown(constants.USAA_AR_ADVISOR_PURPOSE)
+            gr.Markdown(constants.AR_ADVISOR_PURPOSE)
         with gr.Column():
             gr.Markdown(constants.DIRECTIONS_MD)
-            gr.Markdown(constants.USAA_AR_ADVISOR_DIRECTIONS)
+            gr.Markdown(constants.AR_ADVISOR_DIRECTIONS)
     question= gr.Textbox (label=constants.QUESTIONS_PREFIX, value=constants.QUESTIONS_AR_EXPERT)
     chatbot = gr.Chatbot(label=constants.CHAT_BOT, height=constants.CHAT_BOT_HEIGHT)
     msg = gr.Textbox(label=constants.CHAT_BOT_INPUT)    
     commDF = gr.DataFrame(type="pandas", value=pd.DataFrame({"role": [""], "content": [""] }), wrap=True, label=constants.OPENAI_LOG)
     clear = gr.ClearButton([msg, chatbot])
-    msg.submit(respond, [msg, chatbot], [msg, chatbot,commDF])
+    msg.submit(Respond, [msg, chatbot], [msg, chatbot,commDF])
 
 InitDF()

craft_mlt_25k.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a5efbfb48b4081100544e75e1e2b57f8de3d84f213004b14b85fd4b3748db17
+size 83152330

english_g2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2272681d9d67a04e2dff396b6e95077bc19001f8f6d3593c307b9852e1c29e8
+size 15143997

image_to_text/image_to_text.py

@@ -0,0 +1,132 @@
+import os, openai,string
+import pandas as pd
+from pydub import AudioSegment
+from utilities import constants, api_keys, clean_text, prompt_constants
+from textwrap import wrap
+from moviepy.editor import VideoFileClip
+from nltk.tokenize import word_tokenize
+
+openai.api_key = api_keys.APIKeys().get_key('OPENAI_API_KEY')
+key_words=set()
+
+def CompletionEngine(sys_message, user_message, num_tokens, num_results, temperature, topic_model, top_p):
+    return openai.ChatCompletion.create(
+        model=topic_model,
+        messages=[
+            {"role": "system", "content": sys_message},
+            {"role": "user", "content": user_message}, ],
+        max_tokens=num_tokens,
+        n=num_results,
+        temperature=temperature,
+        stop=None,
+        top_p=top_p
+    )
+
+#-------------  #1: strip audio from video, create text from audio using OpenAI whisper-1  ----------------
+def StripAndTranslateAudio(input_file):
+    if not os.path.exists(input_file):
+        return f"{input_file} {constants.FILE_DOES_NOT_EXIST}",[],pd.DataFrame({"role": [""], "content": [""] })   
+    video = VideoFileClip(input_file)
+    audio = video.audio    
+    if not input_file.endswith((".mp4", ".mov", ".avi", ".mkv")):
+        return constants.ANALYSIS_WRONG_FORMAT,[],pd.DataFrame({"role": [""], "content": [""] })        
+    file_name=clean_text.CleanFileName(input_file)
+    transcript=""
+    messages=[]
+    #see if already transcribed, if so, return the transcript
+    if os.path.exists(constants.TRANSCRIPT_PATH+file_name+".txt"):
+        with open(constants.TRANSCRIPT_PATH+file_name+".txt", "r") as f:
+            transcript = f.read()
+            messages.append({"role": "system", "content": f"Sending audio file {file_name} to OpenAI whisper-1"})    
+            messages.append({"role": "transcribe", "content": transcript})               
+    else:
+        audio_file = constants.ORIGINALS_PATH+"audio_" + file_name + '.wav'
+        print(audio_file)    
+        audio.write_audiofile(audio_file) 
+        audio_segment = AudioSegment.from_file(audio_file, format="wav")        
+        chunk_length = 60 * 1000  # 1 minute in milliseconds
+        chunks = [audio_segment[i:i + chunk_length] for i in range(0, len(audio_segment), chunk_length)]        
+        full_transcript = ""
+        for i, chunk in enumerate(chunks):
+            chunk_audio_file = f"{constants.ORIGINALS_PATH}audio_chunk_{i}_{file_name}.wav"
+            chunk.export(chunk_audio_file, format="wav")
+            messages.append({"role": "system", "content": f"Sending audio chunk {i} to OpenAI whisper-1"})            
+            with open(chunk_audio_file, 'rb') as f:
+                chunk_transcript = openai.Audio.transcribe("whisper-1", f)
+                full_transcript += chunk_transcript['text'] + " "
+                messages.append({"role": "transcribe", "content": chunk_transcript['text']})
+            os.remove(chunk_audio_file)          
+        with open(constants.TRANSCRIPT_PATH + file_name + ".txt", "w") as f:
+            f.write(full_transcript)        
+        os.remove(audio_file)        
+        transcript = full_transcript    
+        with open(constants.TRANSCRIPT_PATH + file_name + ".txt", "w") as f:
+            f.write(transcript)
+    return transcript, messages
+
+
+#-------------  2: chunk & process transcripts using OpenAI gpt-3.5-turbo  ----------------
+def SummarizeLargeTranscript(transcript, messages):    
+    while(NeedsChunks(transcript)):
+        text_chunks=CreateChunks(transcript)
+        summarized_text=SummarizeChunks(text_chunks,messages)
+        transcript=summarized_text    
+    return transcript, messages
+
+def NeedsChunks(transcript):
+    return len(transcript) > constants.CHUNK_LENGTH
+
+def CreateChunks(transcript):
+    # Calculate the total length of the string and segment length
+    total_length = len(transcript)
+    segment_length = constants.CHUNK_LENGTH
+    segment_indices = [i for i in range(segment_length - 1, total_length, segment_length)]    
+    text_chunks = []
+    start_idx = 0
+    for end_idx in segment_indices:
+        # Adjust end index backward to find a space or punctuation mark
+        while end_idx > start_idx and transcript[end_idx] not in string.whitespace + string.punctuation:
+            end_idx -= 1
+        if end_idx > start_idx:
+            text_chunks.append(transcript[start_idx:end_idx])
+            start_idx = end_idx + 1  # Skip the space or punctuation
+    if start_idx < total_length:
+        text_chunks.append(transcript[start_idx:])
+    return text_chunks
+
+def SummarizeChunks(text_chunks, messages):
+    summarized_text_list = []
+    for chunk in text_chunks:
+        chunk_summary = SummarizeChunk(chunk, messages)  
+        summarized_text_list.append(chunk_summary)
+    return " ".join(summarized_text_list)
+
+def SummarizeChunk(chunk, messages):
+    chunk = clean_text.CleanText(chunk)
+    completion = CompletionEngine(prompt_constants.ANALYSIS_SYSTEM_PROMPT, chunk, constants.SUMMARY_TOKENS, constants.NUM_RESULTS, constants.TEMP, constants.ANALYSIS_MODEL, constants.TOP_P)
+    summary = completion.choices[0]['message']['content']
+    messages.append({"role": "assistant", "content": summary})
+    return summary
+
+
+#-------------  #3: find topics using OpenAI gpt-3.5-turbo  ----------------
+def FindTopics(transcript, messages):
+    messages.append({"role": "system", "content": prompt_constants.KEYWORD_SYSTEM_PROMPT})    
+    topicCompletion=CompletionEngine(prompt_constants.KEYWORD_SYSTEM_PROMPT, transcript, constants.KEYWORD_TOKENS, constants.NUM_RESULTS, constants.TEMP, constants.ANALYSIS_MODEL, constants.TOP_P)
+    topics=topicCompletion.choices[0]['message']['content']
+    messages.append({"role": "assistant", "content": topics})  
+    return topics, messages
+
+def ProcessAudio(input_file):
+    #1: strip audio from video, create text from audio using OpenAI whisper-1
+    transcript,messages=StripAndTranslateAudio(input_file)
+    messages.append({"role": "system", "content": prompt_constants.ANALYSIS_SYSTEM_PROMPT})
+
+    #2: process transcript using OpenAI gpt-3.5-turbo
+    transcript,messages= SummarizeLargeTranscript(transcript,messages)
+
+    #3: find topics using OpenAI gpt-3.5-turbo
+    topics,messages = FindTopics(transcript,messages)
+
+    df = pd.DataFrame(messages) 
+    return transcript, topics, df

image_to_text/image_to_text_translator.py

@@ -0,0 +1,65 @@
+import easyocr as ocr  #OCR
+from PIL import Image #Image Processing
+import numpy as np #Image Processing 
+import pandas as pd
+import gradio as gr
+from utilities import constants
+
+reader = None
+
+def LoadModel(): 
+    global reader
+    if reader is None:
+        reader = ocr.Reader(['en'],model_storage_directory='.')
+    return reader 
+
+def ProcessImage(image):
+    global reader
+    reader = LoadModel()
+
+    # Debugging lines to find out what 'image' is
+    print(f"Type of image: {type(image)}")
+    if isinstance(image, np.ndarray):
+        print(f"Shape of image array: {image.shape}")
+        print(f"Data type of image array: {image.dtype}")
+
+    if image is not None:
+        if isinstance(image, np.ndarray):
+            input_image = Image.fromarray(image)
+        else:
+            input_image = Image.open(image)
+        result = reader.readtext(np.array(input_image))
+        result_text = []
+        for text in result:
+            result_text.append(text[1])
+    return result_text
+
+def ImageIdentity(image):
+    transcript = ProcessImage(image)
+    return transcript
+
+def Clear():
+    global imageBlock, summaryBlock, analysisDF
+    imageBlock=None
+    return "","",pd.DataFrame({"role": [""], "content": [""] })
+
+with gr.Blocks() as ui:
+    label = gr.Label(show_label=False, value=constants.UI_6, container=False)
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown(constants.PURPOSE_MD)
+            gr.Markdown(constants.AR_ADVISOR_PURPOSE)
+        with gr.Column():
+            gr.Markdown(constants.DIRECTIONS_MD)
+            gr.Markdown(constants.AR_ADVISOR_DIRECTIONS)
+    with gr.Row():
+        imageBlock = gr.Image(label=constants.IMAGE_INPUT, source="upload")
+        summaryBlock= gr.Textbox (label=constants.SUMMARY)
+    with gr.Row():
+        submitBtn=gr.Button(value=constants.EXTRACT_SUMMARY, variant="primary")
+        clearBtn=gr.Button(value=constants.CLEAR)
+    with gr.Row():
+        analysisDF=gr.DataFrame(type="pandas", value=pd.DataFrame({"role": [""], "content": [""] }),
+                               wrap=True, show_label=False, label=constants.OPENAI_LOG)
+    submitBtn.click(ImageIdentity, inputs=[imageBlock], outputs=[summaryBlock])
+    clearBtn.click(Clear, inputs=[], outputs=[summaryBlock,analysisDF])

image_to_text/ui_image_to_text.py

@@ -0,0 +1,53 @@
+import gradio as gr
+import pandas as pd
+import re
+from utilities import constants
+from recording_analysis import recording_analysis
+
+def InitDF():
+    global analysisDF
+    analysisDF=pd.DataFrame({"role": [""], "content": [""] })
+
+def VideoIdentity(video):
+    transcript,topics,df = recording_analysis.ProcessAudio(video)
+    html=RenderBoxes(topics)
+    return transcript, html, df
+
+def RenderBoxes(strings_list):
+    clean_list = [re.sub(r'^\d+\.\s*', '', topic) if re.match(r'^\d+\.\s*', topic) else topic for topic in strings_list.split('\n') if topic.strip()]
+    boxes_html = ""
+    for string in clean_list:
+        boxes_html += f'<div style="background-color: #107276; color: #ffffff; display: inline-block; font-size: 13pt; margin: 5px; padding: 10px;">{string}</div>'
+    return boxes_html
+
+def ClearScreen():
+    global videoBlock, summaryBlock, topics, analysisDF
+    videoBlock=None
+    return "","",pd.DataFrame({"role": [""], "content": [""] })
+
+with gr.Blocks() as ui:
+    label2 = gr.Label(show_label=False, value=constants.RECORDING_ANALYSIS, container=False)
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown(constants.PURPOSE_MD)
+            gr.Markdown(constants.RECORDING_ANALYSIS_PURPOSE)
+        with gr.Column():
+            gr.Markdown(constants.DIRECTIONS_MD)
+            gr.Markdown(value=constants.RECORDING_ANALYSIS_DIRECTIONS)
+
+    with gr.Row():
+        videoBlock = gr.Video(label=constants.VIDEO_INPUT, source="upload")
+        summaryBlock= gr.Textbox (label=constants.SUMMARY)
+    with gr.Column():
+        gr.Markdown(constants.TOPICS_MD)
+        topics= gr.HTML ()    
+    with gr.Row():
+        submitBtn=gr.Button(value=constants.EXTRACT_SUMMARY, variant="primary")
+        clearBtn=gr.Button(value=constants.CLEAR)
+    with gr.Row():
+        analysisDF=gr.DataFrame(type="pandas", value=pd.DataFrame({"role": [""], "content": [""] }),
+                               wrap=True, show_label=False, label=constants.OPENAI_LOG)
+    submitBtn.click(VideoIdentity, inputs=[videoBlock], outputs=[summaryBlock,topics,analysisDF])
+    clearBtn.click(ClearScreen, inputs=[], outputs=[topics,summaryBlock,analysisDF])
+
+InitDF()

recording_analysis/recording_analysis.py

@@ -22,14 +22,7 @@ def CompletionEngine(sys_message, user_message, num_tokens, num_results, tempera
         top_p=top_p
     )
 
-def SummarizeChunk(chunk, messages):
-    chunk = clean_text.CleanText(chunk)
-    completion = CompletionEngine(prompt_constants.ANALYSIS_SYSTEM_PROMPT, chunk, constants.SUMMARY_TOKENS, constants.NUM_RESULTS, constants.TEMP, constants.ANALYSIS_MODEL, constants.TOP_P)
-    summary = completion.choices[0]['message']['content']
-    messages.append({"role": "assistant", "content": summary})
-    return summary
-
-#1: strip audio from video, create text from audio using OpenAI whisper-1
+#-------------  #1: strip audio from video, create text from audio using OpenAI whisper-1  ----------------
 def StripAndTranslateAudio(input_file):
     if not os.path.exists(input_file):
         return f"{input_file} {constants.FILE_DOES_NOT_EXIST}",[],pd.DataFrame({"role": [""], "content": [""] })   
@@ -71,14 +64,13 @@ def StripAndTranslateAudio(input_file):
             f.write(transcript)
     return transcript, messages
 
-#2: chunk & process transcripts using OpenAI gpt-3.5-turbo
+
+#-------------  2: chunk & process transcripts using OpenAI gpt-3.5-turbo  ----------------
 def SummarizeLargeTranscript(transcript, messages):    
     while(NeedsChunks(transcript)):
         text_chunks=CreateChunks(transcript)
         summarized_text=SummarizeChunks(text_chunks,messages)
-        transcript=summarized_text
-    
-    print("transcript: "+transcript)
+        transcript=summarized_text    
     return transcript, messages
 
 def NeedsChunks(transcript):
@@ -109,7 +101,15 @@ def SummarizeChunks(text_chunks, messages):
         summarized_text_list.append(chunk_summary)
     return " ".join(summarized_text_list)
 
-#3: find topics using OpenAI gpt-3.5-turbo
+def SummarizeChunk(chunk, messages):
+    chunk = clean_text.CleanText(chunk)
+    completion = CompletionEngine(prompt_constants.ANALYSIS_SYSTEM_PROMPT, chunk, constants.SUMMARY_TOKENS, constants.NUM_RESULTS, constants.TEMP, constants.ANALYSIS_MODEL, constants.TOP_P)
+    summary = completion.choices[0]['message']['content']
+    messages.append({"role": "assistant", "content": summary})
+    return summary
+
+
+#-------------  #3: find topics using OpenAI gpt-3.5-turbo  ----------------
 def FindTopics(transcript, messages):
     messages.append({"role": "system", "content": prompt_constants.KEYWORD_SYSTEM_PROMPT})    
     topicCompletion=CompletionEngine(prompt_constants.KEYWORD_SYSTEM_PROMPT, transcript, constants.KEYWORD_TOKENS, constants.NUM_RESULTS, constants.TEMP, constants.ANALYSIS_MODEL, constants.TOP_P)

requirements.txt

@@ -22,4 +22,5 @@ inflect==7.0.0
 diffusers==0.10.2
 moviepy==1.0.3
 pdfplumber==0.10.2
-pytz==2023.3
+pytz==2023.3
+easyocr==1.7.1

utilities/constants.py

@@ -13,10 +13,10 @@ RECORDING_ANALYSIS_DIRECTIONS="To get started, upload a video (.mp4) that you'd
 SLACK_SENTIMENT_ANALYSIS_PURPOSE="Generative AI can help you understand the sentiment of your Slack messages. The 'Analyze Sentiment' action parses Slack content and uses OpenAI Chat Completion to score the sentiment of each message. The 'Sentiment' scatterplot shows the distribution of sentiment over the dataset."
 SLACK_SENTIMENT_ANALYSIS_DIRECTIONS="To get started select one of the slack data options in 'Slack Examples' below, you'll see the Slack content in the 'Original' textbox. Next tap 'Analyze Sentiment'. ChatCompletion calls will provide sentiment scores for each message. The results will be displayed in the 'Dataframe' section. Finally, the distribution of sentiment over the dataset is shown in the 'Sentiment' scatterplot. Prompt history will display in the 'OpenAI Communication Log' box, below."
 
-USAA_AR_ADVISOR_PURPOSE = "This component demonstrates how a chatbot can leverage local embeddings to provide context. In this example, embeddings created from USAA's 2022 Member Report provide organizational context for all interactions. This shows how generative AI enables any dataset to be queried and explored via natural language."
-USAA_AR_ADVISOR_DIRECTIONS = "To get started, simply start asking questions. This bot is a USAA AR Advisor and can answer questions about any content from the 2022 Member Report. You can ask questions like: 'What's the best thing that happened last year?' or 'What was the biggest obstancle faced in 2022?'"
+AR_ADVISOR_PURPOSE = "This component demonstrates how a chatbot can leverage local embeddings to provide context. In this example, embeddings were created from USAA's 2022 Member Report provide organizational context for all interactions. This shows how generative AI enables any dataset to be queried and explored via natural language."
+AR_ADVISOR_DIRECTIONS = "To get started, simply start asking questions. This bot has been trainged on USAA's 2022 Member Report and can answer questions about any content from the report. You can ask questions like: 'What's the best thing that happened last year?' or 'What was the biggest obstancle faced in 2022?'"
 
-CHAT_BOT="USAA Advisor Conversation Log"
+CHAT_BOT="Conversation Log"
 CHAT_BOT_INPUT="Ask a Question"
 CHAT_BOT_HEIGHT=800
 
@@ -106,7 +106,8 @@ UI_2="Summary Extraction"
 UI_3="Image Creation"
 UI_4="Recording Analysis"
 UI_5="Slack Sentiment Analysis"
-UI_6="USAA Advisor Chatbot"
+UI_6="Image to Text"
+UI_7="PDF Query Chatbot"
 
 HR_BENEFIT_PKL = 'utilities/data/embeddings/hr_asst_embeddings.pkl'
 HR_BENEFIT_CSV = 'utilities/data/embeddings/hr_asst_data.csv'
@@ -154,6 +155,7 @@ GENERATE_IMAGES="Generate Images"
 GENERATED_IMAGES="Generated Images"
 
 TOPICS_MD="### Topics"
+IMAGE_INPUT="Upload an Image(.png, .jpg, .jpeg)"
 VIDEO_INPUT="Upload a Video(.mp4)"
 EXTRACT_SUMMARY="Extract Summary"
 RECORDING_ANALYSIS="Recording Analysis"