added NLP summary component

app.py

@@ -10,6 +10,7 @@ from prompts import ui_prompt_builder
 from chat_bot import ui_simple_chat
 from slack_sentiment_analysis import ui_sentiment_analysis
 from image_to_text import image_to_text_translator as ui_image_to_text
+from text_analysis import ui_text_analysis
 
 openai.api_key = api_keys.APIKeys().get_key('OPENAI_API_KEY')
 
@@ -20,9 +21,10 @@ video_analysis=ui_recording_analysis.ui
 slack_sentiment=ui_sentiment_analysis.ui
 usaa_advisor=ui_simple_chat.ui
 image_to_text=ui_image_to_text.ui
+text_analysis=ui_text_analysis.ui
 
-ui = gr.TabbedInterface([prompt_builder, summarize, image_generator, video_analysis,slack_sentiment, usaa_advisor], 
-                        (constants.UI_1, constants.UI_2, constants.UI_3,constants.UI_4, constants.UI_5,constants.UI_7), 
+ui = gr.TabbedInterface([prompt_builder, text_analysis, image_generator, video_analysis,slack_sentiment, usaa_advisor, summarize], 
+                        (constants.UI_1, constants.UI_2, constants.UI_3,constants.UI_4, constants.UI_5,constants.UI_7, constants.UI_8), 
                         theme=SoftBlue())
 
-ui.launch()    
+ui.launch()

craft_mlt_25k.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4a5efbfb48b4081100544e75e1e2b57f8de3d84f213004b14b85fd4b3748db17
-size 83152330

english_g2.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e2272681d9d67a04e2dff396b6e95077bc19001f8f6d3593c307b9852e1c29e8
-size 15143997

requirements.txt

@@ -24,4 +24,5 @@ moviepy==1.0.3
 pdfplumber==0.10.2
 pytz==2023.3
 easyocr==1.7.1
-ffmpeg==1.4
+ffmpeg==1.4
+spacy==3.7.2

setup.sh

@@ -0,0 +1 @@
+python -m spacy download en_core_web_sm

text_analysis/text_analysis.py

@@ -0,0 +1,97 @@
+import openai, json, re, random
+import pandas as pd
+from utilities import date_format, prompt_constants
+
+def Completion(slack_message):
+    response = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo", 
+        messages=[
+            {"role": "system", "content": prompt_constants.SLACK_SENTIMENT_SYSTEM_PROMPT},
+            {"role": "user", "content": slack_message} ])
+    print("response")
+    print(response["choices"][0]["message"]["content"])
+    return response["choices"][0]["message"]["content"]
+
+def sanitize_blob(blob_str):
+    return re.sub(r"(?<=: )'", '"', re.sub(r"'(?=:)", '"', blob_str))
+
+def FindScore(response):
+    match = re.search(r"\b(0(\.\d+)?|1(\.0+)?)\b", response)
+    random_offset = round(random.uniform(0.01, .099), 3)
+    if match:
+        value = round(float(match.group(1)), 2)
+        return value + random_offset
+    else:
+        return 0
+    
+def CheckType(response):
+    if isinstance(response, float):
+        return round(response, 2)
+    elif isinstance(response, str):
+        return FindScore(response)
+    
+def ProcessMessage(message, summary_messages, slack_messages, id, parent_user=None ):
+    user = message["user"]
+    message_text = message["text"]
+    timestamp = message["timestamp"]
+    response = Completion(message_text)  # Assuming Completion is defined elsewhere
+    summary_messages.append({"role": "user", "content": message_text})
+    summary_messages.append({"role": "assistant", "content": response})
+
+    sentiment_score = CheckType(response)  # Assuming CheckType is defined elsewhere
+
+    sentiment = "Neutral"
+    if sentiment_score == 0:
+        sentiment = "Undefined"
+    elif 0 < sentiment_score < 0.3:
+        sentiment = "Negative"
+    elif sentiment_score > 0.6:
+        sentiment = "Positive"
+    
+    dateX, timeX, twentyfour_time = date_format.TimeStampToDateAndTime(timestamp)
+
+    message_obj = {
+        "id": id,
+        "user": user,
+        "message": f"{message_text}",
+        "date": dateX +": " +twentyfour_time,        
+        "time": timestamp,
+        "twentyfour_time": twentyfour_time,
+        "sentiment_score": sentiment_score,
+        "sentiment": sentiment,
+        "size": 8,
+        "parent_user": parent_user
+    }
+    id=id+1
+    slack_messages.append(message_obj)
+    # Process nested replies if any
+    if "replies" in message:
+        for reply in message["replies"]:
+            ProcessMessage(reply, summary_messages, slack_messages, id, parent_user=user)
+            id=id+1
+
+def ParseBlobs(blob, summary_messages):
+    global id
+    sanitized_blob = sanitize_blob(blob)
+    try:
+        response_data = json.loads(sanitized_blob)
+    except json.JSONDecodeError:
+        print("Invalid JSON format.")
+        return None
+    
+    slack_messages = []
+    summary_messages.append({"role": "system", "content": prompt_constants.SLACK_SENTIMENT_SYSTEM_PROMPT})
+    for message in response_data["messages"]:
+        ProcessMessage(message,summary_messages,slack_messages, id)
+        id=id+1
+    jsonobj = json.dumps(slack_messages, ensure_ascii=False)
+    return jsonobj,summary_messages
+
+def AnalyzeSentiment(blob):    
+    global id
+    summary_messages = []
+    id=3
+    slack_blobs,summary_messages=ParseBlobs(blob,summary_messages)   
+    df = pd.DataFrame(summary_messages)
+    sentimentDF=pd.read_json(slack_blobs)
+    return df, sentimentDF, id+3

text_analysis/ui_text_analysis.py

@@ -0,0 +1,65 @@
+
+import gradio as gr
+import os
+import base64
+import spacy
+from utilities import constants
+from spacy import displacy
+os.system('python -m spacy download en_core_web_sm')
+nlp = spacy.load("en_core_web_sm")
+
+input_examples=[
+    "Every day may not be good, but there's something good in every day.",
+    "The best preparation for tomorrow is doing your best today.",
+    "Believe you can, and you're halfway there."]
+
+def AnalyzeText(text):
+    doc = nlp(text)
+    svg = displacy.render(doc, style='dep')
+    svg_base64_encoded = base64.b64encode(svg.encode('utf-8')).decode('utf-8')
+    nlp_html = f"""
+    <div style='display: flex; justify-content: center; align-items: center; width: 100%; overflow-x: auto;'>
+        <img src="data:image/svg+xml;base64,{svg_base64_encoded}" 
+            style='display: block; max-width: 100%; height: auto; margin: auto;'/>
+    </div>
+    """
+    pos_count = {
+        "char_count": len(text),
+        "token_count": len(doc)
+    }
+    pos_tokens = []
+    for token in doc:
+        pos_tokens.extend([(token.text, token.pos_), (" ", None)])
+    return pos_tokens, pos_count, nlp_html
+
+def Clear(clearBtn):
+    return(constants.NLP_PROMPT, [], {}, [])
+
+with gr.Blocks() as ui:
+    label = gr.Label(show_label=False, value=constants.TEXT_ANALYSIS, container=False)
+    with gr.Column():
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown(constants.PURPOSE_MD)
+                gr.Markdown(constants.NLP_ANALYSIS_PURPOSE)
+            with gr.Column():
+                gr.Markdown(constants.DIRECTIONS_MD)
+                gr.Markdown(value=constants.NLP_ANALYSIS_DIRECTIONS)
+        with gr.Row():
+            with gr.Column():
+                inputString=gr.Textbox(placeholder=constants.NLP_PROMPT, 
+                                    label="Input Text", lines=3, height="500", 
+                                    interactive=True)
+                with gr.Row():
+                    clearBtn=gr.Button(constants.CLEAR, variant="secondary")
+                    submitBtn=gr.Button(constants.SUBMIT, variant="primary")
+            with gr.Column():
+                posTags=gr.HighlightedText(label=constants.TOKENS)
+                gr.Markdown(constants.NLP_ANALYSIS_MD)
+                posCount=gr.JSON()
+    
+    inputExampleSelect = gr.Examples(input_examples,inputs=[inputString],label="Or select an example." )
+    gr.Markdown(constants.NLP_POS_MAP_MD)      
+    posTokens=gr.HTML()
+    submitBtn.click(AnalyzeText, inputs=[inputString], outputs=[posTags,posCount,posTokens])
+    clearBtn.click(Clear, inputs=[clearBtn], outputs=[inputString,posTags,posCount,posTokens])

utilities/constants.py

@@ -1,3 +1,8 @@
+
+SUMMARY_MD_HEAD="## Summary"
+SUMMARY_MD_SUBHEAD="##### Understand legacy software, complex or technical documents with summarization."
+SUMMARY_MD_DIRECTIONS="Enter text or select one of the Code & Document Examples, below. Then tap Summarize."
+
 PROMPT_DESCRIPTION="Explore how generative AI can be used to create content. Personas let you explore the variety of roles the system can take on. Translation options specify the language for your response. Voice options are an assortment of AI voices that will bring messages to life. Check the Sentiment box to see how the system identifies sentiment from conversation. Check the emotion to see how the sytem detects your emotion."
 PROMPT_DIRECTIONS="To get started, select a Persona, Language, and Voice. If you'd like Sentiment or Emotion analysis, check the respective box. Tap the 'Record from microphone' button and say what's on your mind, tap 'Stop recording' when your done. If you're happy with what you've said, tap 'Submit' to send your request, otherwise tap 'Clear' to record a new message."
 
@@ -13,6 +18,9 @@ RECORDING_ANALYSIS_DIRECTIONS="To get started, upload a video (.mp4) that you'd
 SLACK_SENTIMENT_ANALYSIS_PURPOSE="Generative AI can help you understand the sentiment of your Slack messages. The 'Analyze Sentiment' action parses Slack content and uses OpenAI Chat Completion to score the sentiment of each message. The 'Sentiment' scatterplot shows the distribution of sentiment over the dataset."
 SLACK_SENTIMENT_ANALYSIS_DIRECTIONS="To get started select one of the slack data options in 'Slack Examples' below, you'll see the Slack content in the 'Original' textbox. Next tap 'Analyze Sentiment'. ChatCompletion calls will provide sentiment scores for each message. The results will be displayed in the 'Dataframe' section. Finally, the distribution of sentiment over the dataset is shown in the 'Sentiment' scatterplot. Prompt history will display in the 'OpenAI Communication Log' box, below."
 
+NLP_ANALYSIS_PURPOSE="Natural Language Processing (NLP) enables us to understand the structure and meaning in text. This view provides some understanding of NLP under the hood of Generative AI."
+NLP_ANALYSIS_DIRECTIONS="Start by entering a sentence or paragraph in the 'Text' box. Then tap 'Analyze'. You'll see token counts, part-of-speech tags, and a visual representation of the analysis."
+NLP_PROMPT="Enter text to analyze..."
 AR_ADVISOR_PURPOSE = "This component demonstrates how a chatbot can leverage local embeddings to provide context. In this example, embeddings were created from USAA's 2022 Member Report provide organizational context for all interactions. This shows how generative AI enables any dataset to be queried and explored via natural language."
 AR_ADVISOR_DIRECTIONS = "To get started, simply start asking questions. This bot has been trainged on USAA's 2022 Member Report and can answer questions about any content from the report. You can ask questions like: 'What's the best thing that happened last year?' or 'What was the biggest obstancle faced in 2022?'"
 
@@ -102,12 +110,13 @@ QUESTIONS_AR_EXPERT="How was 2022? What was one of the biggest challenges? What
 
 #tab navigator labels
 UI_1="Prompt Builder"
-UI_2="Summary Extraction"
+UI_8="Summary Extraction"
 UI_3="Image Creation"
 UI_4="Recording Analysis"
 UI_5="Slack Sentiment Analysis"
 UI_6="Image to Text"
 UI_7="PDF Query Chatbot"
+UI_2="NLP Text Analysis"
 
 HR_BENEFIT_PKL = 'utilities/data/embeddings/hr_asst_embeddings.pkl'
 HR_BENEFIT_CSV = 'utilities/data/embeddings/hr_asst_data.csv'
@@ -135,15 +144,14 @@ SUMMARIZATION="Summarization"
 SUMMARIZE="Summarize"
 SUMMARIZATION_TRUNCATION="***Code was truncated. Only first 14000 characters are included in the summary.***\n\n"
 CHAR_COUNT="\n\nNum Characters: "
-
+TOKENS="Tokens"
+NLP_ANALYSIS_MD="### NLP Analysis"
+NLP_POS_MAP_MD="### NLP POS Map"
 FILE_DOES_NOT_EXIST="does not exist."
 ORIGINAL_DOC="Original Document"
 SUMMARIZE_INFO="Pick a document to summarize."
 CODE_DOC_EXAMPLES_MD="### Code & Document Examples"
 
-SUMMARY_MD_HEAD="## Summary"
-SUMMARY_MD_SUBHEAD="##### Understand legacy software, complex or technical documents with summarization."
-SUMMARY_MD_DIRECTIONS="Enter text or select one of the Code & Document Examples, below. Then tap Summarize."
 
 IMAGE_PROMPT="Image Prompt"
 IMAGE_PLACEHOLDER="Enter a description of what you'd like to see."