Upload 12 files

.gitattributes

@@ -1,35 +1,36 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+TEST.mp4 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.env

Gpt4oDemo.py

@@ -0,0 +1,402 @@
+import gradio as gr
+import plotly.graph_objs as go
+import numpy as np
+import time
+from openai import OpenAI
+import os
+from hardCodedData import *
+from Helper import *
+import cv2
+from moviepy.editor import VideoFileClip
+import time
+import base64
+import whisperx
+import gc 
+from moviepy.editor import VideoFileClip
+from dotenv import load_dotenv
+
+load_dotenv()
+
+'''
+Model Information 
+GPT4o
+'''
+
+import openai
+api_key = os.getenv("OPENAI_API_KEY")
+client = openai.OpenAI(
+    api_key=api_key,
+    base_url="https://openai.gateway.salt-lab.org/v1",
+)
+MODEL="gpt-4o"
+
+# Whisperx config
+device = "cpu" 
+batch_size = 16 # reduce if low on GPU mem
+compute_type = "int8" # change to "int8" if low on GPU mem (may reduce accuracy)
+model = whisperx.load_model("large-v2", device, compute_type=compute_type)
+
+'''
+Video
+'''
+video_file = None
+audio_path=None
+base64Frames = []
+transcript=""
+
+def process_video(video_path, seconds_per_frame=2):
+    global base64Frames, audio_path
+    base_video_path, _ = os.path.splitext(video_path)
+
+    video = cv2.VideoCapture(video_path)
+    total_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
+    fps = video.get(cv2.CAP_PROP_FPS)
+    frames_to_skip = int(fps * seconds_per_frame)
+    curr_frame=0
+
+    while curr_frame < total_frames - 1:
+        video.set(cv2.CAP_PROP_POS_FRAMES, curr_frame)
+        success, frame = video.read()
+        if not success:
+            break
+        _, buffer = cv2.imencode(".jpg", frame)
+        base64Frames.append(base64.b64encode(buffer).decode("utf-8"))
+        curr_frame += frames_to_skip
+    video.release()
+
+    audio_path = "./TEST.mp3"
+    clip = VideoFileClip(video_path)
+    clip.audio.write_audiofile(audio_path, bitrate="32k")
+    clip.audio.close()
+    clip.close()
+    # transcribe_video(audio_path)
+    print(f"Extracted {len(base64Frames)} frames")
+    print(f"Extracted audio to {audio_path}")
+    return base64Frames, audio_path
+
+chat_history = []
+# chat_history.append({
+#             "role": "system",
+#             "content": (
+#                     """
+#                         You are an assistant chatbot for a Speech Language Pathologist (SLP). 
+#                         Your task is to help analyze a provided video of a therapy session and answer questions accurately. 
+#                         Provide timestamps for specific events or behaviors mentioned. Conclude each response with possible follow-up questions.
+
+#                         Follow these steps:
+
+#                         1.	Suggest to the user to ask, “To get started, you can try asking me how many people there are in the video.”
+#                         2.  Detect how many people are in the video.
+#                         2.	Suggest to the user to tell you the names of the people in the video, starting from left to right.
+#                         3.	After receiving the names, respond with, “Ok thank you! Now you can ask me any questions about this video.”
+#                         4.	If the user asks about a behavior, respond with, “My understanding of this behavior is [xxx - AI generated output]. Is this a behavior that you want to track? If it is, please define this behavior and tell me more about it so I can analyze it more accurately according to your practice.”
+#                         5.	If you receive names, confirm that these are the names of the people from left to right.
+#                     """
+#             )
+#         })
+
+def transcribe_video(filename):
+    global transcript
+    if not audio_path:
+        raise ValueError("Audio path is None")
+    print(audio_path)
+    audio = whisperx.load_audio(audio_path)
+    result = model.transcribe(audio, batch_size=batch_size)
+
+    model_a, metadata = whisperx.load_align_model(language_code=result["language"], device=device)
+    result = whisperx.align(result["segments"], model_a, metadata, audio, device, return_char_alignments=False)
+
+
+    hf_auth_token = os.getenv("HF_AUTH_TOKEN")
+    diarize_model = whisperx.DiarizationPipeline(use_auth_token=hf_auth_token, device=device)
+
+    diarize_segments = diarize_model(audio)
+
+    dia_result = whisperx.assign_word_speakers(diarize_segments, result)
+
+    for res in dia_result["segments"]:
+        # transcript += "Speaker: " + str(res.get("speaker", None)) + "\n"
+        transcript += "Dialogue: " + str(res["text"].lstrip()) + "\n"
+        transcript += "start: " + str(int(res["start"])) + "\n"
+        transcript += "end: " + str(int(res["end"])) + "\n"
+        transcript += "\n"
+
+    return transcript
+
+
+def handle_video(video=None):
+    global video_file, base64Frames, audio_path, chat_history, transcript
+    
+    if video is None:
+        # Load example video
+        video = "./TEST.mp4"
+   
+    base64Frames, audio_path = process_video(video_path=video, seconds_per_frame=100)
+    chat_history.append({
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "These are the frames from the video."},
+            *map(lambda x: {"type": "image_url", "image_url": {"url": f'data:image/jpg;base64,{x}', "detail": "low"}}, base64Frames)
+        ]
+    })
+
+    if transcript:
+        chat_history[-1]['content'].append({
+            "type": "text", 
+            "text": f"Also, below is the template of transcript from the video:\n"
+                    "Speaker: <the speaker of the dialogue>\n"
+                    "Dialogue: <the text of the dialogue>\n"
+                    "start: <the starting timestamp of the dialogue in the video in second>\n"
+                    "end: <the ending timestamp of the dialogue in the video in second>\n"
+                    f"Transcription: {transcript}"
+        })
+
+    video_file = video
+    return video_file
+
+'''
+Chatbot
+'''
+
+def new_prompt(prompt):
+    global chat_history, video_file
+    chat_history.append({"role": "user","content": prompt,})
+    MODEL="gpt-4o"
+    # print(chat_history)
+    print(transcript)
+    try:
+        if video_file:
+            # Video exists and is processed
+            response = client.chat.completions.create(model=MODEL,messages=chat_history,temperature=0,)
+        else:
+            # No video uploaded yet
+            response = client.chat.completions.create(model=MODEL,messages=chat_history,temperature=0,)
+
+        # Extract the text content from the response and append it to the chat history
+        assistant_message = response.choices[0].message.content
+        chat_history.append({'role': 'model', 'content': assistant_message})
+        print(assistant_message)
+    except Exception as e:
+        print("Error: ",e)
+        assistant_message = "API rate limit has been reached. Please wait a moment and try again."
+        chat_history.append({'role': 'model', 'content': assistant_message})
+
+    # except google.api_core.exceptions.ResourceExhausted:
+    #     assistant_message = "API rate limit has been reached. Please wait a moment and try again."
+    #     chat_history.append({'role': 'model', 'parts': [assistant_message]})
+    # except Exception as e:
+    #     assistant_message = f"An error occurred: {str(e)}"
+    #     chat_history.append({'role': 'model', 'parts': [assistant_message]})
+
+    return chat_history
+
+def user_input(user_message, history):
+    return "", history + [[user_message, None]]
+
+def bot_response(history):
+    user_message = history[-1][0]
+    updated_history = new_prompt(user_message)
+    assistant_message = updated_history[-1]['content']
+    history[-1][1] = assistant_message
+    yield history
+
+
+'''
+Behaivor box
+'''
+initial_behaviors = [
+    ("Initiating Behavioral Request (IBR)", 
+    ("The child's skill in using behavior(s) to elicit aid in obtaining an object, or object related event", 
+    ["00:10", "00:45", "01:30"])),
+
+    ("Initiating Joint Attention (IJA)", 
+    ("The child's skill in using behavior(s) to initiate shared attention to objects or events.", 
+    ["00:15", "00:50", "01:40"])),
+
+    ("Responding to Joint Attention (RJA)", 
+    ("The child's skill in following the examiner’s line of regard and pointing gestures.", 
+    ["00:20", "01:00", "02:00"])),
+
+    ("Initiating Social Interaction (ISI)", 
+    ("The child's skill at initiating turn-taking sequences and the tendency to tease the tester", 
+    ["00:20", "00:50", "02:00"])),
+
+    ("Responding to Social Interaction (RSI)", 
+    ("The child’s skill in responding to turn-taking interactions initiated by the examiner.", 
+    ["00:20", "01:00", "02:00"]))
+]
+
+behaviors = initial_behaviors
+behavior_bank = []
+
+def add_or_update_behavior(name, definition, timestamps, selected_behavior):
+    global behaviors, behavior_bank
+    if selected_behavior:  # Update existing behavior
+        for i, (old_name, _) in enumerate(behaviors):
+            if old_name == selected_behavior:
+                behaviors[i] = (name, (definition, timestamps))
+                break
+        # Update behavior in the bank if it exists
+        behavior_bank = [name if b == selected_behavior else b for b in behavior_bank]
+    else:  # Add new behavior
+        new_behavior = (name, (definition, timestamps))
+        behaviors.append(new_behavior)
+    choices = [b[0] for b in behaviors]
+    return gr.Dropdown(choices=choices, value=None, interactive=True), gr.CheckboxGroup(choices=behavior_bank, value=behavior_bank, interactive=True), "", "", ""
+
+def add_to_behaivor_bank(selected_behavior, checkbox_group_values):
+    global behavior_bank
+    if selected_behavior and selected_behavior not in checkbox_group_values:
+        checkbox_group_values.append(selected_behavior)
+    behavior_bank = checkbox_group_values
+    return gr.CheckboxGroup(choices=checkbox_group_values, value=checkbox_group_values, interactive=True), gr.Dropdown(value=None,interactive=True)
+
+def delete_behavior(selected_behavior, checkbox_group_values):
+    global behaviors, behavior_bank
+    behaviors = [b for b in behaviors if b[0] != selected_behavior]
+    behavior_bank = [b for b in behavior_bank if b != selected_behavior]
+    updated_choices = [b[0] for b in behaviors]
+    updated_checkbox_group = [cb for cb in checkbox_group_values if cb != selected_behavior]
+    return gr.Dropdown(choices=updated_choices, value=None, interactive=True), gr.CheckboxGroup(choices=updated_checkbox_group, value=updated_checkbox_group, interactive=True)
+
+def edit_behavior(selected_behavior):
+    for name, (definition, timestamps) in behaviors:
+        if name == selected_behavior:
+            # Return values to populate textboxes
+            return name, definition, timestamps
+    return "", "", ""
+
+
+welcome_message = """
+Hello! I'm your AI assistant.
+I can help you analyze your video sessions following your instructions.
+To get started, please upload a video or add your behaviors to the Behavior Bank using the Behavior Manager.
+"""
+#If you want to tell me about the people in the video, please name them starting from left to right.
+
+css="""
+    body {
+        background-color: #edf1fa; /* offwhite */
+    }
+    .gradio-container {
+        background-color: #edf1fa; /* offwhite */
+    }
+    .column-form .wrap {
+        flex-direction: column;
+    }
+    .sidebar {
+        background: #ffffff; 
+        padding: 10px; 
+        border-right: 1px solid #dee2e6;
+    }
+    .content {
+        padding: 10px;
+    }
+"""
+
+'''
+Gradio Demo
+'''
+with gr.Blocks(theme='base', css=css, title="Soap.AI") as demo:
+    gr.Markdown("# 🤖 AI-Supported SOAP Generation")
+
+    with gr.Row():
+        with gr.Column():
+            video = gr.Video(label="Video", visible=True, height=360, container=True)
+            with gr.Row():
+                with gr.Column(min_width=1, scale=1):
+                    video_upload_button = gr.Button("Analyze Video", variant="primary")
+                with gr.Column(min_width=1, scale=1):
+                    example_video_button = gr.Button("Load Example Video")
+
+            video_upload_button.click(handle_video, inputs=video, outputs=video)
+            example_video_button.click(handle_video, None, outputs=video)
+
+        with gr.Column():
+            chat_section = gr.Group(visible=True)
+            with chat_section:
+                chatbot = gr.Chatbot(elem_id="chatbot", 
+                                     container=True, 
+                                     likeable=True, 
+                                     value=[[None, welcome_message]],
+                                     avatar_images=(None, "./avatar.webp"))
+                with gr.Row():
+                    txt = gr.Textbox(show_label=False, placeholder="Type here!")
+            with gr.Row():
+                send_btn = gr.Button("Send Message", elem_id="send-btn", variant="primary")
+                clear_btn = gr.Button("Clear Chat", elem_id="clear-btn")
+            
+            with gr.Row():
+                behaivor_bank = gr.CheckboxGroup(label="Behavior Bank", 
+                                                 choices=[], 
+                                                 interactive=True,
+                                                 info="A space to store all the behaviors you want to analyze.")
+                open_sidebar_btn = gr.Button("Show Behavior Manager", scale=0)
+                close_sidebar_btn = gr.Button("Hide Behavior Manager", visible=False, scale=0)
+
+            txt.submit(user_input, [txt, chatbot], [txt, chatbot], queue=False).then(
+                bot_response, chatbot, chatbot)
+            send_btn.click(user_input, [txt, chatbot], [txt, chatbot], queue=False).then(
+                bot_response, chatbot, chatbot)
+            clear_btn.click(lambda: None, None, chatbot, queue=False)
+            
+        # Define a sidebar column that is initially hidden
+        with gr.Column(visible=False, min_width=200, scale=0.5, elem_classes="sidebar") as sidebar:
+            behavior_dropdown = gr.Dropdown(label="Behavior Collection",
+                                            choices=behaviors, 
+                                            interactive=True,
+                                            container=True,
+                                            elem_classes="column-form",
+                                            info="Choose a behavior to add to the bank, edit or remove.")
+            with gr.Row():
+                add_toBank_button = gr.Button("Add Behavior to Bank", variant="primary")
+                edit_button = gr.Button("Edit Behavior")
+                delete_button = gr.Button("Remove Behavior")
+
+            with gr.Row():
+                name_input = gr.Textbox(label="Behavior Name", 
+                                        placeholder="(e.g., IBR)",
+                                        info="The name you give to the specific behavior you're tracking or analyzing.")
+                timestamps_input = gr.Textbox(label="Timestamps MM:SS", 
+                                              placeholder="(e.g., (01:15,01:35) )",
+                                              info="The exact times during a session when you saw the behavior. The first two digits represent minutes and the last two digits represent seconds.")
+                definition_input = gr.Textbox(lines=3,
+                                              label="Behavior Definition", 
+                                              placeholder="(e.g., the child's skill in using behavior(s) to elicit aid in obtaining an object, or object related event)",
+                                              info="Provide a clear definition of the behavior.")
+
+            with gr.Row():
+                submit_button = gr.Button("Save Behavior", variant="primary")
+
+    submit_button.click(fn=add_or_update_behavior, 
+                    inputs=[name_input, definition_input, timestamps_input, behavior_dropdown], 
+                    outputs=[behavior_dropdown, behaivor_bank, name_input, definition_input, timestamps_input])
+
+    add_toBank_button.click(fn=add_to_behaivor_bank, 
+                            inputs=[behavior_dropdown, behaivor_bank], 
+                            outputs=[behaivor_bank, behavior_dropdown])
+
+    delete_button.click(fn=delete_behavior, 
+                        inputs=[behavior_dropdown, behaivor_bank], 
+                        outputs=[behavior_dropdown, behaivor_bank])
+
+    edit_button.click(fn=edit_behavior, 
+                    inputs=[behavior_dropdown], 
+                    outputs=[name_input, definition_input, timestamps_input])
+        
+    # Function to open the sidebar
+    open_sidebar_btn.click(lambda: {
+        open_sidebar_btn: gr.Button(visible=False),
+        close_sidebar_btn: gr.Button(visible=True),
+        sidebar: gr.Column(visible=True)
+    }, outputs=[open_sidebar_btn, close_sidebar_btn, sidebar])
+
+    # Function to close the sidebar
+    close_sidebar_btn.click(lambda: {
+        open_sidebar_btn: gr.Button(visible=True),
+        close_sidebar_btn: gr.Button(visible=False),
+        sidebar: gr.Column(visible=False)
+    }, outputs=[open_sidebar_btn, close_sidebar_btn, sidebar])
+
+# Launch the demo
+demo.launch(share=True)

Helper.py

@@ -0,0 +1,79 @@
+import re
+
+# table 
+css = """
+.scrollable-table {
+    max-height: 300px;
+    overflow-y: auto;
+    border: 1px solid #ccc;
+    padding: 10px;
+    
+}
+.scrollable-table table {
+    width: 100%;
+    border-collapse: collapse;
+    background-color: #FFFFFF;
+    
+}
+.scrollable-table th, .scrollable-table td {
+    border: 1px solid #ddd;
+    padding: 8px;
+    text-align: left;
+    color: #000000
+    
+}
+.scrollable-table th {
+    background-color: #EAF2FF;
+    color: #3366CC;
+}
+"""
+
+def parse_transcript(transcript):
+    # Regex to match the timestamps and the text
+    pattern = re.compile(r'(\d{2}:\d{2})\s+(.+?)(?=\d{2}:\d{2}|$)', re.DOTALL)
+    matches = pattern.findall(transcript)
+    
+    timestamps = []
+    texts = []
+    
+    for match in matches:
+        timestamps.append(match[0])
+        texts.append(match[1].strip())
+    
+    return timestamps, texts
+
+def create_transcript_table(timestamps, transcript_text):
+    table_html = '<div class="scrollable-table">\n'
+    table_html += '<table>\n'
+    table_html += '  <thead>\n'
+    table_html += '    <tr>\n'
+    table_html += '      <th>Timestamp</th>\n'
+    table_html += '      <th>Transcript</th>\n'
+    table_html += '    </tr>\n'
+    table_html += '  </thead>\n'
+    table_html += '  <tbody>\n'
+    for ts, text in zip(timestamps, transcript_text):
+        table_html += '    <tr>\n'
+        table_html += f'      <td>{ts}</td>\n'
+        table_html += f'      <td>{text}</td>\n'
+        table_html += '    </tr>\n'
+    table_html += '  </tbody>\n'
+    table_html += '</table>\n'
+    return table_html
+
+def filter_transcript():
+    timestamps = [
+        "15.0 - 17.0",
+        "38.08 - 39.50"
+    ]
+    transcript_text = [
+        "Sad (prompt; 1st)",
+        "Because he fell (no prompt; 2nd)"
+    ]
+    return timestamps, transcript_text
+
+# Guidance Generation Function
+def generate_guidance():
+    guidance_text = """ Engagement: Student may display behaviors such as rocking when showing engagement.
+    Impact factors: Weather (e.g., raining) can impact student’s performance."""
+    return guidance_text

README.md

@@ -1,13 +1,8 @@
----
-title: SOAP Temp
-emoji: 💬
-colorFrom: yellow
-colorTo: purple
-sdk: gradio
-sdk_version: 4.36.1
-app_file: app.py
-pinned: false
-license: mit
----
-
-An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+---
+title: SOAP
+app_file: newDemo.py
+sdk: gradio
+sdk_version: 4.31.0
+---
+# SOAPdemo
+SOAP demo by Qingxiao/Parisa/Aditya

TEST.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6278941748127fc61cee67d4e7477b842f6890a3df50afcbece9960137352898
+size 22321588

hardCodedData.py

@@ -0,0 +1,98 @@
+# Define chatbot and checkbox options
+First_response = """
+Four people identified in the video. Please select the people you want to analyse:
+"""
+Second_response="""Let’s confirm the learning goals:"""
+
+non_verbal_options = [
+    "Following directions",
+    "On-task percentage",
+    "Elopement attempts",
+    "Falling response",
+    "Overwhelm indicators",
+    "Avoidance behaviors",
+    "Harm attempts",
+    "Eye contact percentage",
+    "Eye contact duration",
+    "Body language",
+    "Stimming behaviors",
+    "Attention tracking",
+    "Whole-body listening",
+    "Gaze tracking",
+    "Inappropriate touching",
+    "Listening behaviors",
+    "Noises or gestures"
+]
+
+verbal_options = [
+    "Target sounds",
+    "Word position",
+    "Sound substitutions",
+    "Articulation frequency",
+    "Mean Length of Utterance (MLU)",
+    "Disfluencies",
+    "Questions asked",
+    "Student answers",
+    "Response time",
+    "Peer responses",
+    "On-topic responses",
+    "Correct response ratio",
+    "Cues needed",
+    "Pragmatic skills",
+    "Conversation volume",
+    "Initiating conversation",
+    "Interruptions",
+    "Following directions",
+    "Answering questions",
+    "Idioms/sarcasm",
+    "Grammar and syntax",
+    "Vocabulary use",
+    "Answering wh-questions"
+]
+
+environmental_options = [
+    "External noises",
+    "Room activity",
+    "Room temperature",
+    "Fluorescent lights",
+    "School vs home",
+    "Session location",
+    "Furniture size",
+]
+
+
+client_options=[
+    "David",
+    "John",
+    "Sam",
+    "Mike",
+]
+
+data_insights = """
+- On 9/17, Mike seemed to be distracted by the lighting in the room, which could be the reason he had more approximate words than correct words.
+- On 9/18, there was a level of outside noise present during the session due to weather. This was also where Mike’s incorrect words were at their highest.
+- On 9/19, Mike performed the best he has yet!
+- Between 9/20 - 9/22, Mike's approximate words have tapered out.
+"""
+
+
+subjective_report = """
+Mike exhibited signs of stress at the start of the session due to the rainy weather, but he maintained engagement and participation throughout. 
+"""
+
+objective_report = """
+- Number of Answers Given: This child named one of the characters in the story as 'Bob.'
+- Cues Required: Responded directly to the SLP’s prompt without requiring additional cues.
+- Observed Behavior: Participation in the naming process indicates engagement, though the child's involvement appears less proactive compared to the middle child.
+'Wh' Questions Addressed: Answered a 'who' question by providing a name for a character.
+- Grammar and Complexity: The response was simple but appropriate for the task at hand.
+- Visual Cues Used: It's not clear from the transcript if the child used visual cues, but the direct response to the SLP's question suggests attentiveness to verbal cues.
+"""
+
+assessment_report = """
+His ability to produce /er/ words when presented with a phonemic cue has also improved from only making 20% of words in yesterday’s session. 
+"""
+
+plan_report = """
+
+"""

newDemo.py

@@ -0,0 +1,311 @@
+import gradio as gr
+import plotly.graph_objs as go
+import numpy as np
+import time
+import google.generativeai as genai
+from hardCodedData import *
+from Helper import *
+import google
+
+'''
+Model Information 
+Gemini 1.5 pro
+'''
+GOOGLE_API_KEY = "api"
+genai.configure(api_key="AIzaSyC6msuJuuRiXTplyOzgnlZchpu5_olBXYs")
+generation_config = genai.GenerationConfig(temperature=0.5)
+
+# Model configuration
+model = genai.GenerativeModel(
+    model_name='gemini-1.5-pro-latest',
+       system_instruction= """
+    You are an assistant chatbot for a Speech Language Pathologist (SLP). 
+    Your task is to help analyze a provided video of a therapy session and answer questions accurately. 
+    Provide timestamps for specific events or behaviors mentioned. Conclude each response with possible follow-up questions.
+
+    Follow these steps:
+
+	1.	Suggest to the user to ask, “To get started, you can try asking me how many people there are in the video.”
+    2.  Detect how many people are in the video.
+	2.	Suggest to the user to tell you the names of the people in the video, starting from left to right.
+	3.	After receiving the names, respond with, “Ok thank you! Now you can ask me any questions about this video.”
+	4.	If the user asks about a behavior, respond with, “My understanding of this behavior is [xxx - AI generated output]. Is this a behavior that you want to track? If it is, please define this behavior and tell me more about it so I can analyze it more accurately according to your practice.”
+	5.	If you receive names, confirm that these are the names of the people from left to right.
+    """
+   )
+
+
+
+'''
+Video
+'''
+video_file = None
+
+def handle_video(video=None):
+    global video_file
+    if video is None:
+        # Load example video
+        video = "./TEST.mp4"
+        isTest = True
+   
+    video_file = genai.upload_file(path=video)
+    
+    while video_file.state.name == "PROCESSING":
+        print(".", end="")
+        time.sleep(10)
+        video_file = genai.get_file(video_file.name)
+
+    if video_file.state.name == "FAILED":
+        raise ValueError(video_file.state.name)
+
+    if isTest:
+        return video
+    else:
+        return video_file
+
+'''
+Chatbot
+'''
+chat_history = []
+def new_prompt(prompt):
+    global chat_history, video_file
+
+    # Append user prompt to chat history
+    chat_history.append({'role': 'user', 'parts': [prompt]})
+    try:
+
+        if video_file:
+            # Video exists and is processed
+            chat_history[-1]['parts'].extend([" from video: ", video_file])
+            response = model.generate_content(chat_history, request_options={"timeout": 600})
+        else:
+            # No video uploaded yet
+            response = model.generate_content(chat_history)
+
+        # Extract the text content from the response and append it to the chat history
+        assistant_message = response.candidates[0].content.parts[0].text
+        chat_history.append({'role': 'model', 'parts': [assistant_message]})
+
+    except google.api_core.exceptions.ResourceExhausted:
+        assistant_message = "API rate limit has been reached. Please wait a moment and try again."
+        chat_history.append({'role': 'model', 'parts': [assistant_message]})
+    except Exception as e:
+        assistant_message = f"An error occurred: {str(e)}"
+        chat_history.append({'role': 'model', 'parts': [assistant_message]})
+
+    return chat_history
+
+def user_input(user_message, history):
+    return "", history + [[user_message, None]]
+
+def bot_response(history):
+    user_message = history[-1][0]
+    print(history)
+    updated_history = new_prompt(user_message)
+    print(updated_history)
+    assistant_message = updated_history[-1]['parts'][0]
+    for i in range(len(assistant_message)):
+        time.sleep(0.05)
+        history[-1][1] = assistant_message[:i+1]
+        yield history
+
+
+'''
+Behaivor box
+'''
+initial_behaviors = [
+    ("Initiating Behavioral Request (IBR)", 
+    ("The child's skill in using behavior(s) to elicit aid in obtaining an object, or object related event", 
+    ["00:10", "00:45", "01:30"])),
+
+    ("Initiating Joint Attention (IJA)", 
+    ("The child's skill in using behavior(s) to initiate shared attention to objects or events.", 
+    ["00:15", "00:50", "01:40"])),
+
+    ("Responding to Joint Attention (RJA)", 
+    ("The child's skill in following the examiner’s line of regard and pointing gestures.", 
+    ["00:20", "01:00", "02:00"])),
+
+    ("Initiating Social Interaction (ISI)", 
+    ("The child's skill at initiating turn-taking sequences and the tendency to tease the tester", 
+    ["00:20", "00:50", "02:00"])),
+
+    ("Responding to Social Interaction (RSI)", 
+    ("The child’s skill in responding to turn-taking interactions initiated by the examiner.", 
+    ["00:20", "01:00", "02:00"]))
+]
+
+behaviors = initial_behaviors
+behavior_bank = []
+
+def add_or_update_behavior(name, definition, timestamps, selected_behavior):
+    global behaviors, behavior_bank
+    if selected_behavior:  # Update existing behavior
+        for i, (old_name, _) in enumerate(behaviors):
+            if old_name == selected_behavior:
+                behaviors[i] = (name, (definition, timestamps))
+                break
+        # Update behavior in the bank if it exists
+        behavior_bank = [name if b == selected_behavior else b for b in behavior_bank]
+    else:  # Add new behavior
+        new_behavior = (name, (definition, timestamps))
+        behaviors.append(new_behavior)
+    choices = [b[0] for b in behaviors]
+    return gr.Dropdown(choices=choices, value=None, interactive=True), gr.CheckboxGroup(choices=behavior_bank, value=behavior_bank, interactive=True), "", "", ""
+
+def add_to_behaivor_bank(selected_behavior, checkbox_group_values):
+    global behavior_bank
+    if selected_behavior and selected_behavior not in checkbox_group_values:
+        checkbox_group_values.append(selected_behavior)
+    behavior_bank = checkbox_group_values
+    return gr.CheckboxGroup(choices=checkbox_group_values, value=checkbox_group_values, interactive=True), gr.Dropdown(value=None,interactive=True)
+
+def delete_behavior(selected_behavior, checkbox_group_values):
+    global behaviors, behavior_bank
+    behaviors = [b for b in behaviors if b[0] != selected_behavior]
+    behavior_bank = [b for b in behavior_bank if b != selected_behavior]
+    updated_choices = [b[0] for b in behaviors]
+    updated_checkbox_group = [cb for cb in checkbox_group_values if cb != selected_behavior]
+    return gr.Dropdown(choices=updated_choices, value=None, interactive=True), gr.CheckboxGroup(choices=updated_checkbox_group, value=updated_checkbox_group, interactive=True)
+
+def edit_behavior(selected_behavior):
+    for name, (definition, timestamps) in behaviors:
+        if name == selected_behavior:
+            # Return values to populate textboxes
+            return name, definition, timestamps
+    return "", "", ""
+
+
+welcome_message = """
+Hello! I'm your AI assistant.
+I can help you analyze your video sessions following your instructions.
+To get started, please upload a video or add your behaviors to the Behavior Bank using the Behavior Manager.
+"""
+#If you want to tell me about the people in the video, please name them starting from left to right.
+
+css="""
+    body {
+        background-color: #edf1fa; /* offwhite */
+    }
+    .gradio-container {
+        background-color: #edf1fa; /* offwhite */
+    }
+    .column-form .wrap {
+        flex-direction: column;
+    }
+    .sidebar {
+        background: #ffffff; 
+        padding: 10px; 
+        border-right: 1px solid #dee2e6;
+    }
+    .content {
+        padding: 10px;
+    }
+"""
+
+'''
+Gradio Demo
+'''
+with gr.Blocks(theme='base', css=css, title="Soap.AI") as demo:
+    gr.Markdown("# 🤖 AI-Supported SOAP Generation")
+
+    with gr.Row():
+        with gr.Column():
+            video = gr.Video(label="Video", visible=True, height=360, container=True)
+            with gr.Row():
+                with gr.Column(min_width=1, scale=1):
+                    video_upload_button = gr.Button("Analyze Video", variant="primary")
+                with gr.Column(min_width=1, scale=1):
+                    example_video_button = gr.Button("Load Example Video")
+
+            video_upload_button.click(handle_video, inputs=video, outputs=video)
+            example_video_button.click(handle_video, None, outputs=video)
+
+        with gr.Column():
+            chat_section = gr.Group(visible=True)
+            with chat_section:
+                chatbot = gr.Chatbot(elem_id="chatbot", 
+                                     container=True, 
+                                     likeable=True, 
+                                     value=[[None, welcome_message]],
+                                     avatar_images=(None, "./avatar.webp"))
+                with gr.Row():
+                    txt = gr.Textbox(show_label=False, placeholder="Type here!")
+            with gr.Row():
+                send_btn = gr.Button("Send Message", elem_id="send-btn", variant="primary")
+                clear_btn = gr.Button("Clear Chat", elem_id="clear-btn")
+            
+            with gr.Row():
+                behaivor_bank = gr.CheckboxGroup(label="Behavior Bank", 
+                                                 choices=[], 
+                                                 interactive=True,
+                                                 info="A space to store all the behaviors you want to analyze.")
+                open_sidebar_btn = gr.Button("Show Behavior Manager", scale=0)
+                close_sidebar_btn = gr.Button("Hide Behavior Manager", visible=False, scale=0)
+
+            txt.submit(user_input, [txt, chatbot], [txt, chatbot], queue=False).then(
+                bot_response, chatbot, chatbot)
+            send_btn.click(user_input, [txt, chatbot], [txt, chatbot], queue=False).then(
+                bot_response, chatbot, chatbot)
+            clear_btn.click(lambda: None, None, chatbot, queue=False)
+            
+        # Define a sidebar column that is initially hidden
+        with gr.Column(visible=False, min_width=200, scale=0.5, elem_classes="sidebar") as sidebar:
+            behavior_dropdown = gr.Dropdown(label="Behavior Collection",
+                                            choices=behaviors, 
+                                            interactive=True,
+                                            container=True,
+                                            elem_classes="column-form",
+                                            info="Choose a behavior to add to the bank, edit or remove.")
+            with gr.Row():
+                add_toBank_button = gr.Button("Add Behavior to Bank", variant="primary")
+                edit_button = gr.Button("Edit Behavior")
+                delete_button = gr.Button("Remove Behavior")
+
+            with gr.Row():
+                name_input = gr.Textbox(label="Behavior Name", 
+                                        placeholder="(e.g., IBR)",
+                                        info="The name you give to the specific behavior you're tracking or analyzing.")
+                timestamps_input = gr.Textbox(label="Timestamps MM:SS", 
+                                              placeholder="(e.g., (01:15,01:35) )",
+                                              info="The exact times during a session when you saw the behavior. The first two digits represent minutes and the last two digits represent seconds.")
+                definition_input = gr.Textbox(lines=3,
+                                              label="Behavior Definition", 
+                                              placeholder="(e.g., the child's skill in using behavior(s) to elicit aid in obtaining an object, or object related event)",
+                                              info="Provide a clear definition of the behavior.")
+
+            with gr.Row():
+                submit_button = gr.Button("Save Behavior", variant="primary")
+
+    submit_button.click(fn=add_or_update_behavior, 
+                    inputs=[name_input, definition_input, timestamps_input, behavior_dropdown], 
+                    outputs=[behavior_dropdown, behaivor_bank, name_input, definition_input, timestamps_input])
+
+    add_toBank_button.click(fn=add_to_behaivor_bank, 
+                            inputs=[behavior_dropdown, behaivor_bank], 
+                            outputs=[behaivor_bank, behavior_dropdown])
+
+    delete_button.click(fn=delete_behavior, 
+                        inputs=[behavior_dropdown, behaivor_bank], 
+                        outputs=[behavior_dropdown, behaivor_bank])
+
+    edit_button.click(fn=edit_behavior, 
+                    inputs=[behavior_dropdown], 
+                    outputs=[name_input, definition_input, timestamps_input])
+        
+    # Function to open the sidebar
+    open_sidebar_btn.click(lambda: {
+        open_sidebar_btn: gr.Button(visible=False),
+        close_sidebar_btn: gr.Button(visible=True),
+        sidebar: gr.Column(visible=True)
+    }, outputs=[open_sidebar_btn, close_sidebar_btn, sidebar])
+
+    # Function to close the sidebar
+    close_sidebar_btn.click(lambda: {
+        open_sidebar_btn: gr.Button(visible=True),
+        close_sidebar_btn: gr.Button(visible=False),
+        sidebar: gr.Column(visible=False)
+    }, outputs=[open_sidebar_btn, close_sidebar_btn, sidebar])
+
+# Launch the demo
+demo.launch(share=True)

requirements.txt

@@ -1 +1,166 @@
-huggingface_hub==0.22.2
+aiofiles==23.2.1
+aiohttp==3.9.5
+aiosignal==1.3.1
+alembic==1.13.2
+altair==5.3.0
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asteroid-filterbanks==0.4.0
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+av==11.0.0
+beautifulsoup4==4.12.3
+certifi==2024.7.4
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+coloredlogs==15.0.1
+colorlog==6.8.2
+contourpy==1.2.1
+ctranslate2==4.3.1
+cycler==0.12.1
+decorator==4.4.2
+distro==1.9.0
+dnspython==2.6.1
+docopt==0.6.2
+einops==0.8.0
+email_validator==2.2.0
+exceptiongroup==1.2.1
+fastapi==0.111.0
+fastapi-cli==0.0.4
+faster-whisper==1.0.0
+ffmpy==0.3.2
+filelock==3.15.4
+flatbuffers==24.3.25
+fonttools==4.53.1
+frozenlist==1.4.1
+fsspec==2024.6.1
+google==3.0.0
+gradio==4.37.2
+gradio_client==1.0.2
+greenlet==3.0.3
+h11==0.14.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.23.4
+humanfriendly==10.0
+HyperPyYAML==1.2.2
+idna==3.7
+imageio==2.34.2
+imageio-ffmpeg==0.5.1
+importlib_resources==6.4.0
+intel-openmp==2021.4.0
+Jinja2==3.1.4
+joblib==1.4.2
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+julius==0.2.7
+kiwisolver==1.4.5
+lazy_loader==0.4
+librosa==0.10.2.post1
+lightning==2.3.3
+lightning-utilities==0.11.3.post0
+llvmlite==0.43.0
+Mako==1.3.5
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.9.1
+mdurl==0.1.2
+mkl==2021.4.0
+moviepy==1.0.3
+mpmath==1.3.0
+msgpack==1.0.8
+multidict==6.0.5
+networkx==3.2.1
+nltk==3.8.1
+numba==0.60.0
+numpy==1.26.4
+omegaconf==2.3.0
+onnxruntime==1.18.1
+openai==1.35.12
+opencv-python==4.10.0.84
+optuna==3.6.1
+orjson==3.10.6
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+platformdirs==4.2.2
+plotly==5.22.0
+pooch==1.8.2
+primePy==1.3
+proglog==0.1.10
+protobuf==5.27.2
+pyannote.audio==3.1.1
+pyannote.core==5.0.0
+pyannote.database==5.1.0
+pyannote.metrics==3.2.1
+pyannote.pipeline==3.0.1
+pycparser==2.22
+pydantic==2.8.2
+pydantic_core==2.20.1
+pydub==0.25.1
+Pygments==2.18.0
+pyparsing==3.1.2
+pyreadline3==3.4.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytorch-lightning==2.3.3
+pytorch-metric-learning==2.5.0
+pytz==2024.1
+PyYAML==6.0.1
+referencing==0.35.1
+regex==2024.5.15
+requests==2.32.3
+rich==13.7.1
+rpds-py==0.19.0
+ruamel.yaml==0.18.6
+ruamel.yaml.clib==0.2.8
+ruff==0.5.1
+safetensors==0.4.3
+scikit-learn==1.5.1
+scipy==1.13.1
+semantic-version==2.10.0
+semver==3.0.2
+sentencepiece==0.2.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+speechbrain==1.0.0
+SQLAlchemy==2.0.31
+starlette==0.37.2
+sympy==1.13.0
+tabulate==0.9.0
+tbb==2021.13.0
+tenacity==8.5.0
+tensorboardX==2.6.2.2
+threadpoolctl==3.5.0
+tokenizers==0.15.2
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.3.1
+torch-audiomentations==0.11.1
+torch-pitch-shift==1.2.4
+torchaudio==2.3.1
+torchmetrics==1.4.0.post0
+tqdm==4.66.4
+transformers==4.39.3
+typer==0.12.3
+typing_extensions==4.12.2
+tzdata==2024.1
+ujson==5.10.0
+urllib3==2.2.2
+uvicorn==0.30.1
+watchfiles==0.22.0
+websockets==11.0.3
+whisperx==3.1.1
+yarl==1.9.4
+zipp==3.19.2

style.css

@@ -0,0 +1,50 @@
+body {
+    font-family: Arial, sans-serif;
+    color: #000; /* Set default text color to black */
+}
+
+.gradio-container {
+    background-color: #1a3e63; /* Background color similar to the image */
+}
+
+.gradio-block, .gradio-column, .gradio-row {
+    background-color: #fff; /* White background for blocks */
+    border-radius: 10px; /* Rounded corners */
+    box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1); /* Subtle shadow for depth */
+    margin: 10px;
+    padding: 20px;
+}
+
+.gradio-markdown {
+    color: #000; /* Text color set to black */
+}
+
+.gradio-button {
+    background-color: #4CAF50; /* Button color */
+    color: white; /* Button text color */
+    border: none;
+    padding: 10px 20px;
+    text-align: center;
+    text-decoration: none;
+    display: inline-block;
+    font-size: 16px;
+    margin: 4px 2px;
+    cursor: pointer;
+    border-radius: 5px; /* Rounded corners */
+}
+
+.gradio-checkbox-group, .gradio-checkbox {
+    color: #000; /* Text color set to black */
+}
+
+.gradio-textbox {
+    color: #000; /* Text color set to black */
+}
+
+.gradio-header {
+    color: #000; /* Text color set to black */
+}
+
+.gradio-container p, .gradio-container h1, .gradio-container h2, .gradio-container h3, .gradio-container h4, .gradio-container h5, .gradio-container h6 {
+    color: #000; /* Ensure all header and paragraph texts are black */
+}