Added Gradio app.py

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.env

app.py

@@ -0,0 +1,570 @@
+# import json
+# import os
+# from pathlib import Path
+# import numpy as np
+# from fastapi import FastAPI, Query
+# from fastapi.responses import FileResponse
+
+# # Use a dedicated library for creating text embeddings
+# from sentence_transformers import SentenceTransformer
+
+# # --- 1. Load the Local Embedding Model ---
+# # This line downloads (first time only) and loads a powerful, lightweight model 
+# # into memory. This is much more efficient than using an API for this task.
+# print("Loading sentence-transformer model...")
+# embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
+# print("Model loaded successfully.")
+
+# # --- 2. Load Image Metadata ---
+# try:
+#     with open("image.json", "r") as f:
+#         data = json.load(f)
+# except FileNotFoundError:
+#     print("Error: image.json not found. Please make sure the file exists.")
+#     exit()
+
+# image_list = []
+# image_dir = Path("images")
+# if not image_dir.exists():
+#     print(f"Error: The '{image_dir}' directory does not exist.")
+#     exit()
+
+# # Prepare list of images and descriptions
+# for page in data.get("pages", []):
+#     for img in page.get("images", []):
+#         description = img.get("description", "")
+#         if not description:
+#             continue
+#         # Match description to a file in the images folder
+#         for img_file in image_dir.iterdir():
+#             if img_file.is_file() and description.lower() in img_file.name.lower():
+#                 image_list.append({"file": str(img_file), "description": description})
+#                 break # Move to the next description once a match is found
+
+# print(f"Found {len(image_list)} images with matching descriptions.")
+
+
+# # --- 3. Function to Get Embeddings Locally ---
+# def get_embedding(text: str) -> np.ndarray:
+#     """
+#     Generates an embedding for the given text using the local SentenceTransformer model.
+#     """
+#     # The model.encode() method directly returns a numpy array. It's fast and local.
+#     return embedding_model.encode(text)
+
+# # --- 4. Precompute Embeddings for All Images ---
+# print("Precomputing embeddings for all image descriptions...")
+# for img in image_list:
+#     # Each description is converted into a numerical vector (embedding)
+#     img["embedding"] = get_embedding(img["description"])
+# print("Embeddings precomputed.")
+
+
+# # --- 5. FastAPI Application ---
+# app = FastAPI(title="Semantic Image Search API")
+
+# def cosine_similarity(vec1: np.ndarray, vec2: np.ndarray) -> float:
+#     """Calculates the cosine similarity between two vectors."""
+#     norm1 = np.linalg.norm(vec1)
+#     norm2 = np.linalg.norm(vec2)
+#     if norm1 == 0 or norm2 == 0:
+#         return 0.0
+#     return np.dot(vec1, vec2) / (norm1 * norm2)
+
+# @app.get("/search_image/")
+# async def search_image(query: str = Query(..., description="Search text")):
+#     # Convert the user's search query into an embedding
+#     query_emb = get_embedding(query)
+    
+#     best_match = None
+#     highest_score = -1.0 # Cosine similarity ranges from -1 to 1
+    
+#     # Compare the query embedding to all precomputed image description embeddings
+#     for img in image_list:
+#         score = cosine_similarity(query_emb, img["embedding"])
+#         if score > highest_score:
+#             highest_score = score
+#             best_match = img
+
+#     if best_match:
+#         print(f"Query: '{query}' -> Found best match: {best_match['file']} with score: {highest_score:.4f}")
+#         return FileResponse(best_match["file"])
+        
+#     return {"error": "No matching image found"}
+
+
+
+# import json
+# import os
+# from pathlib import Path
+# import numpy as np
+# import requests
+# from typing import Optional, List, Tuple, Dict
+
+# import gradio as gr
+# from dotenv import load_dotenv
+# from gradio_client import Client, handle_file
+# from sentence_transformers import SentenceTransformer, util
+
+# # --- 1. SETUP AND MODEL LOADING ---
+# load_dotenv()
+# GROQ_API_KEY = os.getenv("GROQ_API_KEY") # Still used for summarizing results
+
+# if not GROQ_API_KEY:
+#     raise ValueError("GROQ_API_KEY not found. It's needed for summarizing analysis results.")
+
+# print("Loading models and connecting to clients...")
+# # Model for local intent classification and image search
+# embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
+# try:
+#     chatbot_client = Client("Anvit25/LLM_chatbot2")
+#     audio_client = Client("Anvit25/new_audio")
+#     vision_client = Client("Anvit25/vision-classifier")
+#     print("All models and clients loaded successfully.")
+# except Exception as e:
+#     print(f"FATAL: Failed to connect to a Gradio client: {e}")
+#     exit()
+
+# # --- 2. LOAD & PRECOMPUTE DATA FOR LOCAL SEARCH & INTENT ---
+# # Load local image data (same as before)
+# image_list = []
+# # ... (Your image loading and embedding logic is unchanged) ...
+
+# # NEW: Load intents from JSON and pre-compute their embeddings
+# intent_embeddings = {}
+# try:
+#     with open("intents.json", "r") as f:
+#         intents_data = json.load(f)
+#         for intent, phrases in intents_data.items():
+#             intent_embeddings[intent] = {
+#                 "phrases": phrases,
+#                 "embeddings": embedding_model.encode(phrases)
+#             }
+#     print("Local intent classifier loaded successfully.")
+# except FileNotFoundError:
+#     print("FATAL: intents.json not found. This file is required for the local intent classifier.")
+#     exit()
+
+# # --- 3. HELPER FUNCTIONS ---
+
+# def get_user_intent_local(user_query: str) -> dict:
+#     """
+#     Uses SentenceTransformer to classify user intent locally based on intents.json.
+#     """
+#     query_embedding = embedding_model.encode(user_query)
+#     best_match = {"intent": "chat", "score": 0.7, "query": user_query} # Default to chat
+
+#     for intent, data in intent_embeddings.items():
+#         # Calculate cosine similarity between user query and all trigger phrases for an intent
+#         scores = util.cos_sim(query_embedding, data["embeddings"])[0]
+#         max_score = max(scores)
+        
+#         if max_score > best_match["score"]:
+#             best_match["score"] = max_score.item()
+#             best_match["intent"] = intent
+#             # Extract the subject by removing the trigger phrase
+#             best_phrase_index = np.argmax(scores)
+#             trigger_phrase = data["phrases"][best_phrase_index]
+#             subject = user_query.lower().replace(trigger_phrase.lower(), "").strip()
+#             best_match["query"] = subject if subject else user_query
+
+#     print(f"Local Intent Classifier Result: {best_match}")
+#     return best_match
+
+# def summarize_analysis_with_groq(json_result: dict, context: str) -> str:
+#     """
+#     NEW: Takes a JSON/dict result and uses Groq to create a human-readable summary.
+#     """
+#     prompt = f"""
+#     You are a helpful assistant. Based on the following technical analysis from a specialized AI model, provide a friendly and concise summary for the user.
+#     Context: The user asked to '{context}'.
+#     AI Model's Raw JSON Output:
+#     ```json
+#     {json.dumps(json_result, indent=2)}
+#     ```
+#     Your friendly, easy-to-understand summary:
+#     """
+#     try:
+#         response = requests.post(
+#             "https://api.groq.com/openai/v1/chat/completions",
+#             headers={"Authorization": f"Bearer {GROQ_API_KEY}"},
+#             json={"messages": [{"role": "user", "content": prompt}], "model": "llama3-8b-8192"},
+#         )
+#         response.raise_for_status()
+#         return response.json()["choices"][0]["message"]["content"]
+#     except Exception as e:
+#         print(f"Groq summary error: {e}")
+#         return f"I finished the analysis, but had trouble summarizing it. Here is the raw data:\n`{json.dumps(json_result)}`"
+
+
+# # ... (cosine_similarity and find_best_matching_image functions are unchanged) ...
+# def find_best_matching_image(query: str) -> Optional[dict]: # ... (Identical) ...
+#     pass
+# def generate_groq_narrative(user_query: str, search_result: Optional[dict]) -> str: # ... (Identical) ...
+#     pass
+
+# # --- 4. CORE GRADIO LOGIC (UPDATED) ---
+
+# def handle_image_analysis(file_path: str) -> str:
+#     """Analyzes an image and returns a text summary."""
+#     try:
+#         vision_result = vision_client.predict(image=handle_file(file_path), api_name="/predict")
+#         # NEW: Summarize the JSON result
+#         summary = summarize_analysis_with_groq(vision_result, "Analyze this image")
+#         return summary
+#     except Exception as e:
+#         return f"Sorry, I couldn't analyze the image. Error: {e}"
+
+# def handle_audio_analysis(file_path: str) -> str:
+#     """Analyzes audio and returns a text summary."""
+#     try:
+#         prediction_text, _ = audio_client.predict(audio_filepath=handle_file(file_path), api_name="/predict")
+#         return f"The audio analysis result is: **{prediction_text}**"
+#     except Exception as e:
+#         return f"Sorry, I couldn't analyze the audio. Error: {e}"
+
+
+# def chat_interface(user_input: dict, history: List[Tuple[str, str]]):
+#     """
+#     The main function that powers the Gradio chat interface.
+#     It now prioritizes file uploads over text for intent classification.
+#     """
+#     user_text = user_input["text"].strip()
+#     user_files = user_input["files"]
+#     new_history = history or []
+
+#     bot_message = ""
+    
+#     # === Priority 1: Handle file uploads ===
+#     if user_files:
+#         file_path = user_files[0]
+#         # Display the uploaded file in the chat
+#         new_history.append(((file_path,), None))
+        
+#         if file_path.lower().endswith(('.png', '.jpg', '.jpeg', '.webp')):
+#             bot_message = handle_image_analysis(file_path)
+#         elif file_path.lower().endswith(('.wav', '.mp3', '.flac')):
+#             bot_message = handle_audio_analysis(file_path)
+#         else:
+#             bot_message = "I'm not sure how to handle that file type."
+        
+#         new_history[-1] = (new_history[-1][0], bot_message)
+#         return new_history, None
+
+#     # === Priority 2: Handle text-only queries if no files are uploaded ===
+#     if not user_text:
+#         return new_history, None
+
+#     new_history.append((user_text, None))
+#     intent_data = get_user_intent_local(user_text) # Use local classifier
+#     intent = intent_data.get("intent")
+#     query_subject = intent_data.get("query")
+
+#     if intent == "chat":
+#         prediction = chatbot_client.predict(user_input=query_subject, api_name="/chatbot_response")
+#         bot_message = prediction[-1]['content']
+
+#     elif intent == "search_local_image":
+#         found_image = find_best_matching_image(query_subject)
+#         bot_message = generate_groq_narrative(query_subject, found_image)
+#         new_history[-1] = (user_text, bot_message)
+#         if found_image:
+#              new_history.append((None, (found_image['file'],))) # Display image on new line
+#         return new_history, None
+
+#     # For these intents, we just prompt the user to upload a file
+#     elif intent == "request_image_analysis":
+#         bot_message = "Of course. Please upload the image you want me to analyze."
+#     elif intent == "request_audio_analysis":
+#         bot_message = "I'm ready. Please upload the audio file for analysis."
+#     else:
+#         bot_message = "I'm not sure how to handle that. Can you rephrase?"
+
+#     new_history[-1] = (user_text, bot_message)
+#     return new_history, None
+
+
+# # --- 5. GRADIO UI DEFINITION ---
+# with gr.Blocks(theme=gr.themes.Soft(), title="Multi-Modal AI Chatbot") as demo:
+#     gr.Markdown("# Multi-Modal AI Chatbot")
+#     gr.Markdown("I can chat, search for local images, or analyze images and audio you upload.")
+
+#     # CORRECTED LINE: The 'bubble_fn' argument is removed.
+#     chatbot_history = gr.Chatbot(height=600, show_copy_button=True, layout="bubble", render=False)
+    
+#     with gr.Row():
+#         multimodal_textbox = gr.MultimodalTextbox(
+#             file_types=["image", "audio"],
+#             placeholder="Type your message or upload a file...",
+#             submit_btn="Send",
+#             render=False,
+#             autofocus=True
+#         )
+
+#     # Render components after defining the layout
+#     chatbot_history.render()
+#     multimodal_textbox.render()
+
+#     multimodal_textbox.submit(
+#         fn=chat_interface,
+#         inputs=[multimodal_textbox, chatbot_history],
+#         outputs=[chatbot_history, multimodal_textbox]
+#     )
+
+# if __name__ == "__main__":
+#     demo.launch(debug=True)
+
+
+
+
+
+
+
+
+import json
+import os
+from pathlib import Path
+import numpy as np
+import requests
+from typing import Optional, List, Tuple, Dict
+
+import gradio as gr
+from dotenv import load_dotenv
+from gradio_client import Client, handle_file
+from sentence_transformers import SentenceTransformer
+
+# --- 1. SETUP AND MODEL LOADING ---
+load_dotenv()
+GROQ_API_KEY = os.getenv("GROQ_API_KEY") 
+
+if not GROQ_API_KEY:
+    raise ValueError("GROQ_API_KEY not found. It's needed for summarizing analysis results.")
+
+print("Loading models and connecting to clients...")
+embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
+try:
+    chatbot_client = Client("Anvit25/LLM_chatbot2")
+    audio_client = Client("Anvit25/new_audio")
+    vision_client = Client("Anvit25/vision-classifier")
+    print("All models and clients loaded successfully.")
+except Exception as e:
+    print(f"FATAL: Failed to connect to a Gradio client: {e}")
+    exit()
+
+# --- 2. LOAD & PRECOMPUTE DATA FOR LOCAL SEARCH & INTENT ---
+# Load local image data
+image_list = []
+try:
+    with open("image.json", "r") as f:
+        data = json.load(f)
+    image_dir = Path("images")
+    if image_dir.exists():
+        for page in data.get("pages", []):
+            for img in page.get("images", []):
+                description = img.get("description", "")
+                if not description: continue
+                for img_file in image_dir.iterdir():
+                    if img_file.is_file() and description.lower() in img_file.name.lower():
+                        image_list.append({"file": str(img_file), "description": description})
+                        break
+        print(f"Found {len(image_list)} local images for semantic search.")
+        print("Precomputing embeddings for local images...")
+        for img in image_list:
+            img["embedding"] = embedding_model.encode(img["description"])
+        print("Embeddings precomputed.")
+    else:
+        print("Warning: 'images' directory not found.")
+except FileNotFoundError:
+    print("Warning: image.json not found.")
+
+# Load intents from JSON for the new rule-based classifier
+try:
+    with open("intents.json", "r") as f:
+        intents_data = json.load(f)
+    print("Local intent classifier phrases loaded successfully.")
+except FileNotFoundError:
+    print("FATAL: intents.json not found. This file is required.")
+    exit()
+
+# --- 3. HELPER FUNCTIONS ---
+
+def get_user_intent_local(user_query: str) -> dict:
+    """
+    CORRECTED: Uses a robust rule-based check to classify user intent.
+    This is much more reliable than the previous semantic search approach for intents.
+    """
+    lower_query = user_query.lower()
+    
+    # Iterate through intents and their trigger phrases
+    for intent, phrases in intents_data.items():
+        for phrase in phrases:
+            if phrase.lower() in lower_query:
+                # If a trigger phrase is found, identify the intent
+                subject = lower_query.replace(phrase.lower(), "").strip()
+                result = {
+                    "intent": intent,
+                    "query": subject if subject else user_query
+                }
+                print(f"Local Intent Classifier Result: {result}")
+                return result
+
+    # If no specific trigger phrase is found, default to a general chat
+    result = {"intent": "chat", "query": user_query}
+    print(f"Local Intent Classifier Result: {result}")
+    return result
+
+def summarize_analysis_with_groq(json_result: dict, context: str) -> str:
+    """Takes a JSON/dict result and uses Groq to create a human-readable summary."""
+    prompt = f"""
+    You are a helpful assistant. Based on the following technical analysis from a specialized AI model, provide a friendly and concise summary for the user.
+    Context: The user asked to '{context}'.
+    AI Model's Raw JSON Output:
+    ```json
+    {json.dumps(json_result, indent=2)}
+    ```
+    Your friendly, easy-to-understand summary:
+    """
+    try:
+        response = requests.post(
+            "https://api.groq.com/openai/v1/chat/completions",
+            headers={"Authorization": f"Bearer {GROQ_API_KEY}"},
+            json={"messages": [{"role": "user", "content": prompt}], "model": "llama-3.3-70b-versatile"},
+        )
+        response.raise_for_status()
+        return response.json()["choices"][0]["message"]["content"]
+    except Exception as e:
+        print(f"Groq summary error: {e}")
+        return f"I finished the analysis, but had trouble summarizing it. Here is the raw data:\n`{json.dumps(json_result)}`"
+
+def cosine_similarity(vec1, vec2):
+    norm1 = np.linalg.norm(vec1)
+    norm2 = np.linalg.norm(vec2)
+    if norm1 == 0 or norm2 == 0: return 0.0
+    return np.dot(vec1, vec2) / (norm1 * norm2)
+
+def find_best_matching_image(query: str) -> Optional[dict]:
+    if not image_list: return None
+    query_emb = embedding_model.encode(query)
+    best_match = max(image_list, key=lambda img: cosine_similarity(query_emb, img.get("embedding", [])))
+    highest_score = cosine_similarity(query_emb, best_match.get("embedding", []))
+    if best_match and highest_score > 0.4:
+        return best_match
+    return None
+
+def generate_groq_narrative(user_query: str, search_result: Optional[dict]) -> str:
+    if search_result:
+        prompt = f"A user asked to find: '{user_query}'. You found an image described as: '{search_result['description']}'. Craft a short, friendly response."
+    else:
+        prompt = f"A user asked to find: '{user_query}'. You searched but couldn't find a match. Craft a short, polite response."
+    try:
+        response = requests.post(
+            "https://api.groq.com/openai/v1/chat/completions",
+            headers={"Authorization": f"Bearer {GROQ_API_KEY}"},
+            json={"messages": [{"role": "user", "content": prompt}], "model": "llama3-8b-8192"},
+        )
+        response.raise_for_status()
+        return response.json()["choices"][0]["message"]["content"]
+    except Exception as e:
+        print(f"Groq narrative error: {e}")
+        return "I had an issue describing the search result."
+
+# --- 4. CORE GRADIO LOGIC ---
+
+def handle_image_analysis(file_path: str) -> str:
+    try:
+        vision_result = vision_client.predict(image=handle_file(file_path), api_name="/predict")
+        summary = summarize_analysis_with_groq(vision_result, "Analyze this image")
+        return summary
+    except Exception as e:
+        return f"Sorry, I couldn't analyze the image. Error: {e}"
+
+def handle_audio_analysis(file_path: str) -> str:
+    try:
+        prediction_text, _ = audio_client.predict(audio_filepath=handle_file(file_path), api_name="/predict")
+        return f"The audio analysis result is: **{prediction_text}**"
+    except Exception as e:
+        return f"Sorry, I couldn't analyze the audio. Error: {e}"
+
+def chat_interface(user_input: dict, history: List[Tuple[str, str]]):
+    user_text = user_input["text"].strip()
+    user_files = user_input["files"]
+    new_history = history or []
+    
+    # Priority 1: Handle file uploads
+    if user_files:
+        file_path = user_files[0]
+        new_history.append(((file_path,), None))
+        
+        if file_path.lower().endswith(('.png', '.jpg', '.jpeg', '.webp')):
+            bot_message = handle_image_analysis(file_path)
+        elif file_path.lower().endswith(('.wav', '.mp3', '.flac')):
+            bot_message = handle_audio_analysis(file_path)
+        else:
+            bot_message = "I'm not sure how to handle that file type."
+        
+        new_history[-1] = (new_history[-1][0], bot_message)
+        return new_history, None
+
+    # Priority 2: Handle text-only queries
+    if not user_text:
+        return new_history, None
+
+    new_history.append((user_text, None))
+    intent_data = get_user_intent_local(user_text) # Use the NEW, robust classifier
+    intent = intent_data.get("intent")
+    query_subject = intent_data.get("query")
+
+    if intent == "chat":
+        try:
+            prediction = chatbot_client.predict(user_input=query_subject, api_name="/chatbot_response")
+            bot_message = prediction[-1]['content']
+        except Exception as e:
+            print(f"Error calling chatbot client: {e}")
+            bot_message = "I'm sorry, I'm having trouble connecting to my chat brain right now. Please try again."
+
+    elif intent == "search_local_image":
+        found_image = find_best_matching_image(query_subject)
+        bot_message = generate_groq_narrative(query_subject, found_image)
+        new_history[-1] = (user_text, bot_message)
+        if found_image:
+             new_history.append((None, (found_image['file'],)))
+        return new_history, None
+
+    elif intent == "request_image_analysis":
+        bot_message = "Of course. Please upload the image you want me to analyze."
+    elif intent == "request_audio_analysis":
+        bot_message = "I'm ready. Please upload the audio file for analysis."
+    else:
+        bot_message = "I'm not sure how to handle that. Can you rephrase?"
+
+    new_history[-1] = (user_text, bot_message)
+    return new_history, None
+
+
+# --- 5. GRADIO UI DEFINITION ---
+with gr.Blocks(theme=gr.themes.Soft(), title="Multi-Modal AI Chatbot") as demo:
+    gr.Markdown("# Multi-Modal AI Chatbot")
+    gr.Markdown("I can chat, search for local images, or analyze images and audio you upload.")
+
+    chatbot_history = gr.Chatbot(height=600, show_copy_button=True, layout="bubble", render=False)
+    
+    with gr.Row():
+        multimodal_textbox = gr.MultimodalTextbox(
+            file_types=["image", "audio"],
+            placeholder="Type your message or upload a file...",
+            submit_btn="Send",
+            render=False,
+            autofocus=True
+        )
+
+    chatbot_history.render()
+    multimodal_textbox.render()
+
+    multimodal_textbox.submit(
+        fn=chat_interface,
+        inputs=[multimodal_textbox, chatbot_history],
+        outputs=[chatbot_history, multimodal_textbox]
+    )
+
+if __name__ == "__main__":
+    demo.launch(debug=True)

image.json

@@ -0,0 +1,315 @@
+{
+  "document_title": "Samsung Washing Machine Service Manual (WF316LAW Series)",
+  "models_covered": [
+    "WF316LAW", "WF326***", "WF306***", "WF317***", "WF206***", "WF317AAW/XAA", "WF206BNW/XAA"
+  ],
+  "pages": [
+    {
+      "page_number": 1,
+      "title": "Cover Page",
+      "content_summary": "The cover page introduces the washing machine service manual, listing basic models, project names, and key product features.",
+      "images": [
+        {
+          "type": "product_photo",
+          "description": "Image of the silver Samsung WF316LAS washing machine."
+        },
+        {
+          "type": "product_photo",
+          "description": "Image of the champagne/beige Samsung WF316BAC washing machine."
+        },
+        {
+          "type": "product_photo",
+          "description": "Image of the white Samsung WF306LAW washing machine."
+        },
+        {
+          "type": "product_photo",
+          "description": "Image of the white Samsung WF206LNW washing machine."
+        }
+      ],
+      "structured_content": [
+        {
+          "type": "model_info",
+          "heading": "Model Information",
+          "data": {
+            "Basic Model": "WF316LAW",
+            "Model Name": ["<FRONTIER 1 PROJECT>", "WF326***", "WF306***", "WF316***", "WF317***", "<MARS 1 PROJECT>", "WF206***"],
+            "Model Code": ["WF317AAW/XAA (FRONTIER1)", "WF206BNW/XAA (MARS1)"]
+          }
+        },
+        {
+          "type": "feature_list",
+          "heading": "The Feature of Product",
+          "items": [
+            "SilverCare",
+            "SuperSize Capacity",
+            "Direct Drive Motor",
+            "Child Lock",
+            "My Cycle"
+          ]
+        }
+      ]
+    },
+    {
+      "page_number": 2,
+      "title": "Table of Contents (Page 1)",
+      "content_summary": "The first page of the table of contents, covering sections 1 through 7.",
+      "images": [],
+      "structured_content": [
+        {
+          "type": "table_of_contents",
+          "entries": [
+            {"section": "1. PRECAUTIONS", "page": "1-1"},
+            {"section": "2. PRODUCT SPECIFICATIONS", "page": "2-1"},
+            {"section": "3. OPERATING INSTRUCTIONS", "page": "3-1"},
+            {"section": "4. ALIGNMENT AND ADJUSTMENTS", "page": "4-1"},
+            {"section": "5. ASSEMBLY AND DISASSEMBLY", "page": "5-1"},
+            {"section": "6. TROUBLE SHOOTING", "page": "6-1"},
+            {"section": "7. EXPLODED VIEWS AND PARTS LIST", "page": "7-1"}
+          ]
+        }
+      ]
+    },
+    {
+      "page_number": 4,
+      "title": "1-1. Safety Precautions",
+      "content_summary": "A list of 19 critical safety precautions to be followed by service personnel before, during, and after servicing the appliance.",
+      "images": [],
+      "structured_content": [
+        {
+          "type": "safety_list",
+          "items": [
+            "Do not allow the customer to repair the product.",
+            "Disconnect power to the appliance before servicing.",
+            "Do not use multi-plug.",
+            "Check for any damage on power plug or power outlet.",
+            "Make sure to earth the product.",
+            "Do not clean the product with water."
+          ]
+        }
+      ]
+    },
+    {
+      "page_number": 5,
+      "title": "1-2. Precautions Upon Installation",
+      "content_summary": "Visual guide on how to remove shipping bolts and general precautions for installing the washing machine.",
+      "images": [
+        {
+          "type": "instructional_diagram",
+          "description": "A 4-step diagram showing how to remove shipping bolts and cover the holes."
+        },
+        {
+          "type": "instructional_diagram",
+          "description": "Illustrations depicting installation precautions: use two people to move, ensure a firm/level floor, avoid direct sunlight/humidity, provide easy outlet access, avoid freezing temperatures, and keep away from heat sources."
+        }
+      ],
+      "structured_content": []
+    },
+    {
+      "page_number": 10,
+      "title": "2-1. Specifications of Product",
+      "content_summary": "A table detailing the technical specifications of the front-loading washer, including dimensions, weight, capacity, power consumption, and spin speeds for various models.",
+      "images": [],
+      "structured_content": [
+        {
+          "type": "specification_table",
+          "data": [
+            {"Specification": "Height-Overall", "Value": "38 (96.5) Inches (cm)"},
+            {"Specification": "Width", "Value": "27 (68.6) Inches (cm)"},
+            {"Specification": "Depth", "Value": "30.25 (77.0) Inches (cm)"},
+            {"Specification": "Weight", "Value": "89.9 kg"},
+            {"Specification": "Capacity", "Value": "3.29 Cu.ft"},
+            {"Specification": "Power Consumption (Washing)", "Value": "226W"},
+            {"Specification": "Spin Revolution (WF326***)", "Value": "1200rpm"},
+            {"Specification": "Spin Revolution (WF306***)", "Value": "1000rpm"}
+          ]
+        }
+      ]
+    },
+    {
+      "page_number": 14,
+      "title": "3-1. Overview of the Control Panel",
+      "content_summary": "Description of the main control panel functions, including the digital display, temperature, spin, soil level, signal, and SilverCare buttons.",
+      "images": [
+        {
+          "type": "control_panel_diagram",
+          "description": "A detailed diagram of the washing machine's control panel with numbered callouts pointing to different buttons and the display."
+        }
+      ],
+      "structured_content": [
+        {
+          "type": "function_list",
+          "items": [
+            {"number": 1, "name": "Digital graphic display", "description": "Displays remaining wash time, wash information, and error messages."},
+            {"number": 2, "name": "Temperature selection button", "description": "Cycles through temperature options like Extra Hot/Cold, Hot/Cold, etc."},
+            {"number": 3, "name": "Spin selection button", "description": "Cycles through spin speed options like Extra High, High, Medium, Low, No Spin."},
+            {"number": 4, "name": "Soil Level selection button", "description": "Selects washing time based on soil level: Heavy, Normal, Light."},
+            {"number": 6, "name": "SilverCare button", "description": "Activates the silver ion sanitizing feature."}
+          ]
+        }
+      ]
+    },
+    {
+      "page_number": 24,
+      "title": "4-1. General Error Function",
+      "content_summary": "A comprehensive table listing all diagnostic and error codes, their descriptions, and the recommended corrective actions for troubleshooting.",
+      "images": [],
+      "structured_content": [
+        {
+          "type": "error_code_table",
+          "data": [
+            {"LED_Display": "nd", "Diagnostic_Code": 1, "Description": "The water level fails to drop below the Reset Water Level within 15 minutes.", "Corrective_Action": "Go to 'Will Not Drain' Troubleshooting Section."},
+            {"LED_Display": "LO", "Diagnostic_Code": 2, "Description": "Door fails to unlock after 3 attempts.", "Corrective_Action": "Go to 'Will Not Unlock' Troubleshooting Section."},
+            {"LED_Display": "nF", "Diagnostic_Code": 3, "Description": "Filling continues for more than 16 minutes or no change of water level for 3 minutes.", "Corrective_Action": "Go to 'No Water Fill' Troubleshooting Section."},
+            {"LED_Display": "LE", "Diagnostic_Code": 8, "Description": "Water Level Sensor Trouble.", "Corrective_Action": "Go to 'No Water Fill' Troubleshooting Section."},
+            {"LED_Display": "dc", "Diagnostic_Code": 10, "Description": "Unbalance or cabinet bump is detected during final spin.", "Corrective_Action": "Go to 'Wet Clothes' Troubleshooting Section."},
+            {"LED_Display": "tE", "Diagnostic_Code": 29, "Description": "Abnormal high/low temperature or resistance (Thermal sensor or PBA).", "Corrective_Action": "Check Water Temperature. Replace PCB or thermistor."},
+            {"LED_Display": "3E", "Diagnostic_Code": "3E", "Description": "Over-current is detected. Motor won’t turn.", "Corrective_Action": "Evaluate wire harness. Test Motor."},
+            {"LED_Display": "SUdS", "Diagnostic_Code": "-", "Description": "Suds is detected during the washing session.", "Corrective_Action": "Guide a user to reduce amount of detergent usage."}
+          ]
+        }
+      ]
+    },
+    {
+      "page_number": 35,
+      "title": "5-2. Disassembly",
+      "content_summary": "Step-by-step photographic guide for disassembling key components: Top Cover, MEMS Sensor, Water Level Sensor, and Ag Kit.",
+      "images": [
+        { "type": "disassembly_photo", "description": "Photo showing the removal of the Top Cover." },
+        { "type": "disassembly_photo", "description": "Photo showing the location and removal of the MEMS Sensor." },
+        { "type": "disassembly_photo", "description": "Photo showing the disconnection of the Water Level Sensor." },
+        { "type": "disassembly_photo", "description": "Photo showing the removal of the Ag (SilverCare) Kit." }
+      ],
+      "structured_content": [
+        {
+          "type": "disassembly_steps",
+          "part_name": "Top Cover",
+          "steps": ["Unplug the unit.", "Remove screws(2ea) at the back.", "Slide Top Cover back and lift it up."]
+        },
+        {
+          "type": "disassembly_steps",
+          "part_name": "Water Level Sensor",
+          "steps": ["Unplug the unit.", "Remove Top Cover.", "Remove the screw(1ea).", "Disconnect the wire harness.", "Take out Pressure Hose."]
+        }
+      ]
+    },
+    {
+      "page_number": 51,
+      "title": "7-2. THE CONTROL PARTS (WF317AAW/XAA)",
+      "content_summary": "Exploded view diagram of the control panel and dispenser assembly for model WF317AAW/XAA, with part callouts for identification.",
+      "images": [
+        {
+          "type": "exploded_view",
+          "description": "Exploded diagram showing the assembly of the main control panel, knobs, buttons, dispenser drawer, and housing, with labels like C0002, R0014, Y0170."
+        }
+      ],
+      "structured_content": []
+    },
+    {
+      "page_number": 52,
+      "title": "Parts List for Control Parts (WF317AAW/XAA)",
+      "content_summary": "A detailed parts list corresponding to the exploded view on the previous page, providing part numbers, descriptions, and specifications.",
+      "images": [],
+      "structured_content": [
+        {
+          "type": "parts_list_table",
+          "data": [
+            {"Location.No": "C0002", "CODE-NO": "DC97-10513R", "DESCRIPTION": "ASSY-S.PANEL CONTROL"},
+            {"Location.No": "C0029", "CODE-NO": "DC97-10511A", "DESCRIPTION": "ASSY-KNOB ENCODER"},
+            {"Location.No": "R0014", "CODE-NO": "DC97-10336A", "DESCRIPTION": "ASSY-DRAWER"},
+            {"Location.No": "Y0170", "CODE-NO": "MFS-WF317-T0", "DESCRIPTION": "ASSY-PCB PARTS"}
+          ]
+        }
+      ]
+    },
+    {
+      "page_number": 72,
+      "title": "8. BLOCK DIAGRAM",
+      "content_summary": "A high-level block diagram showing the electronic architecture of the washing machine, illustrating power flow (AC/DC) and signal paths between the Main PBA and other modules like the motor, sensors, and display.",
+      "images": [
+        {
+          "type": "block_diagram",
+          "description": "A diagram showing the main PBA connected to Input Voltage, Motor, Drive Circuit, and various other PBAs (Display, AG, MEMS)."
+        }
+      ],
+      "structured_content": []
+    },
+    {
+      "page_number": 74,
+      "title": "9-1. WIRING DIAGRAM",
+      "content_summary": "A complete wiring schematic of the washing machine, detailing all electrical connections between components.",
+      "images": [
+        {
+          "type": "schematic_diagram",
+          "description": "Full wiring diagram showing connections between the main and sub PCBs, motor, sensors, valves, heater, pump, door lock, etc."
+        }
+      ],
+      "structured_content": []
+    },
+    {
+      "page_number": 82,
+      "title": "11-1. MAIN PCB Schematic Diagram",
+      "content_summary": "A detailed, component-level electronic schematic for the Main Printed Circuit Board (PCB).",
+      "images": [
+        {
+          "type": "schematic_diagram",
+          "description": "A complex electronic circuit diagram showing microcontrollers, power circuits, relays, and connectors for the Main PBA."
+        }
+      ],
+      "structured_content": []
+    },
+    {
+      "page_number": 92,
+      "title": "13-1. Model Name Nomenclature",
+      "content_summary": "A flowchart diagram explaining how to decode the Samsung washing machine model numbers.",
+      "images": [
+        {
+          "type": "flowchart",
+          "description": "A diagram breaking down a model number like 'WF326LAS1/XAA' into its constituent parts: Product Type, Capacity, RPM, Year, Dealer, Features, Color, and Service Code."
+        }
+      ],
+      "structured_content": []
+    },
+    {
+      "page_number": 96,
+      "title": "13-5. Q & A",
+      "content_summary": "A Frequently Asked Questions (FAQ) section addressing common user queries about machine operation, such as door unlocking, error codes, and power failures.",
+      "images": [],
+      "structured_content": [
+        {
+          "type": "faq_table",
+          "data": [
+            {"Question": "How long does it take for the door to unlock?", "Answer": "It takes approximately 2-3 seconds for the door to unlock."},
+            {"Question": "What should I do when Information Code ('dc') lights up?", "Answer": "Press the Start/Pause dial and then restart the cycle."},
+            {"Question": "The washer door gets locked after a power failure. How can I open it?", "Answer": "The door will remain locked until power comes back on. The cycle will resume where it left off."}
+          ]
+        }
+      ]
+    },
+    {
+      "page_number": 104,
+      "title": "Back Cover / GSPN Information",
+      "content_summary": "The back cover of the manual, providing a list of websites for the Global Service Partner Network (GSPN) and copyright information.",
+      "images": [
+        {
+          "type": "logo",
+          "description": "Samsung company logo."
+        }
+      ],
+      "structured_content": [
+        {
+          "type": "weblink_table",
+          "heading": "GSPN (Global Service Partner Network)",
+          "data": [
+            {"Area": "North America", "Web Site": "http://service.samsungportal.com"},
+            {"Area": "Europe", "Web Site": "http://europe.samsungportal.com"},
+            {"Area": "Asia", "Web Site": "http://asia.samsungportal.com"}
+          ]
+        },
+        {
+          "type": "legal_notice",
+          "text": "© 2007 Samsung Electronics Co.,Ltd. All rights reserved. Code No.: DC68-02631A-00"
+        }
+      ]
+    }
+  ]
+}

intents.json

@@ -0,0 +1,23 @@
+{
+  "search_local_image": [
+    "show me a picture of",
+    "find an image of",
+    "can I see a photo of",
+    "search for a picture of",
+    "display a picture of"
+  ],
+  "request_image_analysis": [
+    "what do you see in this",
+    "can you analyze this picture",
+    "describe this photo for me",
+    "tell me about this image",
+    "what is in this picture"
+  ],
+  "request_audio_analysis": [
+    "what is this sound",
+    "identify this audio clip",
+    "can you listen to this",
+    "analyze this sound",
+    "what does this audio say"
+  ]
+}

requirements.txt

@@ -0,0 +1,9 @@
+gradio
+pydantic 
+python-dotenv 
+requests
+sentence-transformers 
+torch 
+numpy 
+gradio_client 
+requests