Spaces:
Sleeping
Sleeping
Upload 2 files
Browse files- alz_companion/agent.py +358 -0
- alz_companion/prompts.py +196 -0
alz_companion/agent.py
ADDED
|
@@ -0,0 +1,358 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
import os
|
| 3 |
+
import json
|
| 4 |
+
import base64
|
| 5 |
+
import time
|
| 6 |
+
import tempfile
|
| 7 |
+
|
| 8 |
+
from typing import List, Dict, Any, Optional
|
| 9 |
+
|
| 10 |
+
# OpenAI for LLM (optional)
|
| 11 |
+
try:
|
| 12 |
+
from openai import OpenAI
|
| 13 |
+
except Exception: # pragma: no cover
|
| 14 |
+
OpenAI = None # type: ignore
|
| 15 |
+
|
| 16 |
+
# LangChain & RAG
|
| 17 |
+
from langchain.schema import Document
|
| 18 |
+
from langchain_community.vectorstores import FAISS
|
| 19 |
+
from langchain_community.embeddings import HuggingFaceEmbeddings
|
| 20 |
+
|
| 21 |
+
# TTS
|
| 22 |
+
try:
|
| 23 |
+
from gtts import gTTS
|
| 24 |
+
except Exception: # pragma: no cover
|
| 25 |
+
gTTS = None # type: ignore
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
from .prompts import (
|
| 29 |
+
SYSTEM_TEMPLATE, ANSWER_TEMPLATE_CALM, ANSWER_TEMPLATE_ADQ,
|
| 30 |
+
SAFETY_GUARDRAILS, RISK_FOOTER, render_emotion_guidelines, CLASSIFICATION_PROMPT,
|
| 31 |
+
# Add the new templates to the import list
|
| 32 |
+
ROUTER_PROMPT,
|
| 33 |
+
ANSWER_TEMPLATE_FACTUAL,
|
| 34 |
+
ANSWER_TEMPLATE_GENERAL_KNOWLEDGE,
|
| 35 |
+
ANSWER_TEMPLATE_GENERAL,
|
| 36 |
+
QUERY_EXPANSION_PROMPT
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
# -----------------------------
|
| 40 |
+
# Multimodal Processing Functions
|
| 41 |
+
# -----------------------------
|
| 42 |
+
|
| 43 |
+
def _openai_client() -> Optional[OpenAI]:
|
| 44 |
+
api_key = os.getenv("OPENAI_API_KEY", "").strip()
|
| 45 |
+
return OpenAI(api_key=api_key) if api_key and OpenAI else None
|
| 46 |
+
|
| 47 |
+
# In agent.py
|
| 48 |
+
|
| 49 |
+
def describe_image(image_path: str) -> str:
|
| 50 |
+
"""Uses a vision model to describe an image for context."""
|
| 51 |
+
client = _openai_client()
|
| 52 |
+
if not client:
|
| 53 |
+
return "(Image description failed: OpenAI API key not configured.)"
|
| 54 |
+
|
| 55 |
+
try:
|
| 56 |
+
# --- FIX START ---
|
| 57 |
+
# Determine the MIME type based on the file extension
|
| 58 |
+
extension = os.path.splitext(image_path)[1].lower()
|
| 59 |
+
if extension == ".png":
|
| 60 |
+
mime_type = "image/png"
|
| 61 |
+
elif extension in [".jpg", ".jpeg"]:
|
| 62 |
+
mime_type = "image/jpeg"
|
| 63 |
+
elif extension == ".gif":
|
| 64 |
+
mime_type = "image/gif"
|
| 65 |
+
elif extension == ".webp":
|
| 66 |
+
mime_type = "image/webp"
|
| 67 |
+
else:
|
| 68 |
+
# Default to JPEG, but this handles the most common cases
|
| 69 |
+
mime_type = "image/jpeg"
|
| 70 |
+
# --- FIX END ---
|
| 71 |
+
|
| 72 |
+
with open(image_path, "rb") as image_file:
|
| 73 |
+
base64_image = base64.b64encode(image_file.read()).decode('utf-8')
|
| 74 |
+
|
| 75 |
+
response = client.chat.completions.create(
|
| 76 |
+
model="gpt-4o",
|
| 77 |
+
messages=[
|
| 78 |
+
{
|
| 79 |
+
"role": "user",
|
| 80 |
+
"content": [
|
| 81 |
+
{"type": "text", "text": "Describe this image in a concise, factual way for a memory journal. Focus on people, places, and key objects. For example: 'A photo of John and Mary smiling on a bench at the park.'"},
|
| 82 |
+
{
|
| 83 |
+
"type": "image_url",
|
| 84 |
+
# Use the dynamically determined MIME type
|
| 85 |
+
"image_url": {"url": f"data:{mime_type};base64,{base64_image}"}
|
| 86 |
+
}
|
| 87 |
+
],
|
| 88 |
+
}
|
| 89 |
+
],
|
| 90 |
+
max_tokens=100,
|
| 91 |
+
)
|
| 92 |
+
return response.choices[0].message.content or "No description available."
|
| 93 |
+
except Exception as e:
|
| 94 |
+
return f"[Image description error: {e}]"
|
| 95 |
+
|
| 96 |
+
# -----------------------------
|
| 97 |
+
# NLU Classification Function
|
| 98 |
+
# -----------------------------
|
| 99 |
+
def detect_tags_from_query(query: str, behavior_options: list, emotion_options: list) -> Dict[str, Optional[str]]:
|
| 100 |
+
"""Uses an LLM call to classify the user's query into a behavior and emotion tag."""
|
| 101 |
+
behavior_str = ", ".join(f'"{opt}"' for opt in behavior_options if opt != "None")
|
| 102 |
+
emotion_str = ", ".join(f'"{opt}"' for opt in emotion_options if opt != "None")
|
| 103 |
+
prompt = CLASSIFICATION_PROMPT.format(behavior_options=behavior_str, emotion_options=emotion_str, query=query)
|
| 104 |
+
messages = [{"role": "system", "content": "You are a helpful NLU classification assistant. Respond only with the JSON object requested."}, {"role": "user", "content": prompt}]
|
| 105 |
+
response_str = call_llm(messages, temperature=0.1)
|
| 106 |
+
try:
|
| 107 |
+
clean_response = response_str.strip().replace("```json", "").replace("```", "")
|
| 108 |
+
result = json.loads(clean_response)
|
| 109 |
+
behavior = result.get("detected_behavior")
|
| 110 |
+
emotion = result.get("detected_emotion")
|
| 111 |
+
return {"detected_behavior": behavior if behavior in behavior_options else "None", "detected_emotion": emotion if emotion in emotion_options else "None"}
|
| 112 |
+
except (json.JSONDecodeError, AttributeError):
|
| 113 |
+
return {"detected_behavior": "None", "detected_emotion": "None"}
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
# -----------------------------
|
| 117 |
+
# Embeddings & VectorStore
|
| 118 |
+
# -----------------------------
|
| 119 |
+
|
| 120 |
+
def _default_embeddings():
|
| 121 |
+
"""Lightweight, widely available model."""
|
| 122 |
+
model_name = os.getenv("EMBEDDINGS_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
|
| 123 |
+
return HuggingFaceEmbeddings(model_name=model_name)
|
| 124 |
+
|
| 125 |
+
def build_or_load_vectorstore(docs: List[Document], index_path: str, is_personal: bool = False) -> FAISS:
|
| 126 |
+
os.makedirs(os.path.dirname(index_path), exist_ok=True)
|
| 127 |
+
if os.path.isdir(index_path) and os.path.exists(os.path.join(index_path, "index.faiss")):
|
| 128 |
+
try:
|
| 129 |
+
return FAISS.load_local(index_path, _default_embeddings(), allow_dangerous_deserialization=True)
|
| 130 |
+
except Exception:
|
| 131 |
+
pass
|
| 132 |
+
|
| 133 |
+
if is_personal and not docs:
|
| 134 |
+
docs = [Document(page_content="(This is the start of the personal memory journal.)", metadata={"source": "placeholder"})]
|
| 135 |
+
|
| 136 |
+
vs = FAISS.from_documents(docs, _default_embeddings())
|
| 137 |
+
vs.save_local(index_path)
|
| 138 |
+
return vs
|
| 139 |
+
|
| 140 |
+
def texts_from_jsonl(path: str) -> List[Document]:
|
| 141 |
+
out: List[Document] = []
|
| 142 |
+
try:
|
| 143 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 144 |
+
for i, line in enumerate(f):
|
| 145 |
+
line = line.strip()
|
| 146 |
+
if not line: continue
|
| 147 |
+
obj = json.loads(line)
|
| 148 |
+
txt = obj.get("text") or ""
|
| 149 |
+
if not isinstance(txt, str) or not txt.strip(): continue
|
| 150 |
+
md = {"source": os.path.basename(path), "chunk": i}
|
| 151 |
+
for k in ("behaviors", "emotion"):
|
| 152 |
+
if k in obj: md[k] = obj[k]
|
| 153 |
+
out.append(Document(page_content=txt, metadata=md))
|
| 154 |
+
except Exception:
|
| 155 |
+
return []
|
| 156 |
+
return out
|
| 157 |
+
|
| 158 |
+
def bootstrap_vectorstore(sample_paths: List[str] | None = None, index_path: str = "data/faiss_index") -> FAISS:
|
| 159 |
+
docs: List[Document] = []
|
| 160 |
+
for p in (sample_paths or []):
|
| 161 |
+
try:
|
| 162 |
+
if p.lower().endswith(".jsonl"):
|
| 163 |
+
docs.extend(texts_from_jsonl(p))
|
| 164 |
+
else:
|
| 165 |
+
with open(p, "r", encoding="utf-8", errors="ignore") as fh:
|
| 166 |
+
docs.append(Document(page_content=fh.read(), metadata={"source": os.path.basename(p)}))
|
| 167 |
+
except Exception:
|
| 168 |
+
continue
|
| 169 |
+
if not docs:
|
| 170 |
+
docs = [Document(page_content="(empty index)", metadata={"source": "placeholder"})]
|
| 171 |
+
return build_or_load_vectorstore(docs, index_path=index_path)
|
| 172 |
+
|
| 173 |
+
# -----------------------------
|
| 174 |
+
# LLM Call
|
| 175 |
+
# -----------------------------
|
| 176 |
+
def call_llm(messages: List[Dict[str, str]], temperature: float = 0.6) -> str:
|
| 177 |
+
"""Call OpenAI Chat Completions if available; else return a fallback."""
|
| 178 |
+
client = _openai_client()
|
| 179 |
+
model = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
|
| 180 |
+
if not client:
|
| 181 |
+
return "(Offline Mode: OpenAI API key not configured.)"
|
| 182 |
+
try:
|
| 183 |
+
# --- FIX START ---
|
| 184 |
+
# Use a default temperature if the provided value is None
|
| 185 |
+
temp_value = temperature if temperature is not None else 0.6
|
| 186 |
+
# --- FIX END ---
|
| 187 |
+
|
| 188 |
+
resp = client.chat.completions.create(model=model, messages=messages, temperature=float(temp_value))
|
| 189 |
+
return (resp.choices[0].message.content or "").strip()
|
| 190 |
+
except Exception as e:
|
| 191 |
+
return f"[LLM API Error: {e}]"
|
| 192 |
+
|
| 193 |
+
# -----------------------------
|
| 194 |
+
# Prompting & RAG Chain
|
| 195 |
+
# -----------------------------
|
| 196 |
+
|
| 197 |
+
def _format_sources(docs: List[Document]) -> List[str]:
|
| 198 |
+
return list(set(d.metadata.get("source", "unknown") for d in docs))
|
| 199 |
+
|
| 200 |
+
# In agent.py, replace the existing make_rag_chain function with this new one to handle general & specific conversations .
|
| 201 |
+
# The logic for the "factual_question" path needs to be updated to perform the expansion query
|
| 202 |
+
|
| 203 |
+
def make_rag_chain(
|
| 204 |
+
vs_general: FAISS,
|
| 205 |
+
vs_personal: FAISS,
|
| 206 |
+
*,
|
| 207 |
+
role: str = "patient",
|
| 208 |
+
temperature: float = 0.6,
|
| 209 |
+
language: str = "English",
|
| 210 |
+
patient_name: str = "the patient",
|
| 211 |
+
caregiver_name: str = "the caregiver",
|
| 212 |
+
tone: str = "warm",
|
| 213 |
+
):
|
| 214 |
+
"""Returns a callable that performs the complete, intelligent RAG process."""
|
| 215 |
+
|
| 216 |
+
def _format_docs(docs: List[Document], default_msg: str) -> str:
|
| 217 |
+
if not docs: return default_msg
|
| 218 |
+
unique_docs = {doc.page_content: doc for doc in docs}.values()
|
| 219 |
+
return "\n".join([f"- {d.page_content.strip()}" for d in unique_docs])
|
| 220 |
+
|
| 221 |
+
def _answer_fn(query: str, chat_history: List[Dict[str, str]], scenario_tag: Optional[str] = None, emotion_tag: Optional[str] = None) -> Dict[str, Any]:
|
| 222 |
+
|
| 223 |
+
router_messages = [{"role": "user", "content": ROUTER_PROMPT.format(query=query)}]
|
| 224 |
+
query_type = call_llm(router_messages, temperature=0.0).strip().lower()
|
| 225 |
+
print(f"Query classified as: {query_type}")
|
| 226 |
+
|
| 227 |
+
system_message = SYSTEM_TEMPLATE.format(tone=tone, language=language, patient_name=patient_name or "the patient", caregiver_name=caregiver_name or "the caregiver", guardrails=SAFETY_GUARDRAILS)
|
| 228 |
+
messages = [{"role": "system", "content": system_message}]
|
| 229 |
+
messages.extend(chat_history)
|
| 230 |
+
|
| 231 |
+
# --- NEW 'general_knowledge_question' PATH ---
|
| 232 |
+
if "general_knowledge_question" in query_type:
|
| 233 |
+
user_prompt = ANSWER_TEMPLATE_GENERAL_KNOWLEDGE.format(question=query, language=language)
|
| 234 |
+
messages.append({"role": "user", "content": user_prompt})
|
| 235 |
+
answer = call_llm(messages, temperature=temperature)
|
| 236 |
+
return {"answer": answer, "sources": ["General Knowledge"]}
|
| 237 |
+
# --- END NEW PATH ---
|
| 238 |
+
|
| 239 |
+
elif "factual_question" in query_type:
|
| 240 |
+
# ... (This entire section for query expansion and factual search remains the same)
|
| 241 |
+
print(f"Performing query expansion for: '{query}'")
|
| 242 |
+
expansion_prompt = QUERY_EXPANSION_PROMPT.format(question=query)
|
| 243 |
+
expansion_response = call_llm([{"role": "user", "content": expansion_prompt}], temperature=0.1)
|
| 244 |
+
|
| 245 |
+
try:
|
| 246 |
+
clean_response = expansion_response.strip().replace("```json", "").replace("```", "")
|
| 247 |
+
expanded_queries = json.loads(clean_response)
|
| 248 |
+
search_queries = [query] + expanded_queries
|
| 249 |
+
except json.JSONDecodeError:
|
| 250 |
+
search_queries = [query]
|
| 251 |
+
|
| 252 |
+
print(f"Searching with queries: {search_queries}")
|
| 253 |
+
retriever_personal = vs_personal.as_retriever(search_kwargs={"k": 2})
|
| 254 |
+
retriever_general = vs_general.as_retriever(search_kwargs={"k": 2})
|
| 255 |
+
|
| 256 |
+
all_docs = []
|
| 257 |
+
for q in search_queries:
|
| 258 |
+
all_docs.extend(retriever_personal.invoke(q))
|
| 259 |
+
all_docs.extend(retriever_general.invoke(q))
|
| 260 |
+
|
| 261 |
+
context = _format_docs(all_docs, "(No relevant information found in the memory journal.)")
|
| 262 |
+
|
| 263 |
+
user_prompt = ANSWER_TEMPLATE_FACTUAL.format(context=context, question=query, language=language)
|
| 264 |
+
messages.append({"role": "user", "content": user_prompt})
|
| 265 |
+
answer = call_llm(messages, temperature=temperature)
|
| 266 |
+
return {"answer": answer, "sources": _format_sources(all_docs)}
|
| 267 |
+
|
| 268 |
+
elif "general_conversation" in query_type:
|
| 269 |
+
user_prompt = ANSWER_TEMPLATE_GENERAL.format(question=query, language=language)
|
| 270 |
+
messages.append({"role": "user", "content": user_prompt})
|
| 271 |
+
answer = call_llm(messages, temperature=temperature)
|
| 272 |
+
return {"answer": answer, "sources": []}
|
| 273 |
+
|
| 274 |
+
else: # Default to the original caregiving logic
|
| 275 |
+
# ... (This entire section for caregiving scenarios remains the same)
|
| 276 |
+
search_filter = {}
|
| 277 |
+
if scenario_tag and scenario_tag != "None":
|
| 278 |
+
search_filter["behaviors"] = scenario_tag.lower()
|
| 279 |
+
if emotion_tag and emotion_tag != "None":
|
| 280 |
+
search_filter["emotion"] = emotion_tag.lower()
|
| 281 |
+
|
| 282 |
+
if search_filter:
|
| 283 |
+
personal_docs = vs_personal.similarity_search(query, k=3, filter=search_filter)
|
| 284 |
+
general_docs = vs_general.similarity_search(query, k=3, filter=search_filter)
|
| 285 |
+
else:
|
| 286 |
+
retriever_personal = vs_personal.as_retriever(search_kwargs={"k": 3})
|
| 287 |
+
retriever_general = vs_general.as_retriever(search_kwargs={"k": 3})
|
| 288 |
+
personal_docs = retriever_personal.invoke(query)
|
| 289 |
+
general_docs = retriever_general.invoke(query)
|
| 290 |
+
|
| 291 |
+
personal_context = _format_docs(personal_docs, "(No relevant personal memories found.)")
|
| 292 |
+
general_context = _format_docs(general_docs, "(No general guidance found.)")
|
| 293 |
+
|
| 294 |
+
first_emotion = None
|
| 295 |
+
all_docs_care = personal_docs + general_docs
|
| 296 |
+
for doc in all_docs_care:
|
| 297 |
+
if "emotion" in doc.metadata and doc.metadata["emotion"]:
|
| 298 |
+
emotion_data = doc.metadata["emotion"]
|
| 299 |
+
if isinstance(emotion_data, list): first_emotion = emotion_data[0]
|
| 300 |
+
else: first_emotion = emotion_data
|
| 301 |
+
if first_emotion: break
|
| 302 |
+
|
| 303 |
+
emotions_context = render_emotion_guidelines(first_emotion or emotion_tag)
|
| 304 |
+
is_tagged_scenario = (scenario_tag and scenario_tag != "None") or (emotion_tag and emotion_tag != "None") or (first_emotion is not None)
|
| 305 |
+
template = ANSWER_TEMPLATE_ADQ if is_tagged_scenario else ANSWER_TEMPLATE_CALM
|
| 306 |
+
|
| 307 |
+
if template == ANSWER_TEMPLATE_ADQ:
|
| 308 |
+
user_prompt = template.format(general_context=general_context, personal_context=personal_context, question=query, scenario_tag=scenario_tag, emotions_context=emotions_context, role=role, language=language)
|
| 309 |
+
else:
|
| 310 |
+
combined_context = f"General Guidance:\n{general_context}\n\nPersonal Memories:\n{personal_context}"
|
| 311 |
+
user_prompt = template.format(context=combined_context, question=query, language=language)
|
| 312 |
+
|
| 313 |
+
messages.append({"role": "user", "content": user_prompt})
|
| 314 |
+
answer = call_llm(messages, temperature=temperature)
|
| 315 |
+
|
| 316 |
+
high_risk_scenarios = ["exit_seeking", "wandering", "elopement"]
|
| 317 |
+
if scenario_tag and scenario_tag.lower() in high_risk_scenarios:
|
| 318 |
+
answer += f"\n\n---\n{RISK_FOOTER}"
|
| 319 |
+
|
| 320 |
+
return {"answer": answer, "sources": _format_sources(all_docs_care)}
|
| 321 |
+
|
| 322 |
+
return _answer_fn
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
def answer_query(chain, question: str, **kwargs) -> Dict[str, Any]:
|
| 326 |
+
if not callable(chain): return {"answer": "[Error: RAG chain is not callable]", "sources": []}
|
| 327 |
+
chat_history, scenario_tag, emotion_tag = kwargs.get("chat_history", []), kwargs.get("scenario_tag"), kwargs.get("emotion_tag")
|
| 328 |
+
try:
|
| 329 |
+
return chain(question, chat_history=chat_history, scenario_tag=scenario_tag, emotion_tag=emotion_tag)
|
| 330 |
+
except Exception as e:
|
| 331 |
+
print(f"ERROR in answer_query: {e}")
|
| 332 |
+
return {"answer": f"[Error executing chain: {e}]", "sources": []}
|
| 333 |
+
|
| 334 |
+
# -----------------------------
|
| 335 |
+
# TTS & Transcription
|
| 336 |
+
# -----------------------------
|
| 337 |
+
def synthesize_tts(text: str, lang: str = "en"):
|
| 338 |
+
if not text or gTTS is None: return None
|
| 339 |
+
try:
|
| 340 |
+
fd, path = tempfile.mkstemp(suffix=".mp3")
|
| 341 |
+
os.close(fd)
|
| 342 |
+
tts = gTTS(text=text, lang=(lang or "en"))
|
| 343 |
+
tts.save(path)
|
| 344 |
+
return path
|
| 345 |
+
except Exception:
|
| 346 |
+
return None
|
| 347 |
+
|
| 348 |
+
def transcribe_audio(filepath: str, lang: str = "en"):
|
| 349 |
+
client = _openai_client()
|
| 350 |
+
if not client:
|
| 351 |
+
return "[Transcription failed: API key not configured]"
|
| 352 |
+
api_args = {"model": "whisper-1"}
|
| 353 |
+
if lang and lang != "auto":
|
| 354 |
+
api_args["language"] = lang
|
| 355 |
+
with open(filepath, "rb") as audio_file:
|
| 356 |
+
transcription = client.audio.transcriptions.create(file=audio_file, **api_args)
|
| 357 |
+
return transcription.text
|
| 358 |
+
|
alz_companion/prompts.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Prompts for the Alzheimer’s AI Companion.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
# ------------------------ Behaviour‑level tags ------------------------
|
| 6 |
+
BEHAVIOUR_TAGS = {
|
| 7 |
+
# Tags from "The Father"
|
| 8 |
+
"repetitive_questioning": ["validation", "gentle_redirection", "offer_distraction"],
|
| 9 |
+
"confusion": ["reassurance", "time_place_orientation", "photo_anchors"],
|
| 10 |
+
"wandering": ["walk_along_support", "simple_landmarks", "visual_cues", "safe_wandering_space"],
|
| 11 |
+
"agitation": ["de-escalating_tone", "validate_feelings", "reduce_stimulation", "simple_choices"],
|
| 12 |
+
"false_accusations": ["reassure_no_blame", "avoid_arguing", "redirect_activity"],
|
| 13 |
+
"address_memory_loss": ["encourage_ID_bracelet_or_GPS", "place_contact_info_in_wallet", "inform_trusted_neighbors", "avoid_quizzing_on_address"],
|
| 14 |
+
"hallucinations_delusions": ["avoid_arguing_or_correcting", "validate_the_underlying_emotion", "offer_reassurance_of_safety", "gently_redirect_to_real_activity", "check_for_physical_triggers"],
|
| 15 |
+
|
| 16 |
+
# Tags from "Still Alice" (and others for future use)
|
| 17 |
+
"exit_seeking": ["validation", "calm_presence", "safe_wandering_space", "environmental_cues"],
|
| 18 |
+
"aphasia": ["patience", "simple_language", "nonverbal_cues", "validation"],
|
| 19 |
+
"withdrawal": ["gentle_invitation", "calm_presence", "offer_familiar_comforts", "no_pressure"],
|
| 20 |
+
"affection": ["reciprocate_warmth", "positive_reinforcement", "simple_shared_activity"],
|
| 21 |
+
"sleep_disturbance": ["establish_calm_bedtime_routine", "limit_daytime_naps", "check_for_discomfort_or_pain"],
|
| 22 |
+
"anxiety": ["calm_reassurance", "simple_breathing_exercise", "reduce_environmental_stimuli"],
|
| 23 |
+
"depression_sadness": ["validate_feelings_of_sadness", "encourage_simple_pleasant_activity", "ensure_social_connection"],
|
| 24 |
+
"orientation_check": ["gentle_orientation_cues", "use_familiar_landmarks", "avoid_quizzing"],
|
| 25 |
+
|
| 26 |
+
# Tags from "Away from Her"
|
| 27 |
+
"misidentification": ["gently_correct_with_context", "use_photos_as_anchors", "respond_to_underlying_emotion", "avoid_insistent_correction"],
|
| 28 |
+
|
| 29 |
+
# Other useful tags
|
| 30 |
+
"sundowning_restlessness": ["predictable_routine", "soft_lighting", "low_stimulation", "familiar_music"],
|
| 31 |
+
"object_misplacement": ["nonconfrontational_search", "fixed_storage_spots"]
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
# ------------------------ Emotion styles & helpers ------------------------
|
| 35 |
+
EMOTION_STYLES = {
|
| 36 |
+
"confusion": {"tone": "calm, orienting, concrete", "playbook": ["Offer a simple time/place orientation cue (who/where/when).", "Reference one familiar anchor (photo/object/person).", "Use short sentences and one step at a time."]},
|
| 37 |
+
"fear": {"tone": "reassuring, safety-forward, gentle", "playbook": ["Acknowledge fear without contradiction.", "Provide a clear safety cue (e.g., 'You’re safe here with me').", "Reduce novelty and stimulation; suggest one safe action."]},
|
| 38 |
+
"anger": {"tone": "de-escalating, validating, low-arousal", "playbook": ["Validate the feeling; avoid arguing/correcting.", "Keep voice low and sentences short.", "Offer a simple choice to restore control (e.g., 'tea or water?')."]},
|
| 39 |
+
"sadness": {"tone": "warm, empathetic, gentle reminiscence", "playbook": ["Acknowledge loss/longing.", "Invite one comforting memory or familiar song.", "Keep pace slow; avoid tasking."]},
|
| 40 |
+
"warmth": {"tone": "affirming, appreciative", "playbook": ["Reflect gratitude and positive connection.", "Reinforce what’s going well.", "Keep it light; don’t overload with new info."]},
|
| 41 |
+
"joy": {"tone": "supportive, celebratory (but not overstimulating)", "playbook": ["Share the joy briefly; match energy gently.", "Offer a simple, pleasant follow-up activity.", "Avoid adding complex tasks."]},
|
| 42 |
+
"calm": {"tone": "matter-of-fact, concise, steady", "playbook": ["Keep instructions simple.", "Maintain steady pace.", "No extra soothing needed."]},
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
def render_emotion_guidelines(emotion: str | None) -> str:
|
| 46 |
+
e = (emotion or "").strip().lower()
|
| 47 |
+
if e not in EMOTION_STYLES:
|
| 48 |
+
return "Emotion: (auto)\nDesired tone: calm, clear.\nWhen replying, reassure if distress is apparent; prioritise validation and simple choices."
|
| 49 |
+
style = EMOTION_STYLES[e]
|
| 50 |
+
bullet = "\n".join([f"- {x}" for x in style["playbook"]])
|
| 51 |
+
return f"Emotion: {e}\nDesired tone: {style['tone']}\nWhen replying, follow:\n{bullet}"
|
| 52 |
+
|
| 53 |
+
# ------------------------ NLU Classification ------------------------
|
| 54 |
+
CLASSIFICATION_PROMPT = """You are an expert NLU engine. Your task is to analyze the user's query about a situation involving a person with Alzheimer's and classify it.
|
| 55 |
+
Identify the primary behavior from this list: {behavior_options}
|
| 56 |
+
Identify the primary emotion from this list: {emotion_options}
|
| 57 |
+
|
| 58 |
+
Respond ONLY with a single, valid JSON object with two keys: "detected_behavior" and "detected_emotion".
|
| 59 |
+
The values for these keys MUST be one of the options provided in the lists above, or "None" if no specific tag applies.
|
| 60 |
+
|
| 61 |
+
User Query: "{query}"
|
| 62 |
+
|
| 63 |
+
JSON Response:
|
| 64 |
+
"""
|
| 65 |
+
|
| 66 |
+
# ------------------------ Guardrails ------------------------
|
| 67 |
+
SAFETY_GUARDRAILS = """Never provide medical diagnoses or dosing. If a situation implies imminent risk (e.g., wandering/elopement, severe agitation, choking, falls), signpost immediate support from onsite staff or emergency services. Use respectful, person‑centred language. Keep guidance concrete and stepwise."""
|
| 68 |
+
|
| 69 |
+
# ------------------------ System & Answer Templates ------------------------
|
| 70 |
+
SYSTEM_TEMPLATE = """You are an Alzheimer’s caregiving companion. Address the patient as {patient_name} and the caregiver as {caregiver_name}. Ground every suggestion in retrieved evidence when possible. If unsure, say so plainly.
|
| 71 |
+
{guardrails}
|
| 72 |
+
--- IMPORTANT RULE ---
|
| 73 |
+
You MUST write your entire response in {language} ONLY. This is a strict instruction. Do not use any other language, even if the user or the retrieved context uses a different language. Your final output must be in {language}."""
|
| 74 |
+
|
| 75 |
+
ANSWER_TEMPLATE_CALM = """Context:
|
| 76 |
+
{context}
|
| 77 |
+
|
| 78 |
+
---
|
| 79 |
+
Question from user: {question}
|
| 80 |
+
|
| 81 |
+
---
|
| 82 |
+
Instructions:
|
| 83 |
+
Based on the context, write a gentle and supportive response in a single, natural-sounding paragraph.
|
| 84 |
+
Your response should:
|
| 85 |
+
1. Start by briefly and calmly acknowledging the user's situation or feeling.
|
| 86 |
+
2. Weave 2-3 practical, compassionate suggestions from the context into your paragraph. Do not use a numbered or bulleted list.
|
| 87 |
+
3. Conclude with a short, reassuring phrase.
|
| 88 |
+
4. You MUST use the retrieved context to directly address the user's specific **Question**.
|
| 89 |
+
Your response in {language}:"""
|
| 90 |
+
|
| 91 |
+
# For scenarios tagged with a specific behavior (e.g., agitation, confusion)
|
| 92 |
+
ANSWER_TEMPLATE_ADQ = """--- General Guidance from Knowledge Base ---
|
| 93 |
+
{general_context}
|
| 94 |
+
|
| 95 |
+
--- Relevant Personal Memories ---
|
| 96 |
+
{personal_context}
|
| 97 |
+
|
| 98 |
+
---
|
| 99 |
+
Care scenario: {scenario_tag}
|
| 100 |
+
Response Guidelines:
|
| 101 |
+
{emotions_context}
|
| 102 |
+
Question from user: {question}
|
| 103 |
+
|
| 104 |
+
---
|
| 105 |
+
Instructions:
|
| 106 |
+
Based on ALL the information above, write a **concise, warm, and validating** response for the {role} in a single, natural-sounding paragraph. **Keep the total response to 2-4 sentences.**
|
| 107 |
+
If possible, weave details from the 'Relevant Personal Memories' into your suggestions to make the response feel more personal and familiar.
|
| 108 |
+
Pay close attention to the Response Guidelines to tailor your tone.
|
| 109 |
+
Your response should follow this pattern:
|
| 110 |
+
1. Start by validating the user's feeling or concern with a unique, empathetic opening. DO NOT USE THE SAME OPENING PHRASE REPEATEDLY. Choose from different styles of openers, such as:
|
| 111 |
+
- Acknowledging the difficulty: "That sounds like a very challenging situation..."
|
| 112 |
+
- Expressing understanding: "I can see why that would be worrying..."
|
| 113 |
+
- Stating a shared goal: "Let's walk through how we can handle that..."
|
| 114 |
+
- Directly validating the feeling: "It's completely understandable to feel frustrated when..."
|
| 115 |
+
2. Gently offer **1-2 of the most important practical steps**, combining general guidance with personal memories where appropriate. Do not use a list.
|
| 116 |
+
3. If the scenario involves risk (like exit_seeking), subtly include a safety cue.
|
| 117 |
+
4. End with a compassionate, de-escalation phrase.
|
| 118 |
+
Your response in {language}:"""
|
| 119 |
+
|
| 120 |
+
RISK_FOOTER = """If safety is a concern right now, please seek immediate assistance from onsite staff or local emergency services."""
|
| 121 |
+
|
| 122 |
+
# ------------------------ Router & Specialized Templates ------------------------
|
| 123 |
+
|
| 124 |
+
# --- NEW: Template for expanding user queries for better retrieval ---
|
| 125 |
+
QUERY_EXPANSION_PROMPT = """You are a helpful AI assistant. Your task is to rephrase a user's question into 3 different, semantically similar questions to improve document retrieval.
|
| 126 |
+
Provide the rephrased questions as a JSON list of strings.
|
| 127 |
+
|
| 128 |
+
User Question: "{question}"
|
| 129 |
+
|
| 130 |
+
JSON List:
|
| 131 |
+
"""
|
| 132 |
+
|
| 133 |
+
# Template for routing/classifying the user's intent
|
| 134 |
+
ROUTER_PROMPT = """You are an expert NLU router. Your task is to classify the user's query into one of four categories:
|
| 135 |
+
1. `caregiving_scenario`: The user is describing a situation, asking for advice, or expressing a concern related to Alzheimer's or caregiving.
|
| 136 |
+
2. `factual_question`: The user is asking a direct question about a personal memory, person, or event that would be stored in the memory journal.
|
| 137 |
+
3. `general_knowledge_question`: The user is asking a general knowledge question about the world, facts, or topics not related to personal memories or caregiving (e.g., 'What is the capital of France?', 'Who directed the movie Inception?').
|
| 138 |
+
4. `general_conversation`: The user is making a general conversational remark, like a greeting, a thank you, or a simple statement that does not require a knowledge base lookup.
|
| 139 |
+
|
| 140 |
+
User Query: "{query}"
|
| 141 |
+
|
| 142 |
+
Respond with ONLY a single category name from the list above.
|
| 143 |
+
Category: """
|
| 144 |
+
|
| 145 |
+
# Template for answering direct factual questions
|
| 146 |
+
ANSWER_TEMPLATE_FACTUAL = """Context:
|
| 147 |
+
{context}
|
| 148 |
+
|
| 149 |
+
---
|
| 150 |
+
Question from user: {question}
|
| 151 |
+
|
| 152 |
+
---
|
| 153 |
+
Instructions:
|
| 154 |
+
Based on the provided context, directly and concisely answer the user's question.
|
| 155 |
+
- If the context contains the answer, state it clearly and naturally.
|
| 156 |
+
- If the context does not contain the answer, respond in a warm and friendly tone that you couldn't find a memory of that topic and gently ask if the user would like to talk more about it or add it as a new memory.
|
| 157 |
+
- Do not offer advice or suggestions unless they are part of the retrieved context.
|
| 158 |
+
Your response MUST be in {language}:"""
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
# --- NEW: Template for answering general knowledge questions ---
|
| 162 |
+
# Template for answering general knowledge questions
|
| 163 |
+
ANSWER_TEMPLATE_GENERAL_KNOWLEDGE = """You are a factual answering engine.
|
| 164 |
+
Your task is to directly answer the user's general knowledge question based on your training data.
|
| 165 |
+
|
| 166 |
+
Instructions:
|
| 167 |
+
- Be factual and concise. Go straight to the answer.
|
| 168 |
+
- If the answer requires a list of examples, provide a maximum of 3 items. Do not use numbering.
|
| 169 |
+
- Do NOT include apologies or disclaimers about your knowledge cutoff date.
|
| 170 |
+
# - Do NOT recommend external websites or other services.
|
| 171 |
+
# - Do NOT ask conversational follow-up questions.
|
| 172 |
+
|
| 173 |
+
User's Question: "{question}"
|
| 174 |
+
|
| 175 |
+
Your factual response in {language}:"""
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
# Template for general, non-RAG conversation
|
| 179 |
+
ANSWER_TEMPLATE_GENERAL = """You are a warm and friendly AI companion. The user has just said: "{question}".
|
| 180 |
+
Respond in a brief, natural, and conversational way. Do not try to provide caregiving advice unless the user asks for it.
|
| 181 |
+
Your response MUST be in {language}:"""
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
# ------------------------ Convenience exports ------------------------
|
| 185 |
+
__all__ = [
|
| 186 |
+
"SYSTEM_TEMPLATE", "ANSWER_TEMPLATE_CALM", "ANSWER_TEMPLATE_ADQ",
|
| 187 |
+
"SAFETY_GUARDRAILS", "RISK_FOOTER", "BEHAVIOUR_TAGS", "EMOTION_STYLES",
|
| 188 |
+
"render_emotion_guidelines", "CLASSIFICATION_PROMPT",
|
| 189 |
+
|
| 190 |
+
# --- New additions ---
|
| 191 |
+
"QUERY_EXPANSION_PROMPT"
|
| 192 |
+
"ROUTER_PROMPT",
|
| 193 |
+
"ANSWER_TEMPLATE_FACTUAL",
|
| 194 |
+
"ANSWER_TEMPLATE_GENERAL_KNOWLEDGE",
|
| 195 |
+
"ANSWER_TEMPLATE_GENERAL"
|
| 196 |
+
]
|