Seth0330 commited on
Commit
8e1a014
·
verified ·
1 Parent(s): 0f68c48

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +529 -261
app.py CHANGED
@@ -1,274 +1,542 @@
1
- import os, io, base64, json, time, random
2
- from typing import Optional, Dict, Any, List, Tuple
3
- from urllib.parse import quote_plus
4
-
5
- from fastapi import FastAPI, Request, BackgroundTasks
6
- from fastapi.responses import PlainTextResponse
7
- import httpx
8
- from bs4 import BeautifulSoup
9
  from PIL import Image
 
10
 
11
- from twilio.rest import Client as TwilioClient
 
 
12
 
13
- # OpenAI
14
- from openai import OpenAI
15
- oai_client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
16
-
17
- # LangChain minimal (structured output)
18
- from langchain_openai import ChatOpenAI
19
- from langchain_core.pydantic_v1 import BaseModel, Field
20
- from langchain_core.prompts import ChatPromptTemplate
21
-
22
- # Search tools
23
- from duckduckgo_search import DDGS
24
- try:
25
- from tavily import TavilyClient
26
- _HAS_TAVILY = True
27
- except Exception:
28
- _HAS_TAVILY = False
29
-
30
- app = FastAPI(title="SAVE SMS Webhook (Async Reply)")
31
-
32
- # ---------- Twilio client ----------
33
- TW_SID = os.getenv("TWILIO_ACCOUNT_SID", "")
34
- TW_TOKEN = os.getenv("TWILIO_AUTH_TOKEN", "")
35
- TW_FROM = os.getenv("TWILIO_FROM", "") # e.g., +12175898085
36
- _twilio_ok = bool(TW_SID and TW_TOKEN and TW_FROM)
37
- twilio_client = TwilioClient(TW_SID, TW_TOKEN) if _twilio_ok else None
38
-
39
- # ---------- shared helpers ----------
40
- lc_llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
41
-
42
- MERCHANT_DOMAINS = (
43
- "walmart.ca","realcanadiansuperstore.ca","amazon.ca","metro.ca",
44
- "nofrills.ca","freshco.com","well.ca","costco.ca","iga.net","londondrugs.com"
45
  )
46
 
47
- class Offer(BaseModel):
48
- merchant: str = Field(...)
49
- title: str = Field(...)
50
- price: float = Field(...)
51
- url: str = Field(...)
52
-
53
- extract_prompt = ChatPromptTemplate.from_messages([
54
- ("system","Extract one best CAD offer for the queried item from the page text. "
55
- "Return JSON: merchant,title,price(float),url. If none, return empty with price 0."),
56
- ("human","Query: {query}\nURL: {url}\n--- PAGE TEXT ---\n{text}\n--- END ---")
57
- ])
58
- chain_extract = extract_prompt | lc_llm.with_structured_output(Offer)
59
-
60
- def img_or_pdf_to_image_bytes(data: bytes, filename: str) -> bytes:
61
- name = (filename or "").lower()
62
- if name.endswith((".jpg",".jpeg",".png",".webp")):
63
- img = Image.open(io.BytesIO(data)).convert("RGB")
64
- buf = io.BytesIO(); img.save(buf, format="JPEG", quality=90); return buf.getvalue()
65
- if name.endswith(".pdf"):
66
- try:
67
- img = Image.open(io.BytesIO(data)).convert("RGB")
68
- buf = io.BytesIO(); img.save(buf, format="JPEG", quality=90); return buf.getvalue()
69
- except Exception:
70
- return data
71
- return data
72
-
73
- def b64_data_uri(data: bytes, mime: str) -> str:
74
- return f"data:{mime};base64," + base64.b64encode(data).decode("utf-8")
75
-
76
- def call_openai_vision_for_receipt(image_bytes: bytes) -> Dict[str, Any]:
77
- is_pdf = image_bytes[0:4] == b"%PDF"
78
- mime = "application/pdf" if is_pdf else "image/jpeg"
79
- system = ("You are a strict, no-chitchat receipt parser for Canadian grocery receipts. "
80
- "Return ONLY JSON; prices in CAD.")
81
- user_prompt = """
82
- { "store":{"name":"string","address":"string|null","date":"YYYY-MM-DD|null"},
83
- "items":[{"name":"string","size":"string|null","qty":1,"unit_price":0.00,"line_total":0.00}],
84
- "subtotal":0.00,"tax":0.00,"total":0.00 }
85
- Rules: shopper-friendly names; qty>=1; unit_price before tax; line_total=qty*unit_price; use null if missing.
86
- Return ONLY JSON.
87
- """
88
- resp = oai_client.chat.completions.create(
89
- model="gpt-4o-mini", temperature=0,
90
- messages=[{"role":"system","content":system},
91
- {"role":"user","content":[
92
- {"type":"text","text":user_prompt},
93
- {"type":"image_url","image_url":{"url":b64_data_uri(image_bytes,mime)}}
94
- ]}]
95
- )
96
- s = resp.choices[0].message.content.strip()
97
- if s.startswith("```"):
98
- s = s.split("```",2)[1]
99
- if s.lower().startswith("json"): s = s.split("\n",1)[1]
100
- return json.loads(s)
101
-
102
- def _fallback_store_search_urls(q: str, k: int = 5) -> List[str]:
103
- qenc = quote_plus(q)
104
- urls = [
105
- f"https://www.walmart.ca/search?q={qenc}",
106
- f"https://www.realcanadiansuperstore.ca/search?search-bar={qenc}",
107
- f"https://www.amazon.ca/s?k={qenc}",
108
- f"https://www.metro.ca/en/online-grocery/search?filter.query={qenc}",
109
- f"https://www.nofrills.ca/search?search-bar={qenc}",
110
- f"https://www.freshco.com/en/search?search-bar={qenc}",
111
- f"https://well.ca/searchresult.html?keyword={qenc}",
112
- f"https://www.costco.ca/CatalogSearch?dept=All&keyword={qenc}",
113
- f"https://www.iga.net/en/search?search={qenc}",
114
- f"https://www.londondrugs.com/search?searchTerm={qenc}",
115
- ]
116
- return urls[:k]
117
-
118
- def _search_web(query: str, k: int = 5) -> List[str]:
119
- urls: List[str] = []
120
- if _HAS_TAVILY and os.getenv("TAVILY_API_KEY"):
121
- try:
122
- tv = TavilyClient(os.getenv("TAVILY_API_KEY"))
123
- res = tv.search(query=f"{query} price", search_depth="basic", max_results=k,
124
- include_domains=list(MERCHANT_DOMAINS))
125
- for r in res.get("results", []):
126
- if r.get("url"): urls.append(r["url"])
127
- except Exception: pass
128
- if not urls:
129
- try:
130
- with DDGS() as ddgs:
131
- q = f"{query} price"
132
- for attempt in range(3):
133
- for r in ddgs.text(q, region="ca-en", max_results=k):
134
- u = r.get("href") or r.get("url")
135
- if u: urls.append(u)
136
- if urls: break
137
- time.sleep(0.6*(2**attempt)+random.random()*0.3)
138
- except Exception: urls=[]
139
- urls = [u for u in urls if any(dom in u for dom in MERCHANT_DOMAINS)]
140
- if not urls: urls = _fallback_store_search_urls(query, k=k)
141
- seen, out = set(), []
142
- for u in urls:
143
- if u not in seen: out.append(u); seen.add(u)
144
- return out[:k]
145
-
146
- def _fetch_text(url: str, timeout=15) -> str:
147
  try:
148
- headers = {"User-Agent":"Mozilla/5.0 (compatible; PriceAgent/1.0)"}
149
- with httpx.Client(follow_redirects=True, timeout=timeout) as client:
150
- r = client.get(url, headers=headers)
151
- soup = BeautifulSoup(r.text,"html.parser")
152
- for t in soup(["script","style","noscript"]): t.decompose()
153
- return " ".join(soup.get_text(separator=" ").split())[:12000]
154
- except Exception: return ""
155
-
156
- def langchain_price_lookup(item_name: str) -> Optional[Dict[str, Any]]:
157
- urls = _search_web(item_name, k=5)
158
- best: Optional[Offer] = None
159
- for u in urls:
160
- text = _fetch_text(u)
161
- if not text: continue
162
- try:
163
- offer: Offer = chain_extract.invoke({"query": item_name, "url": u, "text": text})
164
- except Exception:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165
  continue
166
- if not offer or not offer.price or offer.price <= 0: continue
167
- if best is None or offer.price < best.price: best = offer
168
- if not best: return None
169
- return {"title": best.title or item_name, "price": float(best.price),
170
- "source": best.merchant or "Other store", "link": best.url or urls[0]}
171
-
172
- def normalize_query(item: Dict[str, Any]) -> str:
173
- base = item.get("name") or ""; size = item.get("size") or ""
174
- q = f"{base} {size}".strip()
175
- return " ".join([t for t in q.split() if len(t) > 1])
176
-
177
- def research_prices(items: List[Dict[str, Any]], max_items=6) -> List[Dict[str, Any]]:
178
- out=[]
179
- for it in items[:max_items]:
180
- name = normalize_query(it)
181
- if not name: continue
182
- unit = it.get("unit_price")
183
- offer = langchain_price_lookup(name)
184
- if not offer: continue
185
- cheaper = isinstance(unit,(int,float)) and offer["price"] < float(unit)-0.005
186
- out.append({"item_name":it.get("name"),"receipt_unit_price":unit,
187
- "found_price":offer["price"],"found_store":offer["source"],
188
- "found_title":offer["title"],"found_link":offer["link"],
189
- "is_cheaper":cheaper})
190
- time.sleep(0.25)
191
- return out
192
-
193
- def compute_savings(receipt: Dict[str, Any], found: List[Dict[str, Any]]) -> Tuple[float,List[Dict[str, Any]]]:
194
- cheaper=[f for f in found if f.get("is_cheaper")]
195
- s=0.0
196
- for f in cheaper:
197
- try: s += float(f["receipt_unit_price"]) - float(f["found_price"])
198
- except Exception: pass
199
- return round(s,2), cheaper
200
-
201
- def format_five_lines(receipt: Dict[str, Any], savings: float, cheaper_list: List[Dict[str, Any]]) -> str:
202
- store = (receipt.get("store") or {}).get("name") or "your store"
203
- total_val = receipt.get("total") or receipt.get("subtotal")
204
- total_txt = "N/A"
205
- try: total_txt = f"${float(str(total_val).replace('$','').strip()):.2f}"
206
- except Exception: pass
207
- lines = [
208
- f"Receipt read: {store}, total {total_txt}.",
209
- f"I found potential savings of ${savings:.2f} by checking other stores.",
210
- ]
211
- if cheaper_list:
212
- bullets=[]
213
- for f in cheaper_list[:3]:
214
- item=f.get("item_name") or "Item"; shop=f.get("found_store") or "other store"
215
- price=float(f.get("found_price") or 0.0)
216
- bullets.append(f"{item} @ {shop} for ${price:.2f}")
217
- lines.append("Cheaper picks: " + "; ".join(bullets) + ".")
218
- else:
219
- lines.append("No clearly cheaper matches found right now for your items.")
220
- lines.append("Reply 'DEALS' anytime to get weekly picks tailored to your receipts.")
221
- return "\n".join(lines[:5])
222
-
223
- # ---------- background job ----------
224
- def process_and_reply(media_url: str, to_number: str):
225
- if not _twilio_ok:
226
- return
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
227
  try:
228
- with httpx.Client(follow_redirects=True, timeout=20) as client:
229
- r = client.get(media_url); r.raise_for_status()
230
- content = r.content
231
- img_bytes = img_or_pdf_to_image_bytes(content, "mms.jpg")
232
- receipt = call_openai_vision_for_receipt(img_bytes)
233
- items = receipt.get("items") or []
234
- if not items:
235
- msg = "I couldn't read items. Send a clearer photo."
236
- else:
237
- found = research_prices(items)
238
- savings, cheaper = compute_savings(receipt, found)
239
- msg = format_five_lines(receipt, savings, cheaper)
240
- msg = "\n".join(msg.split("\n")[:5])[:1400]
 
 
 
 
 
 
241
  except Exception as e:
242
- msg = f"Processing error: {e}"
 
 
 
 
 
243
 
244
- # outbound SMS
245
  try:
246
- twilio_client.messages.create(to=to_number, from_=TW_FROM, body=msg)
247
- except Exception:
248
- pass
249
-
250
- # ---------- HTTP routes ----------
251
- @app.get("/sms")
252
- async def sms_health():
253
- return PlainTextResponse("SMS webhook is up (POST only).", media_type="text/plain")
254
-
255
- @app.post("/sms")
256
- async def sms_webhook(request: Request, background_tasks: BackgroundTasks):
257
- form = dict(await request.form())
258
- from_number = form.get("From", "")
259
- num_media = int(form.get("NumMedia","0") or "0")
260
- media_url = form.get("MediaUrl0") if num_media > 0 else None
261
-
262
- # quick acknowledgement to beat 15s timeout
263
- if not media_url:
264
- ack = "<Response><Message>Please MMS a clear photo of your grocery receipt to analyze savings.</Message></Response>"
265
- return PlainTextResponse(ack, media_type="application/xml")
266
-
267
- # run heavy work in background, then send outbound SMS
268
- if _twilio_ok and from_number:
269
- background_tasks.add_task(process_and_reply, media_url, from_number)
270
- reply = "<Response><Message>Got it—processing your receipt now. You’ll get a follow-up text shortly.</Message></Response>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
271
  else:
272
- reply = "<Response><Message>Got the image—backend SMS sending is misconfigured. Set TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN, TWILIO_FROM.</Message></Response>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
273
 
274
- return PlainTextResponse(reply, media_type="application/xml")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import io
3
+ import base64
4
+ import textwrap
5
+ import requests
6
+ import streamlit as st
 
 
7
  from PIL import Image
8
+ from openai import OpenAI
9
 
10
+ # =========================
11
+ # CONFIG
12
+ # =========================
13
 
14
+ st.set_page_config(
15
+ page_title="CareCall AI (Canada) - Agentic Health Info",
16
+ page_icon="🩺",
17
+ layout="centered"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  )
19
 
20
+ OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
21
+ if not OPENAI_API_KEY:
22
+ st.error("Missing OPENAI_API_KEY. Please set it in your Space secrets.")
23
+ client = OpenAI(api_key=OPENAI_API_KEY) if OPENAI_API_KEY else None
24
+
25
+ # Models
26
+ VISION_MODEL = "gpt-4.1-mini" # vision-capable
27
+ REASONING_MODEL = "gpt-4.1-mini" # reasoning / agent
28
+ ASR_MODEL = "whisper-1" # transcription
29
+
30
+ # Health Canada Drug Product Database API (public)
31
+ DPD_BASE_URL = "https://health-products.canada.ca/api/drug/drugproduct"
32
+
33
+ # Recalls & Safety Alerts JSON feed (configurable; keep overridable)
34
+ RECALLS_FEED_URL = os.getenv(
35
+ "RECALLS_FEED_URL",
36
+ "https://recalls-rappels.canada.ca/static-data/items-en.json"
37
+ )
38
+
39
+ # CIHI wait-times info (general reference, not live triage)
40
+ WAIT_TIMES_INFO_URL = "https://www.cihi.ca/en/topics/access-and-wait-times"
41
+
42
+ # Postal code to province mapping (by first letter)
43
+ POSTAL_PREFIX_TO_PROVINCE = {
44
+ "A": "Newfoundland and Labrador",
45
+ "B": "Nova Scotia",
46
+ "C": "Prince Edward Island",
47
+ "E": "New Brunswick",
48
+ "G": "Quebec",
49
+ "H": "Quebec",
50
+ "J": "Quebec",
51
+ "K": "Ontario",
52
+ "L": "Ontario",
53
+ "M": "Ontario",
54
+ "N": "Ontario",
55
+ "P": "Ontario",
56
+ "R": "Manitoba",
57
+ "S": "Saskatchewan",
58
+ "T": "Alberta",
59
+ "V": "British Columbia",
60
+ "X": "Northwest Territories / Nunavut",
61
+ "Y": "Yukon"
62
+ }
63
+
64
+
65
+ # =========================
66
+ # HELPERS
67
+ # =========================
68
+
69
+ def safe_get_json(url, params=None, timeout=6):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
  try:
71
+ resp = requests.get(url, params=params, timeout=timeout)
72
+ if resp.ok:
73
+ return resp.json()
74
+ except Exception:
75
+ return None
76
+ return None
77
+
78
+
79
+ def shorten(text, max_chars=400):
80
+ if not text:
81
+ return ""
82
+ text = text.strip()
83
+ if len(text) <= max_chars:
84
+ return text
85
+ return text[: max_chars - 3].rstrip() + "..."
86
+
87
+
88
+ # =========================
89
+ # REGION CONTEXT (POSTAL CODE)
90
+ # =========================
91
+
92
+ def infer_province_from_postal(postal_code: str):
93
+ """
94
+ Very simple FSA-based province inference using the first character.
95
+ """
96
+ if not postal_code:
97
+ return None
98
+ pc = postal_code.strip().upper().replace(" ", "")
99
+ if not pc:
100
+ return None
101
+ first = pc[0]
102
+ return POSTAL_PREFIX_TO_PROVINCE.get(first)
103
+
104
+
105
+ def tool_region_context(postal_code: str) -> str:
106
+ """
107
+ Build region-aware context for the agent:
108
+ - Province guess (if possible)
109
+ - Telehealth-style guidance
110
+ - Where to check local wait times / services
111
+ """
112
+ province = infer_province_from_postal(postal_code)
113
+ if not postal_code:
114
+ base = "No postal code provided. Provide general Canada-wide guidance."
115
+ return base
116
+
117
+ if not province:
118
+ return (
119
+ f"Postal code '{postal_code}' could not be confidently mapped. "
120
+ "Provide generic Canada-wide guidance and suggest checking local provincial health websites."
121
+ )
122
+
123
+ # Generic, safe phrasing without hard-coding all numbers:
124
+ telehealth_note = (
125
+ f"In {province}, guide the user to their official provincial Telehealth or 811-style nurse advice line "
126
+ "and ministry of health website for region-specific instructions."
127
+ )
128
+
129
+ wait_note = (
130
+ "Advise the user that approximate wait-time and access information may be available via their "
131
+ "provincial or regional health authority website, and through CIHI's access and wait-times resources. "
132
+ f"Reference (non-exclusively): {WAIT_TIMES_INFO_URL}"
133
+ )
134
+
135
+ return (
136
+ f"User-provided postal code: '{postal_code}' mapped to: {province}.\n"
137
+ f"{telehealth_note}\n"
138
+ f"{wait_note}"
139
+ )
140
+
141
+
142
+ # =========================
143
+ # TOOL: DRUG PRODUCT LOOKUP (DPD)
144
+ # =========================
145
+
146
+ def tool_lookup_drug_products(candidates):
147
+ if not candidates:
148
+ return "No clear medication names detected that require a drug database lookup."
149
+
150
+ found_lines = []
151
+ for name in candidates[:5]:
152
+ params = {
153
+ "lang": "en",
154
+ "type": "json",
155
+ "brandname": name
156
+ }
157
+ data = safe_get_json(DPD_BASE_URL, params=params)
158
+ if not data:
159
  continue
160
+
161
+ unique_brands = set()
162
+ for item in data[:3]:
163
+ b = (
164
+ item.get("brand_name")
165
+ or item.get("brandname")
166
+ or name
167
+ )
168
+ if b:
169
+ unique_brands.add(b)
170
+
171
+ if unique_brands:
172
+ found_lines.append(
173
+ f"- Found Health Canada Drug Product Database entries related to '{name}' "
174
+ f"(examples: {', '.join(sorted(unique_brands))})."
175
+ )
176
+
177
+ if not found_lines:
178
+ return (
179
+ "No strong matches found in the Drug Product Database for the detected terms. "
180
+ "Users should verify drug details directly via the official Health Canada DPD search."
181
+ )
182
+
183
+ found_lines.append(
184
+ "For definitive product information, ingredients, status, and monographs, "
185
+ "users must consult the official Health Canada Drug Product Database."
186
+ )
187
+ return "\n".join(found_lines)
188
+
189
+
190
+ # =========================
191
+ # TOOL: RECALLS & SAFETY ALERTS
192
+ # =========================
193
+
194
+ def tool_get_recent_recalls_snippet():
195
+ data = safe_get_json(RECALLS_FEED_URL)
196
+ if not data or not isinstance(data, list):
197
+ return (
198
+ "Unable to fetch recent recalls. Direct users to the official Government of Canada "
199
+ "Recalls and Safety Alerts website for up-to-date information."
200
+ )
201
+
202
+ items = data[:5]
203
+ lines = ["Recent recalls and safety alerts (high-level snapshot):"]
204
+ for item in items:
205
+ title = (
206
+ item.get("title")
207
+ or item.get("english_title")
208
+ or "Recall / Alert"
209
+ )
210
+ date = item.get("date_published") or item.get("date") or ""
211
+ category = item.get("category") or item.get("type") or ""
212
+ lines.append(f"- {shorten(title, 90)} ({category}, {date})")
213
+ lines.append(
214
+ "For full details or to check a specific product, direct users to the official Recalls and Safety Alerts portal."
215
+ )
216
+ return "\n".join(lines)
217
+
218
+
219
+ # =========================
220
+ # TOOL: WAIT TIMES AWARENESS (GENERIC)
221
+ # =========================
222
+
223
+ def tool_get_wait_times_awareness():
224
+ return textwrap.dedent(f"""
225
+ Use open Canadian data conceptually:
226
+ - CIHI and several provinces publish indicators and dashboards for emergency room, surgery, and diagnostic wait times.
227
+ - These are averages and not real-time triage.
228
+ Guidance:
229
+ - For non-urgent issues: consider checking local clinic or urgent care centre information online.
230
+ - For urgent symptoms: do not spend time checking wait-time tools; go to the nearest ER or call 911.
231
+ Reference (general): {WAIT_TIMES_INFO_URL}
232
+ """).strip()
233
+
234
+
235
+ # =========================
236
+ # TOOL: CANDIDATE TERM EXTRACTION
237
+ # =========================
238
+
239
+ def extract_candidate_terms(text: str):
240
+ if not text:
241
+ return []
242
+ tokens = []
243
+ for tok in text.split():
244
+ clean = "".join(ch for ch in tok if ch.isalnum())
245
+ if len(clean) > 3 and clean[0].isalpha() and clean[0].isupper():
246
+ tokens.append(clean)
247
+ return sorted(set(tokens))[:10]
248
+
249
+
250
+ # =========================
251
+ # OPENAI: VISION, ASR, AGENT
252
+ # =========================
253
+
254
+ def call_vision_summarizer(image_bytes: bytes) -> str:
255
+ if not client or not image_bytes:
256
+ return ""
257
+
258
+ b64 = base64.b64encode(image_bytes).decode("utf-8")
259
+ prompt = (
260
+ "You are a cautious Canadian health-information assistant.\n"
261
+ "Look at the image and ONLY describe observable details in neutral terms.\n"
262
+ "- If it's a skin issue: describe general appearance (location/size/colour/pattern). Do NOT name diseases.\n"
263
+ "- If it's a pill bottle or box: note visible drug name/strength/marks if legible.\n"
264
+ "- If it's a health letter/report: state the type (e.g., appointment letter, lab report) without exposing identifiers.\n"
265
+ "- If it's irrelevant or unclear: say it is not medically interpretable.\n"
266
+ "Never diagnose. Never prescribe. Keep within ~80-100 words."
267
+ )
268
+
269
  try:
270
+ resp = client.chat.completions.create(
271
+ model=VISION_MODEL,
272
+ messages=[
273
+ {
274
+ "role": "user",
275
+ "content": [
276
+ {"type": "text", "text": prompt},
277
+ {
278
+ "type": "image_url",
279
+ "image_url": {
280
+ "url": f"data:image/jpeg;base64,{b64}"
281
+ },
282
+ },
283
+ ],
284
+ }
285
+ ],
286
+ temperature=0.2,
287
+ )
288
+ return resp.choices[0].message.content.strip()
289
  except Exception as e:
290
+ return f"(Vision analysis unavailable: {e})"
291
+
292
+
293
+ def call_asr(audio_file) -> str:
294
+ if not client or not audio_file:
295
+ return ""
296
 
 
297
  try:
298
+ audio_bytes = audio_file.read()
299
+ bio = io.BytesIO(audio_bytes)
300
+ bio.name = audio_file.name or "voice.wav"
301
+
302
+ transcript = client.audio.transcriptions.create(
303
+ model=ASR_MODEL,
304
+ file=bio
305
+ )
306
+ text = getattr(transcript, "text", None)
307
+ if isinstance(text, str):
308
+ return text.strip()
309
+ return str(transcript)
310
+ except Exception as e:
311
+ return f"(Transcription unavailable: {e})"
312
+
313
+
314
+ def call_reasoning_agent(
315
+ user_text: str,
316
+ voice_text: str,
317
+ vision_summary: str,
318
+ dpd_context: str,
319
+ recalls_context: str,
320
+ wait_times_context: str,
321
+ region_context: str,
322
+ ) -> str:
323
+ if not client:
324
+ return (
325
+ "AI backend not configured. With a valid API key, this agent would combine your inputs and "
326
+ "open-data context into cautious, non-diagnostic guidance."
327
+ )
328
+
329
+ narrative_parts = []
330
+ if user_text.strip():
331
+ narrative_parts.append("Typed description:\n" + user_text.strip())
332
+ if voice_text.strip():
333
+ narrative_parts.append("Transcribed voice note:\n" + voice_text.strip())
334
+ if not narrative_parts:
335
+ narrative_parts.append("No textual description provided.")
336
+ narrative = "\n\n".join(narrative_parts)
337
+
338
+ system_prompt = """
339
+ You are CareCall AI, an agentic Canadian health information assistant.
340
+
341
+ Rules:
342
+ - You DO NOT diagnose.
343
+ - You DO NOT prescribe or give specific dosing.
344
+ - You DO NOT claim exact real-time wait times.
345
+ - You MAY:
346
+ - Summarize concerns neutrally.
347
+ - Map them to broad, possible categories (e.g., irritation, infection, allergy, medication question, admin/letter).
348
+ - Use the provided Drug Database context, Recalls context, wait-time awareness, and region context
349
+ to make the guidance safer and more relevant.
350
+ - Suggest next steps:
351
+ * Self-care for clearly mild issues,
352
+ * Pharmacist / family doctor / walk-in clinic,
353
+ * Provincial Telehealth / 811-style nurse line,
354
+ * ER / 911 if red-flag symptoms are present.
355
+ - Red flags (e.g., chest pain, difficulty breathing, signs of stroke, heavy bleeding, suicidal thoughts,
356
+ severe abdominal pain, high fever with stiff neck, major trauma, rapidly worsening symptoms)
357
+ -> clearly recommend ER/911.
358
+ - Always mention that official sources include:
359
+ Health Canada, Public Health Agency of Canada, CIHI, and the user's provincial health services.
360
+ - Max 350 words. Clear, calm, practical.
361
+ """
362
+
363
+ user_prompt = f"""
364
+ INPUT TO THE AGENT
365
+
366
+ 1) User narrative:
367
+ {narrative}
368
+
369
+ 2) Vision summary:
370
+ {vision_summary or "No image or no usable visual details."}
371
+
372
+ 3) Drug Product Database context:
373
+ {dpd_context}
374
+
375
+ 4) Recalls & Safety Alerts context:
376
+ {recalls_context}
377
+
378
+ 5) Wait-time awareness context:
379
+ {wait_times_context}
380
+
381
+ 6) Region (postal-code-based) context:
382
+ {region_context}
383
+
384
+ TASK
385
+
386
+ Using ONLY the above information:
387
+
388
+ 1. Briefly summarize what the user seems worried about (neutral; no diagnosis).
389
+ 2. Describe, in broad terms, what categories of issues it could relate to, without asserting a specific disease.
390
+ 3. Provide clear next-step options in bullet form, aligned with Canadian context and (if available) their province:
391
+ - When self-care might be reasonable.
392
+ - When to talk to a pharmacist, family doctor, or walk-in clinic.
393
+ - When to call their provincial Telehealth / nurse line.
394
+ - When to bypass everything and go to ER / call 911.
395
+ 4. If any medication/recall hints appear, remind them to confirm via Health Canada resources and follow professional advice.
396
+ 5. End with a concise disclaimer that this is informational only and not medical care.
397
+ """
398
+
399
+ try:
400
+ resp = client.chat.completions.create(
401
+ model=REASONING_MODEL,
402
+ messages=[
403
+ {"role": "system", "content": system_prompt},
404
+ {"role": "user", "content": user_prompt},
405
+ ],
406
+ temperature=0.35,
407
+ )
408
+ return resp.choices[0].message.content.strip()
409
+ except Exception as e:
410
+ return (
411
+ f"(Reasoning agent unavailable: {e})\n\n"
412
+ "Please contact a licensed provider, Telehealth, or emergency services as appropriate."
413
+ )
414
+
415
+
416
+ # =========================
417
+ # STREAMLIT UI
418
+ # =========================
419
+
420
+ st.title("CareCall AI (Canada)")
421
+ st.caption("Agentic, vision + voice + postal-aware health info companion. Not a doctor. Not a diagnosis.")
422
+
423
+ st.markdown(
424
+ """
425
+ This demo:
426
+ - Accepts a photo (rash, pill label, letter, etc.).
427
+ - Accepts a voice note and/or typed description.
428
+ - Optionally uses your **postal code** to tailor guidance to your province.
429
+ - Uses:
430
+ - OpenAI vision (for safe visual summary),
431
+ - Whisper (for transcription),
432
+ - Canadian open-data style tools (Drug Product Database, recalls, wait-time context),
433
+ - A reasoning agent with strict safety rules.
434
+ """
435
+ )
436
+
437
+ # Postal code
438
+ st.markdown("### 1. Postal code (optional, Canada only)")
439
+ postal_code = st.text_input(
440
+ "Enter your Canadian postal code (e.g., M5V 2T6). Leave blank if you prefer not to share.",
441
+ max_chars=7
442
+ )
443
+
444
+ # Image upload
445
+ st.markdown("### 2. Upload an image (optional)")
446
+ image_file = st.file_uploader(
447
+ "Upload a photo (JPG/PNG). Avoid sharing highly identifying or explicit content.",
448
+ type=["jpg", "jpeg", "png"],
449
+ accept_multiple_files=False,
450
+ )
451
+
452
+ image_bytes = None
453
+ if image_file is not None:
454
+ try:
455
+ img = Image.open(image_file)
456
+ st.image(img, caption="Image received", use_column_width=True)
457
+ buf = io.BytesIO()
458
+ img.save(buf, format="JPEG")
459
+ image_bytes = buf.getvalue()
460
+ except Exception as e:
461
+ st.warning(f"Could not read that image: {e}")
462
+ image_bytes = None
463
+
464
+ # Audio upload
465
+ st.markdown("### 3. Add a voice note (optional)")
466
+ audio_file = st.file_uploader(
467
+ "Upload a short voice note (wav/mp3/m4a).",
468
+ type=["wav", "mp3", "m4a"],
469
+ accept_multiple_files=False,
470
+ )
471
+
472
+ # Text description
473
+ user_text = st.text_area(
474
+ "Or type a short description:",
475
+ placeholder='Example: "Red itchy patch on forearm for 3 days, mild burning, no fever, using a new soap."',
476
+ height=120,
477
+ )
478
+
479
+ # Run agent
480
+ if st.button("Run CareCall Agent"):
481
+ if not client:
482
+ st.error("OPENAI_API_KEY is not set. Configure it in your Space settings.")
483
  else:
484
+ with st.spinner("Running vision + voice + open-data tools + reasoning..."):
485
+
486
+ # Vision
487
+ vision_summary = ""
488
+ if image_bytes:
489
+ vision_summary = call_vision_summarizer(image_bytes)
490
+
491
+ # ASR
492
+ voice_text = ""
493
+ if audio_file is not None:
494
+ voice_text = call_asr(audio_file)
495
+
496
+ # Candidates for DPD lookup
497
+ combined_for_terms = " ".join(
498
+ t for t in [user_text or "", voice_text or "", vision_summary or ""] if t
499
+ )
500
+ candidates = extract_candidate_terms(combined_for_terms)
501
+ dpd_context = tool_lookup_drug_products(candidates)
502
+
503
+ # Recalls
504
+ recalls_context = tool_get_recent_recalls_snippet()
505
 
506
+ # Wait-time awareness
507
+ wait_times_context = tool_get_wait_times_awareness()
508
+
509
+ # Region context from postal
510
+ region_context = tool_region_context(postal_code) if postal_code else (
511
+ "No postal code provided. Use Canada-wide guidance and suggest provincial resources."
512
+ )
513
+
514
+ # Final reasoning
515
+ final_answer = call_reasoning_agent(
516
+ user_text=user_text or "",
517
+ voice_text=voice_text or "",
518
+ vision_summary=vision_summary,
519
+ dpd_context=dpd_context,
520
+ recalls_context=recalls_context,
521
+ wait_times_context=wait_times_context,
522
+ region_context=region_context,
523
+ )
524
+
525
+ st.markdown("### CareCall AI Summary (Informational Only)")
526
+ st.write(final_answer)
527
+
528
+ st.markdown(
529
+ """
530
+ ---
531
+ Critical Safety Notice
532
+
533
+ - This tool does not provide medical diagnoses, prescriptions, or emergency triage.
534
+ - It may be incomplete or incorrect and must not replace a doctor, nurse, pharmacist,
535
+ Telehealth service, or emergency care.
536
+ - If you have severe or worsening symptoms (such as chest pain, difficulty breathing,
537
+ signs of stroke, heavy bleeding, suicidal thoughts, major injury, high fever with stiff neck),
538
+ call 911 or go to the nearest emergency department immediately.
539
+ - For dependable information, consult:
540
+ Health Canada, the Public Health Agency of Canada, CIHI, and your provincial/territorial health services.
541
+ """
542
+ )