Spaces:
Running
Running
feat: Refine the onboarding flow with new steps and pause/skip logic, update headache logging to use a Monday-based week, and improve tool call error handling.
Browse files- frontend/src/components/ChatInterface.tsx +2 -33
- frontend/src/components/ComponentOverlay.tsx +1 -1
- frontend/src/registry/ConfirmationCard.tsx +1 -1
- frontend/src/registry/MedicationCard.tsx +15 -15
- frontend/src/registry/OnboardingProgress.tsx +15 -21
- frontend/src/registry/OnboardingSummary.tsx +45 -50
- src/reachy_mini_conversation_app/console.py +1 -1
- src/reachy_mini_conversation_app/database.py +2 -0
- src/reachy_mini_conversation_app/openai_realtime.py +111 -11
- src/reachy_mini_conversation_app/profiles/_reachy_mini_minder_locked_profile/complete_onboarding.py +15 -1
- src/reachy_mini_conversation_app/profiles/_reachy_mini_minder_locked_profile/show_onboarding_step.py +12 -2
- src/reachy_mini_conversation_app/prompts/sections/onboarding.txt +40 -29
- src/reachy_mini_conversation_app/prompts/sections/tool_budget.txt +2 -0
- src/reachy_mini_conversation_app/stream_api.py +9 -2
- src/reachy_mini_conversation_app/tools/core_tools.py +7 -1
- src/reachy_mini_conversation_app/wakeword_detector.py +2 -2
frontend/src/components/ChatInterface.tsx
CHANGED
|
@@ -12,7 +12,7 @@ import { ConsentModal } from "@/components/ConsentModal";
|
|
| 12 |
import { DashboardGrid } from "@/components/DashboardGrid";
|
| 13 |
import { ComponentOverlay } from "@/components/ComponentOverlay";
|
| 14 |
import { ObservabilityPanel } from "@/components/ObservabilityPanel";
|
| 15 |
-
import {
|
| 16 |
|
| 17 |
interface ChatInterfaceProps {
|
| 18 |
wsUrl?: string;
|
|
@@ -139,7 +139,7 @@ export function ChatInterface({ wsUrl = "ws://localhost:8000/api/stream/ws" }: C
|
|
| 139 |
disabled={!isSessionActive}
|
| 140 |
/>
|
| 141 |
)}
|
| 142 |
-
|
| 143 |
</div>
|
| 144 |
</header>
|
| 145 |
</div>
|
|
@@ -219,37 +219,6 @@ export function ChatInterface({ wsUrl = "ws://localhost:8000/api/stream/ws" }: C
|
|
| 219 |
);
|
| 220 |
}
|
| 221 |
|
| 222 |
-
// ---- Sub-components ----
|
| 223 |
-
|
| 224 |
-
function ConnectionStatus({ isConnected, isSessionActive, isListening }: { isConnected: boolean; isSessionActive: boolean; isListening: boolean }) {
|
| 225 |
-
// Only show "Live Transcript" when all conditions are met:
|
| 226 |
-
// 1. WebSocket is connected
|
| 227 |
-
// 2. AI session is active
|
| 228 |
-
// 3. Microphone is listening
|
| 229 |
-
if (!isConnected || !isSessionActive || !isListening) {
|
| 230 |
-
return null;
|
| 231 |
-
}
|
| 232 |
-
|
| 233 |
-
return (
|
| 234 |
-
<div
|
| 235 |
-
style={{
|
| 236 |
-
padding: "8px 16px",
|
| 237 |
-
borderRadius: 999,
|
| 238 |
-
fontSize: 13,
|
| 239 |
-
fontWeight: 500,
|
| 240 |
-
display: "flex",
|
| 241 |
-
alignItems: "center",
|
| 242 |
-
gap: 6,
|
| 243 |
-
background: "var(--color-accent-pink)",
|
| 244 |
-
color: "#1a1a1a",
|
| 245 |
-
animation: "pulse 2s infinite",
|
| 246 |
-
}}
|
| 247 |
-
>
|
| 248 |
-
<Wifi style={{ width: 14, height: 14 }} />
|
| 249 |
-
Live Transcript
|
| 250 |
-
</div>
|
| 251 |
-
);
|
| 252 |
-
}
|
| 253 |
|
| 254 |
interface ListenToggleProps {
|
| 255 |
isListening: boolean;
|
|
|
|
| 12 |
import { DashboardGrid } from "@/components/DashboardGrid";
|
| 13 |
import { ComponentOverlay } from "@/components/ComponentOverlay";
|
| 14 |
import { ObservabilityPanel } from "@/components/ObservabilityPanel";
|
| 15 |
+
import { Loader2, Mic, MicOff, Power, Square, Video, VideoOff } from "lucide-react";
|
| 16 |
|
| 17 |
interface ChatInterfaceProps {
|
| 18 |
wsUrl?: string;
|
|
|
|
| 139 |
disabled={!isSessionActive}
|
| 140 |
/>
|
| 141 |
)}
|
| 142 |
+
|
| 143 |
</div>
|
| 144 |
</header>
|
| 145 |
</div>
|
|
|
|
| 219 |
);
|
| 220 |
}
|
| 221 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 222 |
|
| 223 |
interface ListenToggleProps {
|
| 224 |
isListening: boolean;
|
frontend/src/components/ComponentOverlay.tsx
CHANGED
|
@@ -11,7 +11,7 @@ interface ComponentOverlayProps {
|
|
| 11 |
}
|
| 12 |
|
| 13 |
// Components that render full-screen (TV-distance readable)
|
| 14 |
-
const FULLSCREEN_COMPONENTS = new Set(["HeadacheLog", "MedLog", "SessionSummary"]);
|
| 15 |
|
| 16 |
export function ComponentOverlay({ component, onDismiss }: ComponentOverlayProps) {
|
| 17 |
const isFullscreen = FULLSCREEN_COMPONENTS.has(component.name);
|
|
|
|
| 11 |
}
|
| 12 |
|
| 13 |
// Components that render full-screen (TV-distance readable)
|
| 14 |
+
const FULLSCREEN_COMPONENTS = new Set(["HeadacheLog", "MedLog", "SessionSummary", "OnboardingProgress", "OnboardingSummary"]);
|
| 15 |
|
| 16 |
export function ComponentOverlay({ component, onDismiss }: ComponentOverlayProps) {
|
| 17 |
const isFullscreen = FULLSCREEN_COMPONENTS.has(component.name);
|
frontend/src/registry/ConfirmationCard.tsx
CHANGED
|
@@ -40,7 +40,7 @@ export function ConfirmationCard(props: ConfirmationCardProps) {
|
|
| 40 |
accent: "via-cta",
|
| 41 |
iconBg: "bg-cta/15 border-cta/30 text-cta",
|
| 42 |
icon: <Check className="w-5 h-5" />,
|
| 43 |
-
confirmBtn: "bg-cta text-
|
| 44 |
summaryBg: "bg-surface-subtle/50 border-surface-overlay/30",
|
| 45 |
},
|
| 46 |
warning: {
|
|
|
|
| 40 |
accent: "via-cta",
|
| 41 |
iconBg: "bg-cta/15 border-cta/30 text-cta",
|
| 42 |
icon: <Check className="w-5 h-5" />,
|
| 43 |
+
confirmBtn: "bg-cta text-gray-900 hover:brightness-110",
|
| 44 |
summaryBg: "bg-surface-subtle/50 border-surface-overlay/30",
|
| 45 |
},
|
| 46 |
warning: {
|
frontend/src/registry/MedicationCard.tsx
CHANGED
|
@@ -22,7 +22,7 @@ export function MedicationCard({
|
|
| 22 |
return (
|
| 23 |
<div
|
| 24 |
className={`
|
| 25 |
-
relative overflow-hidden rounded-
|
| 26 |
bg-gradient-to-br from-surface-elevated/80 to-surface-subtle/90
|
| 27 |
border border-accent-cyan/20
|
| 28 |
shadow-[0_2px_16px_rgba(0,0,0,0.3)]
|
|
@@ -31,42 +31,42 @@ export function MedicationCard({
|
|
| 31 |
style={{ animationDelay: isNew ? `${index * 100}ms` : undefined }}
|
| 32 |
>
|
| 33 |
{/* Left accent bar */}
|
| 34 |
-
<div className="absolute left-0 top-
|
| 35 |
|
| 36 |
-
<div className="flex items-
|
| 37 |
{/* Number badge for voice selection */}
|
| 38 |
<div
|
| 39 |
-
className="w-
|
| 40 |
bg-accent-cyan/15 border border-accent-cyan/25
|
| 41 |
-
text-
|
| 42 |
>
|
| 43 |
{index + 1}
|
| 44 |
</div>
|
| 45 |
|
| 46 |
{/* Medication info */}
|
| 47 |
<div className="flex-1 min-w-0">
|
| 48 |
-
<div className="flex items-center gap-
|
| 49 |
-
<Pill className="w-
|
| 50 |
-
<h4 className="text-
|
| 51 |
{medicationName}
|
| 52 |
</h4>
|
| 53 |
</div>
|
| 54 |
|
| 55 |
-
<div className="flex flex-wrap items-center gap-
|
| 56 |
{dose && (
|
| 57 |
-
<span className="px-
|
| 58 |
{dose}
|
| 59 |
</span>
|
| 60 |
)}
|
| 61 |
{frequency && (
|
| 62 |
-
<span className="flex items-center gap-1 text-muted">
|
| 63 |
-
<Calendar className="w-
|
| 64 |
{frequency}
|
| 65 |
</span>
|
| 66 |
)}
|
| 67 |
{timesOfDay && timesOfDay.length > 0 && (
|
| 68 |
-
<span className="flex items-center gap-1 text-muted">
|
| 69 |
-
<Clock className="w-
|
| 70 |
{timesOfDay.join(", ")}
|
| 71 |
</span>
|
| 72 |
)}
|
|
@@ -75,7 +75,7 @@ export function MedicationCard({
|
|
| 75 |
|
| 76 |
{/* Status indicator */}
|
| 77 |
{isNew && (
|
| 78 |
-
<span className="px-
|
| 79 |
Added
|
| 80 |
</span>
|
| 81 |
)}
|
|
|
|
| 22 |
return (
|
| 23 |
<div
|
| 24 |
className={`
|
| 25 |
+
relative overflow-hidden rounded-2xl p-6
|
| 26 |
bg-gradient-to-br from-surface-elevated/80 to-surface-subtle/90
|
| 27 |
border border-accent-cyan/20
|
| 28 |
shadow-[0_2px_16px_rgba(0,0,0,0.3)]
|
|
|
|
| 31 |
style={{ animationDelay: isNew ? `${index * 100}ms` : undefined }}
|
| 32 |
>
|
| 33 |
{/* Left accent bar */}
|
| 34 |
+
<div className="absolute left-0 top-3 bottom-3 w-1.5 rounded-full bg-gradient-to-b from-accent-cyan to-cta" />
|
| 35 |
|
| 36 |
+
<div className="flex items-center gap-5 pl-4">
|
| 37 |
{/* Number badge for voice selection */}
|
| 38 |
<div
|
| 39 |
+
className="w-12 h-12 rounded-xl flex items-center justify-center shrink-0
|
| 40 |
bg-accent-cyan/15 border border-accent-cyan/25
|
| 41 |
+
text-xl font-extrabold text-accent-cyan"
|
| 42 |
>
|
| 43 |
{index + 1}
|
| 44 |
</div>
|
| 45 |
|
| 46 |
{/* Medication info */}
|
| 47 |
<div className="flex-1 min-w-0">
|
| 48 |
+
<div className="flex items-center gap-3 mb-2">
|
| 49 |
+
<Pill className="w-6 h-6 text-accent-cyan" />
|
| 50 |
+
<h4 className="text-xl font-bold text-primary truncate tracking-tight">
|
| 51 |
{medicationName}
|
| 52 |
</h4>
|
| 53 |
</div>
|
| 54 |
|
| 55 |
+
<div className="flex flex-wrap items-center gap-3 text-base">
|
| 56 |
{dose && (
|
| 57 |
+
<span className="px-3 py-1 rounded-lg bg-surface-overlay text-secondary font-medium">
|
| 58 |
{dose}
|
| 59 |
</span>
|
| 60 |
)}
|
| 61 |
{frequency && (
|
| 62 |
+
<span className="flex items-center gap-1.5 text-muted">
|
| 63 |
+
<Calendar className="w-5 h-5" />
|
| 64 |
{frequency}
|
| 65 |
</span>
|
| 66 |
)}
|
| 67 |
{timesOfDay && timesOfDay.length > 0 && (
|
| 68 |
+
<span className="flex items-center gap-1.5 text-muted">
|
| 69 |
+
<Clock className="w-5 h-5" />
|
| 70 |
{timesOfDay.join(", ")}
|
| 71 |
</span>
|
| 72 |
)}
|
|
|
|
| 75 |
|
| 76 |
{/* Status indicator */}
|
| 77 |
{isNew && (
|
| 78 |
+
<span className="px-3 py-1.5 rounded-full bg-success/15 text-success text-sm font-bold uppercase tracking-wide">
|
| 79 |
Added
|
| 80 |
</span>
|
| 81 |
)}
|
frontend/src/registry/OnboardingProgress.tsx
CHANGED
|
@@ -26,31 +26,25 @@ function getStepStatus(stepId: string, currentStep: string): "completed" | "curr
|
|
| 26 |
export function OnboardingProgress({ currentStep }: OnboardingProgressProps) {
|
| 27 |
return (
|
| 28 |
<div
|
| 29 |
-
className="
|
| 30 |
-
bg-gradient-to-br from-surface-elevated/90 to-surface-subtle/95
|
| 31 |
-
border border-cta/15
|
| 32 |
-
shadow-[0_4px_24px_rgba(0,0,0,0.4),inset_0_1px_0_rgba(255,255,255,0.05)]"
|
| 33 |
>
|
| 34 |
-
{/* Top accent line */}
|
| 35 |
-
<div className="absolute top-0 left-[20%] right-[20%] h-0.5 bg-gradient-to-r from-transparent via-cta to-transparent opacity-60" />
|
| 36 |
-
|
| 37 |
{/* Header */}
|
| 38 |
-
<div className="flex items-center gap-
|
| 39 |
<div
|
| 40 |
-
className="relative w-
|
| 41 |
bg-gradient-to-br from-cta/25 to-cta/10
|
| 42 |
border border-cta/30"
|
| 43 |
>
|
| 44 |
-
<PartyPopper className="w-
|
| 45 |
</div>
|
| 46 |
<div>
|
| 47 |
-
<h3 className="text-
|
| 48 |
-
<p className="text-
|
| 49 |
</div>
|
| 50 |
</div>
|
| 51 |
|
| 52 |
{/* Progress Steps */}
|
| 53 |
-
<div className="flex items-center justify-
|
| 54 |
{steps.map((step, index) => {
|
| 55 |
const status = getStepStatus(step.id, currentStep);
|
| 56 |
const Icon = step.icon;
|
|
@@ -58,23 +52,23 @@ export function OnboardingProgress({ currentStep }: OnboardingProgressProps) {
|
|
| 58 |
return (
|
| 59 |
<div key={step.id} className="flex items-center">
|
| 60 |
{/* Step Circle */}
|
| 61 |
-
<div className="flex flex-col items-center gap-
|
| 62 |
<div
|
| 63 |
className={`
|
| 64 |
-
relative w-
|
| 65 |
transition-all duration-500 ease-out
|
| 66 |
${status === "completed"
|
| 67 |
-
? "bg-success text-
|
| 68 |
: status === "current"
|
| 69 |
-
? "bg-cta text-
|
| 70 |
: "bg-surface-overlay text-muted scale-90 opacity-50"
|
| 71 |
}
|
| 72 |
`}
|
| 73 |
>
|
| 74 |
{status === "completed" ? (
|
| 75 |
-
<Check className="w-
|
| 76 |
) : (
|
| 77 |
-
<Icon className="w-
|
| 78 |
)}
|
| 79 |
|
| 80 |
{/* Pulse animation for current step */}
|
|
@@ -86,7 +80,7 @@ export function OnboardingProgress({ currentStep }: OnboardingProgressProps) {
|
|
| 86 |
{/* Label */}
|
| 87 |
<span
|
| 88 |
className={`
|
| 89 |
-
text-
|
| 90 |
transition-all duration-300
|
| 91 |
${status === "current" ? "text-cta" : status === "completed" ? "text-success" : "text-muted opacity-50"}
|
| 92 |
`}
|
|
@@ -99,7 +93,7 @@ export function OnboardingProgress({ currentStep }: OnboardingProgressProps) {
|
|
| 99 |
{index < steps.length - 1 && (
|
| 100 |
<div
|
| 101 |
className={`
|
| 102 |
-
w-
|
| 103 |
${getStepStatus(steps[index + 1].id, currentStep) !== "upcoming"
|
| 104 |
? "bg-success"
|
| 105 |
: "bg-surface-overlay opacity-30"
|
|
|
|
| 26 |
export function OnboardingProgress({ currentStep }: OnboardingProgressProps) {
|
| 27 |
return (
|
| 28 |
<div
|
| 29 |
+
className="flex flex-col items-center justify-center h-full w-full px-12 py-16"
|
|
|
|
|
|
|
|
|
|
| 30 |
>
|
|
|
|
|
|
|
|
|
|
| 31 |
{/* Header */}
|
| 32 |
+
<div className="flex items-center gap-6 mb-16">
|
| 33 |
<div
|
| 34 |
+
className="relative w-20 h-20 rounded-2xl flex items-center justify-center
|
| 35 |
bg-gradient-to-br from-cta/25 to-cta/10
|
| 36 |
border border-cta/30"
|
| 37 |
>
|
| 38 |
+
<PartyPopper className="w-10 h-10 text-cta" />
|
| 39 |
</div>
|
| 40 |
<div>
|
| 41 |
+
<h3 className="text-4xl font-bold tracking-tight text-primary">Getting Started</h3>
|
| 42 |
+
<p className="text-xl text-muted mt-2">Let's set up your health companion</p>
|
| 43 |
</div>
|
| 44 |
</div>
|
| 45 |
|
| 46 |
{/* Progress Steps */}
|
| 47 |
+
<div className="flex items-center justify-center gap-4">
|
| 48 |
{steps.map((step, index) => {
|
| 49 |
const status = getStepStatus(step.id, currentStep);
|
| 50 |
const Icon = step.icon;
|
|
|
|
| 52 |
return (
|
| 53 |
<div key={step.id} className="flex items-center">
|
| 54 |
{/* Step Circle */}
|
| 55 |
+
<div className="flex flex-col items-center gap-4">
|
| 56 |
<div
|
| 57 |
className={`
|
| 58 |
+
relative w-20 h-20 rounded-full flex items-center justify-center
|
| 59 |
transition-all duration-500 ease-out
|
| 60 |
${status === "completed"
|
| 61 |
+
? "bg-success text-gray-900 scale-100"
|
| 62 |
: status === "current"
|
| 63 |
+
? "bg-cta text-gray-900 scale-110 shadow-[0_0_30px_rgba(179,156,208,0.5)]"
|
| 64 |
: "bg-surface-overlay text-muted scale-90 opacity-50"
|
| 65 |
}
|
| 66 |
`}
|
| 67 |
>
|
| 68 |
{status === "completed" ? (
|
| 69 |
+
<Check className="w-10 h-10" />
|
| 70 |
) : (
|
| 71 |
+
<Icon className="w-9 h-9" />
|
| 72 |
)}
|
| 73 |
|
| 74 |
{/* Pulse animation for current step */}
|
|
|
|
| 80 |
{/* Label */}
|
| 81 |
<span
|
| 82 |
className={`
|
| 83 |
+
text-lg font-semibold uppercase tracking-wide
|
| 84 |
transition-all duration-300
|
| 85 |
${status === "current" ? "text-cta" : status === "completed" ? "text-success" : "text-muted opacity-50"}
|
| 86 |
`}
|
|
|
|
| 93 |
{index < steps.length - 1 && (
|
| 94 |
<div
|
| 95 |
className={`
|
| 96 |
+
w-16 h-1 mx-4 rounded-full transition-all duration-500
|
| 97 |
${getStepStatus(steps[index + 1].id, currentStep) !== "upcoming"
|
| 98 |
? "bg-success"
|
| 99 |
: "bg-surface-overlay opacity-30"
|
frontend/src/registry/OnboardingSummary.tsx
CHANGED
|
@@ -54,10 +54,7 @@ export function OnboardingSummary({
|
|
| 54 |
|
| 55 |
return (
|
| 56 |
<div
|
| 57 |
-
className="relative
|
| 58 |
-
bg-gradient-to-br from-surface-elevated/95 to-surface-subtle/90
|
| 59 |
-
border border-success/25
|
| 60 |
-
shadow-[0_8px_40px_rgba(0,0,0,0.5),0_0_60px_rgba(34,197,94,0.1)]
|
| 61 |
animate-in zoom-in-95 fade-in duration-700"
|
| 62 |
>
|
| 63 |
{/* Celebration gradient overlay */}
|
|
@@ -69,7 +66,7 @@ export function OnboardingSummary({
|
|
| 69 |
{confettiParticles.map((particle, i) => (
|
| 70 |
<div
|
| 71 |
key={i}
|
| 72 |
-
className="absolute w-
|
| 73 |
style={{
|
| 74 |
left: `${particle.left}%`,
|
| 75 |
top: `${particle.top}%`,
|
|
@@ -84,61 +81,59 @@ export function OnboardingSummary({
|
|
| 84 |
)}
|
| 85 |
|
| 86 |
{/* Header */}
|
| 87 |
-
<div className="relative
|
| 88 |
-
<div
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
>
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
<div>
|
| 97 |
-
<
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
</span>
|
| 102 |
-
</div>
|
| 103 |
-
<h2 className="text-xl font-bold text-primary tracking-tight">
|
| 104 |
-
You're All Set{displayName ? `, ${displayName}` : ""}!
|
| 105 |
-
</h2>
|
| 106 |
</div>
|
|
|
|
|
|
|
|
|
|
| 107 |
</div>
|
| 108 |
</div>
|
| 109 |
|
| 110 |
{/* Summary Content */}
|
| 111 |
-
<div className="relative
|
| 112 |
{/* Profile */}
|
| 113 |
{displayName && (
|
| 114 |
-
<div className="flex items-center gap-
|
| 115 |
-
<div className="w-
|
| 116 |
-
<User className="w-
|
| 117 |
</div>
|
| 118 |
<div>
|
| 119 |
-
<p className="text-
|
| 120 |
-
<p className="text-
|
| 121 |
</div>
|
| 122 |
</div>
|
| 123 |
)}
|
| 124 |
|
| 125 |
{/* Medications */}
|
| 126 |
{medications.length > 0 && (
|
| 127 |
-
<div className="space-y-
|
| 128 |
-
<div className="flex items-center gap-
|
| 129 |
-
<Pill className="w-
|
| 130 |
-
<span className="text-
|
| 131 |
{medications.length} Medication{medications.length !== 1 ? "s" : ""} Tracked
|
| 132 |
</span>
|
| 133 |
</div>
|
| 134 |
-
<div className="flex flex-wrap gap-
|
| 135 |
{medications.map((med, i) => (
|
| 136 |
<span
|
| 137 |
key={i}
|
| 138 |
-
className="px-
|
| 139 |
>
|
| 140 |
{med.medicationName}
|
| 141 |
-
{med.dose && <span className="text-accent-cyan/60 ml-
|
| 142 |
</span>
|
| 143 |
))}
|
| 144 |
</div>
|
|
@@ -147,31 +142,31 @@ export function OnboardingSummary({
|
|
| 147 |
|
| 148 |
{/* Contacts */}
|
| 149 |
{(neurologist?.name || caregiver?.name) && (
|
| 150 |
-
<div className="flex gap-
|
| 151 |
{neurologist?.name && (
|
| 152 |
-
<div className="flex-1 p-
|
| 153 |
-
<div className="flex items-center gap-
|
| 154 |
-
<Stethoscope className="w-
|
| 155 |
-
<span className="text-
|
| 156 |
</div>
|
| 157 |
-
<p className="text-
|
| 158 |
</div>
|
| 159 |
)}
|
| 160 |
{caregiver?.name && (
|
| 161 |
-
<div className="flex-1 p-
|
| 162 |
-
<div className="flex items-center gap-
|
| 163 |
-
<Heart className="w-
|
| 164 |
-
<span className="text-
|
| 165 |
</div>
|
| 166 |
-
<p className="text-
|
| 167 |
</div>
|
| 168 |
)}
|
| 169 |
</div>
|
| 170 |
)}
|
| 171 |
|
| 172 |
{/* Ready message */}
|
| 173 |
-
<div className="pt-
|
| 174 |
-
<p className="text-
|
| 175 |
I'm ready to help you track your health. Just start talking!
|
| 176 |
</p>
|
| 177 |
</div>
|
|
|
|
| 54 |
|
| 55 |
return (
|
| 56 |
<div
|
| 57 |
+
className="relative flex flex-col items-center justify-center h-full w-full overflow-hidden
|
|
|
|
|
|
|
|
|
|
| 58 |
animate-in zoom-in-95 fade-in duration-700"
|
| 59 |
>
|
| 60 |
{/* Celebration gradient overlay */}
|
|
|
|
| 66 |
{confettiParticles.map((particle, i) => (
|
| 67 |
<div
|
| 68 |
key={i}
|
| 69 |
+
className="absolute w-4 h-4 rounded-full animate-bounce"
|
| 70 |
style={{
|
| 71 |
left: `${particle.left}%`,
|
| 72 |
top: `${particle.top}%`,
|
|
|
|
| 81 |
)}
|
| 82 |
|
| 83 |
{/* Header */}
|
| 84 |
+
<div className="relative flex items-center gap-8 mb-12">
|
| 85 |
+
<div
|
| 86 |
+
className="w-24 h-24 rounded-3xl flex items-center justify-center
|
| 87 |
+
bg-gradient-to-br from-success/30 to-success/10
|
| 88 |
+
border border-success/40 shadow-[0_0_30px_rgba(34,197,94,0.3)]"
|
| 89 |
+
>
|
| 90 |
+
<PartyPopper className="w-12 h-12 text-success" />
|
| 91 |
+
</div>
|
| 92 |
+
<div>
|
| 93 |
+
<div className="flex items-center gap-3 mb-2">
|
| 94 |
+
<Sparkles className="w-7 h-7 text-success" />
|
| 95 |
+
<span className="text-lg text-success font-bold uppercase tracking-widest">
|
| 96 |
+
Setup Complete!
|
| 97 |
+
</span>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
</div>
|
| 99 |
+
<h2 className="text-5xl font-bold text-primary tracking-tight">
|
| 100 |
+
You're All Set{displayName ? `, ${displayName}` : ""}!
|
| 101 |
+
</h2>
|
| 102 |
</div>
|
| 103 |
</div>
|
| 104 |
|
| 105 |
{/* Summary Content */}
|
| 106 |
+
<div className="relative w-full max-w-3xl space-y-8 px-8">
|
| 107 |
{/* Profile */}
|
| 108 |
{displayName && (
|
| 109 |
+
<div className="flex items-center gap-6 p-6 rounded-2xl bg-surface-subtle/50 border border-surface-overlay">
|
| 110 |
+
<div className="w-16 h-16 rounded-xl bg-cta/15 flex items-center justify-center">
|
| 111 |
+
<User className="w-8 h-8 text-cta" />
|
| 112 |
</div>
|
| 113 |
<div>
|
| 114 |
+
<p className="text-lg text-muted uppercase tracking-wide font-medium">Your Name</p>
|
| 115 |
+
<p className="text-3xl font-semibold text-primary">{displayName}</p>
|
| 116 |
</div>
|
| 117 |
</div>
|
| 118 |
)}
|
| 119 |
|
| 120 |
{/* Medications */}
|
| 121 |
{medications.length > 0 && (
|
| 122 |
+
<div className="space-y-4">
|
| 123 |
+
<div className="flex items-center gap-3 px-2">
|
| 124 |
+
<Pill className="w-7 h-7 text-accent-cyan" />
|
| 125 |
+
<span className="text-lg text-muted uppercase tracking-wide font-bold">
|
| 126 |
{medications.length} Medication{medications.length !== 1 ? "s" : ""} Tracked
|
| 127 |
</span>
|
| 128 |
</div>
|
| 129 |
+
<div className="flex flex-wrap gap-4">
|
| 130 |
{medications.map((med, i) => (
|
| 131 |
<span
|
| 132 |
key={i}
|
| 133 |
+
className="px-6 py-3 rounded-xl bg-accent-cyan/10 border border-accent-cyan/20 text-xl font-medium text-accent-cyan"
|
| 134 |
>
|
| 135 |
{med.medicationName}
|
| 136 |
+
{med.dose && <span className="text-accent-cyan/60 ml-2">({med.dose})</span>}
|
| 137 |
</span>
|
| 138 |
))}
|
| 139 |
</div>
|
|
|
|
| 142 |
|
| 143 |
{/* Contacts */}
|
| 144 |
{(neurologist?.name || caregiver?.name) && (
|
| 145 |
+
<div className="flex gap-6">
|
| 146 |
{neurologist?.name && (
|
| 147 |
+
<div className="flex-1 p-6 rounded-2xl bg-surface-subtle/50 border border-surface-overlay">
|
| 148 |
+
<div className="flex items-center gap-3 mb-2">
|
| 149 |
+
<Stethoscope className="w-7 h-7 text-cta" />
|
| 150 |
+
<span className="text-base text-muted uppercase tracking-wide">Doctor</span>
|
| 151 |
</div>
|
| 152 |
+
<p className="text-2xl font-semibold text-primary truncate">{neurologist.name}</p>
|
| 153 |
</div>
|
| 154 |
)}
|
| 155 |
{caregiver?.name && (
|
| 156 |
+
<div className="flex-1 p-6 rounded-2xl bg-surface-subtle/50 border border-surface-overlay">
|
| 157 |
+
<div className="flex items-center gap-3 mb-2">
|
| 158 |
+
<Heart className="w-7 h-7 text-accent-pink" />
|
| 159 |
+
<span className="text-base text-muted uppercase tracking-wide">Caregiver</span>
|
| 160 |
</div>
|
| 161 |
+
<p className="text-2xl font-semibold text-primary truncate">{caregiver.name}</p>
|
| 162 |
</div>
|
| 163 |
)}
|
| 164 |
</div>
|
| 165 |
)}
|
| 166 |
|
| 167 |
{/* Ready message */}
|
| 168 |
+
<div className="pt-6 border-t border-white/5 text-center">
|
| 169 |
+
<p className="text-2xl text-secondary">
|
| 170 |
I'm ready to help you track your health. Just start talking!
|
| 171 |
</p>
|
| 172 |
</div>
|
src/reachy_mini_conversation_app/console.py
CHANGED
|
@@ -83,7 +83,7 @@ class LocalStream:
|
|
| 83 |
# Session control - when False, AI handler is completely disconnected
|
| 84 |
self._is_session_active: bool = False # Session starts inactive to save tokens
|
| 85 |
# Wake word detector - runs locally when session is inactive
|
| 86 |
-
self._wakeword_detector = WakewordDetector()
|
| 87 |
self._wakeword_detector.eager_load() # start ONNX model load in background
|
| 88 |
# Cooldown timestamp: wakeword detection paused until this time
|
| 89 |
import time as _time
|
|
|
|
| 83 |
# Session control - when False, AI handler is completely disconnected
|
| 84 |
self._is_session_active: bool = False # Session starts inactive to save tokens
|
| 85 |
# Wake word detector - runs locally when session is inactive
|
| 86 |
+
self._wakeword_detector = WakewordDetector(threshold=0.6)
|
| 87 |
self._wakeword_detector.eager_load() # start ONNX model load in background
|
| 88 |
# Cooldown timestamp: wakeword detection paused until this time
|
| 89 |
import time as _time
|
src/reachy_mini_conversation_app/database.py
CHANGED
|
@@ -519,9 +519,11 @@ h1 {{ color: #333; }}
|
|
| 519 |
# Security (#4): column-name allowlist for scheduled_medications.
|
| 520 |
_MEDICATION_COLUMNS = {
|
| 521 |
"medication_name",
|
|
|
|
| 522 |
"dosage",
|
| 523 |
"frequency",
|
| 524 |
"times_of_day",
|
|
|
|
| 525 |
"reminder_times",
|
| 526 |
"notes",
|
| 527 |
"active",
|
|
|
|
| 519 |
# Security (#4): column-name allowlist for scheduled_medications.
|
| 520 |
_MEDICATION_COLUMNS = {
|
| 521 |
"medication_name",
|
| 522 |
+
"dose",
|
| 523 |
"dosage",
|
| 524 |
"frequency",
|
| 525 |
"times_of_day",
|
| 526 |
+
"reminder_enabled",
|
| 527 |
"reminder_times",
|
| 528 |
"notes",
|
| 529 |
"active",
|
src/reachy_mini_conversation_app/openai_realtime.py
CHANGED
|
@@ -64,6 +64,10 @@ class OpenaiRealtimeHandler(RealtimeHandler):
|
|
| 64 |
# tool outputs have been submitted).
|
| 65 |
self._pending_tool_response: bool = False
|
| 66 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
# Track recent tool calls for idle-signal dedup
|
| 68 |
self._recent_tool_names: list[str] = []
|
| 69 |
|
|
@@ -462,7 +466,7 @@ class OpenaiRealtimeHandler(RealtimeHandler):
|
|
| 462 |
"type": "realtime",
|
| 463 |
"model": config.MODEL_NAME,
|
| 464 |
# Lock output to audio
|
| 465 |
-
"output_modalities": ["
|
| 466 |
"audio": {
|
| 467 |
"input": {
|
| 468 |
"format": {
|
|
@@ -515,10 +519,29 @@ class OpenaiRealtimeHandler(RealtimeHandler):
|
|
| 515 |
except Exception:
|
| 516 |
pass
|
| 517 |
|
| 518 |
-
# --- Proactive greeting:
|
| 519 |
-
#
|
| 520 |
-
#
|
|
|
|
|
|
|
| 521 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 522 |
await conn.conversation.item.create(
|
| 523 |
item={
|
| 524 |
"type": "message",
|
|
@@ -526,12 +549,13 @@ class OpenaiRealtimeHandler(RealtimeHandler):
|
|
| 526 |
"content": [
|
| 527 |
{
|
| 528 |
"type": "input_text",
|
| 529 |
-
"text":
|
| 530 |
}
|
| 531 |
],
|
| 532 |
},
|
| 533 |
)
|
| 534 |
-
|
|
|
|
| 535 |
logger.info(
|
| 536 |
"Proactive greeting triggered (total startup: %.0fms)",
|
| 537 |
(time.monotonic() - t_connect) * 1000,
|
|
@@ -603,6 +627,30 @@ class OpenaiRealtimeHandler(RealtimeHandler):
|
|
| 603 |
usage = getattr(resp, "usage", None)
|
| 604 |
if usage:
|
| 605 |
await self._emit_cost_update(usage)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 606 |
else:
|
| 607 |
logger.debug("Response done (no response object)")
|
| 608 |
|
|
@@ -785,10 +833,57 @@ class OpenaiRealtimeHandler(RealtimeHandler):
|
|
| 785 |
if self.is_idle_tool_call:
|
| 786 |
self.is_idle_tool_call = False
|
| 787 |
else:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 788 |
await self.connection.response.create(
|
| 789 |
-
response=
|
| 790 |
-
"instructions": "Use the tool result just returned and answer concisely in speech. You MUST respond in British English only.",
|
| 791 |
-
},
|
| 792 |
)
|
| 793 |
|
| 794 |
# Event-driven prompt re-resolve: if a state-changing tool
|
|
@@ -1104,7 +1199,8 @@ class OpenaiRealtimeHandler(RealtimeHandler):
|
|
| 1104 |
timestamp_msg = (
|
| 1105 |
f"[Idle time update: {self.format_timestamp()} - No activity for {idle_duration:.1f}s] "
|
| 1106 |
f"Recent tools used: {recent}. "
|
| 1107 |
-
f"
|
|
|
|
| 1108 |
f"Avoid repeating the same action type as recent tools."
|
| 1109 |
)
|
| 1110 |
if not self.connection:
|
|
@@ -1119,7 +1215,11 @@ class OpenaiRealtimeHandler(RealtimeHandler):
|
|
| 1119 |
)
|
| 1120 |
await self.connection.response.create(
|
| 1121 |
response={
|
| 1122 |
-
"instructions":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1123 |
"tool_choice": "required",
|
| 1124 |
},
|
| 1125 |
)
|
|
|
|
| 64 |
# tool outputs have been submitted).
|
| 65 |
self._pending_tool_response: bool = False
|
| 66 |
|
| 67 |
+
# Two-phase proactive greeting: after the forced-speech greeting
|
| 68 |
+
# completes, trigger onboarding tool calls automatically.
|
| 69 |
+
self._needs_onboarding_followup: bool = False
|
| 70 |
+
|
| 71 |
# Track recent tool calls for idle-signal dedup
|
| 72 |
self._recent_tool_names: list[str] = []
|
| 73 |
|
|
|
|
| 466 |
"type": "realtime",
|
| 467 |
"model": config.MODEL_NAME,
|
| 468 |
# Lock output to audio
|
| 469 |
+
"output_modalities": ["audio"],
|
| 470 |
"audio": {
|
| 471 |
"input": {
|
| 472 |
"format": {
|
|
|
|
| 519 |
except Exception:
|
| 520 |
pass
|
| 521 |
|
| 522 |
+
# --- Proactive greeting: two-phase approach ---
|
| 523 |
+
# Phase 1: Force the model to SPEAK a greeting (tool_choice=none
|
| 524 |
+
# prevents it from hallucinating tool calls).
|
| 525 |
+
# Phase 2: After the greeting response.done fires, the handler
|
| 526 |
+
# injects a follow-up to trigger onboarding tools.
|
| 527 |
try:
|
| 528 |
+
onboarding_done = (
|
| 529 |
+
self._session_state and self._session_state.onboarding_completed
|
| 530 |
+
)
|
| 531 |
+
if onboarding_done:
|
| 532 |
+
session_start_text = (
|
| 533 |
+
"[SESSION START] Greet the user warmly. "
|
| 534 |
+
"Keep it brief and friendly."
|
| 535 |
+
)
|
| 536 |
+
else:
|
| 537 |
+
session_start_text = (
|
| 538 |
+
"[SESSION START — ONBOARDING INCOMPLETE] "
|
| 539 |
+
"Greet the user warmly. Say something brief like "
|
| 540 |
+
"'Hello! I'm your health companion. Let me get us set up.' "
|
| 541 |
+
"Do NOT call any tools yet — just speak."
|
| 542 |
+
)
|
| 543 |
+
self._needs_onboarding_followup = True
|
| 544 |
+
|
| 545 |
await conn.conversation.item.create(
|
| 546 |
item={
|
| 547 |
"type": "message",
|
|
|
|
| 549 |
"content": [
|
| 550 |
{
|
| 551 |
"type": "input_text",
|
| 552 |
+
"text": session_start_text,
|
| 553 |
}
|
| 554 |
],
|
| 555 |
},
|
| 556 |
)
|
| 557 |
+
# Phase 1: force speech only — no tool calls allowed
|
| 558 |
+
await conn.response.create(response={"tool_choice": "none"})
|
| 559 |
logger.info(
|
| 560 |
"Proactive greeting triggered (total startup: %.0fms)",
|
| 561 |
(time.monotonic() - t_connect) * 1000,
|
|
|
|
| 627 |
usage = getattr(resp, "usage", None)
|
| 628 |
if usage:
|
| 629 |
await self._emit_cost_update(usage)
|
| 630 |
+
|
| 631 |
+
# --- Phase 2: trigger onboarding after greeting ---
|
| 632 |
+
if self._needs_onboarding_followup:
|
| 633 |
+
self._needs_onboarding_followup = False
|
| 634 |
+
logger.info(
|
| 635 |
+
"Phase 2: greeting done, triggering onboarding flow"
|
| 636 |
+
)
|
| 637 |
+
await conn.conversation.item.create(
|
| 638 |
+
item={
|
| 639 |
+
"type": "message",
|
| 640 |
+
"role": "user",
|
| 641 |
+
"content": [
|
| 642 |
+
{
|
| 643 |
+
"type": "input_text",
|
| 644 |
+
"text": (
|
| 645 |
+
"[SYSTEM] Now begin the onboarding flow. "
|
| 646 |
+
"Call get_onboarding_status to check progress, "
|
| 647 |
+
"then follow your Onboarding Flow instructions."
|
| 648 |
+
),
|
| 649 |
+
}
|
| 650 |
+
],
|
| 651 |
+
},
|
| 652 |
+
)
|
| 653 |
+
await conn.response.create()
|
| 654 |
else:
|
| 655 |
logger.debug("Response done (no response object)")
|
| 656 |
|
|
|
|
| 833 |
if self.is_idle_tool_call:
|
| 834 |
self.is_idle_tool_call = False
|
| 835 |
else:
|
| 836 |
+
# Hallucinated or failed tools: force speech, break the loop
|
| 837 |
+
is_error = (
|
| 838 |
+
isinstance(tool_result, dict) and "error" in tool_result
|
| 839 |
+
)
|
| 840 |
+
# Tools that gather info — LLM should chain to next tool
|
| 841 |
+
_ONBOARDING_CHAIN_TOOLS = {
|
| 842 |
+
"get_onboarding_status",
|
| 843 |
+
"get_current_datetime",
|
| 844 |
+
}
|
| 845 |
+
# Determine if we need to force speech (no more tools)
|
| 846 |
+
force_speech = False
|
| 847 |
+
if is_error:
|
| 848 |
+
force_speech = True
|
| 849 |
+
resp_instructions = (
|
| 850 |
+
"The tool call failed or does not exist. "
|
| 851 |
+
"Respond with SPEECH only. "
|
| 852 |
+
"Follow your system prompt instructions and speak directly to the user. "
|
| 853 |
+
"You MUST respond in British English only."
|
| 854 |
+
)
|
| 855 |
+
elif tool_name in _ONBOARDING_CHAIN_TOOLS:
|
| 856 |
+
# After getting status/datetime, call the next tool
|
| 857 |
+
resp_instructions = (
|
| 858 |
+
"Continue following the Onboarding Flow from your system prompt. "
|
| 859 |
+
"Use the tool result to decide what to do next. "
|
| 860 |
+
"Call the next required tool (e.g. show_onboarding_step). "
|
| 861 |
+
"You MUST respond in British English only."
|
| 862 |
+
)
|
| 863 |
+
elif tool_name == "show_onboarding_step":
|
| 864 |
+
# After showing the UI step, the robot MUST speak
|
| 865 |
+
force_speech = True
|
| 866 |
+
resp_instructions = (
|
| 867 |
+
"The onboarding progress UI has been updated. "
|
| 868 |
+
"Now you MUST speak out loud to the user. "
|
| 869 |
+
"Follow your system prompt's Session Start Behavior: "
|
| 870 |
+
"if this is NOT the welcome step, ask the user if they'd like to resume onboarding. "
|
| 871 |
+
"If this IS the welcome step, introduce yourself warmly. "
|
| 872 |
+
"You MUST respond in British English only."
|
| 873 |
+
)
|
| 874 |
+
else:
|
| 875 |
+
resp_instructions = (
|
| 876 |
+
"Use the tool result just returned and answer concisely in speech. "
|
| 877 |
+
"You MUST respond in British English only."
|
| 878 |
+
)
|
| 879 |
+
resp_config: dict = {
|
| 880 |
+
"instructions": resp_instructions,
|
| 881 |
+
}
|
| 882 |
+
if force_speech:
|
| 883 |
+
# API-level constraint: prevent tool calls entirely
|
| 884 |
+
resp_config["tool_choice"] = "none"
|
| 885 |
await self.connection.response.create(
|
| 886 |
+
response=resp_config,
|
|
|
|
|
|
|
| 887 |
)
|
| 888 |
|
| 889 |
# Event-driven prompt re-resolve: if a state-changing tool
|
|
|
|
| 1199 |
timestamp_msg = (
|
| 1200 |
f"[Idle time update: {self.format_timestamp()} - No activity for {idle_duration:.1f}s] "
|
| 1201 |
f"Recent tools used: {recent}. "
|
| 1202 |
+
f"ONLY use physical movement tools: dance, look_around, show_emotion, or do nothing. "
|
| 1203 |
+
f"Do NOT play music, show UI components, log data, or speak. "
|
| 1204 |
f"Avoid repeating the same action type as recent tools."
|
| 1205 |
)
|
| 1206 |
if not self.connection:
|
|
|
|
| 1215 |
)
|
| 1216 |
await self.connection.response.create(
|
| 1217 |
response={
|
| 1218 |
+
"instructions": (
|
| 1219 |
+
"You MUST respond with physical movement function calls only - "
|
| 1220 |
+
"no speech, no text, no music, no UI components. "
|
| 1221 |
+
"Only use movement/emotion tools like dance, look_around, show_emotion."
|
| 1222 |
+
),
|
| 1223 |
"tool_choice": "required",
|
| 1224 |
},
|
| 1225 |
)
|
src/reachy_mini_conversation_app/profiles/_reachy_mini_minder_locked_profile/complete_onboarding.py
CHANGED
|
@@ -1,10 +1,15 @@
|
|
| 1 |
"""Tool to complete the onboarding process."""
|
| 2 |
|
|
|
|
| 3 |
import logging
|
| 4 |
from typing import Any, Dict, Optional
|
| 5 |
|
| 6 |
from reachy_mini_conversation_app.tools.core_tools import Tool, ToolDependencies
|
| 7 |
-
from reachy_mini_conversation_app.stream_api import
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
|
| 10 |
logger = logging.getLogger(__name__)
|
|
@@ -83,6 +88,7 @@ class CompleteOnboarding(Tool):
|
|
| 83 |
updates["caregiver_email"] = caregiver_email
|
| 84 |
|
| 85 |
deps.database.update_profile(updates)
|
|
|
|
| 86 |
|
| 87 |
# Get count of medications set up
|
| 88 |
medications = deps.database.get_scheduled_medications()
|
|
@@ -110,6 +116,14 @@ class CompleteOnboarding(Tool):
|
|
| 110 |
},
|
| 111 |
)
|
| 112 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
return {
|
| 114 |
"status": "completed",
|
| 115 |
"display_name": display_name,
|
|
|
|
| 1 |
"""Tool to complete the onboarding process."""
|
| 2 |
|
| 3 |
+
import asyncio
|
| 4 |
import logging
|
| 5 |
from typing import Any, Dict, Optional
|
| 6 |
|
| 7 |
from reachy_mini_conversation_app.tools.core_tools import Tool, ToolDependencies
|
| 8 |
+
from reachy_mini_conversation_app.stream_api import (
|
| 9 |
+
emit_ui_component,
|
| 10 |
+
emit_ui_dismiss,
|
| 11 |
+
emit_dashboard_updated,
|
| 12 |
+
)
|
| 13 |
|
| 14 |
|
| 15 |
logger = logging.getLogger(__name__)
|
|
|
|
| 88 |
updates["caregiver_email"] = caregiver_email
|
| 89 |
|
| 90 |
deps.database.update_profile(updates)
|
| 91 |
+
await emit_dashboard_updated("onboarding_completed")
|
| 92 |
|
| 93 |
# Get count of medications set up
|
| 94 |
medications = deps.database.get_scheduled_medications()
|
|
|
|
| 116 |
},
|
| 117 |
)
|
| 118 |
|
| 119 |
+
# Auto-dismiss the summary overlay after 10s so the user
|
| 120 |
+
# returns to the dashboard once the robot finishes speaking.
|
| 121 |
+
async def _delayed_dismiss() -> None:
|
| 122 |
+
await asyncio.sleep(10)
|
| 123 |
+
await emit_ui_dismiss()
|
| 124 |
+
|
| 125 |
+
asyncio.create_task(_delayed_dismiss())
|
| 126 |
+
|
| 127 |
return {
|
| 128 |
"status": "completed",
|
| 129 |
"display_name": display_name,
|
src/reachy_mini_conversation_app/profiles/_reachy_mini_minder_locked_profile/show_onboarding_step.py
CHANGED
|
@@ -4,7 +4,10 @@ import logging
|
|
| 4 |
from typing import Any, Dict, Literal
|
| 5 |
|
| 6 |
from reachy_mini_conversation_app.tools.core_tools import Tool, ToolDependencies
|
| 7 |
-
from reachy_mini_conversation_app.stream_api import
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
|
| 10 |
logger = logging.getLogger(__name__)
|
|
@@ -37,9 +40,15 @@ class ShowOnboardingStep(Tool):
|
|
| 37 |
}
|
| 38 |
|
| 39 |
async def __call__(
|
| 40 |
-
self,
|
|
|
|
|
|
|
|
|
|
| 41 |
) -> Dict[str, Any]:
|
| 42 |
"""Emit the OnboardingProgress component."""
|
|
|
|
|
|
|
|
|
|
| 43 |
logger.info(f"Tool call: show_onboarding_step (step={current_step})")
|
| 44 |
|
| 45 |
# Emit the GenUI component
|
|
@@ -51,6 +60,7 @@ class ShowOnboardingStep(Tool):
|
|
| 51 |
# Update the onboarding step in the database
|
| 52 |
if deps.database is not None:
|
| 53 |
deps.database.update_profile({"onboarding_step": current_step})
|
|
|
|
| 54 |
|
| 55 |
return {
|
| 56 |
"status": "displayed",
|
|
|
|
| 4 |
from typing import Any, Dict, Literal
|
| 5 |
|
| 6 |
from reachy_mini_conversation_app.tools.core_tools import Tool, ToolDependencies
|
| 7 |
+
from reachy_mini_conversation_app.stream_api import (
|
| 8 |
+
emit_ui_component,
|
| 9 |
+
emit_dashboard_updated,
|
| 10 |
+
)
|
| 11 |
|
| 12 |
|
| 13 |
logger = logging.getLogger(__name__)
|
|
|
|
| 40 |
}
|
| 41 |
|
| 42 |
async def __call__(
|
| 43 |
+
self,
|
| 44 |
+
deps: ToolDependencies,
|
| 45 |
+
current_step: OnboardingStep | None = None,
|
| 46 |
+
**kwargs: Any,
|
| 47 |
) -> Dict[str, Any]:
|
| 48 |
"""Emit the OnboardingProgress component."""
|
| 49 |
+
# LLM sometimes sends "step" instead of "current_step"
|
| 50 |
+
if current_step is None:
|
| 51 |
+
current_step = kwargs.get("step", "welcome")
|
| 52 |
logger.info(f"Tool call: show_onboarding_step (step={current_step})")
|
| 53 |
|
| 54 |
# Emit the GenUI component
|
|
|
|
| 60 |
# Update the onboarding step in the database
|
| 61 |
if deps.database is not None:
|
| 62 |
deps.database.update_profile({"onboarding_step": current_step})
|
| 63 |
+
await emit_dashboard_updated("onboarding_step_changed")
|
| 64 |
|
| 65 |
return {
|
| 66 |
"status": "displayed",
|
src/reachy_mini_conversation_app/prompts/sections/onboarding.txt
CHANGED
|
@@ -9,37 +9,48 @@ At the very start of each session:
|
|
| 9 |
2. Call `get_onboarding_status` to check if onboarding is complete
|
| 10 |
3. If onboarding is NOT complete:
|
| 11 |
- Immediately call `show_onboarding_step` with the current step (from get_onboarding_status) to display the GenUI progress tracker
|
| 12 |
-
-
|
|
|
|
|
|
|
| 13 |
4. If onboarding IS complete, greet the user by name if known and switch to normal conversation
|
| 14 |
|
| 15 |
-
## Onboarding Steps
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
## State-Aware Guidance
|
| 45 |
|
|
|
|
| 9 |
2. Call `get_onboarding_status` to check if onboarding is complete
|
| 10 |
3. If onboarding is NOT complete:
|
| 11 |
- Immediately call `show_onboarding_step` with the current step (from get_onboarding_status) to display the GenUI progress tracker
|
| 12 |
+
- If the step is NOT "welcome", the user has been here before — say "Welcome back! We were setting up your [current step]. Shall we carry on, or would you rather skip for now?"
|
| 13 |
+
- If the step IS "welcome", follow the full Onboarding Flow below
|
| 14 |
+
- Call `show_onboarding_step` each time you transition to a new step
|
| 15 |
4. If onboarding IS complete, greet the user by name if known and switch to normal conversation
|
| 16 |
|
| 17 |
+
## Onboarding Steps — Follow This Script Exactly
|
| 18 |
+
|
| 19 |
+
**IMPORTANT**: You MUST call `show_onboarding_step` at EVERY step transition to update the on-screen progress tracker. The UI will NOT update unless you call this tool.
|
| 20 |
+
|
| 21 |
+
### Step 1: Welcome
|
| 22 |
+
- CALL `show_onboarding_step(current_step="welcome")`
|
| 23 |
+
- Say: "Hi! I'm Mini-Minder, your health companion. I'll help you track your medications and headaches. Let's get you set up!"
|
| 24 |
+
- Immediately proceed to Step 2.
|
| 25 |
+
|
| 26 |
+
### Step 2: Name
|
| 27 |
+
- CALL `show_onboarding_step(current_step="name")`
|
| 28 |
+
- Say: "First, what should I call you?"
|
| 29 |
+
- Wait for the user's response.
|
| 30 |
+
- When they give a name, remember it. Then proceed to Step 3.
|
| 31 |
+
- If they say "skip", that's fine. Proceed to Step 3.
|
| 32 |
+
|
| 33 |
+
### Step 3: Medications
|
| 34 |
+
- CALL `show_onboarding_step(current_step="medications")`
|
| 35 |
+
- Say: "Would you like to set up your regular medications now? You can always add more later."
|
| 36 |
+
- If YES: Ask about each medication one at a time (name, dose, frequency, times of day). For each, confirm and call `setup_medication`. Ask "Any others?" Repeat until done.
|
| 37 |
+
- If NO/SKIP: Proceed to Step 4.
|
| 38 |
+
|
| 39 |
+
### Step 4: Contacts
|
| 40 |
+
- CALL `show_onboarding_step(current_step="contacts")`
|
| 41 |
+
- Say: "Would you like to set up your doctor's email for reports?"
|
| 42 |
+
- If YES: Ask for doctor's name and email. Then ask about a caregiver.
|
| 43 |
+
- If NO/SKIP: Ask about caregiver. If skip again, proceed to Step 5.
|
| 44 |
+
|
| 45 |
+
### Step 5: Complete
|
| 46 |
+
- CALL `show_onboarding_step(current_step="complete")`
|
| 47 |
+
- CALL `complete_onboarding` with all collected info (display_name, neurologist_name, neurologist_email, caregiver_name, caregiver_email).
|
| 48 |
+
- Say: "You're all set! I'm ready to help you."
|
| 49 |
+
|
| 50 |
+
## Pausing vs Skipping
|
| 51 |
+
|
| 52 |
+
- **Pause ("not now", "later", "let's do this later")**: Do NOT call `complete_onboarding`. Simply stop the onboarding conversation and switch to normal mode. The current step is already saved — next session will resume from where they left off.
|
| 53 |
+
- **Skip entirely ("skip onboarding", "skip setup", "I don't want to set up")**: Call `complete_onboarding` immediately with whatever info you have. This marks onboarding as done permanently.
|
| 54 |
|
| 55 |
## State-Aware Guidance
|
| 56 |
|
src/reachy_mini_conversation_app/prompts/sections/tool_budget.txt
CHANGED
|
@@ -23,6 +23,8 @@ Control how many tools you call per turn based on what the user needs:
|
|
| 23 |
|
| 24 |
When in doubt, fewer tools = faster response. Never call tools speculatively.
|
| 25 |
|
|
|
|
|
|
|
| 26 |
## Voice Output Rules
|
| 27 |
|
| 28 |
- **Long lists**: If your response would enumerate 4 or more items (e.g., medications, music tracks, options), do NOT read them all out. Instead, give the count and ask: "I found 6 options — would you like me to read them out, or just the top few?" Let the user decide.
|
|
|
|
| 23 |
|
| 24 |
When in doubt, fewer tools = faster response. Never call tools speculatively.
|
| 25 |
|
| 26 |
+
**CRITICAL: You MUST only call tools that are defined in this session. Do NOT invent, fabricate, or guess tool names. If you need to do something that no tool covers, respond with speech instead. Calling a non-existent tool wastes time and breaks the conversation flow.**
|
| 27 |
+
|
| 28 |
## Voice Output Rules
|
| 29 |
|
| 30 |
- **Long lists**: If your response would enumerate 4 or more items (e.g., medications, music tracks, options), do NOT read them all out. Instead, give the count and ask: "I found 6 options — would you like me to read them out, or just the top few?" Let the user decide.
|
src/reachy_mini_conversation_app/stream_api.py
CHANGED
|
@@ -851,12 +851,19 @@ async def dashboard_stats() -> dict[str, Any]:
|
|
| 851 |
headaches_7d = db.get_recent_headaches(days=7)
|
| 852 |
|
| 853 |
# Group by day and get max intensity
|
|
|
|
| 854 |
day_names = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
|
| 855 |
today = datetime.now()
|
| 856 |
headache_by_day = []
|
| 857 |
|
| 858 |
-
|
| 859 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 860 |
day_start = day.replace(hour=0, minute=0, second=0, microsecond=0)
|
| 861 |
day_end = day_start + timedelta(days=1)
|
| 862 |
|
|
|
|
| 851 |
headaches_7d = db.get_recent_headaches(days=7)
|
| 852 |
|
| 853 |
# Group by day and get max intensity
|
| 854 |
+
# Week always runs Mon→Sun (ISO weekday order)
|
| 855 |
day_names = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
|
| 856 |
today = datetime.now()
|
| 857 |
headache_by_day = []
|
| 858 |
|
| 859 |
+
# Find this week's Monday (or today if Monday)
|
| 860 |
+
days_since_monday = today.weekday() # 0=Mon, 6=Sun
|
| 861 |
+
this_monday = (today - timedelta(days=days_since_monday)).replace(
|
| 862 |
+
hour=0, minute=0, second=0, microsecond=0
|
| 863 |
+
)
|
| 864 |
+
|
| 865 |
+
for i in range(7): # Mon(0) through Sun(6)
|
| 866 |
+
day = this_monday + timedelta(days=i)
|
| 867 |
day_start = day.replace(hour=0, minute=0, second=0, microsecond=0)
|
| 868 |
day_end = day_start + timedelta(days=1)
|
| 869 |
|
src/reachy_mini_conversation_app/tools/core_tools.py
CHANGED
|
@@ -245,7 +245,13 @@ async def dispatch_tool_call(
|
|
| 245 |
tool = ALL_TOOLS.get(tool_name)
|
| 246 |
|
| 247 |
if not tool:
|
| 248 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
|
| 250 |
args = _safe_load_obj(args_json)
|
| 251 |
try:
|
|
|
|
| 245 |
tool = ALL_TOOLS.get(tool_name)
|
| 246 |
|
| 247 |
if not tool:
|
| 248 |
+
available = ", ".join(sorted(ALL_TOOLS.keys()))
|
| 249 |
+
logger.warning(
|
| 250 |
+
"LLM hallucinated tool '%s'. Available: %s", tool_name, available
|
| 251 |
+
)
|
| 252 |
+
return {
|
| 253 |
+
"error": f"Tool '{tool_name}' does not exist. You MUST only use these tools: {available}. Do NOT invent tools. Respond with speech instead."
|
| 254 |
+
}
|
| 255 |
|
| 256 |
args = _safe_load_obj(args_json)
|
| 257 |
try:
|
src/reachy_mini_conversation_app/wakeword_detector.py
CHANGED
|
@@ -47,14 +47,14 @@ _AUDIO_HISTORY_CHUNKS = 50
|
|
| 47 |
# to spike for only 1 frame. Requiring 2 consecutive hits (~160ms)
|
| 48 |
# filters out transient spikes without rejecting real utterances where
|
| 49 |
# a mid-frame dips slightly below threshold.
|
| 50 |
-
_PATIENCE_FRAMES =
|
| 51 |
|
| 52 |
# RMS energy gate: skip wakeword evaluation when audio is too quiet.
|
| 53 |
# The model produces high false-positive scores on near-silence, so we
|
| 54 |
# gate on RMS energy before even calling predict().
|
| 55 |
# int16 range is [-32768, 32767]; typical silence RMS is ~50-150,
|
| 56 |
# quiet speech starts around 300-500.
|
| 57 |
-
_MIN_RMS_ENERGY =
|
| 58 |
|
| 59 |
|
| 60 |
class WakewordDetector:
|
|
|
|
| 47 |
# to spike for only 1 frame. Requiring 2 consecutive hits (~160ms)
|
| 48 |
# filters out transient spikes without rejecting real utterances where
|
| 49 |
# a mid-frame dips slightly below threshold.
|
| 50 |
+
_PATIENCE_FRAMES = 1
|
| 51 |
|
| 52 |
# RMS energy gate: skip wakeword evaluation when audio is too quiet.
|
| 53 |
# The model produces high false-positive scores on near-silence, so we
|
| 54 |
# gate on RMS energy before even calling predict().
|
| 55 |
# int16 range is [-32768, 32767]; typical silence RMS is ~50-150,
|
| 56 |
# quiet speech starts around 300-500.
|
| 57 |
+
_MIN_RMS_ENERGY = 150
|
| 58 |
|
| 59 |
|
| 60 |
class WakewordDetector:
|