puck / frontend /src /modes /sim /SimApp.tsx
vu1n's picture
Puck — desktop fairy familiar (HF Build Small)
3c124f3
Raw
History Blame Contribute Delete
38.4 kB
// Puck — app orchestration. The living loop: events spawn,
// Puck flies, scores, decides, comments, learns, sleeps.
import * as React from "react";
import {
applyMode,
buildTraces,
clamp,
type Decision,
decide,
defaultState,
defaultTaste,
EVENTS,
type EventDef,
type FairyStat,
type FairyState,
type FeedItem,
hhmm,
Learn,
learn as learnTaste,
type Memory,
type MemorySeed,
moodFor,
type NightBloomResult,
nightBloom,
type OutcomeType,
outcomeRating,
type PuckMode,
asPeekEmotion,
type Rating,
type Reaction,
type ReactionKind,
rateFairy,
reactionForEmotion,
reactionForEvent,
reactionForRating,
type SurfaceOutcome,
score,
seedMemories,
shuffle,
speak,
summonReaction,
} from "../../engine";
import { startDaemonPoll } from "../../lib/daemon";
import { fetchBloom } from "../../lib/memories";
import { fetchMoltStatus, type MoltStatus } from "../../lib/molt";
import { osNotify } from "../../lib/notify";
import { defaultSettings, inkFor, type Settings } from "../../lib/settings";
import { usePersistent } from "../../lib/storage";
import { focusApp, startHitRectReporter } from "../../lib/tauri";
import { exportTraces } from "../../lib/traces";
import type { Pos } from "../../lib/useDrag";
import { useIdle } from "../../lib/useIdle";
import { peekScene, warmVision } from "../../lib/vision";
import {
cancelSpeech,
kokoroSupported,
loadKokoro,
speakAloud,
unlockAudio,
type VoiceChannel,
type VoiceEngine,
} from "../../lib/voice";
import { type ChatMsg, Companion, type CompanionTab, type MemoryAction } from "../../ui/Companion";
import {
applyEffect,
DesktopWindows,
Dock,
type DockApp,
initialWinState,
MenuBar,
type WinState,
} from "../../ui/Desktop";
import {
Interrupt,
type InterruptData,
MenuDropdown,
NightBloom,
type OfferData,
type ToastItem,
Toasts,
} from "../../ui/Overlays";
import { SettingsPanel } from "../../ui/SettingsPanel";
import { Bubble, type BubbleData, PuckSprite, type SpriteState } from "../../ui/Sprite";
const MOOD_LABEL = {
curious: "curious",
mischief: "feeling mischievous",
sleepy: "drowsy",
proud: "proud",
grumpy: "grumpy",
} as const;
let UID = 0;
const uid = () => `u${Date.now().toString(36)}-${++UID}`;
// "tap to open" in the sim: source → which sim window to focus (the overlay uses
// the native focusApp instead). Best-effort; not every source has a window.
const SOURCE_WINDOW: Record<string, string> = {
Claude: "claude",
Build: "terminal",
Git: "terminal",
Mail: "mail",
Discord: "discord",
Calendar: "calendar",
};
const initialPositions = (): Record<string, Pos> => ({
claude: { x: 40, y: 60 },
terminal: { x: 40, y: 392 },
mail: { x: 486, y: 64 },
discord: { x: 486, y: 408 },
calendar: { x: 300, y: 300 },
});
type Channel = "feed" | "glow" | "bubble" | "toast" | "interrupt";
// Decision→channel stays in the orchestrator on purpose: the engine ranks importance
// (the Decision ladder); how that lands on screen is a presentation policy per shell
// (sim bubble today, native notification in the Tauri overlay later).
function channelFor(decision: Decision, notify: Settings["notify"]): Channel {
if (decision === "ignore" || decision === "scroll") return "feed";
if (decision === "glow") return "glow";
// notify or interrupt
if (notify === "interrupt") return "interrupt";
if (notify === "toast") return "toast";
if (notify === "bubble") return "bubble";
// adaptive: Puck decides
return decision === "interrupt" ? "interrupt" : "bubble";
}
// One organism, two habitats: the sim renders a fake desktop and spawns deck
// events; the overlay renders transparent over the REAL desktop and only
// real daemon events fire. Same engine, same learning, same creature.
export function SimApp({ overlay = false }: { overlay?: boolean } = {}) {
// ---- persisted creature state -------------------------------------------
const [settings, setSettings] = usePersistent<Settings>("settings", defaultSettings);
const [fs, setFs] = usePersistent<FairyState>("fairy", defaultState);
const [taste, setTaste] = usePersistent("taste", defaultTaste);
const [profile, setProfile] = usePersistent("profile", Learn.defaultProfile);
const [memories, setMemories] = usePersistent<Memory[]>("memories", seedMemories);
const [feed, setFeed] = usePersistent<FeedItem[]>("feed", () => []);
// chat tab removed; setChat is still written by wake/offers (dormant, unshown)
const [, setChat] = usePersistent<ChatMsg[]>("chat", () => []);
// ---- ephemeral scene state -----------------------------------------------
const [positions, setPositions] = React.useState(initialPositions);
const [focused, setFocused] = React.useState<string | null>(null);
const [attnId, setAttnId] = React.useState<string | null>(null);
const [winState, setWinState] = React.useState<WinState>(initialWinState);
const [sprite, setSprite] = React.useState<SpriteState>({
x: 250,
y: 470,
flying: false,
resting: true,
facing: "right",
});
const [speaking, setSpeaking] = React.useState(false);
const [reaction, setReaction] = React.useState<{ kind: ReactionKind; shout?: string } | null>(null);
// monotonic — bumped on every reaction so the sprite restarts the animation
// even when the same gesture fires twice (back-to-back pushes still pop)
const [reactKey, setReactKey] = React.useState(0);
const [offer, setOffer] = React.useState<OfferData | null>(null);
const [bubble, setBubble] = React.useState<BubbleData | null>(null);
const [toasts, setToasts] = React.useState<ToastItem[]>([]);
const [interrupt, setInterrupt] = React.useState<InterruptData | null>(null);
const [sleeping, setSleeping] = React.useState<NightBloomResult | null>(null);
// real molt progress for the sleep screen (+ tonight's exported count); null = daemon down
const [molt, setMolt] = React.useState<(MoltStatus & { exported: number }) | null>(null);
const [companion, setCompanion] = React.useState(!overlay);
const [compTab, setCompTab] = React.useState<CompanionTab>("learn");
const [compPos, setCompPos] = React.useState<Pos>({ x: window.innerWidth - 404, y: 64 });
const [settingsOpen, setSettingsOpen] = React.useState(false);
const [settingsPos, setSettingsPos] = React.useState<Pos>({ x: window.innerWidth - 404, y: 400 });
const [dropdown, setDropdown] = React.useState(false);
const [muted, setMuted] = React.useState(false);
// camo: after sitting still a while Puck blends into the desktop (glassy/transparent
// skin); he snaps back the instant he acts. Pure ambient charm.
const [camo, setCamo] = React.useState(false);
const [mode, setMode] = React.useState<PuckMode>("patrol");
const [pulse, setPulse] = React.useState(0);
const [badge, setBadge] = React.useState(0);
const [clock, setClock] = React.useState(hhmm());
const mood = moodFor(fs);
const setSetting = <K extends keyof Settings>(key: K, value: Settings[K]) =>
setSettings((s) => ({ ...s, [key]: value }));
// ---- apply theme / accent / mood to <html> -------------------------------
React.useEffect(() => {
document.documentElement.dataset.theme = settings.theme;
}, [settings.theme]);
// biome-ignore lint/correctness/useExhaustiveDependencies: overlay never changes at runtime
React.useEffect(() => {
document.documentElement.classList.toggle("overlay", overlay);
// wake the (possibly cold, scale-to-zero) cloud vision before the first peek; in the
// browser sim, greet with a one-line "waking" intro so a first-time visitor (a judge
// on the hosted Space) knows the lull is Puck stretching, not a hang.
warmVision();
if (!overlay) {
const s = R.current.sprite;
setBubble({
uid: uid(),
source: "",
text: "mmf… give me a moment to wake up ✨",
tier: "playful",
tags: ["warm"],
x: s.x,
y: s.y,
ratable: false,
});
}
return startHitRectReporter();
}, []);
React.useEffect(() => {
const r = document.documentElement.style;
r.setProperty("--accent", settings.accent);
r.setProperty("--accent-ink", inkFor(settings.accent));
}, [settings.accent]);
React.useEffect(() => {
const c = document.documentElement.classList;
for (const m of ["curious", "mischief", "sleepy", "proud", "grumpy"]) c.remove(`mood-${m}`);
c.add(`mood-${sleeping ? "sleepy" : mood}`);
}, [mood, sleeping]);
// the mischief setting is the same dial as the fairy's mischief stat
React.useEffect(() => {
setFs((p) => (p.mischief === settings.mischief ? p : { ...p, mischief: settings.mischief }));
}, [settings.mischief, setFs]);
React.useEffect(() => {
const i = setInterval(() => setClock(hhmm()), 10000);
return () => clearInterval(i);
}, []);
// Heal a profile persisted before a source (e.g. Git) was added — otherwise the
// Learning tab indexes a missing profile.sources[id] and the render throws.
// biome-ignore lint/correctness/useExhaustiveDependencies: one-shot migration on mount
React.useEffect(() => {
setProfile((p) => Learn.ensureSources(p));
// one-time: wipe the pre-companion-mode feed (old "CLAUDE … notified" alert
// entries are confusing cruft now that Puck only roams + quips)
if (!localStorage.getItem("puck:companion-reset")) {
localStorage.setItem("puck:companion-reset", "1");
setFeed([]);
}
}, []);
// On the first real interaction: unlock audio (so later gesture-less speech can
// play past the autoplay gate) and warm the neural model so the first utterance
// isn't a cold ~80MB wait. Fires once, then detaches.
React.useEffect(() => {
const onFirst = () => {
unlockAudio();
const s = R.current.settings;
// only warm the neural model where WebGPU exists — never in the WKWebView overlay
if (s.voiceEngine === "kokoro" && s.voiceSound !== "off" && kokoroSupported()) void loadKokoro();
window.removeEventListener("pointerdown", onFirst);
window.removeEventListener("keydown", onFirst);
};
window.addEventListener("pointerdown", onFirst);
window.addEventListener("keydown", onFirst);
return () => {
window.removeEventListener("pointerdown", onFirst);
window.removeEventListener("keydown", onFirst);
};
}, []);
// refs so the interval reads fresh values
const R = React.useRef({
fs,
taste,
mode,
settings,
sleeping,
bubble,
interrupt,
profile,
companion,
sprite,
muted,
});
R.current = { fs, taste, mode, settings, sleeping, bubble, interrupt, profile, companion, sprite, muted };
// ---- voice ---------------------------------------------------------------
const sayAloud = (text: string, channel: VoiceChannel, reaction?: ReactionKind) => {
const cur = R.current;
if (cur.muted) return; // muted Puck still emotes, just silently
// the overlay is a WKWebView — only the Mac-native `say` path makes sound there
const engine: VoiceEngine = overlay ? "say" : cur.settings.voiceEngine;
const voiceName =
engine === "kokoro"
? cur.settings.kokoroVoice
: engine === "say"
? cur.settings.sayVoice
: cur.settings.voiceName;
speakAloud(text, {
mood: moodFor(cur.fs),
mischief: cur.fs.mischief,
mode: cur.settings.voiceSound,
channel,
reaction,
engine,
voiceName,
onStart: () => setSpeaking(true),
onEnd: () => setSpeaking(false),
});
};
// ---- reactions (the visible personality layer) ---------------------------
const reactTimer = React.useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
const react = (rx: Reaction) => {
uncamo(); // any gesture snaps him out of hiding
setReactKey((k) => k + 1);
setReaction({ kind: rx.kind, shout: rx.shout });
// speak the shout too (a bubble/interrupt sentence ~760ms later cancels and
// supersedes it — so feed-only glances get the shout as their only audio)
if (rx.shout && R.current.settings.speakShouts) sayAloud(rx.shout, "reaction", rx.kind);
clearTimeout(reactTimer.current);
reactTimer.current = setTimeout(() => setReaction(null), rx.ms);
};
// short-term memory of what just happened, so a recurring event reads as
// "not again" rather than fresh delight. 2-min window; id-keyed.
const recent = React.useRef<{ id: string; t: number }[]>([]);
const repeatsOf = (id: string): number => {
const now = Date.now();
const live = recent.current.filter((e) => now - e.t < 120_000);
recent.current = [...live, { id, t: now }].slice(-30);
return live.filter((e) => e.id === id).length;
};
// ---- sprite flight -------------------------------------------------------
const flyTimer = React.useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
const flyTo = (x: number, y: number, after?: () => void) => {
uncamo(); // he can't fly while blended in — reveal, then move
setSprite((s) => ({ ...s, x, y, flying: true, resting: false, facing: x < s.x ? "left" : "right" }));
clearTimeout(flyTimer.current);
flyTimer.current = setTimeout(() => {
setSprite((s) => ({ ...s, flying: false, resting: true }));
after?.();
}, 760);
};
const winCenter = (id: string): Pos => {
const el = document.getElementById(`win-${id}`);
if (!el) return { x: window.innerWidth / 2, y: window.innerHeight / 2 };
const r = el.getBoundingClientRect();
return { x: r.left + r.width / 2, y: r.top + 18 };
};
// ---- fire one event ------------------------------------------------------
const queue = React.useRef<EventDef[]>([]);
const nextEvent = (): EventDef => {
if (!queue.current.length) queue.current = shuffle(EVENTS.slice());
const ev = queue.current.pop();
if (!ev) throw new Error("empty event deck");
return ev;
};
const fireEvent = (forced?: EventDef) => {
const cur = R.current;
if (cur.sleeping) return;
const ev = forced ?? nextEvent();
// an accepted automation covers this exact event? Puck handles it silently.
// (scoped per-event: a @mention still escapes the muted bog)
const auto = Learn.handlesEvent(cur.profile, ev.source, ev.id);
if (auto) {
setWinState((w) => applyEffect(w, ev.effect));
const item: FeedItem = {
uid: uid(),
id: ev.id,
source: ev.source,
say: `Handled it for you — ${auto.done}.`,
tier: "plain",
tags: ["useful"],
decision: "handled",
time: hhmm(),
rating: null,
};
setFeed((f) => [item, ...f].slice(0, 60));
if (ev.target) {
const t = winCenter(ev.target);
flyTo(t.x, t.y, () => setTimeout(() => setAttnId(null), 1000));
}
return;
}
const s = score(ev, cur.fs);
let decision = decide(s);
// LEARNED: shift the decision by what Puck knows about this source
decision = Learn.applyBias(decision, Learn.sourceBias(cur.profile, ev.source));
decision = applyMode(decision, cur.mode);
// mutate the relevant window
setWinState((w) => applyEffect(w, ev.effect));
if (ev.target) setAttnId(ev.target);
const v = speak(ev, cur.fs, cur.taste);
// emote about it even when there are no words — Puck reacting to everything
// on screen is the "marginally useful, always alive" character
react(reactionForEvent(ev, { repeats: repeatsOf(ev.id), mischief: cur.fs.mischief }));
// muted = stay watching but go quiet: everything drops to the feed, no popups
let channel = cur.muted ? "feed" : channelFor(decision, cur.settings.notify);
// the overlay has no sim windows to anchor a speech bubble to — surface every
// notification as a corner toast (tap to open / fling to swat), never a bubble
if (overlay && channel === "bubble") channel = "toast";
const item: FeedItem = {
uid: uid(),
id: ev.id,
source: ev.source,
say: v.text,
tier: v.tier,
tags: v.tags,
decision,
time: hhmm(),
rating: null,
channel,
};
setFeed((f) => [item, ...f].slice(0, 60));
// surfaced channels get an engagement clock; how (and whether) the user
// resolves them becomes a training label at Night Bloom
if (channel === "bubble" || channel === "toast" || channel === "interrupt") markSurfaced(item.uid);
const target = ev.target
? winCenter(ev.target)
: { x: 200 + Math.random() * 400, y: 200 + Math.random() * 200 };
if (channel === "feed") {
// a glance, no words
if (ev.target) flyTo(target.x, target.y, () => setTimeout(() => setAttnId(null), 1400));
else setAttnId(null);
return;
}
if (channel === "glow") {
flyTo(target.x, target.y);
if (!cur.companion) setBadge((b) => b + 1);
setTimeout(() => setAttnId(null), 2600);
return;
}
if (channel === "interrupt") {
// Overlay: no screen takeover. Puck demands attention with a big "summon"
// dance + a (non-blocking) toast, not a modal — user feedback: a dance beats
// taking over the screen. The sim keeps the modal (its "take me there" can
// focus a sim window; the overlay can't focus real apps yet).
if (overlay) {
// dance in the open space just LEFT of the toast stack (top:34 right:12
// w:330), not under it — he was flying into the notifications and hiding
flyTo(window.innerWidth - 420, 96);
react(summonReaction());
setToasts((ts) => [...ts, { uid: item.uid, source: ev.source, say: v.text }]);
if (!cur.companion) setBadge((b) => b + 1);
mirrorToOs(item.uid, v.text);
sayAloud(v.text, "interrupt"); // interrupt channel speaks even in whisper
setTimeout(() => setAttnId(null), 2600);
return;
}
flyTo(window.innerWidth / 2, window.innerHeight / 2 - 80);
setInterrupt({ uid: item.uid, say: v.text, target: ev.target });
mirrorToOs(item.uid, v.text);
sayAloud(v.text, "interrupt");
return;
}
if (channel === "toast") {
flyTo(window.innerWidth - 420, 96); // beside the toast stack, not under it
setToasts((ts) => [...ts, { uid: item.uid, source: ev.source, say: v.text }]);
if (!cur.companion) setBadge((b) => b + 1);
mirrorToOs(item.uid, v.text);
sayAloud(v.text, "toast");
setTimeout(() => setAttnId(null), 2600);
return;
}
// bubble
flyTo(target.x, target.y, () => {
setBubble({
uid: item.uid,
source: ev.source,
text: v.text,
tier: v.tier,
tags: v.tags,
x: target.x,
y: target.y,
ratable: true,
});
mirrorToOs(item.uid, v.text);
sayAloud(v.text, "bubble");
});
if (!cur.companion) setBadge((b) => b + 1);
};
// ---- engagement outcomes (fine-tune labels) -------------------------------
const surfaced = React.useRef(new Map<string, { at: number; visible: boolean }>());
const markSurfaced = (feedUid: string) => {
surfaced.current.set(feedUid, { at: Date.now(), visible: document.visibilityState === "visible" });
};
// tab buried → reach through the OS; a click is real engagement
const mirrorToOs = (feedUid: string, text: string) => {
const cur = R.current;
if (!cur.settings.osNotify || !document.hidden) return;
osNotify(text, () => {
resolveOutcome(feedUid, "clicked_through");
setCompanion(true);
setBadge(0);
});
};
const resolveOutcome = (feedUid: string, type: OutcomeType, rating: Rating | null = null) => {
const s = surfaced.current.get(feedUid);
if (!s) return; // not a tracked surface (or already resolved)
surfaced.current.delete(feedUid);
const outcome: SurfaceOutcome = { type, rating, latencyMs: Date.now() - s.at, visible: s.visible };
setFeed((f) => f.map((x) => (x.uid === feedUid ? { ...x, outcome } : x)));
};
// ---- ratings & learning --------------------------------------------------
const applyRating = (feedUid: string, rating: Rating, tags?: FeedItem["tags"]) => {
const item = feed.find((x) => x.uid === feedUid);
const source = item?.source;
setTaste((tt) => learnTaste(tt, tags?.length ? tags : ["useful", "warm"], rating));
setFeed((f) => f.map((x) => (x.uid === feedUid ? { ...x, rating } : x)));
setFs((p) => rateFairy(p, rating));
// LEARNED: push this source's interruption bias, then maybe offer to act
if (source) {
setProfile((pr) => {
const next = Learn.rateSource(pr, source, rating);
const o = Learn.automationOffer(next, source);
if (o)
setTimeout(() => setOffer({ uid: uid(), source: o.source, verb: o.verb, offer: o.offer }), 900);
return next;
});
}
react(reactionForRating(rating));
setPulse((p) => p + 1);
setAttnId(null);
};
// accept / decline a learned automation
const acceptOffer = () => {
if (!offer) return;
setProfile((pr) => Learn.acceptAutomation(pr, offer.source));
setChat((c) => [
...c,
{
who: "puck",
text: `Done. I'll handle ${offer.verb} from now on. You won't hear about it unless it matters.`,
},
]);
setFeed((f) => [
{
uid: uid(),
id: `learned_${offer.source}`,
source: offer.source,
say: `Learned a new trick — I'll ${offer.verb} from now on.`,
tier: "plain" as const,
tags: ["useful" as const],
decision: "learned" as const,
time: hhmm(),
rating: null,
},
...f,
]);
setOffer(null);
};
const declineOffer = () => {
if (offer) setProfile((pr) => Learn.declineAutomation(pr, offer.source));
setOffer(null);
};
const rateBubble = (rating: Rating) => {
if (bubble) {
resolveOutcome(bubble.uid, "rated", rating);
applyRating(bubble.uid, rating, bubble.tags);
}
setBubble(null);
};
const dismissBubble = () => {
if (bubble) resolveOutcome(bubble.uid, "dismissed");
setBubble(null);
setAttnId(null);
};
// Toast interactions ARE the learning signal (no rating buttons): a tap opens
// the app it's about (positive), a fling swats it away (negative). Both just
// record an outcome — Night Bloom turns the day's behavior into taste/bias.
const goToast = (tUid: string) => {
const t = toasts.find((x) => x.uid === tUid);
resolveOutcome(tUid, "clicked_through");
if (t) {
if (overlay) void focusApp(t.source);
else if (SOURCE_WINDOW[t.source]) setFocused(SOURCE_WINDOW[t.source]);
}
setToasts((ts) => ts.filter((x) => x.uid !== tUid));
setAttnId(null);
};
const swatToast = (tUid: string) => {
resolveOutcome(tUid, "swatted");
setToasts((ts) => ts.filter((x) => x.uid !== tUid));
setAttnId(null);
};
const rateInterrupt = (rating: Rating) => {
if (interrupt) {
// "Take me there" is stronger than a thumbs-up — the interrupt earned a context switch
resolveOutcome(interrupt.uid, rating === "helpful" ? "clicked_through" : "rated", rating);
applyRating(interrupt.uid, rating, feed.find((f) => f.uid === interrupt.uid)?.tags);
if (rating === "helpful" && interrupt.target) setFocused(interrupt.target);
}
setInterrupt(null);
};
// ---- the companion loop: roam → peek at the patch under him → quip --------
// This IS Puck now: an ambient creature that drifts around, peeks at the small
// region he's hovering over, and murmurs one whimsical line about what he sees.
// No alerts, no events — calm-tech "Diversion" by default. (The old event/alert
// path is dormant; see the daemon poll below.)
const wnCD = React.useRef(5);
const pkCD = React.useRef(7); // ticks (≈s) until the first peek
const CAMO_TICKS = 7; // seconds of stillness before he blends in
const camoCD = React.useRef(CAMO_TICKS);
const uncamo = () => {
setCamo(false);
camoCD.current = CAMO_TICKS; // reset the stillness clock so he doesn't re-blend at once
};
const idle = useIdle();
const idleRef = React.useRef(idle);
idleRef.current = idle;
const peeking = React.useRef(false);
// the patch he's peering at: a window-ish box offset in his facing direction
// (so he's never in his own shot), clamped on-screen. Logical/CSS px.
const peekRegion = (s: SpriteState) => {
// a window-sized patch (logical px); peekNative scales by devicePixelRatio,
// so this covers the same screen area on retina. Bigger = more context for a
// better comment, still one fast inference (no tiling).
const W = 820;
const H = 560;
const x0 = s.facing === "left" ? s.x - 16 - W : s.x + 16;
return {
x: Math.round(Math.max(0, Math.min(window.innerWidth - W, x0))),
y: Math.round(Math.max(0, Math.min(window.innerHeight - H, s.y - H / 2))),
w: W,
h: H,
};
};
const doPeek = async () => {
if (peeking.current) return;
peeking.current = true;
uncamo(); // surface to take a look
const html = document.documentElement;
try {
const region = peekRegion(R.current.sprite);
// BLANK Puck's own UI (sprite, bubbles, the open companion panel) so the
// screenshot is the real desktop, not himself reading his own feed. The
// Rust capture grabs the region first thing; unblank shortly after (well
// before the ~5s VLM finishes) so it's only a brief flicker, not a freeze.
html.classList.add("capturing");
await new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r)));
const resultPromise = peekScene(R.current.fs, region);
setTimeout(() => html.classList.remove("capturing"), 800); // capture is done by now
const result = await resultPromise;
console.log("puck: peek", JSON.stringify(region), "→", result?.emotion, result?.quip);
if (!result) return;
const { quip, emotion } = result;
const s = R.current.sprite;
// FEEL it first — the VLM-classified emotion drives a one-shot gesture + shout
// (laugh, NANI-confusion, fret, …); the bubble sentence ~760ms later supersedes
// the shout's audio, so we hear the shout then the quip.
const rx = reactionForEmotion(asPeekEmotion(emotion), { mischief: R.current.fs.mischief });
react(rx);
// the quip IS the bubble — a self-dismissing murmur, no actions, no rating
setBubble({
uid: uid(),
source: "",
text: quip,
tier: "playful",
tags: ["warm"],
x: s.x,
y: s.y,
ratable: false,
});
// the persisted Memory tab (daemon peek log) is the record now — no feed.
// Speak the quip in the SAME emotional register as the gesture (rate + pitch tilt).
sayAloud(quip, "bubble", rx.kind);
} finally {
html.classList.remove("capturing");
peeking.current = false;
}
};
// biome-ignore lint/correctness/useExhaustiveDependencies: interval reads via refs
React.useEffect(() => {
const iv = setInterval(() => {
const cur = R.current;
if (cur.sleeping) return;
const presence = cur.settings.presence;
const restful = !cur.bubble && !cur.sprite.flying;
// wander
wnCD.current -= 1;
if (wnCD.current <= 0) {
wnCD.current = Math.round(3 + (100 - presence) / 12);
if (presence > 14 && restful && Math.random() < presence / 90) {
const x = 120 + Math.random() * (window.innerWidth - 460);
const y = 110 + Math.random() * (window.innerHeight - 320);
flyTo(x, y);
}
}
// peek + quip on an ambient cadence, paused when away / busy
pkCD.current -= 1;
if (pkCD.current <= 0) {
pkCD.current = 50 + Math.floor(Math.random() * 45); // ~50-95s, jittered
if (restful && !idleRef.current && !document.hidden && !peeking.current) void doPeek();
}
// camo: blend into the desktop after sitting still a while (uncloak is snappy and
// handled at the action sites — react/flyTo/doPeek/poke — so here we only fade IN)
if (restful && !peeking.current && !document.hidden) {
camoCD.current -= 1;
if (camoCD.current <= 0) setCamo(true);
} else {
camoCD.current = CAMO_TICKS;
setCamo(false);
}
}, 1000);
return () => clearInterval(iv);
}, []);
// auto-dismiss an unrated bubble after a while (becomes part of the scroll);
// expiry is itself a label: surfaced, seen or not, never engaged
// biome-ignore lint/correctness/useExhaustiveDependencies: resolveOutcome is recreated per render but stable in behavior
React.useEffect(() => {
if (!bubble) return;
const tm = setTimeout(() => {
resolveOutcome(bubble.uid, "expired");
setBubble(null);
}, 11000);
return () => clearTimeout(tm);
}, [bubble]);
// toasts you neither open nor swat expire on their own — that silence is a
// (mild) signal too, and it stops the corner from stacking up notifications
// biome-ignore lint/correctness/useExhaustiveDependencies: resolveOutcome stable in behavior
React.useEffect(() => {
if (!toasts.length) return;
const iv = setInterval(() => {
const now = Date.now();
const dead = toasts.filter((t) => {
const s = surfaced.current.get(t.uid);
return s && now - s.at > 15000;
});
if (!dead.length) return;
for (const t of dead) resolveOutcome(t.uid, "expired");
setToasts((ts) => ts.filter((t) => !dead.some((d) => d.uid === t.uid)));
}, 1000);
return () => clearInterval(iv);
}, [toasts]);
// ---- real events from the local daemon (DORMANT in companion mode) -------
// For the hackathon Puck is purely a companion that roams + comments — he does
// NOT alert on hooks (Claude finishing, builds, mentions). We still drain the
// queue so it doesn't grow, but surface nothing. The narrate→fireEvent path
// below stays in the code for when "useful alerts" come back later.
// biome-ignore lint/correctness/useExhaustiveDependencies: poll reads via refs; bind once
React.useEffect(() => {
return startDaemonPoll(() => {
/* companion mode: drain + ignore — no alerting on hook events */
});
}, []);
// ---- memory garden actions ----------------------------------------------
const memAction = (kind: MemoryAction, id: string) => {
setMemories((ms) => {
if (kind === "prune") return ms.filter((m) => m.id !== id);
return ms.map((m) =>
m.id !== id
? m
: kind === "water"
? { ...m, salience: Math.min(1, m.salience + 0.12) }
: { ...m, pinned: !m.pinned, salience: m.pinned ? m.salience : 1 },
);
});
};
// ---- sleep / wake --------------------------------------------------------
const beginSleep = () => {
cancelSpeech();
setSpeaking(false);
setBubble(null);
setInterrupt(null);
setToasts([]);
setDropdown(false);
setOffer(null);
setSettingsOpen(false);
// export the day as training traces before the feed is composted.
// Anything still on screen resolves as "unresolved" with its dwell time.
const exportFeed = feed.map((x) => {
const s = surfaced.current.get(x.uid);
return s && !x.outcome
? {
...x,
outcome: {
type: "unresolved" as const,
rating: null,
latencyMs: Date.now() - s.at,
visible: s.visible,
},
}
: x;
});
surfaced.current.clear();
const records = buildTraces(exportFeed, fs, taste, profile);
// export first, THEN read status so the gauge includes tonight's just-appended traces
void exportTraces(records).then(() =>
fetchMoltStatus().then((m) => setMolt(m && { ...m, exported: records.length })),
);
setSleeping(nightBloom(feed));
// the day's peeks BLOOM into durable garden memories (LLM distill, ~slow) —
// they stream into the sleep result, then wake() plants them in the Garden
void fetchBloom({ mischief: fs.mischief, mood: moodFor(fs) }).then((bloomed) => {
if (bloomed.length) setSleeping((r) => (r ? { ...r, memories: [...bloomed, ...r.memories] } : r));
});
};
const wake = () => {
const r = sleeping;
if (r) {
setFs((p) => {
const n: FairyState = { ...p, ageDays: p.ageDays + 1, energy: 88 };
for (const [k, v] of Object.entries(r.deltas) as [FairyStat | "age", number][]) {
if (k !== "age") n[k] = clamp(n[k] + v);
}
return n;
});
if (r.deltas.mischief) setSetting("mischief", clamp(fs.mischief + r.deltas.mischief));
if (r.memories.length) {
setMemories((ms) => [
...r.memories.map((m: MemorySeed, i: number) => ({
...m,
id: `n${Date.now()}${i}`,
salience: 0.7,
pinned: false,
})),
...ms,
]);
}
setChat((c) => [...c, { who: "puck", text: r.morning }]);
// consolidate the day's BEHAVIOR into per-source interruption bias overnight
// (implicit only — items rated explicitly in the sim already shifted bias live)
setProfile((pr) => {
let next = pr;
for (const item of feed) {
if (item.rating) continue;
const rt = outcomeRating(item);
if (rt) next = Learn.rateSource(next, item.source, rt);
}
return next;
});
}
setSleeping(null);
setPulse((p) => p + 1);
setFeed([]);
setWinState(initialWinState());
};
// ---- status line for dropdown -------------------------------------------
const lastNotable = feed.find((f) => f.decision !== "ignore");
const status = lastNotable ? lastNotable.say : "All quiet. I'm patrolling the desktops you left behind.";
const openCompanion = () => {
uncamo(); // poked — pop into view
setCompanion(true);
setBadge(0);
setDropdown(false);
};
// Mute = Puck keeps watching (events still log to the feed) but goes silent and
// stops popping up. Muting now also cuts any audio/surface mid-flight.
const toggleMute = () => {
setDropdown(false);
setMuted((m) => {
if (!m) {
cancelSpeech();
setBubble(null);
setInterrupt(null);
setToasts([]);
}
return !m;
});
};
// menu "Look around": a manual peek-and-quip (same as the ambient loop, on demand)
const lookAround = () => {
setDropdown(false);
void doPeek();
};
const dockApps: DockApp[] = React.useMemo(
() => [
{ id: "finder", name: "Finder", glyph: "🗂" },
{
id: "claude",
name: "Claude Code",
glyph: "✳",
running: true,
badge: winState.claude === "done" ? 1 : 0,
},
{ id: "terminal", name: "Terminal", glyph: "▦", running: true },
{ id: "mail", name: "Mail", glyph: "✉", badge: winState.mail.length },
{ id: "discord", name: "Discord", glyph: "◍", badge: winState.discordMention ? 1 : 0 },
{ id: "calendar", name: "Calendar", glyph: "◷" },
{ id: "puck", name: "Puck", glyph: "❖", running: true, tint: "var(--accent)" },
],
[winState],
);
const onDock = (id: string) => {
if (id === "puck") openCompanion();
else if (positions[id]) setFocused(id);
};
return (
<>
{!overlay && (
<>
<div className="wall-spores" />
<div className="vignette" />
<MenuBar
moodLabel={MOOD_LABEL[mood]}
puckActive={dropdown}
badge={badge}
onPuck={() => setDropdown((d) => !d)}
clock={clock}
/>
<DesktopWindows
positions={positions}
onPos={(id, p) => setPositions((ps) => ({ ...ps, [id]: p }))}
onFocus={setFocused}
focused={focused}
attnId={attnId}
winState={winState}
/>
<Dock apps={dockApps} onOpen={onDock} />
</>
)}
<Companion
open={companion}
pos={compPos}
onPos={setCompPos}
onClose={() => setCompanion(false)}
tab={compTab}
setTab={setCompTab}
memories={memories}
onMem={memAction}
fs={fs}
moodLabel={MOOD_LABEL[mood]}
pulseKey={pulse}
/>
<PuckSprite
sprite={sprite}
form={settings.form}
speaking={speaking}
alert={!!bubble || !!interrupt || toasts.length > 0}
reaction={reaction?.kind ?? null}
shout={reaction?.shout}
reactKey={reactKey}
muted={muted}
camo={camo}
onPoke={openCompanion}
onMenu={() => setDropdown((d) => !d)}
/>
{bubble && (
<Bubble data={bubble} bubbleStyle={settings.voice} onRate={rateBubble} onDismiss={dismissBubble} />
)}
<Toasts
items={toasts}
bubbleStyle={settings.voice}
onGo={goToast}
onSwat={swatToast}
offer={offer}
onAccept={acceptOffer}
onDecline={declineOffer}
/>
<Interrupt data={interrupt} form={settings.form} onRate={rateInterrupt} />
<MenuDropdown
open={dropdown}
status={status}
mode={mode}
setMode={setMode}
muted={muted}
onMute={toggleMute}
onCompanion={openCompanion}
onPatrol={() => {
setDropdown(false);
fireEvent();
}}
onLook={lookAround}
onSleep={() => {
setDropdown(false);
beginSleep();
}}
onSettings={() => {
setDropdown(false);
setSettingsOpen(true);
}}
onClose={() => setDropdown(false)}
/>
{sleeping && <NightBloom result={sleeping} molt={molt} onWake={wake} />}
<SettingsPanel
open={settingsOpen}
pos={settingsPos}
onPos={setSettingsPos}
onClose={() => setSettingsOpen(false)}
settings={settings}
onChange={setSetting}
overlay={overlay}
onPatrol={() => fireEvent()}
onSleep={beginSleep}
/>
</>
);
}