| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <title>Documentation — Voice Form Assistant Web App</title> |
| <link rel="preconnect" href="https://fonts.googleapis.com"> |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> |
| <link href="https://fonts.googleapis.com/css2?family=Sora:wght@500;600;700&family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet"> |
| <style> |
| :root{ |
| --ink:#12181F; --ink-mid:#1E2731; --ink-line:rgba(255,255,255,0.09); |
| --paper:#FBF9F5; --paper-line:rgba(18,24,31,0.10); |
| --text-on-ink:#F3F1EC; --text-muted-on-ink:#8D97A3; |
| --text-on-paper:#12181F; --text-muted-on-paper:#5B6672; |
| --amber:#E8A33D; --amber-dim:rgba(232,163,61,0.15); |
| --teal:#2FA6A0; --teal-dim:rgba(47,166,160,0.13); |
| --danger:#D9694F; --danger-dim:rgba(217,105,79,0.12); |
| --radius:12px; --radius-sm:8px; |
| --shadow:0 10px 30px rgba(18,24,31,0.10); |
| } |
| *{box-sizing:border-box;} |
| html{scroll-behavior:smooth;} |
| body{margin:0; background:var(--paper); color:var(--text-on-paper); font-family:'Inter',sans-serif; font-size:16px; line-height:1.65; -webkit-font-smoothing:antialiased;} |
| h1,h2,h3,h4{font-family:'Sora',sans-serif; line-height:1.2; margin:0 0 0.5em 0; letter-spacing:-0.01em;} |
| p{margin:0 0 1em 0;} |
| a{color:var(--teal);} |
| code,.mono{font-family:'JetBrains Mono',monospace;} |
| code.inline{background:var(--teal-dim); color:#1a6b66; padding:1px 7px; border-radius:5px; font-size:0.87em;} |
| ::selection{background:var(--amber); color:var(--ink);} |
| |
| #progress{position:fixed; top:0; left:0; height:3px; width:0%; background:linear-gradient(90deg,var(--amber),var(--teal)); z-index:200; transition:width .1s linear;} |
| |
| |
| .hero{background:var(--ink); color:var(--text-on-ink); padding:96px 32px 64px 32px; position:relative; overflow:hidden;} |
| .hero::before{content:""; position:absolute; top:-30%; right:-10%; width:480px; height:480px; border-radius:50%; background:radial-gradient(circle,var(--amber-dim),transparent 70%);} |
| .hero::after{content:""; position:absolute; bottom:-40%; left:-5%; width:420px; height:420px; border-radius:50%; background:radial-gradient(circle,var(--teal-dim),transparent 70%);} |
| .hero-inner{max-width:960px; margin:0 auto; position:relative;} |
| .eyebrow{display:flex; align-items:center; gap:10px; color:var(--amber); font-size:0.78rem; letter-spacing:0.14em; text-transform:uppercase; margin-bottom:20px;} |
| .eyebrow .dot{width:8px; height:8px; border-radius:50%; background:var(--amber); box-shadow:0 0 0 4px var(--amber-dim);} |
| .hero h1{font-size:clamp(2.1rem,4.6vw,3.2rem); max-width:18ch; color:var(--text-on-ink);} |
| .hero .lede{font-size:1.12rem; color:var(--text-muted-on-ink); max-width:64ch; margin-top:18px;} |
| .stat-strip{display:flex; gap:34px; flex-wrap:wrap; margin-top:44px; padding-top:30px; border-top:1px solid var(--ink-line);} |
| .stat-strip .stat b{display:block; font-family:'Sora',sans-serif; font-size:1.5rem; color:var(--text-on-ink);} |
| .stat-strip .stat span{font-size:0.8rem; color:var(--text-muted-on-ink);} |
| |
| |
| .shell{max-width:1180px; margin:0 auto; padding:0 32px; display:grid; grid-template-columns:230px minmax(0,1fr); gap:56px;} |
| @media (max-width:900px){.shell{grid-template-columns:1fr;}} |
| .toc{position:sticky; top:28px; align-self:start; padding-top:56px;} |
| .toc-title{font-size:0.72rem; letter-spacing:0.12em; text-transform:uppercase; color:var(--text-muted-on-paper); margin-bottom:14px;} |
| .toc ul{list-style:none; margin:0; padding:0;} |
| .toc a{display:block; font-size:0.83rem; color:var(--text-muted-on-paper); text-decoration:none; padding:7px 0 7px 15px; border-left:2px solid var(--paper-line); transition:.15s;} |
| .toc a.sub{padding-left:27px; font-size:0.78rem;} |
| .toc a:hover{color:var(--text-on-paper);} |
| .toc a.active{color:var(--teal); border-left-color:var(--teal); font-weight:600;} |
| @media (max-width:900px){.toc{position:static; padding-top:32px;}} |
| |
| main{padding-top:56px; padding-bottom:120px; min-width:0;} |
| .part-header{margin:88px 0 36px 0; padding-bottom:18px; border-bottom:1px solid var(--paper-line);} |
| .part-header:first-child{margin-top:0;} |
| .part-eyebrow{color:var(--teal); font-size:0.76rem; letter-spacing:0.12em; text-transform:uppercase; margin-bottom:10px; font-weight:600;} |
| .part-header h2{font-size:1.9rem;} |
| .part-header .sub{color:var(--text-muted-on-paper); font-size:1.02rem; max-width:66ch;} |
| |
| |
| .panel{background:#fff; border:1px solid var(--paper-line); border-radius:var(--radius); padding:24px 26px; margin:18px 0;} |
| .panel.dark{background:var(--ink); color:var(--text-on-ink); border:none;} |
| .panel.dark p, .panel.dark li{color:var(--text-muted-on-ink);} |
| .panel.dark h3, .panel.dark h4{color:var(--text-on-ink);} |
| .panel h4{font-family:'JetBrains Mono',monospace; font-size:0.98rem; color:var(--teal); margin-bottom:12px; font-weight:600;} |
| .panel.dark h4{color:var(--amber);} |
| |
| .callout{border-left:3px solid var(--teal); background:#fff; border:1px solid var(--paper-line); border-left-width:3px; border-radius:0 var(--radius) var(--radius) 0; padding:16px 20px; margin:20px 0; font-size:0.95rem;} |
| .callout .label{font-size:0.72rem; text-transform:uppercase; letter-spacing:0.09em; color:var(--teal); margin-bottom:8px; display:block; font-weight:700;} |
| .callout.warn{border-left-color:var(--danger);} |
| .callout.warn .label{color:var(--danger);} |
| .callout.amber{border-left-color:var(--amber);} |
| .callout.amber .label{color:#b8791f;} |
| |
| pre{background:#0F1720; border-radius:var(--radius); padding:16px 18px; overflow-x:auto; margin:16px 0; font-size:0.84rem; color:#dfe6ee;} |
| pre code{color:inherit; line-height:1.6;} |
| |
| |
| .pipeline{display:flex; flex-direction:column; gap:0; margin:28px 0;} |
| .pipe-stage{display:grid; grid-template-columns:34px 1fr; gap:16px; position:relative; padding-bottom:34px;} |
| .pipe-stage:last-child .pipe-line{display:none;} |
| .pipe-dot{width:34px; height:34px; border-radius:50%; background:var(--teal-dim); color:var(--teal); display:flex; align-items:center; justify-content:center; font-family:'JetBrains Mono',monospace; font-size:0.78rem; font-weight:700; flex-shrink:0; z-index:1;} |
| .pipe-dot.amber{background:var(--amber-dim); color:#b8791f;} |
| .pipe-line{position:absolute; left:17px; top:34px; bottom:0; width:2px; background:var(--paper-line);} |
| .pipe-content h4{font-family:'Sora',sans-serif; font-size:1.02rem; color:var(--text-on-paper); margin-bottom:4px; font-weight:600;} |
| .pipe-content .pipe-file{font-family:'JetBrains Mono',monospace; font-size:0.78rem; color:var(--text-muted-on-paper); margin-bottom:6px;} |
| .pipe-content p{font-size:0.92rem; color:var(--text-muted-on-paper); margin-bottom:0;} |
| |
| |
| .concept{display:grid; grid-template-columns:160px 1fr; gap:20px; padding:20px 0; border-top:1px solid var(--paper-line);} |
| .concept:last-child{border-bottom:1px solid var(--paper-line);} |
| .concept .term{font-family:'Sora',sans-serif; font-weight:600; color:var(--text-on-paper); font-size:0.98rem;} |
| .concept .term .tag{display:block; font-family:'JetBrains Mono',monospace; font-size:0.7rem; color:var(--amber); margin-top:6px; font-weight:500;} |
| @media (max-width:640px){.concept{grid-template-columns:1fr; gap:6px;}} |
| |
| |
| table{width:100%; border-collapse:collapse; margin:18px 0; font-size:0.9rem;} |
| th{text-align:left; font-family:'Sora',sans-serif; font-weight:600; color:var(--text-muted-on-paper); font-size:0.7rem; text-transform:uppercase; letter-spacing:0.07em; padding:0 14px 10px 0; border-bottom:1px solid var(--paper-line);} |
| td{padding:12px 14px 12px 0; border-bottom:1px solid var(--paper-line); vertical-align:top;} |
| td.k{color:var(--teal); font-family:'JetBrains Mono',monospace; font-size:0.83rem; white-space:nowrap;} |
| |
| |
| .bug-timeline{display:flex; flex-direction:column; gap:0; margin:24px 0;} |
| .bug-item{display:grid; grid-template-columns:26px 1fr; gap:16px; position:relative; padding-bottom:30px;} |
| .bug-item:last-child .bug-line{display:none;} |
| .bug-line{position:absolute; left:12px; top:26px; bottom:0; width:2px; background:var(--paper-line);} |
| .bug-marker{width:26px; height:26px; border-radius:50%; background:var(--danger-dim); color:var(--danger); display:flex; align-items:center; justify-content:center; font-size:0.7rem; font-weight:700; flex-shrink:0; z-index:1;} |
| .bug-body{background:#fff; border:1px solid var(--paper-line); border-radius:var(--radius-sm); padding:16px 18px;} |
| .bug-body .bug-title{font-family:'Sora',sans-serif; font-weight:600; font-size:0.95rem; margin-bottom:6px;} |
| .bug-body .bug-meta{font-size:0.72rem; color:var(--text-muted-on-paper); text-transform:uppercase; letter-spacing:0.06em; margin-bottom:10px;} |
| .bug-body p{font-size:0.9rem; margin-bottom:8px; color:var(--text-muted-on-paper);} |
| .bug-body p:last-child{margin-bottom:0;} |
| .bug-body .fix{color:#1a6b66;} |
| |
| |
| .badge{display:inline-block; font-size:0.7rem; padding:2px 10px; border-radius:20px; border:1px solid var(--paper-line); color:var(--text-muted-on-paper); margin-right:6px;} |
| .badge.teal{border-color:rgba(47,166,160,0.4); color:var(--teal);} |
| .badge.amber{border-color:rgba(232,163,61,0.5); color:#b8791f;} |
| ul.plain{padding-left:20px; margin:12px 0;} |
| ul.plain li{margin-bottom:8px; font-size:0.95rem;} |
| .divider{height:1px; background:var(--paper-line); margin:64px 0;} |
| .grid-2{display:grid; grid-template-columns:1fr 1fr; gap:16px;} |
| @media (max-width:640px){.grid-2{grid-template-columns:1fr;}} |
| footer{text-align:center; padding:44px 24px 70px 24px; color:var(--text-muted-on-paper); font-size:0.82rem; border-top:1px solid var(--paper-line);} |
| </style> |
| </head> |
| <body> |
|
|
| <div id="progress"></div> |
|
|
| <div class="hero"> |
| <div class="hero-inner"> |
| <div class="eyebrow"><span class="dot"></span>Project Documentation</div> |
| <h1>Voice Form Assistant — Web App</h1> |
| <p class="lede">A complete record of what this project is, how every piece works, every concept |
| it demonstrates, and the real bugs found and fixed while building it. Written as a reference |
| to actually understand the system, not just a summary to skim before a pitch.</p> |
| <div class="stat-strip"> |
| <div class="stat"><b>4</b><span>Forms, one shared pipeline</span></div> |
| <div class="stat"><b>31</b><span>Total form fields across all 4</span></div> |
| <div class="stat"><b>9</b><span>Backend Python modules</span></div> |
| <div class="stat"><b>79</b><span>Automated test checks, all passing</span></div> |
| <div class="stat"><b>100%</b><span>Local — no cloud AI APIs</span></div> |
| </div> |
| </div> |
| </div> |
|
|
| <div class="shell"> |
| <nav class="toc"> |
| <div class="toc-title">Contents</div> |
| <ul id="toc-list"> |
| <li><a href="#overview">Overview</a></li> |
| <li><a href="#architecture">Architecture</a></li> |
| <li><a href="#forms">The four forms</a></li> |
| <li><a href="#backend">Backend walkthrough</a></li> |
| <li><a href="#backend-app" class="sub">app.py</a></li> |
| <li><a href="#backend-pipeline" class="sub">The AI pipeline files</a></li> |
| <li><a href="#backend-schema" class="sub">Schema & validation</a></li> |
| <li><a href="#frontend">Frontend walkthrough</a></li> |
| <li><a href="#concepts">Concepts explained</a></li> |
| <li><a href="#testing">Testing philosophy</a></li> |
| <li><a href="#bugs">Bugs found & fixed</a></li> |
| <li><a href="#setup">Setup & running</a></li> |
| <li><a href="#limitations">Known limitations</a></li> |
| <li><a href="#pitch">Pitch & resume notes</a></li> |
| </ul> |
| </nav> |
|
|
| <main> |
|
|
| |
| <section id="overview" class="part-header" style="margin-top:0;"> |
| <div class="part-eyebrow">Overview</div> |
| <h2>What this project actually is</h2> |
| <p class="sub">A Flask web app that lets someone fill out any of four different forms either by |
| typing, or by having a real spoken conversation with an AI assistant that listens, understands, |
| validates, and fills the form in for them — live, in the browser.</p> |
| </section> |
|
|
| <div class="panel"> |
| <p>Every form offers two paths that collect <b>exactly the same data</b> through |
| <b>exactly the same validation rules</b>: a normal manual form, and a voice assistant. |
| The voice path is the actual point of the project — it demonstrates a full local AI pipeline |
| doing something concrete and demoable: turning a spoken sentence into a validated, structured |
| field value, live, with no cloud APIs involved anywhere.</p> |
| <p style="margin-bottom:0;">The project exists in two layers that matter for understanding it: |
| a <b>form-agnostic dialogue engine</b> (extraction, validation, state tracking, conversation |
| flow) that has no idea what form it's filling out, and a thin <b>web layer</b> (Flask routes, |
| browser mic/speaker, session handling) that adapts that engine to run in a browser instead of |
| a terminal. That separation is what let four different forms exist with zero duplicated logic.</p> |
| </div> |
|
|
| |
| <section id="architecture" class="part-header"> |
| <div class="part-eyebrow">Architecture</div> |
| <h2>How a spoken sentence becomes a filled-in field</h2> |
| <p class="sub">One full turn of the voice conversation, start to finish.</p> |
| </section> |
|
|
| <div class="pipeline"> |
| <div class="pipe-stage"> |
| <div class="pipe-dot amber">1</div> |
| <div class="pipe-line"></div> |
| <div class="pipe-content"> |
| <h4>Browser records your voice</h4> |
| <div class="pipe-file">static/voice_assistant.js — MediaRecorder API</div> |
| <p>Click the mic, speak, click again. The browser's own microphone access |
| (<code class="inline">getUserMedia</code>) records a short audio clip client-side — |
| the server never touches your microphone directly.</p> |
| </div> |
| </div> |
| <div class="pipe-stage"> |
| <div class="pipe-dot amber">2</div> |
| <div class="pipe-line"></div> |
| <div class="pipe-content"> |
| <h4>Audio is uploaded to the server</h4> |
| <div class="pipe-file">app.py — POST /api/voice/<session_id>/turn</div> |
| <p>The recorded clip is sent as a multipart file upload. Flask saves it to a temp file |
| and hands it to the transcription step.</p> |
| </div> |
| </div> |
| <div class="pipe-stage"> |
| <div class="pipe-dot">3</div> |
| <div class="pipe-line"></div> |
| <div class="pipe-content"> |
| <h4>Speech becomes text</h4> |
| <div class="pipe-file">stt.py — faster-whisper (local Whisper model)</div> |
| <p>A local Whisper model transcribes the clip. Silence/noise is filtered before decoding, |
| and the model is given a short hint about what kind of answer to expect (an email, a |
| date, a yes/no) based on which field was just asked — both measurably improve accuracy |
| on short spoken answers.</p> |
| </div> |
| </div> |
| <div class="pipe-stage"> |
| <div class="pipe-dot">4</div> |
| <div class="pipe-line"></div> |
| <div class="pipe-content"> |
| <h4>The LLM figures out what you meant</h4> |
| <div class="pipe-file">extractor.py — local Ollama call</div> |
| <p>The transcript is sent to a local LLM with a strict instruction: read this and return |
| <em>only</em> JSON containing any of the form's field values mentioned. Nothing is |
| guessed — a field is only filled if it was actually said.</p> |
| </div> |
| </div> |
| <div class="pipe-stage"> |
| <div class="pipe-dot">5</div> |
| <div class="pipe-line"></div> |
| <div class="pipe-content"> |
| <h4>The value is validated and remembered</h4> |
| <div class="pipe-file">state_manager.py + validators.py</div> |
| <p>Every extracted value passes through a type-specific validator (real email, valid date, |
| sensible number...) before being accepted into the form's running state.</p> |
| </div> |
| </div> |
| <div class="pipe-stage"> |
| <div class="pipe-dot">6</div> |
| <div class="pipe-line"></div> |
| <div class="pipe-content"> |
| <h4>The assistant decides what to say next</h4> |
| <div class="pipe-file">dialogue_manager.py</div> |
| <p>A strict priority check, every turn: was the last answer invalid → ask again; is a |
| required field still missing → ask for it; was something already-confirmed just |
| corrected → re-show the summary; is everything filled → confirm; did they confirm → done.</p> |
| </div> |
| </div> |
| <div class="pipe-stage"> |
| <div class="pipe-dot">7</div> |
| <div class="pipe-line"></div> |
| <div class="pipe-content"> |
| <h4>The reply is spoken back</h4> |
| <div class="pipe-file">tts.py — Piper (or the browser's own voice as fallback)</div> |
| <p>The response text is synthesized into audio server-side and sent back as base64 WAV. |
| If no local voice is configured, the frontend automatically falls back to the browser's |
| built-in speech synthesis instead — the assistant always talks, one way or another.</p> |
| </div> |
| </div> |
| <div class="pipe-stage"> |
| <div class="pipe-dot">8</div> |
| <div class="pipe-content"> |
| <h4>The field tracker updates live</h4> |
| <div class="pipe-file">static/voice_assistant.js — renders the JSON response</div> |
| <p>The whole state of the form — what's filled, what's current, what's still pending — |
| comes back in the same JSON response and re-renders instantly next to the conversation. |
| This is the moment that actually demonstrates the pipeline working, without anyone |
| needing to understand Whisper, Ollama, or Piper at all.</p> |
| </div> |
| </div> |
| </div> |
|
|
| <div class="callout"> |
| <span class="label">Why this matters architecturally</span> |
| Every one of these eight steps is a separate, swappable file. <code class="inline">dialogue_manager.py</code> |
| has no idea whether its input came from a browser mic or a keyboard, and no idea whether its |
| output gets spoken or printed — it only ever handles plain text in, plain text out. That's what |
| let the exact same dialogue engine run as a CLI tool first, then get wrapped in a web layer |
| later, without a single line of the actual conversation logic changing. |
| </div> |
|
|
| |
| <section id="forms" class="part-header"> |
| <div class="part-eyebrow">The Product</div> |
| <h2>Four forms, one engine, zero duplicated logic</h2> |
| <p class="sub">Every form is a plain data declaration in <code class="inline">forms.py</code> — |
| nothing else in the app knows or cares which one is active.</p> |
| </section> |
|
|
| <table> |
| <tr><th>Form</th><th>Collects</th><th>Fields</th><th>Field types used</th></tr> |
| <tr> |
| <td class="k">job_application</td> |
| <td>Contact details, experience, availability, relocation willingness</td> |
| <td>8 (6 required)</td> |
| <td>string, email, phone, integer, date, boolean</td> |
| </tr> |
| <tr> |
| <td class="k">school_admission</td> |
| <td>Student and guardian details, grade applying for</td> |
| <td>8 (6 required)</td> |
| <td>string, date, email, phone, boolean</td> |
| </tr> |
| <tr> |
| <td class="k">medical_intake</td> |
| <td>Patient details, reason for visit, insurance, appointment date</td> |
| <td>8 (6 required)</td> |
| <td>string, date, phone, email, boolean</td> |
| </tr> |
| <tr> |
| <td class="k">support_request</td> |
| <td>Contact details, product/issue, priority level</td> |
| <td>7 (5 required)</td> |
| <td>string, email, phone</td> |
| </tr> |
| </table> |
|
|
| <div class="panel"> |
| <h4>What adding a fifth form actually requires</h4> |
| <pre><code>NEW_FORM: List[FormField] = [ |
| FormField("field_name", "human label", "string", True, "What should I ask?"), |
| # ... |
| ] |
|
|
| FORMS["new_form_id"] = FormDefinition( |
| id="new_form_id", title="...", description="...", icon="...", fields=NEW_FORM |
| )</code></pre> |
| <p style="margin-bottom:0;">That's it. No route changes, no template changes, no dialogue logic |
| changes. Every page, every API endpoint, and the entire voice pipeline operate on "whatever |
| fields this form declares" — this is the single design decision that made four forms cost |
| barely more than one.</p> |
| </div> |
|
|
| |
| <section id="backend" class="part-header"> |
| <div class="part-eyebrow">Backend</div> |
| <h2>File-by-file walkthrough</h2> |
| <p class="sub">Nine Python modules, each responsible for exactly one part of the pipeline.</p> |
| </section> |
|
|
| <div id="backend-app" class="panel dark"> |
| <h4>app.py — the web layer</h4> |
| <p>The only file that knows it's a website. Owns every Flask route, in-memory session storage |
| (<code class="inline">VOICE_SESSIONS</code>, <code class="inline">SUBMISSIONS</code>), and the |
| JSON contract the frontend talks to. Two route groups:</p> |
| <ul class="plain"> |
| <li><b>Page routes</b> — form selection, mode choice, the manual form (validated server-side |
| with the exact same <code class="inline">validators.py</code> the voice path uses), and the |
| success page.</li> |
| <li><b>Voice API</b> — <code class="inline">POST /api/voice/<form_id>/start</code> creates |
| a <code class="inline">DialogueManager</code> for that form and returns the opening question; |
| <code class="inline">POST /api/voice/<session_id>/turn</code> handles every subsequent |
| turn: transcribe → extract → validate → decide → synthesize, all in one request.</li> |
| </ul> |
| <p style="margin-bottom:0;">Two helper functions do a lot of the real work: |
| <code class="inline">_synthesize_safe()</code> never lets a missing Piper voice crash a |
| response, and <code class="inline">_progress()</code> builds the live field-tracker data sent |
| to the frontend every turn.</p> |
| </div> |
|
|
| <div id="backend-pipeline"> |
| <h3 style="margin-top:44px;">The AI pipeline files</h3> |
|
|
| <div class="panel"> |
| <h4>stt.py — speech to text</h4> |
| <p style="margin-bottom:0;">A thin wrapper around <code class="inline">faster-whisper</code>. |
| Lazily loads the model once and reuses it (loading is the expensive part — every request |
| after the first is fast). <code class="inline">transcribe()</code> takes any audio file |
| faster-whisper's decoder handles — including the browser's webm/opus recordings directly, |
| via the bundled PyAV decoder, no separate ffmpeg install needed.</p> |
| </div> |
|
|
| <div class="panel"> |
| <h4>extractor.py — structured extraction</h4> |
| <p>This is the "understanding" step. Instead of hoping the LLM's free-text reply happens to |
| contain the right information, the prompt explicitly forces JSON-only output matching the |
| active form's schema — the same idea as "function calling" in hosted LLM APIs, done manually |
| via prompting since this runs through a local Ollama model.</p> |
| <pre><code>try: |
| raw_response = call_llm(prompt) |
| except requests.exceptions.RequestException: |
| return {} # Ollama down or unreachable -> "nothing extracted", not a crash |
|
|
| try: |
| return json.loads(_strip_json_fences(raw_response)) |
| except json.JSONDecodeError: |
| return {} # malformed response -> same graceful fallback</code></pre> |
| <p style="margin-bottom:0;">Also owns <code class="inline">is_llm_reachable()</code> — a cheap |
| connectivity check the web app uses to show a clear warning banner if Ollama isn't running, |
| instead of the conversation just silently never advancing.</p> |
| </div> |
|
|
| <div class="panel"> |
| <h4>tts.py — text to speech</h4> |
| <p style="margin-bottom:0;">Mirrors <code class="inline">stt.py</code>'s structure: lazy-loaded |
| Piper voice, cached after first use. <code class="inline">synthesize_to_wav_bytes()</code> |
| renders straight to an in-memory buffer — no temp file ever touches disk for the outgoing |
| audio, since it only needs to become base64 in a JSON response.</p> |
| </div> |
| </div> |
|
|
| <div id="backend-schema"> |
| <h3 style="margin-top:44px;">Schema, state, and dialogue</h3> |
|
|
| <div class="panel"> |
| <h4>form_schema.py + forms.py — the form as data</h4> |
| <p style="margin-bottom:0;"><code class="inline">FormField</code> is a small dataclass — name, |
| type, required, the question to ask. <code class="inline">forms.py</code> is a plain registry |
| mapping form ids to a list of these. Nothing here is a route or a template — it's pure data, |
| which is exactly why every other layer of the app can be generic.</p> |
| </div> |
|
|
| <div class="panel"> |
| <h4>validators.py — one function per data type</h4> |
| <p style="margin-bottom:0;">Six validators (string, email, phone, integer, date, boolean), |
| each returning the identical shape: <code class="inline">(is_valid, cleaned_value, |
| error_message)</code>. That consistency is what lets a single dispatch table call the right |
| one without a long if/elif chain, and it's the exact same logic used by both the voice path |
| and the manual form — validating a spoken email and a typed email go through the same code.</p> |
| </div> |
|
|
| <div class="panel"> |
| <h4>state_manager.py — the conversation's memory</h4> |
| <p style="margin-bottom:0;"><code class="inline">FormState</code> tracks every field's value, |
| which optional fields were explicitly skipped, and — critically — |
| <code class="inline">update()</code> reports which fields <em>actually changed</em> value |
| this turn, not just which were mentioned. That distinction is what makes corrections work: |
| restating an answer doesn't trigger anything, but a genuine change re-opens a confirmation |
| that was already given.</p> |
| </div> |
|
|
| <div class="panel"> |
| <h4>dialogue_manager.py — orchestration</h4> |
| <p style="margin-bottom:0;">A small finite state machine, checked in strict priority order |
| every turn (see the architecture diagram above, step 6). Also owns the optional natural |
| LLM-phrasing layer — templates are always the source of truth for <em>what</em> gets said; |
| the LLM, when enabled, only ever reword them, and any phrasing failure falls straight back |
| to the plain template.</p> |
| </div> |
| </div> |
|
|
| |
| <section id="frontend" class="part-header"> |
| <div class="part-eyebrow">Frontend</div> |
| <h2>Templates, styling, and the live conversation UI</h2> |
| <p class="sub">Server-rendered Jinja2 pages, one focused piece of JavaScript for the voice |
| screen, no frontend framework or build step.</p> |
| </section> |
|
|
| <div class="grid-2"> |
| <div class="panel"> |
| <h4>Pages (Jinja2 templates)</h4> |
| <p style="margin-bottom:0; font-size:0.92rem;"> |
| <code class="inline">base.html</code> — shared shell + step indicator · |
| <code class="inline">index.html</code> — form selection cards · |
| <code class="inline">choose_mode.html</code> — manual vs. voice · |
| <code class="inline">manual_form.html</code> — inputs generated from the schema, one loop, |
| any form · <code class="inline">voice_assistant.html</code> — the conversation screen · |
| <code class="inline">success.html</code> — submission summary |
| </p> |
| </div> |
| <div class="panel"> |
| <h4>Design system</h4> |
| <p style="margin-bottom:0; font-size:0.92rem;">Two-tone by intent: dark <b>ink</b> surfaces for |
| anything "live" (the voice conversation, the recording state), calm <b>paper</b> surfaces for |
| anything structural (forms, lists, summaries). Amber marks in-progress/attention state, teal |
| marks completion — color carries real meaning throughout, not decoration.</p> |
| </div> |
| </div> |
|
|
| <div class="panel dark"> |
| <h4>The signature screen: the live field tracker</h4> |
| <p>Split layout on <code class="inline">voice_assistant.html</code>: conversation on the left, |
| a running list of every field on the right. Each turn's JSON response includes a |
| <code class="inline">progress</code> array — every field's current status |
| (<code class="inline">pending</code> / <code class="inline">current</code> / |
| <code class="inline">filled</code> / <code class="inline">skipped</code>) and value if any — |
| and <code class="inline">renderTracker()</code> in <code class="inline">voice_assistant.js</code> |
| redraws it after every single turn.</p> |
| <p style="margin-bottom:0;">This is deliberately the centerpiece of the whole UI. It's the one |
| thing that makes "the AI understood what I said and filled in the right field" viscerally |
| obvious to someone watching, without them needing to know anything about the three models |
| running underneath.</p> |
| </div> |
|
|
| <div class="panel"> |
| <h4>Graceful degradation, twice, in the same file</h4> |
| <p><code class="inline">voice_assistant.js</code>'s <code class="inline">speak()</code> function |
| plays real Piper audio when it's available; when it isn't, it silently calls the browser's own |
| <code class="inline">speechSynthesis</code> API instead. Separately, an |
| <code class="inline">llm-banner</code> element shows a clear warning if Ollama becomes |
| unreachable mid-conversation — added specifically after watching a real conversation |
| silently "get stuck" with no visible explanation of why.</p> |
| <p style="margin-bottom:0;">Both are the same underlying philosophy applied at the UI layer: a |
| missing optional dependency should degrade visibly and gracefully, never fail silently and |
| never crash the experience.</p> |
| </div> |
|
|
| |
| <section id="concepts" class="part-header"> |
| <div class="part-eyebrow">Reference</div> |
| <h2>Every core concept, explained</h2> |
| </section> |
|
|
| <div class="concept"> |
| <div class="term">Local speech-to-text<span class="tag">stt.py</span></div> |
| <div><p style="margin:0;">Converting spoken audio into text using a model that runs entirely on |
| your own machine (faster-whisper) instead of a cloud API. Matters here for both privacy and |
| for the "100% local, no API keys" pitch — nothing you say ever leaves the machine running |
| the Flask server.</p></div> |
| </div> |
| <div class="concept"> |
| <div class="term">Structured extraction<span class="tag">extractor.py</span></div> |
| <div><p style="margin:0;">Forcing an LLM to return data matching a schema instead of free-flowing |
| text, by being extremely explicit in the prompt about the exact output format and providing a |
| concrete example. The same idea as "function calling" in hosted APIs, implemented manually here |
| since the model runs through Ollama rather than a service with native tool-calling support.</p></div> |
| </div> |
| <div class="concept"> |
| <div class="term">Local text-to-speech<span class="tag">tts.py</span></div> |
| <div><p style="margin:0;">Piper, a fast local neural TTS engine, converts the assistant's text |
| reply into audio server-side. Chosen specifically for being fast enough to feel conversational |
| rather than optimizing purely for voice quality.</p></div> |
| </div> |
| <div class="concept"> |
| <div class="term">Finite state machine<span class="tag">dialogue_manager.py</span></div> |
| <div><p style="margin:0;">A system that's always in exactly one of a fixed set of states, with |
| clear rules for what happens next. The whole conversation is one: currently-asking, invalid-answer, |
| confirming, or done — checked in the same strict priority order every single turn, which is |
| what makes the flow predictable and debuggable rather than a tangle of conditionals.</p></div> |
| </div> |
| <div class="concept"> |
| <div class="term">Schema-driven design<span class="tag">forms.py</span></div> |
| <div><p style="margin:0;">Declaring the shape of your data once, as pure data, and writing every |
| other layer of the system to act generically on that description rather than hardcoding |
| knowledge about any specific form. This single decision is why four forms exist for barely |
| more effort than one.</p></div> |
| </div> |
| <div class="concept"> |
| <div class="term">Graceful degradation<span class="tag">throughout</span></div> |
| <div><p style="margin:0;">Every dependency that can fail — Ollama being down, a Piper voice not |
| being installed, a malformed LLM response, an empty transcription — is designed to degrade to |
| a visible, safe fallback rather than crash the request. This shows up at least five separate |
| times across this project, each one added after actually watching it fail during development.</p></div> |
| </div> |
| <div class="concept"> |
| <div class="term">REST JSON contract<span class="tag">app.py ↔ voice_assistant.js</span></div> |
| <div><p style="margin:0;">The frontend and backend agree on a fixed shape for every voice API |
| response (<code class="inline">response_text</code>, <code class="inline">audio_base64</code>, |
| <code class="inline">progress</code>, <code class="inline">is_complete</code>, |
| <code class="inline">llm_available</code>...). Keeping this contract explicit and stable is |
| what let the frontend be built and reasoned about independently of the Python behind it.</p></div> |
| </div> |
| <div class="concept"> |
| <div class="term">Browser-native audio I/O<span class="tag">voice_assistant.js</span></div> |
| <div><p style="margin:0;">The <code class="inline">MediaRecorder</code> API records the mic |
| client-side; an <code class="inline"><audio></code> element (or |
| <code class="inline">speechSynthesis</code>) plays the reply client-side. The server never |
| needs direct hardware audio access at all — a meaningful simplification over the CLI version |
| of this same pipeline, which needed <code class="inline">sounddevice</code> and a real local |
| microphone on whatever machine ran it.</p></div> |
| </div> |
|
|
| |
| <section id="testing" class="part-header"> |
| <div class="part-eyebrow">Quality</div> |
| <h2>Testing philosophy: mock the model, test the wiring</h2> |
| <p class="sub">79 checks across two files, all passing, none of them requiring Ollama, a |
| downloaded Whisper model, a Piper voice, or a browser.</p> |
| </section> |
|
|
| <div class="panel"> |
| <p>Every test replaces the actual AI calls (<code class="inline">extract_fields</code>, |
| <code class="inline">transcribe</code>, <code class="inline">synthesize_to_wav_bytes</code>) |
| with small, deterministic fakes — the same pattern used consistently across this whole |
| project. This isolates the thing actually being tested (the routing, the session handling, |
| the state machine, the validation) from the thing that can't be tested this way (real model |
| output quality, which only a human listening to real audio can judge).</p> |
| <p style="margin-bottom:0;"><code class="inline">tests/test_app.py</code> uses Flask's real test |
| client against real routes — including a full multi-turn conversation carried through an |
| actual multipart audio upload, all four forms confirmed to independently produce a working |
| opening question, and the manual-submission flow followed all the way through to actually |
| rendering the success page (not just checking a redirect status — see the bug log below for |
| exactly why that distinction mattered).</p> |
| </div> |
|
|
| |
| <section id="bugs" class="part-header"> |
| <div class="part-eyebrow">Battle Log</div> |
| <h2>Real bugs found and fixed</h2> |
| <p class="sub">Documented deliberately, not swept under the rug — finding and fixing these is |
| the actual engineering work this project demonstrates.</p> |
| </section> |
|
|
| <div class="bug-timeline"> |
| <div class="bug-item"> |
| <div class="bug-marker">1</div> |
| <div class="bug-line"></div> |
| <div class="bug-body"> |
| <div class="bug-title">Schema silently never passed to extraction</div> |
| <div class="bug-meta">dialogue_manager.py · found while adding the 2nd–4th forms</div> |
| <p>The extraction call never actually passed the active form's schema, so it silently |
| always extracted against a hardcoded default form regardless of which one was in use. |
| Invisible with only one form ever built — exactly the class of bug that only surfaces |
| once a second real case exists to compare against.</p> |
| <p class="fix">Fixed by requiring schema explicitly rather than defaulting it, and covered |
| by a regression test that checks all four forms independently.</p> |
| </div> |
| </div> |
| <div class="bug-item"> |
| <div class="bug-marker">2</div> |
| <div class="bug-line"></div> |
| <div class="bug-body"> |
| <div class="bug-title">An unreachable LLM crashed the whole conversation</div> |
| <div class="bug-meta">extractor.py</div> |
| <p>A connection failure to Ollama was raised uncaught, unlike every other failure mode in |
| this project, which all degrade to "nothing extracted" instead of crashing.</p> |
| <p class="fix">Wrapped the LLM call in the same defensive pattern already used for |
| malformed JSON responses.</p> |
| </div> |
| </div> |
| <div class="bug-item"> |
| <div class="bug-marker">3</div> |
| <div class="bug-line"></div> |
| <div class="bug-body"> |
| <div class="bug-title">The success page crashed on every single submission</div> |
| <div class="bug-meta">templates/success.html · classic Jinja2 gotcha</div> |
| <p>The template used <code class="inline">submission.values</code> — but Python dicts have a |
| real built-in method called <code class="inline">.values()</code>, so Jinja silently |
| resolved that instead of the intended dictionary key. The first version of the test suite |
| didn't catch it either, because it only checked the redirect status without ever actually |
| rendering the page.</p> |
| <p class="fix">Renamed the key to <code class="inline">answers</code> everywhere to eliminate |
| the whole class of collision, and added a test that actually renders the success page.</p> |
| </div> |
| </div> |
| <div class="bug-item"> |
| <div class="bug-marker">4</div> |
| <div class="bug-line"></div> |
| <div class="bug-body"> |
| <div class="bug-title">Two projects merged into one folder broke both</div> |
| <div class="bug-meta">packaging, not code</div> |
| <p>A zip combined this web app with an earlier CLI-only version of the same pipeline. They |
| share filenames (<code class="inline">dialogue_manager.py</code>, |
| <code class="inline">state_manager.py</code>...) with incompatible contents, and |
| separately, Flask's <code class="inline">templates/</code>/<code class="inline">static/</code> |
| folder convention had been flattened during zipping — breaking every page and every |
| static asset at once.</p> |
| <p class="fix">Rebuilt the correct structure from the same files (content was untouched), |
| verified by extracting the corrected zip fresh and re-running the full test suite against |
| that exact extraction.</p> |
| </div> |
| </div> |
| <div class="bug-item"> |
| <div class="bug-marker">5</div> |
| <div class="bug-body"> |
| <div class="bug-title">A down Ollama looked like a frozen app, not an error</div> |
| <div class="bug-meta">app.py + voice_assistant.js · found from a real terminal log</div> |
| <p>Once bug #2 was fixed, an unreachable LLM correctly stopped crashing things — but now it |
| failed <em>too</em> quietly: the conversation just kept re-asking the same question with |
| no visible explanation anywhere in the UI.</p> |
| <p class="fix">Added <code class="inline">is_llm_reachable()</code>, threaded into every |
| voice API response as <code class="inline">llm_available</code>, and a clear red banner in |
| the UI the moment it's false.</p> |
| </div> |
| </div> |
| </div> |
|
|
| |
| <section id="setup" class="part-header"> |
| <div class="part-eyebrow">Practical</div> |
| <h2>Setup & running</h2> |
| <p class="sub">Full detail lives in README.md — this is the short version.</p> |
| </section> |
|
|
| <div class="panel"> |
| <pre><code>pip install -r requirements.txt |
|
|
| # Terminal 2 -- local LLM, used for understanding speech |
| ollama pull llama3.2 |
| ollama serve |
|
|
| # voices/en_US-lessac-medium.onnx + .onnx.json |
| # from huggingface.co/rhasspy/piper-voices (optional -- |
| # falls back to the browser's own voice if skipped) |
|
|
| python app.py |
| # -> http://localhost:5000</code></pre> |
| </div> |
|
|
| |
| <section id="limitations" class="part-header"> |
| <div class="part-eyebrow">Honesty</div> |
| <h2>Known limitations</h2> |
| <p class="sub">Stated plainly, the same way every fallback in this project is — say these |
| before anyone else points them out.</p> |
| </section> |
|
|
| <ul class="plain"> |
| <li><b>In-memory sessions.</b> Fine for a local demo; restarting the server drops in-progress |
| conversations. Moving to Redis or a database wouldn't require changing |
| <code class="inline">DialogueManager</code> or <code class="inline">FormState</code> at all.</li> |
| <li><b>Click-to-talk, not fully hands-free.</b> A deliberate tradeoff for demo reliability over |
| the fully automatic voice-activity-detection built in an earlier CLI-only version of this |
| pipeline — a misfiring auto-stop mid-pitch is a worse failure mode than one extra click.</li> |
| <li><b>No accounts or real persistence.</b> Submissions live in memory to demonstrate the flow |
| works, not as a production form backend.</li> |
| <li><b>Depends on Ollama and Piper configured locally.</b> Not something a shipped product would |
| ask an end user to do — in a real deployment these would run server-side, invisible to the |
| user, exactly as they're already architected here (the browser never talks to either directly).</li> |
| </ul> |
|
|
| |
| <section id="pitch" class="part-header"> |
| <div class="part-eyebrow">Reference</div> |
| <h2>Pitch & resume notes</h2> |
| </section> |
|
|
| <ul class="plain"> |
| <li><b>The strongest live-demo moment</b> is the field tracker filling in in real time as you |
| talk — lead with that, not an explanation of the tech underneath.</li> |
| <li><b>"How hard would it be to add our form?"</b> is answered directly by |
| <code class="inline">forms.py</code> — pull it up and show how short one form declaration is.</li> |
| <li><b>Built a fully local, three-model voice pipeline</b> — Whisper for transcription, a local |
| LLM for structured extraction via prompt-based function calling, Piper for synthesis — with |
| zero paid API dependency anywhere in the stack.</li> |
| <li><b>Designed for graceful degradation</b> as a first-class concern, not an afterthought: every |
| external dependency (the LLM, the voice model, the transcription) has a visible, tested |
| fallback path rather than a crash.</li> |
| <li><b>Found and fixed five real bugs</b> through actual testing and real usage, not just |
| written-and-assumed-correct code — including a subtle Jinja2/Python dict-method collision that |
| crashed every single form submission before it was caught.</li> |
| </ul> |
|
|
| </main> |
| </div> |
|
|
| <footer> |
| Voice Form Assistant — Web App · Documentation generated as a project reference. |
| </footer> |
|
|
| <script> |
| const progress = document.getElementById('progress'); |
| window.addEventListener('scroll', () => { |
| const h = document.documentElement; |
| const scrolled = (h.scrollTop) / (h.scrollHeight - h.clientHeight) * 100; |
| progress.style.width = scrolled + '%'; |
| }); |
| |
| const tocLinks = document.querySelectorAll('.toc a'); |
| const idToLink = {}; |
| tocLinks.forEach(link => { idToLink[link.getAttribute('href').slice(1)] = link; }); |
| |
| const observedIds = ['overview','architecture','forms','backend','backend-app','backend-pipeline', |
| 'backend-schema','frontend','concepts','testing','bugs','setup','limitations','pitch']; |
| const observed = observedIds.map(id => document.getElementById(id)).filter(Boolean); |
| |
| const observer = new IntersectionObserver((entries) => { |
| entries.forEach(entry => { |
| const link = idToLink[entry.target.id]; |
| if (!link) return; |
| if (entry.isIntersecting) { |
| tocLinks.forEach(l => l.classList.remove('active')); |
| link.classList.add('active'); |
| } |
| }); |
| }, { rootMargin: '-10% 0px -70% 0px', threshold: 0 }); |
| |
| observed.forEach(el => observer.observe(el)); |
| </script> |
|
|
| </body> |
| </html> |
|
|
