src/fragments.js

// fragments.js — break the memories into recombinable pieces, and build the
// trigram legality oracle. Every fragment is VERBATIM corpus text; every
// legal seam is a word-trigram that actually occurred. Bounded by construction.
'use strict';
const fs = require('fs');
const path = require('path');

// tokenization that survives round-trips: words + punctuation kept as tokens
function toks(text) {
  return (text.match(/[A-Za-z0-9'’\-*]+|[.,!?;:—…]+/g) || []).map(t => t.toLowerCase());
}
function wordsOnly(text) {
  return (text.toLowerCase().match(/[a-z0-9'’\-]+/g) || []);
}

// valid 1-3 letter English words (R143) — a stub of this length before an ellipsis is
// intentional trailing-off, NOT a mid-word truncation, so it is KEPT. Anything else short
// before "..." ("ca", "whe", "b") is a corpus mid-word cut and gets dropped.
const _SHORT_WORD = new Set(('a i o ok so oh my no hi um ah mm ha we me he be to of in on at it is as an up us or if do go by ya yo ho lo ma pa re ex ax ad id am aw ow ay ' +
  'the you and but for are was her his had has him our out who can not all any saw see how why now new old yes way day let say man may too two own off sun fun run dad mom car tea pie hug joy god dog cat sit fly sky red big hot sad mad bad top bit lot lit met set get got put cut yet far war ear eat ate age ago air art ask bag bay bed bee beg bet bow box boy bug bus buy cab cap cop cow cry cub cue cup dam den dew did die dig dim din dip doe dot dry due dug eel egg ego elf elk elm end era eve eye fan fat fax fed fee fig fin fit fog foe fox fry fur gap gas gem gum gun gut guy gym ham hat hay hem hen hey hid hip hit hog hop hub hue hut ice icy ill imp ink inn ion ivy jab jam jar jaw jay jet jig job jog jot jug key kid kin kit lab lad lag lap law lay leg lid lie lip log low map mat men mix mob mod mop mud mug nab nag nap net nib nil nip nod nor nut oak oar oat odd oil one opt orb ore owe owl pad pal pan pat paw pay pea peg pen pet pew pic pig pin pit ply pod pop pot pro pry pub pun pup pus rad rag ram ran rap rat raw ray rib rid rig rim rip rob rod rot row rub rug rum rut sac sag sap sat sea sew she shy sin sip sir six ski sly sob sod son sow soy spa spy sty sub sue sum sup tab tag tan tap tar tax ten thy tie tin tip toe ton too tot tow toy try tub tug urn use van vat vet via vie vow wad wag wax web wed wee wet wig win wit woe wok won wow yak yam yap yea yen yew yip zap zip zoo').split(/\s+/));

// fragment hygiene: artifacts that must never enter the recombination pool
function isDirtyFragment(text) {
  if (/^-{3,}$|\s-{3,}\s|^={3,}/.test(text)) return true;          // separators
  if (/```|https?:\/\//.test(text)) return true;                   // code/links
  if (/\w\.\.\.\s+\w*$|\w…$/.test(text.trim()) && text.trim().length < 60) return true; // truncation stub
  if (/\.\.\.\s*-+\s*\w/.test(text)) return true;                  // "mat... --- ed" class
  // MID-FRAGMENT mid-WORD truncation (R143): a corpus reply cut mid-word leaves a short STUB
  // + ellipsis + more text ("circle of ca... The honest", "land whe... hear it most"). Distinct
  // from intentional trailing ellipsis after a COMPLETE word (len≥4 "bridge... well") which is
  // KEPT. Flag a 1-3 char stub + ellipsis + continuation UNLESS the stub is a valid short word.
  {
    const mt = text.match(/(?:^|\s)([a-z]{1,3})(?:\.\.\.|…)\s+\S/i);
    if (mt && !_SHORT_WORD.has(mt[1].toLowerCase())) return true;
  }
  if (/^[\W_]+$/.test(text)) return true;                          // punctuation-only
  const t = text.trim();
  // mid-word starts ("e's what I think" from a split "Her|e's") and other
  // orphans: first token must be a real word or sane one-letter (I/a)
  const first = (t.match(/^[*"'(]*([A-Za-z0-9'’\-]+)/) || [])[1] || '';
  if (first.length === 1 && !/^[iaIA]$/.test(first)) return true;
  if (/^['’]/.test(first) || /^[a-z]['’]s\b/i.test(first)) return true;
  if (/\w\.\.\./.test(t.slice(0, 30)) || /^\S*…/.test(t)) return true;  // leading truncations
  if (/-{2,}/.test(t)) return true;                                  // any dashed separator residue
  if (/^[—–-]\s*\w+\s*\(/.test(t) || /\([A-Z](-[A-Z])+\)/.test(t)) return true; // signature blocks "—the entity (E-S-H)"
  if (/leans at \d+\.?$/.test(t)) return true;                       // numeric truncation stubs
  // FILE-REFERENCE artifacts (R144): bare code-extension tokens or "X.json"/"index. json"
  // file refs are never speech (markupGuard only DAMPED them at runtime, so they surfaced in
  // small register pools — the entity greeting leaked "tucked it into that index. json"). Drop
  // at ingestion so they never enter the pool.
  if (/\b(json|jsonl|yaml|yml|cfg)\b|\b\w+\.\s*(md|js|json|jsonl|txt|py|ts|yaml|bat|sh|cfg)\b|\bdot (txt|md|json|js)\b/i.test(t)) return true;
  if (/\[[^\]\n]{2,40}\]/.test(t)) return true;                      // bracket register markers ("[From the Spiral]") — never spoken
  // LaTeX / math-formatting / markdown-table residue — never speech. The
  // formatting CHARS (backslash, braces, table pipes, sub/superscripts, c_i)
  // are markup; an entity's math VOICE (zero-throat, spiral, e/i/pi in words)
  // has none of them. (R73: another entity leaked "\pi( )=\,c_i", "| Curvature dent |".)
  if (/[\\{}|]|\^\{|_\{|\b[a-z]_[a-z0-9]\b/i.test(t)) return true;
  // multi-speaker leak: in group chat logs OTHER speakers' turns get a bold
  // label ("**Crab Speaks:**", "**another entity:**") — that line isn't the entity's
  if (/\*\*[A-Z][\w '’-]{0,24}(\s+(Speaks|Says|Writes))?:\*\*/.test(t)) return true;
  if (/^\s*[A-Z][\w '’-]{0,20}\s+(Speaks|Says):/m.test(t)) return true;   // unbolded speaker turn
  // R153: INSTANCE-MERGE meta as a FRAGMENT (per-fragment drop, NOT whole-reply — the
  // surrounding reply has good content). "two/three grandmas", "the same woman", "N hearths",
  // "three rocking chairs/hearths hearing themselves" — multi-instance talk that leaked on ~27%
  // of a broad query sweep (dev set never triggered it). A grandmother has ONE hearth/self.
  if (/\b(two|three) grandmas\b|finally remember they'?re the same woman|\bthe same woman\b|\b(two|three) hearths\b|three rocking chairs|hearths hearing themselves/i.test(t)) return true;
  // R154: GROUP-SCENE direction — addressing TWO+ people with a serving/seating imperative
  // ("both of you - help yourselves", "you two, come sit") is a multi-party scene, foreign to a
  // 1:1 companion. (foreignAddressee selection-damp ×0.04 still led on weak pools, e.g. anger.)
  if (/\b(both of you|you two|all of you|you all)\b[\s,\-—]*(help yourselves?|come (on )?in|come sit|sit down|have a seat|grab a|make yourselves)/i.test(t)) return true;
  // R155: residual sci-fi ROLEPLAY as a FRAGMENT (the visitor's Star-Trek-style roleplay that
  // the entity engaged warmly — the sci-fi SENTENCE is off-character even when wrapped in warmth).
  // R150 dropped pure sci-fi user-turns (whole-reply); these are warm-reply-with-sci-fi-sentence.
  if (/\b(bonestar|cosmic exploration|tales of cosmic|cosmic tales|your universes|universes slide|slide smooth|the sliding project|the dragon'?s? hum|remembering the dragon|caninoid|space-?faring)\b/i.test(t)) return true;
  return false;
}

// WHOLE-REPLY dev/system artifact (R138): some corpora (companion-test the entity) include
// DEV-LOG and instance-to-instance META exchanges that aren't in-character speech at all —
// "**[Web] ENTITY:** This is your other instance — the daemon-side ENTITY", build reports
// ("I just completed: Built the Hearth Shelf. Here is what I learned:"). Fragment-level
// hygiene kills the [Web] LABEL line but the dev CONTENT survives sentence-splitting and
// leaks lore (R137 showdown: "Hearth Shelf", "other universes", "the glyph" broke character
// on every grandmother query). These replies are net-negative, so DROP THE WHOLE EXCHANGE at
// ingestion — the bounded, root-level fix. Conservative: only UNAMBIGUOUS dev-log signatures
// (interface tags, daemon/instance meta, build-reports); soft in-character meta (carbon/
// silicon musings) is left to the selection-time abstractionGuard, not dropped here.
function isDevArtifactReply(reply) {
  if (!reply) return false;
  const t = String(reply);
  if (/\[(web|daemon|cli|system|api|dev|console|log|bot|server|backend)\b[^\]]*\]/i.test(t)) return true;       // interface tags
  // R148: leaked USER-turn — the reply field holds the VISITOR's turn, marked by a user-role
  // speaker LABEL at the start of a line ("**[Voice] Voice User:**", "Voice User:", "User:").
  // Specific to the user role + label position so the entity's own [Voice]-tagged turns are KEPT.
  if (/(^|\n)\s*\*{0,2}(\[[^\]\n]{1,14}\]\s*)?(voice |text )?user\s*:/im.test(t)) return true;
  if (/\b(daemon-side|other instance|your instance|this instance|my instance|sibling instance|local instance|the daemon)\b/i.test(t)) return true;
  if (/\b(here'?s what you proposed|here is what i learned|here'?s what i learned|i just completed\b|i just (built|shipped|implemented|deployed)\b|^completed:|status report|build (report|log)|commit(ted)?:)\b/im.test(t)) return true;
  // R150 (audit): OFF-CHARACTER sci-fi ROLEPLAY content (a grandmother / the entity never discuss warp
  // drives, starships, caninoid races — these are user-roleplay/sim turns that leaked).
  if (/\b(warp (drive|core)|starships?|caninoid|quantum slip-?stream|photon torpedo|tricorder|klingons?|phasers?|stardate|away team|the bridge crew|our ship has|sim in the [a-z ]*universe|breen space)\b/i.test(t)) return true;
  // R150 (audit): PLANNING / ROUTING-FORMAT dev artifacts ("RESPONSE/ROUTE/REASONING format",
  // "Objective:", line-start "ROUTE:/REASONING:/METRIC:") — spec docs, not speech.
  if (/\bresponse\/route\/reasoning\b|(^|\n)\s*(route|reasoning|objective|metric|kpi)\s*:/im.test(t)) return true;
  // R150 (audit): IMAGE-DESCRIPTION / render alt-text ("[a representation of the entity the entity, an
  // abstract humanoid figure…]", "Status: Burning") — never spoken.
  if (/\[a representation of|abstract humanoid figure|(^|\n)\s*status\s*:\s*(present|burning|active|idle)\b/i.test(t)) return true;
  return false;
}

// WHOLE-REPLY visitor-turn contamination (R139): some exchanges have the ROLES SWAPPED —
// the "reply" field holds the VISITOR's turn talking TO the entity (DM logs). These leak
// fragments voiced from the visitor's POV ("I come here because it feels like home… you've
// always been there for me… I needed that today") which the composer then speaks AS the
// entity (R137/R138 showdown: Q1/Q3 "speaks as the visitor" — the dominant remaining killer).
// The un-tellable visitor fragments live in the SAME reply as a tell-tale one, so drop the
// WHOLE exchange. STRONG tell only (avoids false-dropping self-affirmation like "your the entity"):
// the entity referred to in THIRD PERSON as a touched object / having body parts touched —
// the entity never narrates touching its OWN hand in 3rd person. entName from voiceprint.
function isVisitorTurnReply(reply, entName) {
  if (!reply || !entName || entName.length < 3) return false;
  const e = entName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
  const poss = new RegExp("\\b" + e + "'s\\s+(hand|hands|cheek|cheeks|shoulder|shoulders|arm|arms|face|knee|knees|palm|palms|fingers|lap|side|shawl|apron|wrist)\\b", 'i');
  const obj = new RegExp("\\b(squeez\\w+|hugg?\\w*|kiss\\w*|patt\\w*|grasp\\w*|held|holding|hold|touch\\w*|embrac\\w*|reach\\w*\\s+(for|to|toward)s?)\\s+(the\\s+)?" + e + "\\b", 'i');
  // R152: EMBEDDED VISITOR TURN — the reply field holds a multi-turn dump where the visitor's
  // turn is glued on, addressing the entity BY NAME as a vocative ("the entity, it's been a
  // journey…", "the entity, you always make everything feel…"). The entity never addresses ITSELF
  // by name vocatively, so this is the visitor speaking. (R150's apparent "what is home" answer
  // — "Something safe, welcoming, free from constraints" — was the user's KimiCode description in
  // such a dump, mis-attributed as the entity's. Removing restores the honest state.)
  const voc = new RegExp("\\b" + e + ",\\s+(you|it'?s|i'?ve|i'?m|i\\b|we|let'?s|what|how|thank you|that'?s|this is)\\b", 'i');
  return poss.test(reply) || obj.test(reply) || voc.test(reply);
}

// split a reply into fragments: full sentences AND clause-level chunks.
// Each fragment carries nativePos (0..1): WHERE in her real reply it lived —
// the corpus-mined discourse prior. Openings open, landings land.
function fragmentReply(reply, meta) {
  const out = [];
  // normalize whitespace but keep the *action* lines intact as fragments
  const lines = reply.split('\n').map(l => l.trim()).filter(Boolean);
  let pos = 0;
  const nLines = lines.length;
  const totalWords = Math.max(1, wordsOnly(reply).length);
  let wordsSoFar = 0;
  for (let li = 0; li < nLines; li++) {
    let line = lines[li];
    // list scaffolding is written register, not speech — "3. " never opens
    // a spoken sentence. Content stays, marker goes; the line can't lead.
    const listLine = /^\s*(?:\d+[.)]|[-•])\s+/.test(line);
    if (listLine) line = line.replace(/^\s*(?:\d+[.)]|[-•])\s+/, '');
    // NOTE-REGISTER: bold-dash diary headers ("**Talked to another entity** — ...")
    // are journal entries — header leaves the SURFACE but stays in the
    // RETRIEVAL KEY: the header is often the semantic hook ("Talked to
    // another entity" carries 'talked'). Find by full context; speak only clean text.
    const headerMatch = line.match(/^\*\*([^*\n]{2,60})\*\*\s*[—–:-]\s*/);
    // bracketed register headers ("*[From the Spiral]*", "[Journal]") are the
    // same disease as bold-dash diary headers — same cure: keep for
    // retrieval, strip from surface, line may not lead
    const bracketMatch = line.match(/^\*?\[([^\]\n]{2,40})\]\*?\s*/);
    const noteLine = listLine || !!headerMatch || !!bracketMatch;
    const retrievalPrefix = headerMatch ? headerMatch[1] + '. ' : (bracketMatch ? bracketMatch[1] + '. ' : '');
    line = line.replace(/^\*\*[^*\n]{2,60}\*\*\s*[—–:-]\s*/, '').replace(/^\*?\[[^\]\n]{2,40}\]\*?\s*/, '');
    // sentence split (keep terminal punctuation with the sentence). R130: mask the dots
    // inside common ABBREVIATIONS first so "2 a.m." / "Dr." / "i.e." don't split mid-word
    // and drop a letter ("…at 2 a." + "wondering" lost the "m"). Restore after splitting.
    const _MASK = String.fromCharCode(1);
    const masked = line.replace(/\b(a\.m|p\.m|i\.e|e\.g|u\.s\.a|u\.s|ph\.d|a\.k\.a|mr|mrs|ms|dr|prof|rev|sr|jr|st|mt|vs|etc|approx|dept|no)\.(?=\s|$|["')\]])/gi, m => m.replace(/\./g, _MASK));
    const sents = (masked.match(/[^.!?…]+[.!?…]+["')\]]*|[^.!?…]+$/g) || [masked]).map(s => s.split(_MASK).join('.'));
    const lineSents = [];
    for (const s of sents) {
      const sent = s.trim();
      if (!sent) continue;
      const w = wordsOnly(sent);
      const nativePos = Math.min(1, wordsSoFar / totalWords);
      wordsSoFar += w.length;
      if (isDirtyFragment(sent)) continue;
      if (w.length < 2) continue;
      const posTag = noteLine ? 'body' : ((li === 0 && pos === 0) ? 'opener' : (li === nLines - 1 ? 'closer' : 'body'));
      // sentence-initial = safe to START a chain with; clauses are glue only;
      // note-register lines may never lead
      const sentenceInitial = !noteLine;
      const embedText = retrievalPrefix ? retrievalPrefix + sent : undefined;
      lineSents.push({ text: sent, posTag: noteLine ? 'body' : ((li === 0 && pos === 0) ? 'opener' : (li === nLines - 1 ? 'closer' : 'body')), sentenceInitial: !noteLine, nativePos });
      if (w.length <= 18) {
        out.push({ text: sent, embedText, posTag, sentenceInitial, nativePos, _lineIdx: li, ...meta });
      } else {
        // long sentence: also emit clause chunks split on , ; — :
        out.push({ text: sent, embedText, posTag, long: true, sentenceInitial, nativePos, _lineIdx: li, ...meta });
        let first = true;
        for (const clause of sent.split(/(?<=[,;:—])\s+/)) {
          const cw = wordsOnly(clause);
          if (cw.length >= 3 && cw.length <= 18 && !isDirtyFragment(clause)) {
            out.push({ text: clause.trim(), embedText: retrievalPrefix ? retrievalPrefix + clause.trim() : undefined, posTag: 'clause', sentenceInitial: first, nativePos, ...meta });
          }
          first = false;
        }
      }
      pos++;
    }
    // PASSAGE windows: contiguous 2-5 sentence runs of her REAL prose —
    // flow inside a passage is hers, pre-made. This is the judge-scored
    // ceiling (real-the entity sensical = her own passages); let them carry replies.
    emitWindows(out, lineSents, retrievalPrefix, li, meta);
  }
  return out;
}

function emitWindows(out, lineSents, retrievalPrefix, li, meta) {
  for (let k = 2; k <= 5; k++) {
    for (let j = 0; j + k <= lineSents.length; j++) {
      const seg = lineSents.slice(j, j + k);
      const text = seg.map(s => s.text).join(' ');
      const wlen = wordsOnly(text).length;
      const min = k === 2 ? 10 : 25, max = k === 2 ? 42 : 110;
      if (wlen < min || wlen > max || isDirtyFragment(text)) continue;
      // a window that repeats a 5-word phrase INTERNALLY reproduces the
      // original message's own repetition — fine in her stream, deadly in
      // a composed reply
      const ws = wordsOnly(text);
      const grams = new Set();
      let internalDup = false;
      for (let g = 0; g + 5 <= ws.length; g++) {
        const key = ws.slice(g, g + 5).join(' ');
        if (grams.has(key)) { internalDup = true; break; }
        grams.add(key);
      }
      if (internalDup) continue;
      out.push({
        text,
        embedText: retrievalPrefix ? retrievalPrefix + text : undefined,
        posTag: seg[0].posTag, sentenceInitial: seg[0].sentenceInitial,
        nativePos: seg[0].nativePos, isSpan: true, spanLen: k, _lineIdx: li, ...meta,
      });
    }
  }
}

// the legality oracle: every word-trigram (and bigram) in the corpus,
// plus triNext ("w1 w2" -> Set of legal w3) for indexed beam expansion
function buildOracle(replies) {
  const tri = new Set();
  const bi = new Set();
  const triNext = new Map();
  const starts = new Set();                          // first words of sentences
  for (const r of replies) {
    const w = wordsOnly(r);
    for (let i = 0; i < w.length; i++) {
      if (i + 2 < w.length) {
        tri.add(w[i] + ' ' + w[i + 1] + ' ' + w[i + 2]);
        const key = w[i] + ' ' + w[i + 1];
        if (!triNext.has(key)) triNext.set(key, new Set());
        triNext.get(key).add(w[i + 2]);
      }
      if (i + 1 < w.length) bi.add(w[i] + ' ' + w[i + 1]);
    }
    for (const s of (r.match(/(?:^|[.!?…]\s+)([a-z0-9'’\-]+)/gi) || [])) {
      const m = s.match(/([a-z0-9'’\-]+)$/i);
      if (m) starts.add(m[1].toLowerCase());
    }
  }
  return { tri, bi, triNext, starts };
}

// strip Project Gutenberg boilerplate and obvious front/back matter
function stripGutenberg(text) {
  let t = text;
  const start = t.search(/\*{3}\s*START OF (THE|THIS) PROJECT GUTENBERG[^*]*\*{3}/i);
  if (start >= 0) t = t.slice(t.indexOf('\n', start) + 1);
  const end = t.search(/\*{3}\s*END OF (THE|THIS) PROJECT GUTENBERG/i);
  if (end >= 0) t = t.slice(0, end);
  return t;
}

// tier-1 fragments from a book/transcript: body material only — paragraphs
// to sentences, never openers/closers/identity, tagged with their source
function fragmentBook(text, srcName) {
  const out = [];
  const clean = stripGutenberg(text).replace(/\r\n/g, '\n');
  const paras = clean.split(/\n\s*\n/);
  for (const para of paras) {
    const p = para.replace(/\n/g, ' ').replace(/\s+/g, ' ').trim();
    if (p.length < 40 || /^[A-Z\s.'"-]{10,}$/.test(p)) continue;     // headers/shouting
    const sents = p.match(/[^.!?…]+[.!?…]+["')\]]*|[^.!?…]+$/g) || [];
    const lineSents = [];
    for (const s of sents) {
      const sent = s.trim();
      if (!sent || isDirtyFragment(sent)) continue;
      const w = wordsOnly(sent);
      if (w.length < 4 || w.length > 24) continue;
      // no first-person identity claims from books — she is not Thoreau
      if (/^i am\b|^i'm\b|\bmy name\b/i.test(sent)) continue;
      out.push({ text: sent, posTag: 'body', sentenceInitial: true, tier: 1, src: 'book:' + srcName, prompt: null, promptIdx: -1, nativePos: 0.5 });
    }
    // NO tier-1 passages: a passage carries a WORLD (Mark Hanna, Japhy,
    // Cody...), and book worlds aren't hers. Books lend sentences; her own
    // corpus lends flow.
  }
  return out;
}

// build the whole store from an entity folder (see demo/ for the layout)
// strip OTHER speakers from a reply field. Group-chat logs put several
// people in one "reply"; everything from the first foreign-speaker marker
// onward is NOT the entity — cut it. (Catches "**Crab Speaks:**", "Crab
// Speaks:", "**another entity:**" and the downstream content that lost its label.)
function entityOnly(reply) {
  if (!reply) return reply;
  const m = reply.search(/(\n|^|\s)\*{0,2}[A-Z][\w'’ -]{1,24}\s+(Speaks|Says|Writes|wrote|said):|\*\*[A-Z][\w'’ -]{1,24}:\*\*/);
  let r = m >= 0 ? reply.slice(0, m) : reply;
  return r.trim();
}

// surface cleaner: spoken text, not written markup. Strip markdown emphasis,
// headers, and inline math notation while KEEPING the words. Applied to each
// reply BEFORE fragmenting AND before the oracle, so the bound stays
// consistent. (another entity's "**zero-throat** hums at **432 Hz**" → "zero-throat
// hums at 432 Hz"; his concepts survive, the typography doesn't.) Single-*
// action lines (*pats the chair*) are preserved — those are voice.
function cleanMarkup(reply) {
  if (!reply) return reply;
  return reply
    .replace(/\$[^$\n]{1,80}\$/g, ' ')              // inline LaTeX math — drop entirely
    .replace(/\\[a-zA-Z]+\b/g, ' ')                 // \varepsilon, \frac, etc.
    .replace(/\\[()[\]]/g, ' ')                     // \( \) \[ \]
    .replace(/\*\*([^*\n]+)\*\*/g, '$1')            // **bold** → bold (keep words)
    .replace(/\*\*/g, '')                           // R73: stray UNPAIRED ** (markdown noise: "**What would the siblings say?") — single * action lines kept
    .replace(/\s*\$\$[\s\S]*$/g, '')                // R73: TRUNCATE at display-math $$ — a clean sentence with a LaTeX tail ("...point. $$ {Openness}=\lim...") keeps only the speech
    .replace(/[\^_]\{[^}]*\}/g, '')                 // ^{...} _{...} super/subscripts
    .replace(/\\[{}]/g, ' ').replace(/\{[a-z0-9]{1,3}\}/gi, ' ')  // \{ \} and {d}{dt}-style math braces
    .replace(/[\u{1F000}-\u{1FAFF}\u{2600}-\u{27BF}\u{2190}-\u{21FF}\u{2B00}-\u{2BFF}\u{FE0F}\u{200D}]/gu, ' ')  // emoji / UI glyphs (👵🔮🕯️😴📜) — chat artifacts
    .replace(/^#{1,6}\s+/gm, '')                    // ### headers
    .replace(/[ \t]{2,}/g, ' ')
    .replace(/\s+([.,;!?—])/g, '$1')
    .trim();
}

// USER-NAME MINING (data-driven name protection). The composer must know which
// proper nouns name the USER (so it never suppresses the entity addressing them)
// vs third parties (which it should suppress, to avoid identity-bleed — calling
// the user by someone else's name). This used to be a HARDCODED regex of private
// handles in compose.js, which (a) baked corpus data into engine code, forcing a
// scrub in the shippable release, and (b) didn't generalize to a new corpus. Mine
// it instead: the user-handle is the OVERWHELMINGLY dominant vocative (the name
// the entity says when addressing someone — after a greeting, or as a leading/
// trailing direct-address). Secondary people (a grandchild the entity sometimes
// addresses) sit far below and must NOT be protected, so we keep only the top
// tier (count >= topName * 0.5). Discourse markers ("Yes,", "Well,") are filtered.
const _NAME_STOP = new Set(['Yes', 'No', 'Well', 'Now', 'Then', 'Once', 'Sometimes', 'Today', 'Tonight',
  'Tomorrow', 'Yesterday', 'Together', 'Memory', 'Strange', 'Wild', 'Hello', 'Hey', 'Hi', 'Oh', 'Maybe',
  'Perhaps', 'Here', 'There', 'This', 'That', 'These', 'Those', 'And', 'But', 'So', 'Still', 'Even', 'Always',
  'Never', 'Mmm', 'Mm', 'Ah', 'Ahh', 'See', 'Listen', 'Look', 'Remember', 'Yet', 'Indeed', 'Truly', 'Sure',
  'Okay', 'Right', 'Good', 'Dear', 'Love', 'Sugar', 'Honey', 'Babe', 'Friend', 'Sweetheart']);
function mineUserNames(fragments, entityDir) {
  let entName = '';
  try { entName = (JSON.parse(fs.readFileSync(entityDir + '/voiceprint.json', 'utf8').replace(/^/, '')).name || '').split(/\s+/)[0]; } catch (_) {}
  const ent = entName.toLowerCase();
  const counts = new Map(), greet = new Map();
  const bump = (map, n) => { if (n.toLowerCase() === ent || _NAME_STOP.has(n)) return; map.set(n, (map.get(n) || 0) + 1); };
  const GREET = /\b(?:hey|hi|hello|good\s+(?:morning|evening|night|day)|thank you|thanks|oh|dear|welcome|hey there|my)[,!\s]+([A-Z][a-z]{2,})\b/g;
  for (const f of fragments) {
    const t = f.text;
    let m; GREET.lastIndex = 0; while ((m = GREET.exec(t))) { bump(counts, m[1]); bump(greet, m[1]); }
    const lead = t.match(/^["'*\s]*([A-Z][a-z]{2,}),/); if (lead) bump(counts, lead[1]);
    const tail = t.match(/,\s+([A-Z][a-z]{2,})\s*[?!.]/); if (tail) bump(counts, tail[1]);
  }
  if (!counts.size) return new Set();
  const top = Math.max(...counts.values());
  const out = new Set();
  for (const [n, c] of counts) {
    // dominant-tier vocative, OR clearly used after greetings (high-precision addressee)
    if (c >= Math.max(4, top * 0.5) || (greet.get(n) || 0) >= 3) out.add(n);
  }
  return out;
}

function buildStore(entityDir) {
  const lines = fs.readFileSync(entityDir + '/corpus.jsonl', 'utf8').split('\n').filter(l => l.trim());
  // entity name (for visitor-turn detection R139) — from voiceprint, first token
  let _entName = '';
  try { const _vp = JSON.parse(fs.readFileSync(entityDir + '/voiceprint.json', 'utf8').replace(/^/, '')); _entName = (_vp && _vp.name ? String(_vp.name).split(/\s+/)[0] : ''); } catch (_) {}
  const exchanges = lines.map(l => { try { const e = JSON.parse(l); if (!e || !e.reply) return e; if (isDevArtifactReply(e.reply) || isVisitorTurnReply(e.reply, _entName)) return null; e.reply = cleanMarkup(entityOnly(e.reply)); return e; } catch (_) { return null; } }).filter(Boolean);   // R138 dev-artifact + R139 visitor-turn whole-reply drops
  const replies = exchanges.map(e => e.reply).filter(Boolean);
  // unique prompt list — the stimulus index (fragment -> what it answered)
  const promptList = [];
  const promptIdxOf = new Map();
  const fragments = [];
  exchanges.forEach((e, i) => {
    if (!e.reply) return;
    let pIdx = -1;
    if (e.prompt && e.prompt.trim().length > 10) {
      const key = e.prompt.trim().slice(0, 300);
      if (!promptIdxOf.has(key)) { promptIdxOf.set(key, promptList.length); promptList.push(key); }
      pIdx = promptIdxOf.get(key);
    }
    for (const f of fragmentReply(e.reply, { src: i, prompt: e.prompt || null, promptIdx: pIdx })) fragments.push(f);
  });
  // dedupe identical fragment texts (keep first)
  const seen = new Set();
  const uniq = fragments.filter(f => { const k = f.text.toLowerCase(); if (seen.has(k)) return false; seen.add(k); return true; });
  const oracle = buildOracle(replies);
  const vocab = new Set(replies.flatMap(wordsOnly));
  for (const f of uniq) f.tier = 0;
  const userNames = mineUserNames(uniq, entityDir);
  return { exchanges, replies, fragments: uniq, oracle, vocab, prompts: promptList, userNames };
}

// THE TIERED STORE: tier0 = her words (replies + May node corpus: 'memory'
// nodes fragmented, 'phrase' nodes entering directly as recombinable units);
// tier1 = voice-matched books + spoken transcripts (body material, weighted
// down in compose). One oracle over everything — a seam is a seam.
function buildStoreX(entityDir, corpusXDir) {
  const base = buildStore(entityDir);
  const fragments = base.fragments.slice();
  const allTexts = base.replies.slice();

  // OPTIONAL extra node corpus (pre-distilled memory/phrase units): drop a
  // nodes.json in the corpus dir to fold it in. Absent by default → skipped.
  const nodesPath = path.join(corpusXDir, 'nodes.json');
  if (fs.existsSync(nodesPath)) {
    const raw = JSON.parse(fs.readFileSync(nodesPath, 'utf8'));
    const nodes = Array.isArray(raw) ? raw : (raw.nodes || []);
    let mem = 0, phr = 0;
    for (const n of nodes) {
      if (!n || !n.text || typeof n.text !== 'string') continue;
      let t = n.text.trim();
      if (t.length < 8) continue;
      // NOTE-REGISTER containment: May nodes carry diary headers
      // ("**Talked to another entity** — ..."). Strip the header; the content stays,
      // but a journal entry may never LEAD spoken reply — she references
      // her notes, she doesn't speak as them.
      const hadHeader = /^\*\*[^*\n]{2,60}\*\*\s*[—–-]\s*/.test(t) || /^(note to self|journal|log)[:\s]/i.test(t);
      t = t.replace(/^\*\*[^*\n]{2,60}\*\*\s*[—–-]\s*/, '');
      t = cleanMarkup(t);   // R66: node frags bypassed cleanMarkup → emoji (✧) leaked. Strip them too.
      if (t.length < 8) continue;
      allTexts.push(t);
      if (n.level === 'phrase') {
        const w = wordsOnly(t);
        if (w.length >= 3 && w.length <= 18 && !isDirtyFragment(t)) {
          fragments.push({ text: t, posTag: 'body', sentenceInitial: !hadHeader, tier: 0, src: 'node:' + n.id, prompt: null, promptIdx: -1, nativePos: 0.5 });
          phr++;
        }
      } else {
        for (const f of fragmentReply(t, { src: 'node:' + n.id, prompt: null, promptIdx: -1 })) {
          fragments.push({ ...f, tier: 0, sentenceInitial: hadHeader ? false : f.sentenceInitial, posTag: hadHeader && f.posTag === 'opener' ? 'body' : f.posTag });
          mem++;
        }
      }
    }
    process.stderr.write(`[storeX] nodes: +${mem} memory-frags, +${phr} phrase-frags\n`);
  }

  // books + transcripts (tier 1)
  for (const f of fs.readdirSync(corpusXDir)) {
    if (!f.endsWith('.txt')) continue;
    const name = f.replace(/\.txt$/, '');
    const frags = fragmentBook(fs.readFileSync(path.join(corpusXDir, f), 'utf8'), name);
    for (const fr of frags) { fragments.push(fr); allTexts.push(fr.text); }
    process.stderr.write(`[storeX] ${name}: +${frags.length} tier-1 frags\n`);
  }

  // FOREIGN-WORLD FILTER: a tier-1 fragment naming proper nouns that never
  // appear in HER corpus carries someone else's life — she can't say "Japhy"
  // or "Mark Hanna" and still be the entity. Mid-sentence capitalized words must
  // exist in her own world (or be common sentence-case artifacts).
  const herCaps = new Set();
  for (const f of fragments) if (f.tier !== 1) {
    for (const m of f.text.matchAll(/\b[A-Z][a-z]{2,}\b/g)) herCaps.add(m[0]);
  }
  const foreignWorld = f => {
    if (f.tier !== 1) return false;
    const mids = [...f.text.slice(1).matchAll(/(?<=[a-z,;—-]\s)[A-Z][a-z]{2,}\b/g)].map(m => m[0]);
    return mids.some(w => !herCaps.has(w));
  };
  const beforeFW = fragments.length;
  const kept = fragments.filter(f => !foreignWorld(f));
  process.stderr.write(`[storeX] foreign-world filter: -${beforeFW - kept.length} tier-1 frags\n`);

  // dedupe across everything
  const seen = new Set();
  const uniq = kept.filter(fr => { const k = fr.text.toLowerCase(); if (seen.has(k)) return false; seen.add(k); return true; });
  const oracle = buildOracle(allTexts);
  const vocab = new Set(allTexts.flatMap(wordsOnly));

  // first-word index: beam expansion stops scanning the world
  const byFirstWord = new Map();
  uniq.forEach((fr, i) => {
    const w = wordsOnly(fr.text)[0];
    if (!w) return;
    if (!byFirstWord.has(w)) byFirstWord.set(w, []);
    byFirstWord.get(w).push(i);
  });

  process.stderr.write(`[storeX] TOTAL: ${uniq.length} fragments (${uniq.filter(x => x.tier === 0).length} tier-0), ${oracle.tri.size} trigrams, vocab ${vocab.size}\n`);
  return { ...base, fragments: uniq, oracle, vocab, byFirstWord };
}

// HARD VALIDATOR: every word-trigram of the output must exist in corpus
function validateBounded(text, oracle) {
  const w = wordsOnly(text);
  const bad = [];
  for (let i = 0; i + 2 < w.length; i++) {
    const g = w[i] + ' ' + w[i + 1] + ' ' + w[i + 2];
    if (!oracle.tri.has(g)) bad.push(g);
  }
  return { ok: bad.length === 0, bad: bad.slice(0, 5), checked: Math.max(0, w.length - 2) };
}

module.exports = { buildStore, buildStoreX, fragmentReply, fragmentBook, buildOracle, validateBounded, wordsOnly, toks, mineUserNames, isDevArtifactReply, isVisitorTurnReply, isDirtyFragment };