tweaked the coaching logic-v3

.gitignore

@@ -1,5 +1,8 @@
-# Local data and indexes (generated from transcripts)
-data/
+# Local data (generated); ship compact index + transcripts for Spaces
+data/*
+!data/index.npz
+!data/transcripts/
+!data/transcripts/**
 
 # Virtual environments
 .venv/

app.py

@@ -140,7 +140,7 @@ _warmup_error: str | None = None
 def _ensure_index():
     if not Path(INDEX_PATH).exists():
         raise FileNotFoundError(
-            f"Index not found at {INDEX_PATH}. Run `witgym-index` or bundle data/index.json in the Space."
+            f"Index not found at {INDEX_PATH}. Run `witgym-index` or bundle data/index.npz in the Space."
         )
 
 

data/transcripts/README.md

@@ -0,0 +1,25 @@
+Transcript Format (one scene per non-comment line):
+SHOW|CHARACTER|SETUP|RESPONSE|ARCHETYPE|TENSION|VIOLATION_DISTANCE|WHY_IT_WORKS
+
+ARCHETYPE values:
+  status_assertion    — claiming higher status than reality supports
+  self_delusion       — gap between self-image and reality exposed
+  power_inversion     — low-status person turns out to be right
+  anxiety_escalation  — small worry spirals to absurd conclusion
+  social_fail         — attempt at social impression backfires
+  misplaced_conf      — confident about entirely the wrong thing
+
+TENSION values:
+  social_embarrass    — threat of public embarrassment
+  existential         — existential dread about something mundane
+  status_threat       — someone's status is at risk
+  identity_expose     — true identity or gap is revealed
+  logic_collapse      — the logic of the situation collapses
+
+VIOLATION_DISTANCE values:
+  mild     — gentle, familiar, warm recognition
+  moderate — clear incongruity, still comfortable
+  sharp    — pointed, slightly uncomfortable but benign
+
+All fields must be pipe-separated on a single line.
+Lines starting with # are comments.

data/transcripts/office_extended.txt

@@ -0,0 +1,43 @@
+# The Office Extended — 30 additional curated scenes
+# Format: SHOW|CHARACTER|SETUP|RESPONSE|ARCHETYPE|TENSION|VIOLATION_DISTANCE|WHY_IT_WORKS
+#
+# ── STATUS_ASSERTION ──────────────────────────────────────────────────────────
+The Office|Michael Scott|Michael hosts a diversity day seminar and insists on running it himself because he knows more about diversity than the corporate trainer|Would you please call me by my new name, Prison Mike.|status_assertion|social_embarrass|sharp|Claims cultural authority through a made-up persona — the assertion is that impersonating a stereotype demonstrates understanding of it
+The Office|Michael Scott|Michael tells the camera he could have been a professional basketball player if he were taller and faster|I'm fast. I am very fast. I'm like Forrest Gump, except I'm not an idiot.|status_assertion|identity_expose|moderate|Claims athletic superiority immediately undercut by the self-comparison — the violation is the qualifier that defeats the claim
+The Office|Michael Scott|Michael insists he is the only one in the office who truly understands women after reading half of a magazine article|I have a lot of lady friends. Women love me. I am basically one of them.|status_assertion|identity_expose|sharp|Claims insider membership of a group by declaration — the evidence offered proves only the distance between him and the claim
+The Office|Dwight Schrute|Dwight tells the camera he would be the last person standing in any apocalypse scenario because of his survival training on the beet farm|In the event of an apocalypse I will be the last human standing. I have prepared. I have protein.|status_assertion|existential|moderate|Claims survivalist superiority through irrelevant credentials — the violation is the gap between catastrophic scenario and agricultural preparedness
+The Office|Michael Scott|Michael tells the whole office he is writing a screenplay that will be better than everything Hollywood has produced in the last decade|I am writing a movie. It is called Threat Level Midnight. It is going to be a classic.|status_assertion|social_embarrass|moderate|Self-declared classic before creation — the violation is treating aspiration as achievement
+#
+# ── SELF_DELUSION_EXPOSED ─────────────────────────────────────────────────────
+The Office|Michael Scott|Michael explains why he is not bald he simply has a very low hairline and that is different|It's not a bald spot. It's a solar panel for a sex machine.|self_delusion|identity_expose|sharp|Reframes a physical reality as a feature — the violation is the self-image is maintained only through metaphor that makes it worse
+The Office|Michael Scott|Michael gives a motivational speech to a struggling salesman explaining that he himself has never had a bad day at work|I have never had a bad day. I wake up, I think, today is going to be the best day of my life. And then it is.|self_delusion|logic_collapse|moderate|Claims emotional invincibility in a job that visibly defeats him daily — the violation is the assertion contradicts everything the audience has witnessed
+The Office|Kevin Malone|Kevin explains to the camera why he is actually quite good with numbers because he can do small ones very fast|I am actually very good at math. Small numbers. Like two plus two.|self_delusion|social_embarrass|mild|Claims competence in a domain while simultaneously demonstrating its limits — the violation is the claim and the evidence are in the same sentence
+The Office|Andy Bernard|Andy explains that he never gets angry anymore since he went to anger management and the camera shows him immediately crushing a can with his fist|I am a completely changed person. I don't get angry. I am at peace.|self_delusion|identity_expose|sharp|Self-assessment of transformation contradicted instantly by behaviour — the violation is the action and the claim are simultaneous
+The Office|Michael Scott|Michael explains he is an excellent driver and tells the camera about the three minor accidents he has had as evidence|Three fender benders. Zero at-fault. Well, one. But she was on her phone.|self_delusion|logic_collapse|moderate|Claims driving excellence while enumerating evidence of the opposite — the violation is the self-exoneration undermines rather than supports the claim
+#
+# ── POWER_INVERSION ───────────────────────────────────────────────────────────
+The Office|Ryan Howard|Ryan is asked for his business opinion by Michael and gives one clear sentence that the entire office ignores except the camera|That is just not how supply chains work.|power_inversion|status_threat|mild|The lowest-status person in the room states the only accurate thing said all meeting — the violation is correctness lands with zero authority
+The Office|Toby Flenderson|Toby raises a legitimate HR concern in a meeting. Michael immediately says he is not part of this family and moves on.|Michael, I just need to flag that—|power_inversion|identity_expose|moderate|The institutional authority is systematically excluded by the social authority — the violation is that Toby is right and it changes nothing
+The Office|Stanley Hudson|Stanley ignores Michael completely during an important announcement and returns to his crossword puzzle without looking up|Did you hear a word I just said?|power_inversion|status_threat|sharp|Total non-engagement from a subordinate — the power inversion is that Michael needs Stanley's attention more than Stanley needs Michael's approval
+The Office|Jim Halpert|Jim explains quietly to the camera why the entire scheme Michael has proposed will fail within twenty minutes|That is the most optimistic version of events.|power_inversion|logic_collapse|mild|Prophetic understatement from the person with no formal power — the violation is that calm accuracy is more devastating than any argument
+The Office|Oscar Martinez|Oscar tries to explain basic accounting to Michael using smaller and smaller words until Michael nods and announces he understood the first time|I see. Yes. Obviously. That is what I said.|power_inversion|logic_collapse|moderate|The expert's knowledge is rejected by the non-expert who claims to have had it first — the violation is the power to declare understanding belongs to the wrong person
+#
+# ── ANXIETY_ESCALATION ────────────────────────────────────────────────────────
+The Office|Michael Scott|Michael hears the company might be sold and immediately starts planning which personal items he will take when he is escorted out of the building|I am going to need a box. A big one. For my World's Best Boss mug. And my diploma.|anxiety_escalation|existential|moderate|Catastrophic outcome treated as settled fact — the violation is the planning is more detailed than the threat is confirmed
+The Office|Michael Scott|Michael gets a small medical diagnosis and immediately begins writing farewell letters to everyone in the office|I want people to remember me as I was. Young. Vital. A man in his prime.|anxiety_escalation|existential|sharp|Minor health news escalated to farewell mode — the violation is the gap between the diagnosis and the response is wider than the room
+The Office|Dwight Schrute|Dwight discovers a bat in the office and immediately implements a full building evacuation protocol he wrote himself at home|This is not a drill. The bat is real. The bat has disease. We are all exposed.|anxiety_escalation|existential|sharp|Single low-threat animal becomes civilizational emergency — the violation is the protocol escalates faster than any information justifies
+The Office|Michael Scott|Michael is told his car needs a minor repair and begins calculating how he will commute if the car never works again|What if the car is gone forever. What do I do then. I cannot walk. My legs are not built for it.|anxiety_escalation|logic_collapse|moderate|Minor mechanical issue collapsed into permanent lifestyle crisis — the violation is zero intermediate steps between problem and worst case
+#
+# ── SOCIAL_PERFORMANCE_FAIL ───────────────────────────────────────────────────
+The Office|Michael Scott|Michael tries to give a speech at Phyllis's wedding and makes the entire event about himself within the first thirty seconds|Love. That is what I bring to this marriage. Not as a groom. But as the man who brought these two people together. Really, this is my wedding too.|social_fail|social_embarrass|sharp|Attempts to honour others, immediately absorbs the occasion — the violation is every move toward generosity curves back to self-reference
+The Office|Michael Scott|Michael tries to conduct a performance review and cannot bring himself to say anything negative so tells everyone they are the best|Everyone here is the best at what they do. You are all tens.|social_fail|status_threat|moderate|Authority figure cannot perform the core function of authority — the violation is that the review is inverted into flattery to protect the reviewer
+The Office|Michael Scott|Michael organises a casino night to raise money for charity and spends all the charity money at his own casino|I lost the money. But I had a great time.|social_fail|logic_collapse|sharp|The mechanism for helping others is used to harm the cause — the violation is the social performance completely defeats its stated purpose
+The Office|Michael Scott|Michael attempts to do comedy at the office Christmas party and the only person laughing is himself|Okay. Here is one. Why did the accountant cross the road. To get to the other spreadsheet. Get it.|social_fail|social_embarrass|moderate|Performs comedy to a room that does not respond and continues without pausing — the violation is the absence of feedback registers as encouragement
+The Office|Michael Scott|Michael tries to comfort Pam after a hard day by telling her a long story about himself having a harder day|I understand. I have been there. Let me tell you about the time I had a worse version of this exact thing happen to me.|social_fail|social_embarrass|moderate|Attempts empathy and immediately replaces it with autobiography — the violation is that comfort is redirected to compete with the person being comforted
+#
+# ── MISPLACED_CONFIDENCE ──────────────────────────────────────────────────────
+The Office|Michael Scott|Michael confidently negotiates a business deal with zero preparation and announces he is a natural born closer before the meeting|I am a master negotiator. I once talked my way out of a speeding ticket by crying. That is negotiation.|misplaced_conf|logic_collapse|sharp|Claims expert status in a domain using evidence that disproves expertise — the violation is the example undermines the claim it was offered to support
+The Office|Michael Scott|Michael decides to take the office on a wilderness survival team-building day despite never having been outdoors|I grew up in the wild. Well, the suburbs. But there were raccoons.|misplaced_conf|status_threat|moderate|Claims outdoor authority through proximity to nature at maximum remove — the violation is suburbs and wilderness are treated as equivalent credentials
+The Office|Dwight Schrute|Dwight explains to the camera that he can read any person within six seconds of meeting them and proceeds to misread every person in the episode|I have never been wrong about a person. In my life. Not once.|misplaced_conf|identity_expose|sharp|Claims perfect social intelligence immediately before demonstrating its complete absence — the violation is the confidence rating and the evidence are in direct opposition
+The Office|Michael Scott|Michael decides he will cook for the whole office and announces his dish is better than anything a restaurant could produce before anyone has tasted it|I have been told I am an excellent cook. By myself, but also by people who were hungry.|misplaced_conf|social_embarrass|moderate|Self-certification of cooking skill with a supporting witness who was hunger-motivated — the violation is the evidence is explicitly compromised by the person offering it
+The Office|Michael Scott|Michael announces he can run a 5K without training because he once walked a very long distance in a parking lot|Running is basically just fast walking. And I am a great walker.|misplaced_conf|logic_collapse|moderate|Reduces a distinct physical discipline to a thing he already claims competence in — the violation is the equivalence is structurally convenient and factually wrong

data/transcripts/seed_scenes.txt

@@ -0,0 +1,45 @@
+# WitGym Transcript Format: SHOW|CHARACTER|SETUP|RESPONSE|ARCHETYPE|TENSION|VIOLATION_DISTANCE|WHY_IT_WORKS
+# ARCHETYPE values: status_assertion, self_delusion, power_inversion, anxiety_escalation, social_fail, misplaced_conf
+# TENSION values: social_embarrass, existential, status_threat, identity_expose, logic_collapse
+# VIOLATION_DISTANCE values: mild, moderate, sharp
+# NOTE: Office-only. All non-Office entries removed.
+#
+# ── STATUS_ASSERTION ──────────────────────────────────────────────────────────
+The Office|Michael Scott|Michael announces he has declared bankruptcy by simply walking out and yelling the word|I DECLARE BANKRUPTCY! You can't just say the word bankruptcy, Michael.|status_assertion|logic_collapse|sharp|Claims authority over a legal process by performative declaration — the violation is that status is being performed as if it creates reality
+The Office|Michael Scott|Michael tells the camera he is the best boss in the world, citing his own World's Best Boss mug which he bought himself|I bought it for myself because no one else was going to get it for me.|status_assertion|social_embarrass|moderate|Self-awarded status revealed — the violation is the logic gap between claiming recognition and being the source of that recognition
+The Office|Michael Scott|Michael claims to HR that he already apologised for a comment he made, when in fact he apologised for being caught, not for the comment|I said I was sorry that you feel that way. That is an apology.|status_assertion|identity_expose|sharp|The apology is redefined so it covers the speaker's feelings, not the harm done — the violation is the sincerity of the reframing
+The Office|Dwight Schrute|Dwight lectures a coworker about bears, explaining the proper response to a bear attack in a meeting about workplace safety|Bears. Beets. Battlestar Galactica.|status_assertion|social_embarrass|mild|Dwight asserts expert-level authority in an entirely wrong context — the violation is confident irrelevance
+The Office|Dwight Schrute|Dwight insists he cannot be outsmarted by Jim because he has taken an IQ test and the results showed he is in the 99th percentile of people who have taken that particular test|I outsmarted you and I am doing it right now.|status_assertion|status_threat|moderate|Claims intellectual dominance while being visibly outmaneuvered — the violation is the confidence gap between self-assessment and observable reality
+#
+# ── SELF_DELUSION_EXPOSED ─────────────────────────────────────────────────────
+The Office|Michael Scott|Michael explains to the camera why he is not like a boss at all, he is more like a friend, a best friend|Would I rather be feared or loved? Easy. Both. I want people to be afraid of how much they love me.|self_delusion|identity_expose|sharp|The self-image is exposed as incoherent by its own logic — wanting to be feared and loved simultaneously reveals the delusion
+The Office|Michael Scott|Michael gives career advice to a young employee, explaining that he succeeded by never giving up and always trusting his gut|I am Beyoncé, always.|self_delusion|identity_expose|sharp|The self-identification is so grandiose and so specific that it exposes the gap between self-image and reality instantly
+The Office|Andy Bernard|Andy reveals he went to Cornell and drops it into every conversation unprompted while the others already know|I went to Cornell. Ever heard of it?|self_delusion|identity_expose|moderate|Status credential deployed repeatedly at people who are already aware of it — the violation is the gap between how impressive Andy thinks it is and how tired everyone else is of hearing it
+The Office|Kevin Malone|Kevin explains to the camera a complex system he devised to remember people's names, and the system is simply to repeat the name back to them and then forget it|I have a system. It's called Kevin's system.|self_delusion|logic_collapse|mild|The system is described with elaborate confidence and is just a normal thing that does not work — the delusion is in the labeling and ownership, not the method
+The Office|Ryan Howard|Ryan pitches a business plan for a new social media platform to Michael, using terms he clearly learned that morning|It's not a website. It's a place where connections happen organically. It's a lifestyle.|self_delusion|social_embarrass|moderate|The jargon is assembled correctly but is being used by someone who does not understand it — the violation is the confidence in borrowed vocabulary
+#
+# ── POWER_INVERSION ───────────────────────────────────────────────────────────
+The Office|Jim Halpert|Jim stares directly at the camera after Michael explains something confidently that is completely wrong|That is not how any of this works.|power_inversion|social_embarrass|mild|The low-status observer's silent acknowledgment to the audience inverts the authority structure without confrontation
+The Office|Pam Beesly|Michael asks the office if anyone thinks his idea is bad. Pam quietly raises her hand.|Is anyone opposed to this? ... Pam. Why.|power_inversion|status_threat|moderate|The least powerful person in the room is the only honest one — the violation is that her objection lands while everyone else's silence is complicit
+The Office|Toby Flenderson|Michael announces a new policy and Toby explains that HR already told him this policy was illegal three months ago|HR actually flagged this in March. I sent an email.|power_inversion|status_threat|sharp|The person actively despised by the boss is the only one with institutional memory and legal knowledge — the power inverts through the inconvenience of being correct
+The Office|Jim Halpert|Dwight gives a speech about loyalty and sacrifice and Jim responds with a single sentence that disproves the entire premise|You locked yourself out of the building yesterday.|power_inversion|logic_collapse|sharp|An argument about grand principle is demolished by a specific recent failure — the violation is that the counterexample is undeniable and mundane
+#
+# ── ANXIETY_ESCALATION ────────────────────────────────────────────────────────
+The Office|Michael Scott|Michael is told the branch might be closed and spends the morning planning his speech for the company's eventual bankruptcy hearing|I just want to be prepared. When Dunder Mifflin goes under, people are going to need answers.|anxiety_escalation|existential|moderate|The anxiety leaps over the probable outcome to a catastrophic one and begins solving for that instead — the violation is treating the worst case as the only case
+The Office|Kevin Malone|Kevin is asked what he thinks will happen at the company meeting. He says the worst possible outcome with total calm.|I think we're all going to die.|anxiety_escalation|existential|moderate|Delivered without affect — the violation is that the most extreme conclusion is stated with the same tone as a weather report
+The Office|Michael Scott|Michael is told his expense report has a minor error and immediately interprets this as the beginning of a formal investigation into his entire career|This is how it starts. One form, and then it unravels.|anxiety_escalation|identity_expose|sharp|A minor administrative error is recast as existential exposure — the violation is the speed of the catastrophic interpretation
+The Office|Dwight Schrute|Dwight explains the likelihood of a fire drill actually resulting in deaths as he conducts a surprise fire drill that he himself has made extremely dangerous|Fact: in the event of a real fire, every second counts. I may have made this too real.|anxiety_escalation|existential|sharp|The anxiety about disaster is the cause of the disaster — the violation is the absence of irony in the delivery of this revelation
+#
+# ── SOCIAL_PERFORMANCE_FAIL ───────────────────────────────────────────────────
+The Office|Michael Scott|Michael tries to do an impression of a black coworker to show he is comfortable with all cultures|I am not Mr. Scott. I am the original fun-uncle.|social_fail|social_embarrass|sharp|The performance of comfort and inclusion reveals profound discomfort and exclusion — the violation is that the attempt makes everything worse
+The Office|Michael Scott|Michael tries to fire someone and keeps taking it back because he cannot handle their reaction|You know what, you're fired. No. No wait. Come back.|social_fail|status_threat|moderate|The authority figure cannot perform authority — the violation is the reversal mid-firing
+The Office|Michael Scott|Michael delivers a eulogy at a funeral and pivots it into a story about himself within two sentences|He was like a father to me. Which is funny because my actual father was never around.|social_fail|social_embarrass|sharp|The social form of grief is performed for approximately one sentence before self-interest takes over — the violation is the transparency of the pivot
+The Office|Andy Bernard|Andy tries to impress a client by singing a song he wrote about their company, and the song has the client's name wrong|Sabre! Sa-bre! We make the things that make you— wait, how do you say it?|social_fail|social_embarrass|sharp|The performance of preparation reveals the absence of preparation — the violation lands mid-performance with no recovery
+The Office|Michael Scott|Michael gives a motivational speech to the office about never giving up, right after announcing he is cancelling the charity fun run because it is too hard|Do you know what they say about tough times? They end. And then sometimes they come back. But that's later.|social_fail|identity_expose|moderate|The speech directly contradicts the action taken moments ago — the violation is the complete absence of self-awareness about the contradiction
+#
+# ── MISPLACED_CONFIDENCE ──────────────────────────────────────────────────────
+The Office|Dwight Schrute|Dwight explains to the camera why he is the perfect candidate for manager, citing his expertise in beet farming, Judo, and fear|I have been training for this my entire life. I am ready to lead.|misplaced_conf|status_threat|moderate|The evidence offered for leadership readiness is entirely inapplicable to the role — the violation is the complete sincerity of the irrelevant credentials
+The Office|Michael Scott|Michael explains to the camera why he is excellent at interviews, right before giving the worst interview ever seen on camera|I am very good at reading people. It's a gift.|misplaced_conf|social_embarrass|sharp|Confidence in the exact skill that the next 30 seconds will disprove — the violation is the timing of the claim relative to the evidence
+The Office|Michael Scott|Michael confidently explains a negotiation strategy he read about in a book, right before the negotiation begins, and then does none of it|You always go in with your best offer first. That way they see you mean business. Okay. So my opening offer is... free.|misplaced_conf|logic_collapse|sharp|The strategy is explained correctly and then immediately abandoned — the violation is the speed of the collapse between stated plan and action
+The Office|Dwight Schrute|Dwight claims to be an expert on the law after reading one document online and begins advising coworkers on their legal rights|I have done extensive legal research on this matter. Wikipedia confirmed everything.|misplaced_conf|identity_expose|moderate|Expertise is claimed based on the shallowest possible source, named without embarrassment — the violation is the total confidence in the inadequacy of the evidence
+The Office|Ryan Howard|Ryan confidently predicts which startup idea will succeed, and it is the same idea that already failed publicly three months earlier|It's all about timing. And timing is everything right now.|misplaced_conf|logic_collapse|moderate|Confidence about a domain in which the speaker has a documented recent failure — the violation is the absence of any memory of that failure

scripts/eval.py

@@ -34,7 +34,7 @@ TEST_INPUTS = [
 ]
 
 
-def run_eval(tag: str, index_path: str = "data/index.json") -> None:
+def run_eval(tag: str, index_path: str = "data/index.npz") -> None:
     """Run all 5 test inputs, save results snapshot."""
     from witgym.engine import WitGymEngine
 
@@ -144,7 +144,7 @@ def main():
     parser = argparse.ArgumentParser(description="WitGym closed-loop evaluator")
     parser.add_argument("--tag", choices=["before", "after"], help="Run eval and save snapshot with this tag")
     parser.add_argument("--compare", action="store_true", help="Compare before vs after snapshots")
-    parser.add_argument("--index", default="data/index.json")
+    parser.add_argument("--index", default="data/index.npz")
     args = parser.parse_args()
 
     if args.compare:

witgym/config.py

@@ -38,7 +38,7 @@ RETRIEVE_POOL_SIZE = 12
 ENABLE_CROSS_ENCODER_RERANK = True
 RERANK_MODEL_ID = "cross-encoder/ettin-reranker-32m-v1"
 RERANK_DEVICE = "cpu"  # keep 9B on MPS; rerank on CPU to avoid unified-memory spikes
-INDEX_PATH = "data/index.json"
+INDEX_PATH = "data/index.npz"
 TRANSCRIPT_DIR = "data/transcripts"
 
 # Generation guards

witgym/indexer.py

@@ -1,4 +1,4 @@
-"""Offline indexer — parse transcript files, embed comedy metadata, persist to JSON.
+"""Offline indexer — parse transcript files, embed comedy metadata, persist to NPZ.
 
 Implements Principle 2: indexes on abstract comedy fields, NOT raw text.
 The embedding is of: "{archetype} {tension_type} {setup_summary}"
@@ -51,7 +51,7 @@ def _make_index_text(scene: TranscriptScene) -> str:
 
 
 def build_index(transcript_dir: str = config.TRANSCRIPT_DIR, index_path: str = config.INDEX_PATH):
-    """Parse all .txt files, embed comedy metadata strings, save index JSON."""
+    """Parse all .txt files, embed comedy metadata strings, save compressed index."""
     from sentence_transformers import SentenceTransformer
 
     transcript_path = Path(transcript_dir)
@@ -89,15 +89,15 @@ def build_index(transcript_dir: str = config.TRANSCRIPT_DIR, index_path: str = c
         show_progress_bar=True,
     )
 
-    # Persist to JSON
-    index_data = {
-        "scenes": [s.model_dump() for s in all_scenes],
-        "embeddings": embeddings.tolist(),
-        "index_texts": index_texts,
-    }
+    # Persist as compressed NPZ (~6 MB vs ~37 MB JSON float bloat; fits HF 10 MiB git limit)
     os.makedirs(os.path.dirname(index_path) if os.path.dirname(index_path) else ".", exist_ok=True)
-    with open(index_path, "w", encoding="utf-8") as f:
-        json.dump(index_data, f)
+    scenes_json = json.dumps([s.model_dump() for s in all_scenes]).encode("utf-8")
+    np.savez_compressed(
+        index_path,
+        embeddings=embeddings.astype(np.float32),
+        scenes_json=scenes_json,
+        index_texts=np.array(index_texts, dtype=object),
+    )
 
     logger.success(f"Index saved to {index_path} ({len(all_scenes)} scenes)")
     return all_scenes

witgym/main.py

@@ -70,7 +70,7 @@ def _show_debug_panel(result):
 def chat_cmd():
     parser = argparse.ArgumentParser(description="WitGym — conversational comedy engine")
     parser.add_argument("--debug", action="store_true", help="Show metadata, retrieved scenes, and all candidates")
-    parser.add_argument("--index", default="data/index.json", help="Path to the index file")
+    parser.add_argument("--index", default="data/index.npz", help="Path to the index file")
     args = parser.parse_args()
 
     _setup_logging(args.debug)
@@ -123,7 +123,7 @@ def chat_cmd():
 def index_cmd():
     parser = argparse.ArgumentParser(description="WitGym — build RAG index from transcript files")
     parser.add_argument("--transcripts", default="data/transcripts", help="Directory containing .txt transcript files")
-    parser.add_argument("--output", default="data/index.json", help="Output index path")
+    parser.add_argument("--output", default="data/index.npz", help="Output index path")
     parser.add_argument("--debug", action="store_true")
     args = parser.parse_args()
 

witgym/retriever.py

@@ -4,6 +4,7 @@ Implements Principle 2: query is the abstract comedy metadata, not raw text.
 Returns analogous situations (same violation type), not similar words.
 """
 import json
+import os
 from typing import List
 from loguru import logger
 import numpy as np
@@ -15,20 +16,32 @@ _reranker_cache = None
 
 
 def load_index(index_path: str = config.INDEX_PATH) -> dict:
-    """Load index from JSON. Caches in memory."""
+    """Load index from NPZ (or legacy JSON). Caches in memory."""
     global _index_cache
     if _index_cache is not None:
         return _index_cache
 
-    with open(index_path, "r", encoding="utf-8") as f:
-        data = json.load(f)
-
-    # Reconstruct TranscriptScene objects and numpy embeddings
-    scenes = [TranscriptScene.model_validate(s) for s in data["scenes"]]
-    embeddings = np.array(data["embeddings"], dtype=np.float32)
+    path = index_path
+    if path.endswith(".json") and not os.path.exists(path):
+        npz_fallback = path.removesuffix(".json") + ".npz"
+        if os.path.exists(npz_fallback):
+            path = npz_fallback
+
+    if path.endswith(".npz"):
+        archive = np.load(path, allow_pickle=True)
+        scenes = [
+            TranscriptScene.model_validate(s)
+            for s in json.loads(archive["scenes_json"].tobytes().decode("utf-8"))
+        ]
+        embeddings = np.asarray(archive["embeddings"], dtype=np.float32)
+    else:
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        scenes = [TranscriptScene.model_validate(s) for s in data["scenes"]]
+        embeddings = np.array(data["embeddings"], dtype=np.float32)
 
     _index_cache = {"scenes": scenes, "embeddings": embeddings}
-    logger.info(f"Index loaded: {len(scenes)} scenes")
+    logger.info(f"Index loaded: {len(scenes)} scenes from {path}")
     return _index_cache