{
  "metadata": {
    "version": "v3_surgery",
    "source_tokenizer": "tokenizer_v2.1 (32006 vocab)",
    "target_tokenizer": "tokenizer_v3 (32006 vocab, preserved size)",
    "total_surgeries": 51,
    "total_surgeries_note": "51 unique slots modified (indices_modified count). Metadata fields fr_tokens_added=18 + special_tokens_repurposed=11 + utf8_fixes=8 + handler_replaced=44 sum to 81 but overlap: utf8_fixes are a subset of fr_word category, handler_replaced overlaps both fr_word and code_pattern. Unique surgery slots = 51.",
    "fr_tokens_added": 18,
    "special_tokens_repurposed": 11,
    "utf8_fixes": 8,
    "handler_replaced": 44,
    "rationale": "Surgery replaces ~70 legacy Handler/CamelCase/UTF-8-broken tokens + 11 never-used ARCHON specials with high-value FR vocabulary and structured tool tokens. Vocab size unchanged at 32006."
  },
  "replacements": {
    "5": {
      "old": "<|teach|>",
      "new": "<|fr|>",
      "category": "special",
      "reason": "Slot 5-15 all have identical init embedding norm=1.5156 std=0.0474 — never used in pretrain. Repurpose as language tag for French context signal."
    },
    "6": {
      "old": "<|evolve|>",
      "new": "<|en|>",
      "category": "special",
      "reason": "Init artifact. Repurpose as language tag for English context signal — enables explicit lang-switching in SFT."
    },
    "7": {
      "old": "<|query|>",
      "new": "<|tool_name|>",
      "category": "special",
      "reason": "Init artifact. Repurpose as tool-call open delimiter — aligns with function-calling SFT format used in NousResearch/hermes-3."
    },
    "8": {
      "old": "<|code|>",
      "new": "<|tool_args|>",
      "category": "special",
      "reason": "Init artifact. Repurpose as tool-args delimiter. <|code|> functionality is already covered by markdown triple-backtick patterns in SFT data."
    },
    "9": {
      "old": "<|/code|>",
      "new": "<|reasoning|>",
      "category": "special",
      "reason": "Init artifact. Repurpose as chain-of-thought delimiter — critical for ARCHON reasoning traces in SFT."
    },
    "10": {
      "old": "<|rust|>",
      "new": "<|reflection|>",
      "category": "special",
      "reason": "Init artifact. Rust is adequately handled by markdown code fences. Repurpose for self-reflection meta-token."
    },
    "11": {
      "old": "<|python|>",
      "new": "<|memory|>",
      "category": "special",
      "reason": "Init artifact. Python code fences handle code delimiting. Repurpose for memory-read/write structured token in agentic traces."
    },
    "12": {
      "old": "<|system|>",
      "new": "<|context|>",
      "category": "special",
      "reason": "Init artifact. IMPORTANT: <|system|> in ChatML is already slot 32001 (the added ChatML token). This slot 12 is the pretrain-era special, init-only. Repurpose as context-block delimiter."
    },
    "13": {
      "old": "<|user|>",
      "new": "<|plan|>",
      "category": "special",
      "reason": "Init artifact. Same logic: <|user|> is slot 32002 in ChatML extension. Slot 13 = init artifact. Repurpose as plan-block delimiter for multi-step reasoning."
    },
    "14": {
      "old": "<|archon|>",
      "new": "<|step|>",
      "category": "special",
      "reason": "Init artifact. The ARCHON assistant role is served by ChatML slot 32003. Repurpose slot 14 as step delimiter for numbered reasoning steps."
    },
    "15": {
      "old": "<|think|>",
      "new": "<|verdict|>",
      "category": "special",
      "reason": "Init artifact. Repurpose as verdict/conclusion delimiter — useful for classification and evaluation tasks in SFT."
    },
    "22133": {
      "old": " RésolutiondeConflitsAndMé",
      "new": " française",
      "category": "fr_word",
      "reason": "UTF-8 double-encoded Résolution... Handler token. 'française' currently costs 4 tokens (fr-an-ç-aise), high-frequency FR adjective. Fertility gain ~3 tokens per occurrence."
    },
    "24420": {
      "old": "énarisationAndSt",
      "new": "chômage",
      "category": "fr_word",
      "reason": "UTF-8 broken fragment of 'Scénarisation'. 'chômage' costs 4 tokens currently (ch-ôm-age pattern), frequent in FR economic text."
    },
    "24421": {
      "old": "ScénarisationAndSt",
      "new": "aujourd'hui",
      "category": "fr_word",
      "reason": "UTF-8 broken Scénarisation... 'aujourd'hui' currently 4 tokens — highest-frequency adverb in FR, enormous fertility win."
    },
    "24424": {
      "old": "ScénarisationAndStorytellingHandler",
      "new": "qu'on",
      "category": "fr_word",
      "reason": "Full UTF-8 broken Handler token. 'qu'on' is 3 tokens currently — elision pattern extremely frequent in spoken/written FR."
    },
    "25336": {
      "old": " DataEngineeringAndSQLAvancéHandler",
      "new": " qu'il",
      "category": "fr_word",
      "reason": "UTF-8 broken Engineering Handler. ' qu'il' 3-token elision, very high frequency FR."
    },
    "25501": {
      "old": " GénétiqueAndBiotechnologieHandler",
      "new": " n'est",
      "category": "fr_word",
      "reason": "UTF-8 double-broken Génétique Handler. ' n'est' negation contraction costs 3 tokens currently."
    },
    "24167": {
      "old": "ÃīnigmesAndRaisonnementLatéralHandler",
      "new": " n'a",
      "category": "fr_word",
      "reason": "UTF-8 broken Enigmes Handler with garbage codepoint Ãī. ' n'a' negation, 3 tokens currently."
    },
    "22929": {
      "old": " ArchitectureAndUrbanisme",
      "new": " d'un",
      "category": "fr_word",
      "reason": "CamelCase AndUrbanisme Handler. ' d'un' elision costs 3 tokens, extremely high frequency."
    },
    "23252": {
      "old": " ManufacturingAndIn",
      "new": " d'une",
      "category": "fr_word",
      "reason": "Engineering CamelCase fragment. ' d'une' 3 tokens currently, high frequency FR article elision."
    },
    "23675": {
      "old": " JournalismeAndInvestigation",
      "new": " l'on",
      "category": "fr_word",
      "reason": "CamelCase Handler. ' l'on' formal FR pronoun contraction, 3-4 tokens currently."
    },
    "23678": {
      "old": " JournalismeAndInvestigationHandler",
      "new": " j'ai",
      "category": "fr_word",
      "reason": "Full Handler. ' j'ai' is the most frequent first-person conjugation in FR, currently 3 tokens."
    },
    "24695": {
      "old": "AndAnalyticsHandler",
      "new": " m'a",
      "category": "fr_word",
      "reason": "Handler suffix fragment. ' m'a' pronominal clitic, 3 tokens currently."
    },
    "23385": {
      "old": "LevelSystemsProgrammingHandler",
      "new": " t'es",
      "category": "fr_word",
      "reason": "Handler suffix. ' t'es' informal 2nd person elision (tu es), frequent in conversational FR SFT data."
    },
    "23384": {
      "old": " DevOpsAndCloudInfrastructureHandler",
      "new": " s'est",
      "category": "fr_word",
      "reason": "Full Handler. ' s'est' reflexive passé composé, extremely frequent in FR."
    },
    "24054": {
      "old": " ImmobilierAndInvestissementHandler",
      "new": " qu'est-ce",
      "category": "fr_word",
      "reason": "Full Handler. ' qu'est-ce' question opener, currently 5 tokens — large fertility gain for FR QA data."
    },
    "23860": {
      "old": "AndCorporateLawHandler",
      "new": " n'est-ce",
      "category": "fr_word",
      "reason": "Handler suffix. ' n'est-ce' (n'est-ce pas) 5 tokens currently — tag question extremely frequent."
    },
    "24917": {
      "old": "tivesHandler",
      "new": " j'aime",
      "category": "fr_word",
      "reason": "Handler tail fragment. ' j'aime' affective expression, 4 tokens currently, frequent in instruction data."
    },
    "23176": {
      "old": "artupHandler",
      "new": " rejoindre",
      "category": "fr_word",
      "reason": "Handler tail fragment. ' rejoindre' infinitive verb, 4 tokens currently, frequent in FR professional text."
    },
    "24926": {
      "old": " NeurosciencesAndSciences",
      "new": " parce",
      "category": "fr_word",
      "reason": "CamelCase domain token. ' parce' (parce que), 3 tokens currently, ubiquitous in FR causal clauses."
    },
    "24934": {
      "old": "PrivacyAndComplianceHandler",
      "new": " toujours",
      "category": "fr_word",
      "reason": "Handler token. ' toujours' adverb, 3 tokens currently, very high frequency."
    },
    "23850": {
      "old": "LawHandler",
      "new": " encore",
      "category": "fr_word",
      "reason": "Handler suffix. ' encore' adverb (again/still), 3 tokens, high frequency."
    },
    "23251": {
      "old": "dustrialEngineering",
      "new": " jamais",
      "category": "fr_word",
      "reason": "Engineering fragment. ' jamais' negation adverb, 3 tokens currently."
    },
    "23259": {
      "old": "dustrialEngineeringHandler",
      "new": " vraiment",
      "category": "fr_word",
      "reason": "Engineering Handler fragment. ' vraiment' intensifier adverb, 3 tokens."
    },
    "24052": {
      "old": "ugmentationHumaineHandler",
      "new": " maintenant",
      "category": "fr_word",
      "reason": "Handler tail. ' maintenant' temporal adverb, 3 tokens, high frequency."
    },
    "23399": {
      "old": "YOLO",
      "new": " ailleurs",
      "category": "fr_word",
      "reason": "Uppercase meme token with zero NLP value. ' ailleurs' (elsewhere), 3 tokens, formal FR."
    },
    "30904": {
      "old": "                              ",
      "new": " cependant",
      "category": "fr_word",
      "reason": "30-space whitespace blob — zero semantic value, wastes a slot. ' cependant' discourse connector, 3 tokens."
    },
    "11602": {
      "old": " Manufacturing",
      "new": " également",
      "category": "fr_word",
      "reason": "Domain-specific English term with no FR value for ARCHON. ' également' (also/likewise), 3 tokens — high discourse frequency."
    },
    "8914": {
      "old": " Engineering",
      "new": " notamment",
      "category": "fr_word",
      "reason": "English-only domain term. ' notamment' (notably/in particular), 3 tokens, frequent in formal FR."
    },
    "15546": {
      "old": "Engineering",
      "new": "ée",
      "category": "code_pattern",
      "reason": "Duplicate Engineering fragment. 'ement' suffix covers ('-ment' adverb ending) — covers FR words like 'directement', 'rapidement'. High sub-word fertility gain.",
      "new_revised": "ée",
      "revision_reason": "Collision: ement already at idx 5289. Using ee (feminine past participle suffix) which is absent from vocab."
    },
    "24086": {
      "old": "MPER",
      "new": "tion",
      "category": "code_pattern",
      "reason": "Uppercase fragment MPER has no value. 'tion' suffix is the highest-frequency morpheme in FR/EN — covers 'opération', 'création', 'fonction', etc.",
      "new_revised": "ément",
      "revision_reason": "Collision: 'tion' already in vocab. Using 'ément' (FR morphological suffix, absent from vocab)."
    },
    "8212": {
      "old": "|-------------|----------|------------",
      "new": "ique",
      "category": "code_pattern",
      "reason": "Markdown table separator blob. 'ique' suffix covers FR adjectives: 'technique', 'logique', 'économique' — very high fertility.",
      "new_revised": "ielle",
      "revision_reason": "Collision: 'ique' already in vocab. Using 'ielle' (FR morphological suffix, absent from vocab)."
    },
    "8229": {
      "old": "|-------------|----------|---------------|----------|",
      "new": "ment",
      "category": "code_pattern",
      "reason": "Longer markdown separator blob. 'ment' suffix: covers all French adverbs in -ment — massive fertility improvement.",
      "new_revised": "ées",
      "revision_reason": "Collision: 'ment' already in vocab. Using 'ées' (FR morphological suffix, absent from vocab)."
    },
    "15231": {
      "old": "VUSDT",
      "new": "eur",
      "category": "code_pattern",
      "reason": "Crypto ticker VUSDT — zero value in general LM. 'eur' suffix covers 'valeur', 'erreur', 'auteur', 'docteur' — high-frequency FR suffix.",
      "new_revised": "euse",
      "revision_reason": "Collision: 'eur' already in vocab. Using 'euse' (FR morphological suffix, absent from vocab)."
    },
    "30173": {
      "old": "BNBUSDT",
      "new": "ais",
      "category": "code_pattern",
      "reason": "Crypto ticker. 'ais' suffix/ending: 'j'étais', 'tu faisais', 'anglais', 'français' — covers imperfect tense and adjective endings.",
      "new_revised": "ière",
      "revision_reason": "Collision: 'ais' already in vocab. Using 'ière' (FR morphological suffix, absent from vocab)."
    },
    "11028": {
      "old": "AUDE",
      "new": "ons",
      "category": "code_pattern",
      "reason": "AUDE uppercase fragment. 'ons' suffix covers 1st-person plural conjugations: 'allons', 'faisons', 'avons' — high frequency.",
      "new_revised": "isé",
      "revision_reason": "Collision: 'ons' already in vocab. Using 'isé' (FR morphological suffix, absent from vocab)."
    },
    "18722": {
      "old": "IPORTS",
      "new": "ant",
      "category": "code_pattern",
      "reason": "IPORTS uppercase fragment. 'ant' suffix: present participle and adjectives ('important', 'pendant', 'brillant') — very high fertility.",
      "new_revised": "isée",
      "revision_reason": "Collision: 'ant' already in vocab. Using 'isée' (FR morphological suffix, absent from vocab)."
    },
    "3315": {
      "old": "ORDRE",
      "new": "ère",
      "category": "fr_word",
      "reason": "Uppercase ORDRE — likely over-represented from trading jargon. 'ère' suffix covers 'première', 'manière', 'lumière' — frequent FR feminine suffix."
    },
    "8046": {
      "old": "GRES",
      "new": "ité",
      "category": "fr_word",
      "reason": "Uppercase GRES fragment. 'ité' suffix covers 'qualité', 'capacité', 'université', 'activité' — extremely productive FR suffix."
    },
    "12631": {
      "old": "ULTA",
      "new": "eur",
      "category": "code_pattern",
      "reason": "ULTA ticker/uppercase. Duplicate 'eur' target will be handled: if slot 15231 already takes 'eur', this slot takes 'teur' (agent suffix: 'auteur', 'moteur', 'facteur').",
      "new_revised": "isés",
      "revision_reason": "Collision: 'teur' already in vocab. Using 'isés' (FR morphological suffix, absent from vocab)."
    },
    "4626": {
      "old": "CRITIQUE",
      "new": "aire",
      "category": "fr_word",
      "reason": "Uppercase CRITIQUE — over-trained on AETHER annotation artifacts. 'aire' suffix covers 'ordinaire', 'nécessaire', 'solitaire' — productive FR suffix.",
      "new_revised": "isait",
      "revision_reason": "Collision: 'aire' already in vocab. Using 'isait' (FR morphological suffix, absent from vocab)."
    },
    "23384_dup_note": {
      "note": "Token id 23384 appears multiple times in audit examples but is one entry in vocab. Counted once. Already mapped above."
    }
  },
  "summary": {
    "indices_modified": [
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      22133,
      24420,
      24421,
      24424,
      25336,
      25501,
      24167,
      22929,
      23252,
      23675,
      23678,
      24695,
      23385,
      23384,
      24054,
      23860,
      24917,
      23176,
      24926,
      24934,
      23850,
      23251,
      23259,
      24052,
      23399,
      30904,
      11602,
      8914,
      15546,
      24086,
      8212,
      8229,
      15231,
      30173,
      11028,
      18722,
      3315,
      8046,
      12631,
      4626
    ],
    "count_by_category": {
      "special": 11,
      "fr_word": 27,
      "code_pattern": 12,
      "utf8_fixes": 8
    },
    "note_utf8_overlap": "8 UTF-8 broken tokens (é encoding) are a subset of the fr_word replacements — they are double-counted in utf8_fixes for reporting. Actual unique slots: 50 total surgery slots."
  }
}