| { |
| "metadata": { |
| "version": "v3_surgery", |
| "source_tokenizer": "tokenizer_v2.1 (32006 vocab)", |
| "target_tokenizer": "tokenizer_v3 (32006 vocab, preserved size)", |
| "total_surgeries": 51, |
| "total_surgeries_note": "51 unique slots modified (indices_modified count). Metadata fields fr_tokens_added=18 + special_tokens_repurposed=11 + utf8_fixes=8 + handler_replaced=44 sum to 81 but overlap: utf8_fixes are a subset of fr_word category, handler_replaced overlaps both fr_word and code_pattern. Unique surgery slots = 51.", |
| "fr_tokens_added": 18, |
| "special_tokens_repurposed": 11, |
| "utf8_fixes": 8, |
| "handler_replaced": 44, |
| "rationale": "Surgery replaces ~70 legacy Handler/CamelCase/UTF-8-broken tokens + 11 never-used ARCHON specials with high-value FR vocabulary and structured tool tokens. Vocab size unchanged at 32006." |
| }, |
| "replacements": { |
| "5": { |
| "old": "<|teach|>", |
| "new": "<|fr|>", |
| "category": "special", |
| "reason": "Slot 5-15 all have identical init embedding norm=1.5156 std=0.0474 — never used in pretrain. Repurpose as language tag for French context signal." |
| }, |
| "6": { |
| "old": "<|evolve|>", |
| "new": "<|en|>", |
| "category": "special", |
| "reason": "Init artifact. Repurpose as language tag for English context signal — enables explicit lang-switching in SFT." |
| }, |
| "7": { |
| "old": "<|query|>", |
| "new": "<|tool_name|>", |
| "category": "special", |
| "reason": "Init artifact. Repurpose as tool-call open delimiter — aligns with function-calling SFT format used in NousResearch/hermes-3." |
| }, |
| "8": { |
| "old": "<|code|>", |
| "new": "<|tool_args|>", |
| "category": "special", |
| "reason": "Init artifact. Repurpose as tool-args delimiter. <|code|> functionality is already covered by markdown triple-backtick patterns in SFT data." |
| }, |
| "9": { |
| "old": "<|/code|>", |
| "new": "<|reasoning|>", |
| "category": "special", |
| "reason": "Init artifact. Repurpose as chain-of-thought delimiter — critical for ARCHON reasoning traces in SFT." |
| }, |
| "10": { |
| "old": "<|rust|>", |
| "new": "<|reflection|>", |
| "category": "special", |
| "reason": "Init artifact. Rust is adequately handled by markdown code fences. Repurpose for self-reflection meta-token." |
| }, |
| "11": { |
| "old": "<|python|>", |
| "new": "<|memory|>", |
| "category": "special", |
| "reason": "Init artifact. Python code fences handle code delimiting. Repurpose for memory-read/write structured token in agentic traces." |
| }, |
| "12": { |
| "old": "<|system|>", |
| "new": "<|context|>", |
| "category": "special", |
| "reason": "Init artifact. IMPORTANT: <|system|> in ChatML is already slot 32001 (the added ChatML token). This slot 12 is the pretrain-era special, init-only. Repurpose as context-block delimiter." |
| }, |
| "13": { |
| "old": "<|user|>", |
| "new": "<|plan|>", |
| "category": "special", |
| "reason": "Init artifact. Same logic: <|user|> is slot 32002 in ChatML extension. Slot 13 = init artifact. Repurpose as plan-block delimiter for multi-step reasoning." |
| }, |
| "14": { |
| "old": "<|archon|>", |
| "new": "<|step|>", |
| "category": "special", |
| "reason": "Init artifact. The ARCHON assistant role is served by ChatML slot 32003. Repurpose slot 14 as step delimiter for numbered reasoning steps." |
| }, |
| "15": { |
| "old": "<|think|>", |
| "new": "<|verdict|>", |
| "category": "special", |
| "reason": "Init artifact. Repurpose as verdict/conclusion delimiter — useful for classification and evaluation tasks in SFT." |
| }, |
| "22133": { |
| "old": " RésolutiondeConflitsAndMé", |
| "new": " française", |
| "category": "fr_word", |
| "reason": "UTF-8 double-encoded Résolution... Handler token. 'française' currently costs 4 tokens (fr-an-ç-aise), high-frequency FR adjective. Fertility gain ~3 tokens per occurrence." |
| }, |
| "24420": { |
| "old": "énarisationAndSt", |
| "new": "chômage", |
| "category": "fr_word", |
| "reason": "UTF-8 broken fragment of 'Scénarisation'. 'chômage' costs 4 tokens currently (ch-ôm-age pattern), frequent in FR economic text." |
| }, |
| "24421": { |
| "old": "ScénarisationAndSt", |
| "new": "aujourd'hui", |
| "category": "fr_word", |
| "reason": "UTF-8 broken Scénarisation... 'aujourd'hui' currently 4 tokens — highest-frequency adverb in FR, enormous fertility win." |
| }, |
| "24424": { |
| "old": "ScénarisationAndStorytellingHandler", |
| "new": "qu'on", |
| "category": "fr_word", |
| "reason": "Full UTF-8 broken Handler token. 'qu'on' is 3 tokens currently — elision pattern extremely frequent in spoken/written FR." |
| }, |
| "25336": { |
| "old": " DataEngineeringAndSQLAvancéHandler", |
| "new": " qu'il", |
| "category": "fr_word", |
| "reason": "UTF-8 broken Engineering Handler. ' qu'il' 3-token elision, very high frequency FR." |
| }, |
| "25501": { |
| "old": " GénétiqueAndBiotechnologieHandler", |
| "new": " n'est", |
| "category": "fr_word", |
| "reason": "UTF-8 double-broken Génétique Handler. ' n'est' negation contraction costs 3 tokens currently." |
| }, |
| "24167": { |
| "old": "ÃīnigmesAndRaisonnementLatéralHandler", |
| "new": " n'a", |
| "category": "fr_word", |
| "reason": "UTF-8 broken Enigmes Handler with garbage codepoint Ãī. ' n'a' negation, 3 tokens currently." |
| }, |
| "22929": { |
| "old": " ArchitectureAndUrbanisme", |
| "new": " d'un", |
| "category": "fr_word", |
| "reason": "CamelCase AndUrbanisme Handler. ' d'un' elision costs 3 tokens, extremely high frequency." |
| }, |
| "23252": { |
| "old": " ManufacturingAndIn", |
| "new": " d'une", |
| "category": "fr_word", |
| "reason": "Engineering CamelCase fragment. ' d'une' 3 tokens currently, high frequency FR article elision." |
| }, |
| "23675": { |
| "old": " JournalismeAndInvestigation", |
| "new": " l'on", |
| "category": "fr_word", |
| "reason": "CamelCase Handler. ' l'on' formal FR pronoun contraction, 3-4 tokens currently." |
| }, |
| "23678": { |
| "old": " JournalismeAndInvestigationHandler", |
| "new": " j'ai", |
| "category": "fr_word", |
| "reason": "Full Handler. ' j'ai' is the most frequent first-person conjugation in FR, currently 3 tokens." |
| }, |
| "24695": { |
| "old": "AndAnalyticsHandler", |
| "new": " m'a", |
| "category": "fr_word", |
| "reason": "Handler suffix fragment. ' m'a' pronominal clitic, 3 tokens currently." |
| }, |
| "23385": { |
| "old": "LevelSystemsProgrammingHandler", |
| "new": " t'es", |
| "category": "fr_word", |
| "reason": "Handler suffix. ' t'es' informal 2nd person elision (tu es), frequent in conversational FR SFT data." |
| }, |
| "23384": { |
| "old": " DevOpsAndCloudInfrastructureHandler", |
| "new": " s'est", |
| "category": "fr_word", |
| "reason": "Full Handler. ' s'est' reflexive passé composé, extremely frequent in FR." |
| }, |
| "24054": { |
| "old": " ImmobilierAndInvestissementHandler", |
| "new": " qu'est-ce", |
| "category": "fr_word", |
| "reason": "Full Handler. ' qu'est-ce' question opener, currently 5 tokens — large fertility gain for FR QA data." |
| }, |
| "23860": { |
| "old": "AndCorporateLawHandler", |
| "new": " n'est-ce", |
| "category": "fr_word", |
| "reason": "Handler suffix. ' n'est-ce' (n'est-ce pas) 5 tokens currently — tag question extremely frequent." |
| }, |
| "24917": { |
| "old": "tivesHandler", |
| "new": " j'aime", |
| "category": "fr_word", |
| "reason": "Handler tail fragment. ' j'aime' affective expression, 4 tokens currently, frequent in instruction data." |
| }, |
| "23176": { |
| "old": "artupHandler", |
| "new": " rejoindre", |
| "category": "fr_word", |
| "reason": "Handler tail fragment. ' rejoindre' infinitive verb, 4 tokens currently, frequent in FR professional text." |
| }, |
| "24926": { |
| "old": " NeurosciencesAndSciences", |
| "new": " parce", |
| "category": "fr_word", |
| "reason": "CamelCase domain token. ' parce' (parce que), 3 tokens currently, ubiquitous in FR causal clauses." |
| }, |
| "24934": { |
| "old": "PrivacyAndComplianceHandler", |
| "new": " toujours", |
| "category": "fr_word", |
| "reason": "Handler token. ' toujours' adverb, 3 tokens currently, very high frequency." |
| }, |
| "23850": { |
| "old": "LawHandler", |
| "new": " encore", |
| "category": "fr_word", |
| "reason": "Handler suffix. ' encore' adverb (again/still), 3 tokens, high frequency." |
| }, |
| "23251": { |
| "old": "dustrialEngineering", |
| "new": " jamais", |
| "category": "fr_word", |
| "reason": "Engineering fragment. ' jamais' negation adverb, 3 tokens currently." |
| }, |
| "23259": { |
| "old": "dustrialEngineeringHandler", |
| "new": " vraiment", |
| "category": "fr_word", |
| "reason": "Engineering Handler fragment. ' vraiment' intensifier adverb, 3 tokens." |
| }, |
| "24052": { |
| "old": "ugmentationHumaineHandler", |
| "new": " maintenant", |
| "category": "fr_word", |
| "reason": "Handler tail. ' maintenant' temporal adverb, 3 tokens, high frequency." |
| }, |
| "23399": { |
| "old": "YOLO", |
| "new": " ailleurs", |
| "category": "fr_word", |
| "reason": "Uppercase meme token with zero NLP value. ' ailleurs' (elsewhere), 3 tokens, formal FR." |
| }, |
| "30904": { |
| "old": " ", |
| "new": " cependant", |
| "category": "fr_word", |
| "reason": "30-space whitespace blob — zero semantic value, wastes a slot. ' cependant' discourse connector, 3 tokens." |
| }, |
| "11602": { |
| "old": " Manufacturing", |
| "new": " également", |
| "category": "fr_word", |
| "reason": "Domain-specific English term with no FR value for ARCHON. ' également' (also/likewise), 3 tokens — high discourse frequency." |
| }, |
| "8914": { |
| "old": " Engineering", |
| "new": " notamment", |
| "category": "fr_word", |
| "reason": "English-only domain term. ' notamment' (notably/in particular), 3 tokens, frequent in formal FR." |
| }, |
| "15546": { |
| "old": "Engineering", |
| "new": "ée", |
| "category": "code_pattern", |
| "reason": "Duplicate Engineering fragment. 'ement' suffix covers ('-ment' adverb ending) — covers FR words like 'directement', 'rapidement'. High sub-word fertility gain.", |
| "new_revised": "ée", |
| "revision_reason": "Collision: ement already at idx 5289. Using ee (feminine past participle suffix) which is absent from vocab." |
| }, |
| "24086": { |
| "old": "MPER", |
| "new": "tion", |
| "category": "code_pattern", |
| "reason": "Uppercase fragment MPER has no value. 'tion' suffix is the highest-frequency morpheme in FR/EN — covers 'opération', 'création', 'fonction', etc.", |
| "new_revised": "ément", |
| "revision_reason": "Collision: 'tion' already in vocab. Using 'ément' (FR morphological suffix, absent from vocab)." |
| }, |
| "8212": { |
| "old": "|-------------|----------|------------", |
| "new": "ique", |
| "category": "code_pattern", |
| "reason": "Markdown table separator blob. 'ique' suffix covers FR adjectives: 'technique', 'logique', 'économique' — very high fertility.", |
| "new_revised": "ielle", |
| "revision_reason": "Collision: 'ique' already in vocab. Using 'ielle' (FR morphological suffix, absent from vocab)." |
| }, |
| "8229": { |
| "old": "|-------------|----------|---------------|----------|", |
| "new": "ment", |
| "category": "code_pattern", |
| "reason": "Longer markdown separator blob. 'ment' suffix: covers all French adverbs in -ment — massive fertility improvement.", |
| "new_revised": "ées", |
| "revision_reason": "Collision: 'ment' already in vocab. Using 'ées' (FR morphological suffix, absent from vocab)." |
| }, |
| "15231": { |
| "old": "VUSDT", |
| "new": "eur", |
| "category": "code_pattern", |
| "reason": "Crypto ticker VUSDT — zero value in general LM. 'eur' suffix covers 'valeur', 'erreur', 'auteur', 'docteur' — high-frequency FR suffix.", |
| "new_revised": "euse", |
| "revision_reason": "Collision: 'eur' already in vocab. Using 'euse' (FR morphological suffix, absent from vocab)." |
| }, |
| "30173": { |
| "old": "BNBUSDT", |
| "new": "ais", |
| "category": "code_pattern", |
| "reason": "Crypto ticker. 'ais' suffix/ending: 'j'étais', 'tu faisais', 'anglais', 'français' — covers imperfect tense and adjective endings.", |
| "new_revised": "ière", |
| "revision_reason": "Collision: 'ais' already in vocab. Using 'ière' (FR morphological suffix, absent from vocab)." |
| }, |
| "11028": { |
| "old": "AUDE", |
| "new": "ons", |
| "category": "code_pattern", |
| "reason": "AUDE uppercase fragment. 'ons' suffix covers 1st-person plural conjugations: 'allons', 'faisons', 'avons' — high frequency.", |
| "new_revised": "isé", |
| "revision_reason": "Collision: 'ons' already in vocab. Using 'isé' (FR morphological suffix, absent from vocab)." |
| }, |
| "18722": { |
| "old": "IPORTS", |
| "new": "ant", |
| "category": "code_pattern", |
| "reason": "IPORTS uppercase fragment. 'ant' suffix: present participle and adjectives ('important', 'pendant', 'brillant') — very high fertility.", |
| "new_revised": "isée", |
| "revision_reason": "Collision: 'ant' already in vocab. Using 'isée' (FR morphological suffix, absent from vocab)." |
| }, |
| "3315": { |
| "old": "ORDRE", |
| "new": "ère", |
| "category": "fr_word", |
| "reason": "Uppercase ORDRE — likely over-represented from trading jargon. 'ère' suffix covers 'première', 'manière', 'lumière' — frequent FR feminine suffix." |
| }, |
| "8046": { |
| "old": "GRES", |
| "new": "ité", |
| "category": "fr_word", |
| "reason": "Uppercase GRES fragment. 'ité' suffix covers 'qualité', 'capacité', 'université', 'activité' — extremely productive FR suffix." |
| }, |
| "12631": { |
| "old": "ULTA", |
| "new": "eur", |
| "category": "code_pattern", |
| "reason": "ULTA ticker/uppercase. Duplicate 'eur' target will be handled: if slot 15231 already takes 'eur', this slot takes 'teur' (agent suffix: 'auteur', 'moteur', 'facteur').", |
| "new_revised": "isés", |
| "revision_reason": "Collision: 'teur' already in vocab. Using 'isés' (FR morphological suffix, absent from vocab)." |
| }, |
| "4626": { |
| "old": "CRITIQUE", |
| "new": "aire", |
| "category": "fr_word", |
| "reason": "Uppercase CRITIQUE — over-trained on AETHER annotation artifacts. 'aire' suffix covers 'ordinaire', 'nécessaire', 'solitaire' — productive FR suffix.", |
| "new_revised": "isait", |
| "revision_reason": "Collision: 'aire' already in vocab. Using 'isait' (FR morphological suffix, absent from vocab)." |
| }, |
| "23384_dup_note": { |
| "note": "Token id 23384 appears multiple times in audit examples but is one entry in vocab. Counted once. Already mapped above." |
| } |
| }, |
| "summary": { |
| "indices_modified": [ |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 22133, |
| 24420, |
| 24421, |
| 24424, |
| 25336, |
| 25501, |
| 24167, |
| 22929, |
| 23252, |
| 23675, |
| 23678, |
| 24695, |
| 23385, |
| 23384, |
| 24054, |
| 23860, |
| 24917, |
| 23176, |
| 24926, |
| 24934, |
| 23850, |
| 23251, |
| 23259, |
| 24052, |
| 23399, |
| 30904, |
| 11602, |
| 8914, |
| 15546, |
| 24086, |
| 8212, |
| 8229, |
| 15231, |
| 30173, |
| 11028, |
| 18722, |
| 3315, |
| 8046, |
| 12631, |
| 4626 |
| ], |
| "count_by_category": { |
| "special": 11, |
| "fr_word": 27, |
| "code_pattern": 12, |
| "utf8_fixes": 8 |
| }, |
| "note_utf8_overlap": "8 UTF-8 broken tokens (é encoding) are a subset of the fr_word replacements — they are double-counted in utf8_fixes for reporting. Actual unique slots: 50 total surgery slots." |
| } |
| } |
