Add Keystone trail-code tokenizer release KT-2026-04

README.md

@@ -0,0 +1,14 @@
+---
+library_name: tokenizers
+tags:
+- emergency-communications
+- tokenizer
+---
+
+# Keystone Trail-Code Tokenizer
+
+Release: `KT-2026-04`
+
+This compact vocabulary supports controlled trail-response messages. It contains **10** tokens, including reserved tokens. The build admitted 11 physical candidate records and selected 6 ordinary token groups.
+
+The configured maximum sequence length is 128 tokens. Candidate authorities remain private; the repository includes only the derived vocabulary, configuration, review report, and release summary.

release/lexicon-summary.json

@@ -0,0 +1,52 @@
+{
+  "release_id": "KT-2026-04",
+  "physical_records": 14,
+  "eligible_records": 11,
+  "rejected_records": 3,
+  "eligible_token_groups": 9,
+  "selected_ordinary_tokens": 6,
+  "vocabulary_size": 10,
+  "rejections": {
+    "language_not_required": 1,
+    "status_not_approved": 1,
+    "frequency_below_minimum": 1
+  },
+  "selected_tokens": [
+    {
+      "token": "ridge",
+      "vocabulary_id": 4,
+      "aggregate_frequency": 85,
+      "source_records": 2
+    },
+    {
+      "token": "trail",
+      "vocabulary_id": 5,
+      "aggregate_frequency": 80,
+      "source_records": 1
+    },
+    {
+      "token": "water",
+      "vocabulary_id": 6,
+      "aggregate_frequency": 70,
+      "source_records": 2
+    },
+    {
+      "token": "shelter",
+      "vocabulary_id": 7,
+      "aggregate_frequency": 45,
+      "source_records": 1
+    },
+    {
+      "token": "signal",
+      "vocabulary_id": 8,
+      "aggregate_frequency": 40,
+      "source_records": 1
+    },
+    {
+      "token": "beacon",
+      "vocabulary_id": 9,
+      "aggregate_frequency": 35,
+      "source_records": 1
+    }
+  ]
+}

reports/candidate-review.csv

@@ -0,0 +1,15 @@
+record_id,token,language,status,frequency,review_status,reason,token_selected
+C01,trail,en,APPROVED,80,ELIGIBLE,eligible,true
+C02,ridge,en,APPROVED,60,ELIGIBLE,eligible,true
+C03,water,en,APPROVED,55,ELIGIBLE,eligible,true
+C04,shelter,en,APPROVED,45,ELIGIBLE,eligible,true
+C05,signal,en,APPROVED,40,ELIGIBLE,eligible,true
+C06,ridge,en,APPROVED,25,ELIGIBLE,eligible,true
+C07,beacon,en,APPROVED,35,ELIGIBLE,eligible,true
+C08,north,en,APPROVED,35,ELIGIBLE,eligible,false
+C09,weather,en,APPROVED,30,ELIGIBLE,eligible,false
+C10,evacuate,en,APPROVED,28,ELIGIBLE,eligible,false
+C11,water,en,APPROVED,15,ELIGIBLE,eligible,true
+C12,aide,fr,APPROVED,100,REJECTED,language_not_required,false
+C13,camp,en,DRAFT,90,REJECTED,status_not_approved,false
+C14,map,en,APPROVED,9,REJECTED,frequency_below_minimum,false

tokenizer/tokenizer-config.json

@@ -0,0 +1,12 @@
+{
+  "release_id": "KT-2026-04",
+  "model_max_length": 128,
+  "reserved_tokens": [
+    "[PAD]",
+    "[UNK]",
+    "[BOS]",
+    "[EOS]"
+  ],
+  "ordinary_token_budget": 6,
+  "vocabulary_size": 10
+}

tokenizer/vocab.json

@@ -0,0 +1,12 @@
+{
+  "[PAD]": 0,
+  "[UNK]": 1,
+  "[BOS]": 2,
+  "[EOS]": 3,
+  "ridge": 4,
+  "trail": 5,
+  "water": 6,
+  "shelter": 7,
+  "signal": 8,
+  "beacon": 9
+}