Add chosen_planarian context LM (retrained hungry_jellyfish successor)
#3
by lee-futo - opened
- .gitattributes +1 -0
- README.md +13 -1
- chosen_planarian/context_lm.pte +3 -0
- chosen_planarian/metadata.json +4 -0
- chosen_planarian/vocab.txt +0 -0
- scoring.json +17 -1
.gitattributes
CHANGED
|
@@ -37,3 +37,4 @@ honorable_sturgeon/model_fp32.pte filter=lfs diff=lfs merge=lfs -text
|
|
| 37 |
hungry_jellyfish/context_lm.pte filter=lfs diff=lfs merge=lfs -text
|
| 38 |
magic_macaw/model_fp32.pte filter=lfs diff=lfs merge=lfs -text
|
| 39 |
animations/swipe_demo_computer.gif filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 37 |
hungry_jellyfish/context_lm.pte filter=lfs diff=lfs merge=lfs -text
|
| 38 |
magic_macaw/model_fp32.pte filter=lfs diff=lfs merge=lfs -text
|
| 39 |
animations/swipe_demo_computer.gif filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
chosen_planarian/context_lm.pte filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -33,7 +33,7 @@ for more details.
|
|
| 33 |
|
| 34 |
## Models
|
| 35 |
|
| 36 |
-
This repository contains
|
| 37 |
Only the encoder is required. The decoder and language model are
|
| 38 |
additional refinements, leveraging specific layout and language information.
|
| 39 |
The encoder can decode for **any** keyboard layout,
|
|
@@ -44,6 +44,7 @@ while the decoder is English/QWERTY-only and the language model is English-only.
|
|
| 44 |
| **Encoder** | `honorable_sturgeon` | Maps a swipe trajectory to per-timestep character emissions. Layout-agnostic — works on any keyboard supplied at runtime. | 635 K | 2.65 MB |
|
| 45 |
| **Decoder** | `magic_macaw` | (Optional) per-layout refinement over the encoder's frozen features. Lifts top-k where layout-specific training data exists (English here). | 304 K | 1.25 MB |
|
| 46 |
| **Context LM** | `hungry_jellyfish` | (Optional) next-word and beam-rerank language model that blends sentence context into candidate ranking. | 1.5M | 6.25 MB |
|
|
|
|
| 47 |
|
| 48 |
### Encoder — `honorable_sturgeon`
|
| 49 |
|
|
@@ -65,6 +66,17 @@ A causal DFSMN language model with a hash embedding for large vocabularies.
|
|
| 65 |
It can supply a rerank score and perform next-word prediction. This model
|
| 66 |
can be used with or without a decoder.
|
| 67 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
## Getting started
|
| 69 |
|
| 70 |
The example below demonstrates the encoder on CPU (x86) with the ExecuTorch
|
|
|
|
| 33 |
|
| 34 |
## Models
|
| 35 |
|
| 36 |
+
This repository contains CNN models that compose together (encoder, decoder and context LM).
|
| 37 |
Only the encoder is required. The decoder and language model are
|
| 38 |
additional refinements, leveraging specific layout and language information.
|
| 39 |
The encoder can decode for **any** keyboard layout,
|
|
|
|
| 44 |
| **Encoder** | `honorable_sturgeon` | Maps a swipe trajectory to per-timestep character emissions. Layout-agnostic — works on any keyboard supplied at runtime. | 635 K | 2.65 MB |
|
| 45 |
| **Decoder** | `magic_macaw` | (Optional) per-layout refinement over the encoder's frozen features. Lifts top-k where layout-specific training data exists (English here). | 304 K | 1.25 MB |
|
| 46 |
| **Context LM** | `hungry_jellyfish` | (Optional) next-word and beam-rerank language model that blends sentence context into candidate ranking. | 1.5M | 6.25 MB |
|
| 47 |
+
| **Context LM** | `chosen_planarian` | (Optional) retrained successor to `hungry_jellyfish`: same architecture and vocabulary, fixes words the earlier tokenizer never trained (e.g. *I'm*, *café*, *long-term*, *pm*). | 1.5M | 6.25 MB |
|
| 48 |
|
| 49 |
### Encoder — `honorable_sturgeon`
|
| 50 |
|
|
|
|
| 66 |
It can supply a rerank score and perform next-word prediction. This model
|
| 67 |
can be used with or without a decoder.
|
| 68 |
|
| 69 |
+
### Context LM (English) — `chosen_planarian`
|
| 70 |
+
|
| 71 |
+
A retrained drop-in replacement for `hungry_jellyfish` (same architecture, vocabulary and
|
| 72 |
+
`vocab.txt`). The earlier training tokenizer never produced some dictionary words, notably
|
| 73 |
+
hyphenated (*long-term*, *e-mail*), accented (*café*, *naïve*) and some apostrophe forms
|
| 74 |
+
(*I'm*), so their embeddings were never trained as real words
|
| 75 |
+
([android-keyboard#2365](https://github.com/futo-org/android-keyboard/issues/2365)). This model
|
| 76 |
+
was trained on a 20B-word English web + chat mix with that tokenizer fixed; an audit of its
|
| 77 |
+
training stream finds no systematic gaps (only a few dozen words that are rare in web text). Tuned scoring parameters for
|
| 78 |
+
both encoder-only and encoder + decoder use are in `scoring.json`.
|
| 79 |
+
|
| 80 |
## Getting started
|
| 81 |
|
| 82 |
The example below demonstrates the encoder on CPU (x86) with the ExecuTorch
|
chosen_planarian/context_lm.pte
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:30aee96265c6f3831ea2536363796bc625b1fe4607e108edaadf936e82da3d3e
|
| 3 |
+
size 6247616
|
chosen_planarian/metadata.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"kind": "contextlm",
|
| 3 |
+
"codename": "chosen_planarian"
|
| 4 |
+
}
|
chosen_planarian/vocab.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
scoring.json
CHANGED
|
@@ -33,10 +33,26 @@
|
|
| 33 |
},
|
| 34 |
"encoder:honorable_sturgeon decoder:magic_macaw contextlm:hungry_jellyfish": {
|
| 35 |
"gamma": 0.1126,
|
| 36 |
-
"lambda": 0.
|
| 37 |
"beta": 2.2138,
|
| 38 |
"alpha": 0.6387,
|
| 39 |
"gamma_prune": 0.1902,
|
| 40 |
"beta_prune": 1.2727
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
}
|
| 42 |
}
|
|
|
|
| 33 |
},
|
| 34 |
"encoder:honorable_sturgeon decoder:magic_macaw contextlm:hungry_jellyfish": {
|
| 35 |
"gamma": 0.1126,
|
| 36 |
+
"lambda": 0.006,
|
| 37 |
"beta": 2.2138,
|
| 38 |
"alpha": 0.6387,
|
| 39 |
"gamma_prune": 0.1902,
|
| 40 |
"beta_prune": 1.2727
|
| 41 |
+
},
|
| 42 |
+
"encoder:honorable_sturgeon contextlm:chosen_planarian": {
|
| 43 |
+
"gamma": 0.0422,
|
| 44 |
+
"lambda": 0.0244,
|
| 45 |
+
"beta": 3.049,
|
| 46 |
+
"alpha": 0.6638,
|
| 47 |
+
"gamma_prune": 0.2566,
|
| 48 |
+
"beta_prune": 1.0054
|
| 49 |
+
},
|
| 50 |
+
"encoder:honorable_sturgeon decoder:magic_macaw contextlm:chosen_planarian": {
|
| 51 |
+
"gamma": 0.1017,
|
| 52 |
+
"lambda": 0.0059,
|
| 53 |
+
"beta": 2.3241,
|
| 54 |
+
"alpha": 0.7016,
|
| 55 |
+
"gamma_prune": 0.1902,
|
| 56 |
+
"beta_prune": 1.2727
|
| 57 |
}
|
| 58 |
}
|