mlboydaisuke commited on
Commit
1f29a6f
·
verified ·
1 Parent(s): e5b6c73

manifest: Android measured rows from the Galaxy S26 (GPU + same-device CPU control) and the Pixel 8a, with an Android recommendation where the pair supports one

Browse files
Files changed (1) hide show
  1. litertlm_manifest.json +56 -3
litertlm_manifest.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
- "manifest_schema": "0.1.0",
3
  "repo": "mlboydaisuke/S1-mini-LiteRT",
4
- "generated": "2026-08-25",
5
  "generator": "make_manifest.py",
6
  "model": {
7
  "display_name": "S1-mini (LiteRT-LM)",
@@ -22,7 +22,14 @@
22
  "start": "<think>",
23
  "end": "</think>"
24
  }
25
- }
 
 
 
 
 
 
 
26
  }
27
  },
28
  "variants": [
@@ -51,6 +58,20 @@
51
  "gpu"
52
  ],
53
  "default_backend": "cpu",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
  "requirements": {
55
  "platform_notes": [
56
  "Context (KV cache) 4096; prefill signatures 1..1024",
@@ -147,6 +168,38 @@
147
  "date": "2026-08-25",
148
  "source": "on-device harness, single run; device reported thermal state 'serious' — read as a floor.",
149
  "peak_mem_mb": 1427
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
150
  }
151
  ],
152
  "known_issues": [
 
1
  {
2
+ "manifest_schema": "0.1.2",
3
  "repo": "mlboydaisuke/S1-mini-LiteRT",
4
+ "generated": "2026-09-05",
5
  "generator": "make_manifest.py",
6
  "model": {
7
  "display_name": "S1-mini (LiteRT-LM)",
 
22
  "start": "<think>",
23
  "end": "</think>"
24
  }
25
+ },
26
+ "channels": [
27
+ {
28
+ "name": "thought",
29
+ "start": "<think>",
30
+ "end": "</think>"
31
+ }
32
+ ]
33
  }
34
  },
35
  "variants": [
 
58
  "gpu"
59
  ],
60
  "default_backend": "cpu",
61
+ "recommended": [
62
+ {
63
+ "platform": "android",
64
+ "device_class": "midrange",
65
+ "backend": "cpu",
66
+ "reason": "measured on one Pixel 8a with one binary (2026-08-25, 205-token prompt, single run each): GPU 434 tok/s prefill / 12.3 decode against CPU 40.4 / 5.8 - the GPU is faster, but on this phone it deterministically drops a date separator on number-dense input ('March 3rd, 2026.' -> 'March 32026.') while the CPU output is byte-identical to macOS on all 12 test cases. For a text-normalization model exactness is the job, so CPU is the default here; choose gpu when time-to-first-token matters more (0.67 s vs 6.46 s). Same-device pair taken, so the speed ordering is measured, not assumed"
67
+ },
68
+ {
69
+ "platform": "android",
70
+ "device_class": "flagship",
71
+ "backend": "gpu",
72
+ "reason": "measured on one Galaxy S26 with one binary (S7 Android backfill 2026-09-05, 205-token prompt, 2 cold runs per backend, cooled before each run): gpu decodes 25.13-25.8 tok/s against cpu's 12.29 (2.1x), prefill 940.7-1027.5 vs 166.7-175.3 tok/s, peak 1323 vs 2129 MB, engine init 6.3 vs 2.2 s. Same-device CPU control taken - a measured win for gpu on decode. On the Pixel 8a's Mali GPU this model deterministically dropped a date separator on number-dense input; that 12-case exactness check was not repeated on this Adreno GPU, so where exactness is the job prefer cpu and treat gpu as the throughput path"
73
+ }
74
+ ],
75
  "requirements": {
76
  "platform_notes": [
77
  "Context (KV cache) 4096; prefill signatures 1..1024",
 
168
  "date": "2026-08-25",
169
  "source": "on-device harness, single run; device reported thermal state 'serious' — read as a floor.",
170
  "peak_mem_mb": 1427
171
+ },
172
+ {
173
+ "device": "Galaxy S26 (SM-S942Q, Qualcomm SM8850, Adreno)",
174
+ "backend": "gpu",
175
+ "runtime": "litert_lm_advanced_main v0.16.0, LiteRT CL delegate",
176
+ "prompt_tokens": 254,
177
+ "decode_tokens": 195,
178
+ "prefill_tps": "940.7-1027.5",
179
+ "decode_tps": "25.13-25.8",
180
+ "ttft_s": "0.29-0.31",
181
+ "load_s": 6.3,
182
+ "peak_memory_mb": 1323,
183
+ "cache": "no",
184
+ "runs": 2,
185
+ "date": "2026-09-05",
186
+ "source": "S7 Android backfill: 205-token prompt with --benchmark, 2 cold runs (compile/weight caches deleted between runs, device cooled below 42 C before each run); GPU gate full LITERT_CL delegation (15474 ops across 12 subgraphs), generation verified on the gate prompt; the cpu row of the same date is the same-device control"
187
+ },
188
+ {
189
+ "device": "Galaxy S26 (SM-S942Q, Qualcomm SM8850, Adreno)",
190
+ "backend": "cpu",
191
+ "runtime": "litert_lm_advanced_main v0.16.0 (XNNPACK CPU)",
192
+ "prompt_tokens": 254,
193
+ "decode_tokens": 195,
194
+ "prefill_tps": "166.7-175.3",
195
+ "decode_tps": 12.29,
196
+ "ttft_s": "1.53-1.61",
197
+ "load_s": 2.2,
198
+ "peak_memory_mb": 2129,
199
+ "cache": "no",
200
+ "runs": 2,
201
+ "date": "2026-09-05",
202
+ "source": "S7 Android backfill: same-device CPU control - identical binary, prompt file and flags, only --backend differs; 205-token prompt with --benchmark, 2 cold runs (compile/weight caches deleted between runs, device cooled below 42 C before each run); same-device control for the gpu row of the same date"
203
  }
204
  ],
205
  "known_issues": [