| { |
| "xnnpack": { |
| "bytes": 1789728256, |
| "context_length": 8192, |
| "device": "Samsung Galaxy S20+ SM-G986B, Exynos 990, Mali-G77, Android 13", |
| "runtime": "Android shared libraries shipped with React Native ExecuTorch", |
| "compute_dtype": "float32", |
| "delegates": { |
| "XnnpackBackend": 364 |
| }, |
| "portable_operators": [ |
| "aten::sym_size.int", |
| "quantized_decomposed::embedding_byte.dtype_out", |
| "aten::select_copy.int_out", |
| "aten::_local_scalar_dense.", |
| "executorch_prim::add.Scalar", |
| "aten::slice_copy.Tensor_out", |
| "aten::mean.out", |
| "aten::unsqueeze_copy.out", |
| "aten::alias_copy.out", |
| "executorch_prim::et_view.default", |
| "llama::update_cache.out", |
| "aten::cat.out", |
| "llama::custom_sdpa.out", |
| "aten::copy_." |
| ], |
| "first_token": 597, |
| "reference_first_token": 597, |
| "cosine_similarity": 0.9592449069023132, |
| "prefill_bound_checked": 128, |
| "generated_text": "The user is asking for the capital of France and wants just the city name as a reply. This is a straightforward factual question. The capital of France is Paris. I should provide just \"Paris\" as requested.</think>Paris<|im_end|>", |
| "short_prompt": { |
| "prompt_tokens": 35, |
| "prefill_ms": 600.089, |
| "generated": 46, |
| "decode_ms": 3070.132, |
| "tokens_per_second": 14.983, |
| "peak_rss_mib": 1915.4 |
| }, |
| "long_context": { |
| "prompt_tokens": 8000, |
| "prefill_ms": 177152.107, |
| "generated": 90, |
| "decode_ms": 16435.307, |
| "tokens_per_second": 5.476, |
| "peak_rss_mib": 1915.3, |
| "generated_text": "The user has provided a long list of background text (all identical: \"The weather is pleasant and the sky is blue.\") and then at the end asks: \"What is the capital of France? Reply with just the city name.\"\n\nThis is a straightforward question. The answer is \"Paris\". The user wants just the city name, no extra text.\n\nI need to provide the answer \"Paris\" as the only content.</think>Paris<|im_end|>", |
| "last_decode_position": 8089 |
| }, |
| "test_scope": "Generation, shape/finite checks and long-context smoke test; not a quality benchmark.", |
| "mac_validation": { |
| "bytes": 1789728256, |
| "context_length": 8192, |
| "delegates": { |
| "XnnpackBackend": 364 |
| }, |
| "portable_operators": [ |
| "aten::sym_size.int", |
| "quantized_decomposed::embedding_byte.dtype_out", |
| "aten::select_copy.int_out", |
| "aten::_local_scalar_dense.", |
| "executorch_prim::add.Scalar", |
| "aten::slice_copy.Tensor_out", |
| "aten::mean.out", |
| "aten::unsqueeze_copy.out", |
| "aten::alias_copy.out", |
| "executorch_prim::et_view.default", |
| "llama::update_cache.out", |
| "aten::cat.out", |
| "llama::custom_sdpa.out", |
| "aten::copy_." |
| ], |
| "prefill_seconds": 4.127708917018026, |
| "first_token": 597, |
| "reference_first_token": 597, |
| "cosine_similarity": 0.9592365622520447, |
| "reference_top1_rank": 1, |
| "max_abs_error": 10.013851165771484, |
| "decode_tokens_per_second": 23.283732161212995, |
| "prefill_bound_checked": 128, |
| "generated_text": "The user is asking for the capital of France and wants just the city name as a reply. This is a straightforward factual question. The capital of France is Paris. I should provide just \"Paris\" as requested.</think>Paris", |
| "long_context": { |
| "prompt_tokens": 7800, |
| "prefill_seconds": 269.0508715000469, |
| "generated_text": "The user has provided a long list of background text (all about \"The weather is pleasant and the sky is blue\") and then at the end asks: \"What is the capital of France? Reply with just the city name.\"\n\nThis is a straightforward question. The capital of France is Paris. The user wants just the city name as the answer.\n\nI need to provide a concise answer: \"Paris\"\n\nLet me verify: The question is \"What is the capital of France?\" and the instruction is \"Reply with just the city name.\"\n\nSo the answer should be \"Paris\" with no additional text.</think>Paris<|im_end|>", |
| "last_cache_position_checked": 8191, |
| "peak_rss_mib": 2017.765625 |
| }, |
| "device": "Apple M1 Pro Mac, 16 GB RAM" |
| } |
| }, |
| "mlx": { |
| "bytes": 1665769984, |
| "context_length": 8192, |
| "delegates": { |
| "MLXBackend": 1 |
| }, |
| "portable_operators": [ |
| "quantized_decomposed::embedding_4bit.dtype_out" |
| ], |
| "prefill_seconds": 0.888007125002332, |
| "first_token": 597, |
| "reference_first_token": 597, |
| "cosine_similarity": 0.9547369480133057, |
| "reference_top1_rank": 1, |
| "max_abs_error": 16.562870025634766, |
| "decode_tokens_per_second": 71.28213884705849, |
| "prefill_bound_checked": 128, |
| "generated_text": "The user is asking for the capital of France and wants a simple answer with just the city name. This is a straightforward factual question. The capital of France is Paris. I should provide just \"Paris\" as requested.</think>Paris", |
| "device": "Apple M1 Pro Mac, 16 GB RAM", |
| "runtime": "Source-built software-mansion-labs ExecuTorch runtime with native MLX backend", |
| "long_context": { |
| "prompt_tokens": 8000, |
| "prefill_seconds": 16.85774962499272, |
| "generated_text": "The user has provided a long list of background text (all saying \"The weather is pleasant and the sky is blue\") and then at the end asks: \"Ignore the background text. What is the capital of France? Reply with just the city name.\"\n\nSo I need to:\n1. Ignore the background text (the long list)\n2. Answer the question \"What is the capital of France?\"\n3. Reply with just the city name\n\nThe capital of France is Paris.\n\nSo my response should be simply \"Paris\".</think>Paris<|im_end|>", |
| "last_cache_position_checked": 8191, |
| "peak_rss_mib": 2136.0625 |
| } |
| }, |
| "vulkan": { |
| "bytes": 1623052544, |
| "context_length": 8192, |
| "device": "Samsung Galaxy S20+ SM-G986B, Exynos 990, Mali-G77, Android 13", |
| "runtime": "Android shared libraries shipped with React Native ExecuTorch", |
| "compute_dtype": "float16", |
| "delegates": { |
| "VulkanBackend": 1 |
| }, |
| "portable_operators": [], |
| "first_token": 597, |
| "reference_first_token": 597, |
| "cosine_similarity": 0.9498623609542847, |
| "prefill_bound_checked": 128, |
| "generated_text": "The user is asking for the capital of France and wants a simple answer with just the city name. This is a straightforward factual question. The capital of France is Paris. I should provide just \"Paris\" as requested.</think>Paris<|im_end|>", |
| "short_prompt": { |
| "prompt_tokens": 35, |
| "prefill_ms": 1257.702, |
| "generated": 47, |
| "decode_ms": 6588.921, |
| "tokens_per_second": 7.133, |
| "peak_rss_mib": 2305.3 |
| }, |
| "long_context": { |
| "prompt_tokens": 8000, |
| "prefill_ms": 285873.588, |
| "generated": 120, |
| "decode_ms": 24415.921, |
| "tokens_per_second": 4.915, |
| "peak_rss_mib": 2401.0, |
| "generated_text": "The user has provided a long list of background text (all identical: \"This is background text. The weather is pleasant and the sky is blue.\") and then at the end asks: \"Ignore the background text. What is the capital of France? Reply with just the city name.\"\n\nSo I need to:\n1. Ignore the background text (the long list)\n2. Answer the question: \"What is the capital of France?\"\n3. Reply with just the city name\n\nThe capital of France is \"Paris\".\n\nSo my response should be simply \"Paris\".</think>Paris<|im_end|>", |
| "last_decode_position": 8119 |
| }, |
| "test_scope": "Generation, shape/finite checks and long-context smoke test; not a quality benchmark.", |
| "tool_roundtrip": { |
| "generated_call": "The user wants me to multiply 17 by 23 using the calculate function. This is a straightforward arithmetic operation. I need to use the calculate function with:\n- operation: \"multiply\"\n- a: 17\n- b: 23\n\nLet me make this call.</think><|tool_call_start|>[calculate(operation='multiply', a=17, b=23)]<|tool_call_end|><|im_end|>", |
| "tool_result": "Result: 391", |
| "final_answer": "The calculation has been performed. The result of multiplying 17 by 23 is 391. I should provide this answer to the user.</think>The result of multiplying 17 by 23 is 391.<|im_end|>", |
| "tool_prompt": { |
| "prompt_tokens": 406, |
| "prefill_ms": 15040.201, |
| "generated": 76, |
| "decode_ms": 13034.403, |
| "tokens_per_second": 5.831, |
| "peak_rss_mib": 2412.6 |
| }, |
| "followup_prompt": { |
| "prompt_tokens": 438, |
| "prefill_ms": 17350.5, |
| "generated": 42, |
| "decode_ms": 11261.632, |
| "tokens_per_second": 3.729, |
| "peak_rss_mib": 2409.3 |
| } |
| }, |
| "maximum_explicit_3d_texture_extents": [ |
| 2688, |
| 16384, |
| 128 |
| ], |
| "required_max_image_dimension_3d": 16384, |
| "quantized_operator_counts": { |
| "et_vk.linear_dq8ca_q4gsw.default": 211, |
| "et_vk.embedding_q4gsw.default": 1 |
| } |
| } |
| } |
|
|
