| { |
| "compute": { |
| "environment": "Linux x86_64 CPU with mlx[cpu]; no paid cloud Mac used.", |
| "estimated_incremental_cost_usd": 0.0 |
| }, |
| "end_to_end_extraction": { |
| "chunk_lengths": [ |
| 50 |
| ], |
| "input_html": "<html><body><article><h1>Apple MLX conversion</h1><p>This article explains how to convert a EuroBERT content extraction model to MLX format.</p></article></body></html>", |
| "num_blocks": 2, |
| "variants": { |
| "4bit": { |
| "html": "<html><body><article><h1 _item_id=\"1\">Apple MLX conversion</h1><p _item_id=\"2\">This article explains how to convert a EuroBERT content extraction model to MLX format.</p></article></body></html>", |
| "labels": { |
| "1": "main", |
| "2": "main" |
| }, |
| "latency_ms": 77852.023473999, |
| "markdown": "<html><body><article><h1 _item_id=\"1\">Apple MLX conversion</h1><p _item_id=\"2\">This article explains how to convert a EuroBERT content extraction model to MLX format.</p></article></body></html>", |
| "non_empty": true, |
| "predictions": [ |
| 1, |
| 1 |
| ] |
| }, |
| "8bit": { |
| "html": "<html><body><article><h1 _item_id=\"1\">Apple MLX conversion</h1><p _item_id=\"2\">This article explains how to convert a EuroBERT content extraction model to MLX format.</p></article></body></html>", |
| "labels": { |
| "1": "main", |
| "2": "main" |
| }, |
| "latency_ms": 78333.6143399938, |
| "markdown": "<html><body><article><h1 _item_id=\"1\">Apple MLX conversion</h1><p _item_id=\"2\">This article explains how to convert a EuroBERT content extraction model to MLX format.</p></article></body></html>", |
| "non_empty": true, |
| "predictions": [ |
| 1, |
| 1 |
| ] |
| }, |
| "bf16": { |
| "html": "<html><body><article><h1 _item_id=\"1\">Apple MLX conversion</h1><p _item_id=\"2\">This article explains how to convert a EuroBERT content extraction model to MLX format.</p></article></body></html>", |
| "labels": { |
| "1": "main", |
| "2": "main" |
| }, |
| "latency_ms": 5469.0545329940505, |
| "markdown": "<html><body><article><h1 _item_id=\"1\">Apple MLX conversion</h1><p _item_id=\"2\">This article explains how to convert a EuroBERT content extraction model to MLX format.</p></article></body></html>", |
| "non_empty": true, |
| "predictions": [ |
| 1, |
| 1 |
| ] |
| } |
| } |
| }, |
| "load_checks": { |
| "4bit": { |
| "loaded": true, |
| "logits_dtype": "mlx.core.bfloat16", |
| "logits_shape": [ |
| 1, |
| 3, |
| 2 |
| ] |
| }, |
| "8bit": { |
| "loaded": true, |
| "logits_dtype": "mlx.core.bfloat16", |
| "logits_shape": [ |
| 1, |
| 3, |
| 2 |
| ] |
| }, |
| "bf16": { |
| "loaded": true, |
| "logits_dtype": "mlx.core.bfloat16", |
| "logits_shape": [ |
| 1, |
| 3, |
| 2 |
| ] |
| } |
| }, |
| "model_dir": "hf_out", |
| "numerical_accuracy": { |
| "test_inputs": [ |
| "A", |
| "B", |
| "C" |
| ], |
| "token_shape": [ |
| 3, |
| 2 |
| ], |
| "torch_reference": { |
| "attention": "eager", |
| "dtype": "float32", |
| "latency_ms": 50.09009799687192 |
| }, |
| "variants": { |
| "4bit": { |
| "latency_ms": 9323.177823971491, |
| "max_abs_diff": 2.2551989555358887, |
| "mean_abs_diff": 1.1897996664047241 |
| }, |
| "8bit": { |
| "latency_ms": 9380.578643002082, |
| "max_abs_diff": 1.2797489166259766, |
| "mean_abs_diff": 0.542361319065094 |
| }, |
| "bf16": { |
| "latency_ms": 716.8781639775261, |
| "max_abs_diff": 0.04523277282714844, |
| "mean_abs_diff": 0.019181733950972557 |
| } |
| } |
| }, |
| "source_model": "feyninc/pulpie-orange-small", |
| "variants": [ |
| "bf16", |
| "8bit", |
| "4bit" |
| ] |
| } |
|
|
