Spaces:
Running
Running
Download index.html from sanskar003/SAM3-Model-File-Map: direct link, hf CLI and curl.
- Browser
- Download file 49.3 kB
-
https://huggingface.co/spaces/sanskar003/SAM3-Model-File-Map/resolve/main/index.html
- Command line
-
hf download hf://spaces/sanskar003/SAM3-Model-File-Map/index.html
-
curl -L -o index.html https://huggingface.co/spaces/sanskar003/SAM3-Model-File-Map/resolve/main/index.html
49.3 kB
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>SAM3 Model File Map</title> | |
| <link rel="preconnect" href="https://fonts.googleapis.com"> | |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> | |
| <link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=IBM+Plex+Sans+Condensed:wght@600;700&family=IBM+Plex+Sans:wght@400;500;600&display=swap" rel="stylesheet"> | |
| <style> | |
| /* Layout idea: one reference column; the 4.0 GB -> 1.47 GB journey first, then what loads when, | |
| the stage 2 -> 3 diagram, VRAM detail, accuracy, files, tensors, method, how to run. */ | |
| /* Palette: #171c24 background (darkened from #232931), #393e46 surface, #4ecca3 accent, | |
| #eeeeee text. Dark only. Washes are each color mixed 16% over the background; chart | |
| series are validated in-band shades (teal, amber, violet) with dark labels. */ | |
| :root { | |
| color-scheme: dark; | |
| --bg: #171c24; | |
| --surface: #393e46; | |
| --ink: #eeeeee; | |
| --ink-2: #bfc3c9; | |
| --ink-3: #a5abb3; | |
| --line: #4b515a; | |
| --accent: #4ecca3; | |
| --accent-wash: #203838; | |
| --hot: #f2946a; | |
| --hot-wash: #3a2f2f; | |
| --ok: #4ecca3; --ok-wash: #203838; | |
| --amber: #f0c060; --amber-wash: #3a362e; | |
| --stop: #f6948c; --stop-wash: #3b2f35; | |
| --violet: #b9a6ff; --violet-wash: #313247; | |
| --grey: #c3c8ce; --grey-wash: #2c323a; | |
| --s1: #33ac87; --s2: #c38824; --s3: #9a82db; | |
| --neutral: #868d97; | |
| --f-display: "IBM Plex Sans Condensed", "Arial Narrow", system-ui, sans-serif; | |
| --f-body: "IBM Plex Sans", system-ui, -apple-system, "Segoe UI", sans-serif; | |
| --f-mono: "IBM Plex Mono", ui-monospace, "SFMono-Regular", Menlo, Consolas, monospace; | |
| } | |
| * { box-sizing: border-box; } | |
| html { -webkit-text-size-adjust: 100%; } | |
| body { | |
| margin: 0; background: var(--bg); color: var(--ink); | |
| font-family: var(--f-body); font-size: 15.5px; line-height: 1.55; | |
| } | |
| .wrap { max-width: 1100px; margin: 0 auto; padding: 40px 24px 72px; display: grid; gap: 56px; } | |
| .wrap { grid-template-columns: minmax(0, 1fr); } | |
| section, figure, .runs, .run, .codebox, .two > *, .tiles > *, .stages > *, .groups > * { min-width: 0; } | |
| h1, h2, h3 { font-family: var(--f-display); text-wrap: balance; margin: 0; line-height: 1.15; } | |
| h1 { font-size: 2.6rem; font-weight: 700; letter-spacing: -0.01em; } | |
| h2 { font-size: 1.6rem; font-weight: 700; } | |
| h3 { font-size: 1.05rem; font-weight: 600; letter-spacing: 0.01em; } | |
| p { margin: 0; max-width: 70ch; } | |
| code, .mono { font-family: var(--f-mono); font-size: 0.88em; } | |
| a { color: var(--accent); } | |
| :focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; border-radius: 4px; } | |
| section { display: grid; gap: 18px; } | |
| .sec-head { display: grid; gap: 8px; } | |
| .eyebrow { font-family: var(--f-mono); font-size: 0.78rem; color: var(--ink-2); letter-spacing: 0.02em; } | |
| .lede { font-size: 1.12rem; color: var(--ink-2); max-width: 64ch; } | |
| header { display: grid; gap: 14px; } | |
| /* stat tiles */ | |
| .tiles { display: grid; grid-template-columns: repeat(auto-fit, minmax(210px, 1fr)); gap: 14px; } | |
| .tile { background: var(--surface); border: 1px solid var(--line); border-radius: 10px; padding: 16px 18px; display: grid; gap: 4px; } | |
| .tile .label { font-size: 0.85rem; color: var(--ink-2); } | |
| .tile .value { font-family: var(--f-body); font-weight: 600; font-size: 2rem; line-height: 1.1; } | |
| .tile .note { font-size: 0.85rem; color: var(--ink-2); } | |
| .tile.key { border-color: var(--accent); box-shadow: inset 4px 0 0 var(--accent); } | |
| /* load-group cards */ | |
| .groups { display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); gap: 14px; } | |
| .group { background: var(--surface); border: 1px solid var(--line); border-radius: 10px; padding: 16px 18px 14px; display: grid; gap: 12px; align-content: start; border-top: 4px solid var(--tone); } | |
| .group.live { --tone: var(--accent); } | |
| .group.once { --tone: var(--amber); } | |
| .group.build { --tone: var(--ink-3); } | |
| .group.retired { --tone: var(--stop); } | |
| .group ul { list-style: none; margin: 0; padding: 0; display: grid; gap: 10px; } | |
| .group li { display: grid; gap: 2px; } | |
| .group li .f { font-family: var(--f-mono); font-size: 0.8rem; font-weight: 500; overflow-wrap: anywhere; } | |
| .group li .d { font-size: 0.88rem; color: var(--ink-2); } | |
| .group .foot { font-size: 0.85rem; color: var(--ink); border-top: 1px solid var(--line); padding-top: 10px; } | |
| .facts { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 2px; } | |
| .fact { font-size: 0.78rem; font-variant-numeric: tabular-nums; background: var(--grey-wash); color: var(--ink); border-radius: 999px; padding: 1px 8px; } | |
| /* chips */ | |
| .chip { display: inline-block; font-size: 0.74rem; font-weight: 600; letter-spacing: 0.02em; border-radius: 999px; padding: 1px 8px; white-space: nowrap; } | |
| .chip.live { color: var(--accent); background: var(--accent-wash); } | |
| .chip.once { color: var(--amber); background: var(--amber-wash); } | |
| .chip.build { color: var(--grey); background: var(--grey-wash); } | |
| .chip.retired { color: var(--stop); background: var(--stop-wash); } | |
| .chip.new { color: var(--ok); background: var(--ok-wash); } | |
| .chip.changed { color: var(--violet); background: var(--violet-wash); } | |
| .chip.same { color: var(--grey); background: var(--grey-wash); } | |
| /* panels / figures */ | |
| .panel { background: var(--surface); border: 1px solid var(--line); border-radius: 12px; padding: 20px; } | |
| figure { margin: 0; display: grid; gap: 12px; } | |
| figcaption { font-size: 0.9rem; color: var(--ink-2); max-width: 80ch; } | |
| .scroll-x { overflow-x: auto; } | |
| /* diagram */ | |
| .diagram { display: block; width: 100%; min-width: 780px; height: auto; color: var(--ink-2); font-family: var(--f-body); } | |
| .d-box { fill: var(--surface); stroke: var(--ink-3); stroke-width: 1; } | |
| .d-frame { fill: none; stroke: var(--ink-3); stroke-width: 1; } | |
| .d-hot { fill: var(--hot-wash); } | |
| .d-hotbox { fill: var(--hot-wash); stroke: var(--hot); stroke-width: 1.2; } | |
| .d-div { stroke: var(--ink-3); stroke-dasharray: 3 3; } | |
| .d-rule { stroke: var(--line); } | |
| .d-t { fill: var(--ink); font-size: 13px; font-weight: 600; } | |
| .d-m { font-family: var(--f-mono); font-weight: 500; font-size: 12.5px; } | |
| .d-s { fill: var(--ink-2); font-size: 11.5px; } | |
| .d-l { fill: var(--ink-2); font-size: 11px; } | |
| .d-arrow { stroke: currentColor; stroke-width: 1.5; fill: none; } | |
| .d-panel { fill: var(--ink); font-family: var(--f-display); font-size: 13px; font-weight: 700; letter-spacing: 0.08em; } | |
| .d-offline { fill: none; stroke: var(--ink-3); stroke-dasharray: 6 5; } | |
| .d-live { fill: var(--accent-wash); stroke: var(--accent); stroke-width: 1.5; } | |
| .d-group { fill: var(--ink-2); font-size: 11px; font-weight: 600; letter-spacing: 0.06em; } | |
| .d-group-live { fill: var(--accent); } | |
| /* chart */ | |
| .legend { display: flex; flex-wrap: wrap; gap: 8px 20px; font-size: 0.86rem; color: var(--ink); } | |
| .legend span { display: inline-flex; align-items: center; gap: 8px; } | |
| .sw { width: 12px; height: 12px; border-radius: 3px; display: inline-block; } | |
| .sw.s1 { background: var(--s1); } .sw.s2 { background: var(--s2); } .sw.s3 { background: var(--s3); } | |
| .sw.ghost { border: 1.5px dashed var(--ink-3); } | |
| .plot { display: grid; grid-template-columns: 112px 1fr; row-gap: 18px; position: relative; padding: 8px 0 0; } | |
| .rlabel { display: grid; align-content: center; font-size: 0.86rem; color: var(--ink-2); line-height: 1.25; } | |
| .rlabel b { color: var(--ink); font-weight: 600; font-variant-numeric: tabular-nums; } | |
| .track { position: relative; height: 28px; display: flex; z-index: 1; } | |
| .seg { height: 100%; display: flex; align-items: center; padding-left: 8px; font-size: 0.78rem; font-weight: 600; color: var(--bg); font-variant-numeric: tabular-nums; border-right: 2px solid var(--surface); cursor: default; outline-offset: 2px; } | |
| .seg.s1 { background: var(--s1); } .seg.s2 { background: var(--s2); } .seg.s3 { background: var(--s3); } | |
| .seg.end { border-radius: 0 4px 4px 0; border-right: 0; } | |
| .seg.ghost { background: transparent; border: 1.5px dashed var(--ink-3); border-radius: 0 4px 4px 0; color: var(--ink-2); font-weight: 500; white-space: nowrap; margin-left: 2px; } | |
| .gridlines { grid-column: 2; grid-row: 1 / span 2; position: relative; pointer-events: none; } | |
| .gridlines i { position: absolute; top: -6px; bottom: -6px; width: 1px; background: var(--line); } | |
| .axis { grid-column: 2; position: relative; height: 18px; font-size: 0.76rem; color: var(--ink-3); font-variant-numeric: tabular-nums; } | |
| .axis span { position: absolute; transform: translateX(-50%); white-space: nowrap; } | |
| .axis span:first-child { transform: none; } | |
| .axis span:last-child { transform: translateX(-100%); } | |
| .tip { position: fixed; z-index: 10; pointer-events: none; background: var(--ink); color: var(--bg); font-size: 0.8rem; padding: 6px 9px; border-radius: 6px; max-width: 260px; opacity: 0; transition: opacity 0.12s; } | |
| .tip.on { opacity: 1; } | |
| /* tables */ | |
| .table-wrap { overflow-x: auto; background: var(--surface); border: 1px solid var(--line); border-radius: 12px; } | |
| table { width: 100%; border-collapse: collapse; font-size: 0.88rem; } | |
| th, td { text-align: left; padding: 10px 14px; border-bottom: 1px solid var(--line); vertical-align: top; } | |
| thead th { font-size: 0.76rem; text-transform: uppercase; letter-spacing: 0.06em; color: var(--ink-2); font-weight: 600; background: var(--bg); } | |
| tbody tr:last-child td, tbody tr:last-child th { border-bottom: 0; } | |
| td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } | |
| tr.total td, tr.total th { font-weight: 600; background: var(--bg); } | |
| tr.saved td, tr.saved th { font-weight: 600; color: var(--ok); } | |
| tr.group-row th { background: var(--bg); font-size: 0.76rem; text-transform: uppercase; letter-spacing: 0.06em; color: var(--ink-2); } | |
| td .f, th .f { font-family: var(--f-mono); font-size: 0.84rem; font-weight: 500; white-space: nowrap; } | |
| .files { min-width: 980px; } | |
| .code-t { min-width: 760px; } | |
| .vram-t { min-width: 680px; } | |
| .acc-t { min-width: 520px; } | |
| .better { color: var(--ok); font-weight: 600; } | |
| /* notes */ | |
| ul.notes { margin: 0; padding-left: 20px; display: grid; gap: 6px; color: var(--ink-2); font-size: 0.9rem; max-width: 86ch; } | |
| /* tensor cards */ | |
| .tensors { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 420px), 1fr)); gap: 14px; } | |
| .tcard { background: var(--surface); border: 1px solid var(--line); border-radius: 10px; padding: 14px 16px; display: grid; gap: 10px; align-content: start; } | |
| .tcard .hd { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; justify-content: space-between; } | |
| .tcard .hd .f { font-family: var(--f-mono); font-size: 0.86rem; font-weight: 500; } | |
| .io { display: grid; grid-template-columns: auto 1fr; gap: 4px 10px; font-family: var(--f-mono); font-size: 0.78rem; } | |
| .io .dir { color: var(--ink-3); } | |
| .io .nm { color: var(--ink); overflow-wrap: anywhere; } | |
| .io .nm em { font-style: normal; color: var(--ink-2); } | |
| /* steps */ | |
| ol.steps { margin: 0; padding: 0; list-style: none; counter-reset: s; display: grid; gap: 14px; max-width: 86ch; } | |
| ol.steps li { counter-increment: s; display: grid; grid-template-columns: 34px 1fr; gap: 12px; } | |
| ol.steps li::before { content: counter(s); font-family: var(--f-display); font-weight: 700; font-size: 0.95rem; width: 28px; height: 28px; border-radius: 50%; background: var(--accent-wash); color: var(--accent); display: grid; place-items: center; } | |
| ol.steps b { font-weight: 600; } | |
| ol.steps p { color: var(--ink-2); } | |
| ol.steps .t { color: var(--ink); } | |
| /* run */ | |
| .runs { display: grid; gap: 14px; } | |
| .run { background: var(--surface); border: 1px solid var(--line); border-radius: 10px; padding: 14px 16px; display: grid; gap: 10px; } | |
| .run .when { display: flex; flex-wrap: wrap; align-items: baseline; gap: 10px; } | |
| .run .when h3 { font-size: 1rem; } | |
| .run .when span { font-size: 0.85rem; color: var(--ink-2); } | |
| .codebox { position: relative; } | |
| pre { margin: 0; background: var(--bg); border: 1px solid var(--line); border-radius: 8px; padding: 12px 14px; overflow-x: auto; font-family: var(--f-mono); font-size: 0.82rem; line-height: 1.6; color: var(--ink); } | |
| pre .c { color: var(--ink-3); } | |
| .copy { position: absolute; top: 8px; right: 8px; font: 600 0.74rem var(--f-body); color: var(--ink-2); background: var(--surface); border: 1px solid var(--line); border-radius: 6px; padding: 3px 9px; cursor: pointer; } | |
| .copy:hover { color: var(--ink); border-color: var(--ink-3); } | |
| .checklist { margin: 0; padding: 0; list-style: none; display: grid; gap: 8px; max-width: 86ch; } | |
| .checklist li { display: grid; grid-template-columns: 20px 1fr; gap: 10px; font-size: 0.92rem; } | |
| .checklist li::before { content: ""; width: 14px; height: 14px; margin-top: 4px; border: 1.5px solid var(--ink-3); border-radius: 3px; } | |
| .two { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 14px; } | |
| footer { font-size: 0.82rem; color: var(--ink-3); border-top: 1px solid var(--line); padding-top: 16px; } | |
| @media (max-width: 520px) { | |
| .wrap { padding: 28px 16px 56px; gap: 44px; } | |
| h1 { font-size: 2rem; } | |
| .plot { grid-template-columns: 84px 1fr; } | |
| } | |
| @media (prefers-reduced-motion: reduce) { .tip { transition: none; } } | |
| /* --- Space additions: journey section --- */ | |
| .tiles { grid-template-columns: 2fr 1fr 1fr; } | |
| .tile.key .value { font-size: 2.3rem; } | |
| .stages { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 14px; } | |
| .stage { background: var(--surface); border: 1px solid var(--line); border-radius: 10px; padding: 16px 18px; display: grid; gap: 8px; align-content: start; } | |
| .stage.current { border-color: var(--accent); box-shadow: inset 0 4px 0 var(--accent); } | |
| .stage .num { font-family: var(--f-mono); font-size: 0.76rem; color: var(--ink-2); } | |
| .stage .repo { font-family: var(--f-mono); font-size: 0.8rem; overflow-wrap: anywhere; } | |
| .stage p { font-size: 0.9rem; color: var(--ink-2); } | |
| .stage .vram { display: grid; gap: 2px; border-top: 1px solid var(--line); padding-top: 10px; font-size: 0.85rem; color: var(--ink-2); } | |
| .stage .vram b { font-size: 1.25rem; color: var(--ink); font-weight: 600; } | |
| .journey .plot { grid-template-columns: 190px 1fr; } | |
| .seg.over { background: var(--neutral); } | |
| .sw.over { background: var(--neutral); } | |
| .axis.centered span:last-child { transform: translateX(-50%); } | |
| @media (max-width: 760px) { | |
| .tiles { grid-template-columns: 1fr; } | |
| .journey .plot { grid-template-columns: 112px 1fr; } | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <main class="wrap"> | |
| <header> | |
| <div class="eyebrow">SAM3 for fixed prompt lists · ONNX + TensorRT 10.3 · measured on RTX 4070 · Oct 2026</div> | |
| <h1>SAM3 model file map</h1> | |
| <p class="lede">How SAM3 went from 4.0 GB of VRAM to 1.47 GB while it runs, with the model itself down from 3.7 GB to 1.2 GB: which model files exist, which ones stay loaded, and which run only once.</p> | |
| <div class="tiles" aria-label="Summary"> | |
| <div class="tile key"><span class="label">GPU memory held by SAM3 · 4K frames</span><span class="value">3.7 GB → 1.2 GB</span><span class="note">Model memory 3,766 → 1,262 MiB. Whole process in nvidia-smi, CUDA context included: 4,028 → 1,474 MiB.</span></div> | |
| <div class="tile"><span class="label">Parameters on the GPU</span><span class="value">481M</span><span class="note">was 835M with the stage-2 split</span></div> | |
| <div class="tile"><span class="label">Detections that differ from FP32</span><span class="value">1 of 72</span><span class="note">stage-2 split: 8 of 72</span></div> | |
| </div> | |
| </header> | |
| <section id="journey"> | |
| <div class="sec-head"> | |
| <h2>The journey</h2> | |
| <p>Three stages, one workload: a fixed list of prompts, one image at a time.</p> | |
| </div> | |
| <div class="stages"> | |
| <article class="stage"> | |
| <span class="num">STAGE 1</span> | |
| <h3>SAM3 base</h3> | |
| <a class="repo" href="https://huggingface.co/facebook/sam3" target="_blank" rel="noopener">facebook/sam3</a> | |
| <p>Meta's PyTorch model in FP16, 840.4M parameters. Every prompt runs the whole network, and every mask is resized to the full frame on the GPU.</p> | |
| <div class="vram"><b>4,028 MiB</b>whole process at 4K · 3,766 MiB model memory</div> | |
| </article> | |
| <article class="stage"> | |
| <span class="num">STAGE 2</span> | |
| <h3>Split encoder–decoder</h3> | |
| <a class="repo" href="https://huggingface.co/sanskar003/SAM3-Split-Encoder-Decoder" target="_blank" rel="noopener">sanskar003/SAM3-Split-Encoder-Decoder</a> | |
| <p>Built from Embedl's INT8 ONNX export (<a href="https://huggingface.co/embedl/sam3" target="_blank" rel="noopener">embedl/sam3</a>). The vision encoder runs once per image, the decoder once per prompt. The decoder still carries the 353M-parameter text encoder.</p> | |
| <div class="vram"><b>1,872 MiB</b>whole process · 1,664 MiB model memory</div> | |
| </article> | |
| <article class="stage current"> | |
| <span class="num">STAGE 3 · THIS RELEASE</span> | |
| <h3>Text-free decoder + prompt bank</h3> | |
| <a class="repo" href="https://huggingface.co/sanskar003/SAM3-Split-TextFree" target="_blank" rel="noopener">sanskar003/SAM3-Split-TextFree</a> | |
| <p>The text encoder runs once, offline, on CPU, and its outputs are saved to a prompt bank. The GPU keeps the vision encoder and a 26M-parameter decoder.</p> | |
| <div class="vram"><b>1,474 MiB</b>whole process · 1,262 MiB model memory</div> | |
| </article> | |
| </div> | |
| <figure class="panel journey"> | |
| <div class="legend"> | |
| <span><i class="sw s1"></i>Model memory: engines, working memory, I/O buffers</span> | |
| <span><i class="sw over"></i>CUDA context and runtime</span> | |
| </div> | |
| <div class="plot"> | |
| <div class="gridlines" aria-hidden="true" style="grid-row: 1 / span 3"> | |
| <i style="left:0%"></i><i style="left:22.22%"></i><i style="left:44.44%"></i><i style="left:66.67%"></i><i style="left:88.89%"></i> | |
| </div> | |
| <div class="rlabel" style="grid-row:1">1 · SAM3 base<b>4,028 MiB</b></div> | |
| <div class="track" style="grid-column:2; grid-row:1"> | |
| <span class="seg s1" style="width:83.69%" tabindex="0" data-tip="Stage 1 · model memory: 3,766 MiB held by PyTorch (FP16 weights 1,603 MiB + activations and full-frame masks)" aria-label="Stage 1, model memory: 3,766 MiB">3,766</span> | |
| <span class="seg over end" style="width:5.82%" tabindex="0" data-tip="Stage 1 · CUDA context and runtime: 262 MiB" aria-label="Stage 1, CUDA context and runtime: 262 MiB"></span> | |
| </div> | |
| <div class="rlabel" style="grid-row:2">2 · Split<b>1,872 MiB</b></div> | |
| <div class="track" style="grid-column:2; grid-row:2"> | |
| <span class="seg s1" style="width:36.98%" tabindex="0" data-tip="Stage 2 · model memory: 1,664 MiB (encoder 768 + decoder 896)" aria-label="Stage 2, model memory: 1,664 MiB">1,664</span> | |
| <span class="seg over end" style="width:4.62%" tabindex="0" data-tip="Stage 2 · CUDA context and runtime: 208 MiB" aria-label="Stage 2, CUDA context and runtime: 208 MiB"></span> | |
| </div> | |
| <div class="rlabel" style="grid-row:3">3 · Text-free<b>1,474 MiB</b></div> | |
| <div class="track" style="grid-column:2; grid-row:3"> | |
| <span class="seg s1" style="width:28.04%" tabindex="0" data-tip="Stage 3 · model memory: 1,262 MiB (encoder 768 + decoder 494)" aria-label="Stage 3, model memory: 1,262 MiB">1,262</span> | |
| <span class="seg over end" style="width:4.71%" tabindex="0" data-tip="Stage 3 · CUDA context and runtime: 212 MiB" aria-label="Stage 3, CUDA context and runtime: 212 MiB"></span> | |
| </div> | |
| <div class="axis centered" style="grid-row:4" aria-hidden="true"> | |
| <span style="left:0%">0</span><span style="left:22.22%">1,000</span><span style="left:44.44%">2,000</span><span style="left:66.67%">3,000</span><span style="left:88.89%">4,000 MiB</span> | |
| </div> | |
| </div> | |
| <figcaption>Whole-process VRAM of the running app in nvidia-smi, in MiB: RTX 4070 (16 GB), one 3840×2160 image with 4 prompts. Stage 1: official <code>facebook/sam3</code> weights in PyTorch 2.6, FP16. Stages 2 and 3: TensorRT 10.3 engines built <code>--fp16 --int8</code>.</figcaption> | |
| </figure> | |
| <div class="two"> | |
| <div class="table-wrap"> | |
| <table class="acc-t"> | |
| <thead><tr><th>Input frame</th><th class="num">1 · SAM3 base</th><th class="num">2 · Split</th><th class="num">3 · Text-free</th></tr></thead> | |
| <tbody> | |
| <tr><th scope="row">1006×679</th><td class="num">2,428 MiB</td><td class="num">1,872 MiB</td><td class="num">1,474 MiB</td></tr> | |
| <tr><th scope="row">1920×1080</th><td class="num">2,794 MiB</td><td class="num">1,872 MiB</td><td class="num">1,474 MiB</td></tr> | |
| <tr><th scope="row">3840×2160</th><td class="num">4,028 MiB</td><td class="num">1,872 MiB</td><td class="num better">1,474 MiB</td></tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| <div class="panel" style="display:grid; gap:10px; align-content:start"> | |
| <h3>Why frame size only matters for SAM3 base</h3> | |
| <p>The official post-processing resizes every predicted mask to the full frame on the GPU, and PyTorch keeps that peak memory reserved. The split pipelines take a fixed 924×924 input and resize masks on the CPU, so their GPU memory is the same for any frame.</p> | |
| </div> | |
| </div> | |
| </section> | |
| <section id="loading"> | |
| <div class="sec-head"> | |
| <h2>What loads, and when</h2> | |
| <p>The production app runs a fixed list of prompts on one image at a time: one vision encode, then one decode per prompt. With that workload every file falls into one of four groups.</p> | |
| </div> | |
| <div class="groups"> | |
| <article class="group live"> | |
| <h3>Loaded the whole time · production</h3> | |
| <ul> | |
| <li><span class="f">sam3_encoder.engine</span><span class="d">Vision encoder. Runs once per image.</span> | |
| <span class="facts"><span class="fact">455.2M params</span><span class="fact">768 MiB VRAM</span></span></li> | |
| <li><span class="f">sam3_decoder_notext.engine</span><span class="d">Detector head: fusion encoder, DETR decoder, mask head. Runs once per prompt.</span> | |
| <span class="facts"><span class="fact">26.0M params</span><span class="fact">494 MiB VRAM</span></span></li> | |
| <li><span class="f">prompt_bank.npz</span><span class="d">Precomputed text embeddings, read into host RAM at startup. Holds no weights.</span> | |
| <span class="facts"><span class="fact">~33 KB per prompt</span><span class="fact">0 MiB VRAM</span></span></li> | |
| </ul> | |
| <div class="foot">Model memory: <b>1,262 MiB</b>. Whole process: 1,474 MiB.</div> | |
| </article> | |
| <article class="group once"> | |
| <h3>Loaded once · offline, on CPU</h3> | |
| <ul> | |
| <li><span class="f">sam3_text_encoder.onnx</span><span class="d">Text encoder. Runs in ONNX Runtime on CPU to build the prompt bank.</span> | |
| <span class="facts"><span class="fact">353.4M params</span><span class="fact">1.41 GB file</span><span class="fact">0 MiB VRAM</span></span></li> | |
| <li><span class="f">facebook/sam3 tokenizer</span><span class="d">Turns each prompt into 32 token ids.</span></li> | |
| </ul> | |
| <div class="foot">Run again only when the prompt list changes. It takes seconds and needs no GPU.</div> | |
| </article> | |
| <article class="group build"> | |
| <h3>Build time only</h3> | |
| <ul> | |
| <li><span class="f">sam3_encoder.onnx</span><span class="d">Source of <code>sam3_encoder.engine</code>.</span></li> | |
| <li><span class="f">sam3_decoder_notext.onnx</span><span class="d">Source of <code>sam3_decoder_notext.engine</code>.</span></li> | |
| </ul> | |
| <div class="foot">Engines are tied to a GPU model. Build them on the production GPU.</div> | |
| </article> | |
| <article class="group retired"> | |
| <h3>Not needed in production</h3> | |
| <ul> | |
| <li><span class="f">sam3_decoder.onnx</span><span class="d">The stage-2 decoder, text encoder inside. Re-ran the text encoder for every prompt.</span> | |
| <span class="facts"><span class="fact">379.4M params</span><span class="fact">896 MiB VRAM as an engine</span></span></li> | |
| </ul> | |
| <div class="foot">Kept in the repo for verification and rollback.</div> | |
| </article> | |
| </div> | |
| </section> | |
| <section id="pipeline"> | |
| <div class="sec-head"> | |
| <h2>Stage 2 → stage 3: how the pipeline changed</h2> | |
| </div> | |
| <figure class="panel"> | |
| <div class="scroll-x"> | |
| <svg class="diagram" viewBox="0 0 1000 480" role="img" aria-label="Stage 2: the decoder engine holds the 353M-parameter text encoder and re-runs it for every prompt. Stage 3: the text encoder runs once offline on CPU and writes prompt_bank.npz; the production app loads only the vision encoder and a 26M-parameter decoder, which reads the prompt bank."> | |
| <defs> | |
| <marker id="ah" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"> | |
| <path d="M0,0 L10,5 L0,10 z" fill="currentColor"/> | |
| </marker> | |
| </defs> | |
| <text x="20" y="22" class="d-panel">STAGE 2 · SPLIT ENCODER–DECODER</text> | |
| <rect x="20" y="36" width="130" height="56" rx="6" class="d-box"/> | |
| <text x="85" y="60" class="d-t" text-anchor="middle">Image</text> | |
| <text x="85" y="78" class="d-s" text-anchor="middle">1×3×924×924</text> | |
| <line x1="150" y1="64" x2="194" y2="64" class="d-arrow" marker-end="url(#ah)"/> | |
| <rect x="196" y="36" width="220" height="56" rx="6" class="d-box"/> | |
| <text x="306" y="60" class="d-t d-m" text-anchor="middle">sam3_encoder</text> | |
| <text x="306" y="78" class="d-s" text-anchor="middle">vision encoder · 455M params</text> | |
| <line x1="416" y1="64" x2="534" y2="64" class="d-arrow" marker-end="url(#ah)"/> | |
| <text x="476" y="56" class="d-l" text-anchor="middle">3 feature maps</text> | |
| <rect x="536" y="36" width="250" height="132" rx="6" class="d-box"/> | |
| <path d="M536,100 H786 V162 a6,6 0 0 1 -6,6 H542 a6,6 0 0 1 -6,-6 Z" class="d-hot"/> | |
| <rect x="536" y="36" width="250" height="132" rx="6" class="d-frame"/> | |
| <line x1="536" y1="100" x2="786" y2="100" class="d-div"/> | |
| <text x="661" y="60" class="d-t d-m" text-anchor="middle">sam3_decoder</text> | |
| <text x="661" y="78" class="d-s" text-anchor="middle">detector head · 26M params</text> | |
| <text x="661" y="128" class="d-t" text-anchor="middle">text encoder · 353M params</text> | |
| <text x="661" y="148" class="d-s" text-anchor="middle">re-runs for every prompt</text> | |
| <line x1="786" y1="64" x2="830" y2="64" class="d-arrow" marker-end="url(#ah)"/> | |
| <rect x="832" y="36" width="148" height="56" rx="6" class="d-box"/> | |
| <text x="906" y="60" class="d-t" text-anchor="middle">Detections</text> | |
| <text x="906" y="78" class="d-s" text-anchor="middle">boxes · masks · scores</text> | |
| <rect x="20" y="112" width="130" height="56" rx="6" class="d-box"/> | |
| <text x="85" y="136" class="d-t" text-anchor="middle">Prompt</text> | |
| <text x="85" y="154" class="d-s" text-anchor="middle">"person"</text> | |
| <line x1="150" y1="140" x2="194" y2="140" class="d-arrow" marker-end="url(#ah)"/> | |
| <rect x="196" y="112" width="220" height="56" rx="6" class="d-box"/> | |
| <text x="306" y="136" class="d-t" text-anchor="middle">Tokenizer</text> | |
| <text x="306" y="154" class="d-s" text-anchor="middle">32 token ids</text> | |
| <line x1="416" y1="140" x2="534" y2="140" class="d-arrow" marker-end="url(#ah)"/> | |
| <text x="476" y="132" class="d-l d-m" text-anchor="middle">tokenized_text</text> | |
| <line x1="10" y1="192" x2="990" y2="192" class="d-rule"/> | |
| <text x="20" y="214" class="d-panel">STAGE 3 · TEXT-FREE DECODER + PROMPT BANK</text> | |
| <rect x="10" y="226" width="980" height="96" rx="10" class="d-offline"/> | |
| <text x="24" y="246" class="d-group">OFFLINE · RUNS ONCE ON CPU, THEN NEVER LOADED</text> | |
| <rect x="20" y="256" width="130" height="52" rx="6" class="d-box"/> | |
| <text x="85" y="278" class="d-t" text-anchor="middle">Your prompts</text> | |
| <text x="85" y="296" class="d-s" text-anchor="middle">fixed list</text> | |
| <line x1="150" y1="282" x2="194" y2="282" class="d-arrow" marker-end="url(#ah)"/> | |
| <rect x="196" y="256" width="220" height="52" rx="6" class="d-hotbox"/> | |
| <text x="306" y="278" class="d-t d-m" text-anchor="middle">sam3_text_encoder</text> | |
| <text x="306" y="296" class="d-s" text-anchor="middle">353M params · ONNX Runtime</text> | |
| <line x1="416" y1="282" x2="534" y2="282" class="d-arrow" marker-end="url(#ah)"/> | |
| <text x="476" y="274" class="d-l" text-anchor="middle">text embeddings</text> | |
| <rect x="536" y="256" width="250" height="52" rx="6" class="d-box"/> | |
| <text x="661" y="278" class="d-t d-m" text-anchor="middle">prompt_bank.npz</text> | |
| <text x="661" y="296" class="d-s" text-anchor="middle">about 33 KB per prompt</text> | |
| <text x="806" y="278" class="d-s">re-run only when the</text> | |
| <text x="806" y="296" class="d-s">prompt list changes</text> | |
| <line x1="661" y1="308" x2="661" y2="382" class="d-arrow" marker-end="url(#ah)"/> | |
| <text x="672" y="338" class="d-l">loaded at startup into host RAM</text> | |
| <rect x="10" y="348" width="980" height="120" rx="10" class="d-live"/> | |
| <text x="24" y="368" class="d-group d-group-live">PRODUCTION · GPU · LOADED THE WHOLE TIME</text> | |
| <rect x="20" y="384" width="130" height="56" rx="6" class="d-box"/> | |
| <text x="85" y="408" class="d-t" text-anchor="middle">Image</text> | |
| <text x="85" y="426" class="d-s" text-anchor="middle">one at a time</text> | |
| <line x1="150" y1="412" x2="194" y2="412" class="d-arrow" marker-end="url(#ah)"/> | |
| <rect x="196" y="384" width="220" height="56" rx="6" class="d-box"/> | |
| <text x="306" y="408" class="d-t d-m" text-anchor="middle">sam3_encoder</text> | |
| <text x="306" y="426" class="d-s" text-anchor="middle">455M params · once per image</text> | |
| <line x1="416" y1="412" x2="534" y2="412" class="d-arrow" marker-end="url(#ah)"/> | |
| <text x="476" y="404" class="d-l" text-anchor="middle">3 feature maps</text> | |
| <rect x="536" y="384" width="250" height="56" rx="6" class="d-box"/> | |
| <text x="661" y="408" class="d-t d-m" text-anchor="middle">sam3_decoder_notext</text> | |
| <text x="661" y="426" class="d-s" text-anchor="middle">26M params · once per prompt</text> | |
| <line x1="786" y1="412" x2="830" y2="412" class="d-arrow" marker-end="url(#ah)"/> | |
| <rect x="832" y="384" width="148" height="56" rx="6" class="d-box"/> | |
| <text x="906" y="408" class="d-t" text-anchor="middle">Detections</text> | |
| <text x="906" y="426" class="d-s" text-anchor="middle">boxes · masks · scores</text> | |
| </svg> | |
| </div> | |
| <figcaption>The text encoder is shaded orange in both halves. In stage 2 it lived inside the decoder and re-ran for every prompt, although the prompts never change. In stage 3 it runs once, offline, on CPU. Its outputs go to <code>prompt_bank.npz</code>, and production keeps only the vision encoder and a 26M-parameter decoder on the GPU.</figcaption> | |
| </figure> | |
| </section> | |
| <section id="vram"> | |
| <div class="sec-head"> | |
| <h2>Where stage 3 saved 402 MiB</h2> | |
| <p>Memory each engine holds once loaded, measured as the drop in free VRAM. The whole saving is weights: the text encoder left the GPU, and nothing else moved.</p> | |
| </div> | |
| <figure class="panel" id="vram-chart"> | |
| <div class="legend"> | |
| <span><i class="sw s1"></i>Vision encoder engine (all of it)</span> | |
| <span><i class="sw s2"></i>Decoder weights</span> | |
| <span><i class="sw s3"></i>Decoder working memory + I/O buffers</span> | |
| <span><i class="sw ghost"></i>Freed</span> | |
| </div> | |
| <div class="plot"> | |
| <div class="gridlines" aria-hidden="true"> | |
| <i style="left:0%"></i><i style="left:25%"></i><i style="left:50%"></i><i style="left:75%"></i><i style="left:100%"></i> | |
| </div> | |
| <div class="rlabel" style="grid-row:1">Stage 2<b>1,664 MiB</b></div> | |
| <div class="track" style="grid-column:2; grid-row:1"> | |
| <span class="seg s1" style="width:38.4%" tabindex="0" data-tip="Stage 2 · vision encoder engine: 768 MiB (46% of 1,664)" aria-label="Stage 2, vision encoder engine: 768 MiB">768</span> | |
| <span class="seg s2" style="width:22.5%" tabindex="0" data-tip="Stage 2 · decoder weights: 450 MiB (27% of 1,664). Text encoder + detector head." aria-label="Stage 2, decoder weights: 450 MiB">450</span> | |
| <span class="seg s3 end" style="width:22.3%" tabindex="0" data-tip="Stage 2 · decoder working memory 303 + I/O buffers 143 = 446 MiB (27% of 1,664)" aria-label="Stage 2, decoder working memory and I/O buffers: 446 MiB">446</span> | |
| </div> | |
| <div class="rlabel" style="grid-row:2">Stage 3<b>1,262 MiB</b></div> | |
| <div class="track" style="grid-column:2; grid-row:2"> | |
| <span class="seg s1" style="width:38.4%" tabindex="0" data-tip="Stage 3 · vision encoder engine: 768 MiB (61% of 1,262). Unchanged." aria-label="Stage 3, vision encoder engine: 768 MiB">768</span> | |
| <span class="seg s2" style="width:2.45%" tabindex="0" data-tip="Stage 3 · decoder weights: 49 MiB (4% of 1,262). Detector head only." aria-label="Stage 3, decoder weights: 49 MiB"></span> | |
| <span class="seg s3 end" style="width:22.25%" tabindex="0" data-tip="Stage 3 · decoder working memory 302 + I/O buffers 143 = 445 MiB (35% of 1,262)" aria-label="Stage 3, decoder working memory and I/O buffers: 445 MiB">445</span> | |
| <span class="seg ghost" style="width:20.1%" tabindex="0" data-tip="Freed: 402 MiB. 401 MiB of decoder weights + 1 MiB of working memory." aria-label="Freed: 402 MiB">−402 freed</span> | |
| </div> | |
| <div class="axis" style="grid-row:3" aria-hidden="true"> | |
| <span style="left:0%">0</span><span style="left:25%">500</span><span style="left:50%">1,000</span><span style="left:75%">1,500</span><span style="left:100%">2,000 MiB</span> | |
| </div> | |
| </div> | |
| <figcaption>Model memory of the two engines, in MiB, CUDA context excluded. Only the decoder weights changed: 450 → 49 MiB. Every value is also in the table below.</figcaption> | |
| </figure> | |
| <div class="table-wrap"> | |
| <table class="vram-t"> | |
| <thead><tr><th>Engine</th><th class="num">Params</th><th class="num">Weights</th><th class="num">Working memory</th><th class="num">I/O buffers</th><th class="num">Total in VRAM</th></tr></thead> | |
| <tbody> | |
| <tr><th scope="row"><span class="f">sam3_encoder</span> <span class="chip same">both</span></th><td class="num">455.2M</td><td class="num">529 MiB</td><td class="num">140 MiB</td><td class="num">99 MiB</td><td class="num">768 MiB</td></tr> | |
| <tr><th scope="row"><span class="f">sam3_decoder</span> <span class="chip retired">stage 2</span></th><td class="num">379.4M</td><td class="num">450 MiB</td><td class="num">303 MiB</td><td class="num">143 MiB</td><td class="num">896 MiB</td></tr> | |
| <tr><th scope="row"><span class="f">sam3_decoder_notext</span> <span class="chip new">stage 3</span></th><td class="num">26.0M</td><td class="num">49 MiB</td><td class="num">302 MiB</td><td class="num">143 MiB</td><td class="num">494 MiB</td></tr> | |
| <tr class="total"><th scope="row">Stage 2 total</th><td class="num">834.6M</td><td class="num">979 MiB</td><td class="num">443 MiB</td><td class="num">242 MiB</td><td class="num">1,664 MiB</td></tr> | |
| <tr class="total"><th scope="row">Stage 3 total</th><td class="num">481.2M</td><td class="num">578 MiB</td><td class="num">442 MiB</td><td class="num">242 MiB</td><td class="num">1,262 MiB</td></tr> | |
| <tr class="saved"><th scope="row">Saved</th><td class="num">353.4M</td><td class="num">401 MiB</td><td class="num">1 MiB</td><td class="num">0 MiB</td><td class="num">402 MiB (−24%)</td></tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| <ul class="notes"> | |
| <li><b>Weights</b> = measured total minus working memory and I/O buffers, so it includes a few MiB of TensorRT overhead.</li> | |
| <li><b>Working memory</b> is TensorRT's activation scratch space. It barely changed: the mask head (200 queries at 264×264) dominates it, and text activations at 32 tokens are tiny.</li> | |
| <li><b>I/O buffers</b> are the engines' input and output tensors. <code>pred_masks</code> alone is 200×264×264 float32, 53 MiB.</li> | |
| <li>Adding the CUDA context and runtime (about 210 MiB, the same in both stages) gives the whole-process figures in the journey chart: 1,872 and 1,474 MiB.</li> | |
| </ul> | |
| </section> | |
| <section id="accuracy"> | |
| <div class="sec-head"> | |
| <h2>Accuracy and speed</h2> | |
| <p>Both TensorRT pipelines were compared with the FP32 model: <code>sam3_decoder.onnx</code> run in full precision by ONNX Runtime, on the same image features. 1 test image, 7 prompts, 72 reference detections at score > 0.25. A sanity check, not a benchmark.</p> | |
| </div> | |
| <div class="two"> | |
| <div class="table-wrap"> | |
| <table class="acc-t"> | |
| <thead><tr><th>Measure</th><th class="num">Stage 2</th><th class="num">Stage 3</th></tr></thead> | |
| <tbody> | |
| <tr><th scope="row">Detections that differ from FP32</th><td class="num">8 of 72</td><td class="num better">1 of 72</td></tr> | |
| <tr><th scope="row">Mean score error on detections</th><td class="num">0.0135</td><td class="num better">0.0036</td></tr> | |
| <tr><th scope="row">Mean mask IoU vs FP32</th><td class="num">0.9855</td><td class="num better">0.9898</td></tr> | |
| <tr><th scope="row">Decode, per prompt</th><td class="num">32.2 ms</td><td class="num">30.5 ms</td></tr> | |
| <tr><th scope="row">1 encode + 4 decodes, with CPU post-processing</th><td class="num">270 ms</td><td class="num">260 ms</td></tr> | |
| <tr><th scope="row">Tokenizer in the production process</th><td class="num">yes</td><td class="num better">no</td></tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| <div class="panel" style="display:grid; gap:12px; align-content:start"> | |
| <h3>Why stage 3 is more accurate</h3> | |
| <p>In the INT8/FP16 engine, the text encoder's output was about 30% off the FP32 value (mean absolute error 0.24 on features of mean magnitude 0.81). That error reached every prediction. The text encoder now runs only once, so the prompt bank is computed in FP32 and that error is gone.</p> | |
| <h3>Why speed barely changed</h3> | |
| <p>At 32 tokens the text encoder was cheap to run. Its cost was the memory it held, which is what stage 3 removed.</p> | |
| </div> | |
| </div> | |
| </section> | |
| <section id="files"> | |
| <div class="sec-head"> | |
| <h2>Every file</h2> | |
| <p>Four ONNX files ship in the model repo. You build the two engines and the prompt bank yourself. Disk sizes in MB; VRAM in MiB.</p> | |
| </div> | |
| <div class="table-wrap"> | |
| <table class="files"> | |
| <thead><tr><th>File</th><th>Comes from</th><th class="num">On disk</th><th class="num">Params</th><th>What it is</th><th>When it's used</th></tr></thead> | |
| <tbody> | |
| <tr><th scope="row"><span class="f">sam3_encoder.onnx</span></th><td>model repo</td><td class="num">1,821</td><td class="num">455.2M</td><td>Vision encoder. Image in, 3 feature maps out.</td><td><span class="chip build">build</span> Source of the encoder engine.</td></tr> | |
| <tr><th scope="row"><span class="f">sam3_decoder_notext.onnx</span></th><td>model repo</td><td class="num">104</td><td class="num">26.0M</td><td>Detector head: fusion encoder, DETR decoder, mask head. Feature maps and text embeddings in, detections out.</td><td><span class="chip build">build</span> Source of the decoder engine.</td></tr> | |
| <tr><th scope="row"><span class="f">sam3_text_encoder.onnx</span></th><td>model repo</td><td class="num">1,414</td><td class="num">353.4M</td><td>Text encoder with its 1024→256 projection. Token ids in, <code>text_features</code> and <code>text_padding_mask</code> out.</td><td><span class="chip once">offline once</span> Builds the prompt bank, on CPU.</td></tr> | |
| <tr><th scope="row"><span class="f">sam3_decoder.onnx</span></th><td>model repo (stage 2)</td><td class="num">1,518</td><td class="num">379.4M</td><td>Stage-2 decoder: text encoder and detector head in one graph.</td><td><span class="chip retired">not in production</span> Verification and rollback.</td></tr> | |
| <tr><th scope="row"><span class="f">sam3_encoder.engine</span></th><td>you build it</td><td class="num">512</td><td class="num">455.2M</td><td>TensorRT build (<code>--fp16 --int8</code>). Runs once per image.</td><td><span class="chip live">production</span> Loaded the whole time.</td></tr> | |
| <tr><th scope="row"><span class="f">sam3_decoder_notext.engine</span></th><td>you build it</td><td class="num">44</td><td class="num">26.0M</td><td>TensorRT build (<code>--fp16 --int8</code>). Runs once per prompt.</td><td><span class="chip live">production</span> Loaded the whole time.</td></tr> | |
| <tr><th scope="row"><span class="f">prompt_bank.npz</span></th><td>you create it</td><td class="num">0.033 / prompt</td><td class="num">none</td><td>For each prompt: the string, its 32 token ids, <code>text_features</code> and <code>text_padding_mask</code>. Plain arrays, not tied to a GPU.</td><td><span class="chip live">production</span> Host RAM, read at startup.</td></tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| </section> | |
| <section id="tensors"> | |
| <div class="sec-head"> | |
| <h2>Tensor interfaces</h2> | |
| <p>Every model's outputs carry the same names as the next model's inputs, so tensors pass straight through by name.</p> | |
| </div> | |
| <div class="tensors"> | |
| <div class="tcard"> | |
| <div class="hd"><span class="f">sam3_encoder</span><span class="chip live">production</span></div> | |
| <div class="io"> | |
| <span class="dir">in</span><span class="nm">image <em>float32 [1,3,924,924]</em></span> | |
| <span class="dir">out</span><span class="nm">/Add_179_output_0 <em>float32 [1,4356,256]</em></span> | |
| <span class="dir">out</span><span class="nm">/fused_229/Conv_output_0 <em>float32 [1,256,264,264]</em></span> | |
| <span class="dir">out</span><span class="nm">/fused_232/Conv_output_0 <em>float32 [1,256,132,132]</em></span> | |
| </div> | |
| </div> | |
| <div class="tcard"> | |
| <div class="hd"><span class="f">sam3_decoder_notext</span><span class="chip live">production</span></div> | |
| <div class="io"> | |
| <span class="dir">in</span><span class="nm">the 3 feature maps</span> | |
| <span class="dir">in</span><span class="nm">text_features <em>float32 [32,1,256]</em></span> | |
| <span class="dir">in</span><span class="nm">text_padding_mask <em>bool [1,32]</em></span> | |
| <span class="dir">out</span><span class="nm">pred_logits <em>float32 [1,200,1]</em></span> | |
| <span class="dir">out</span><span class="nm">pred_boxes <em>float32 [1,200,4], normalized cx, cy, w, h</em></span> | |
| <span class="dir">out</span><span class="nm">presence_logit_dec <em>float32 [1,1]</em></span> | |
| <span class="dir">out</span><span class="nm">pred_masks <em>float32 [1,200,264,264]</em></span> | |
| </div> | |
| </div> | |
| <div class="tcard"> | |
| <div class="hd"><span class="f">sam3_text_encoder</span><span class="chip once">offline once</span></div> | |
| <div class="io"> | |
| <span class="dir">in</span><span class="nm">tokenized_text <em>int64 [1,32], zero-padded</em></span> | |
| <span class="dir">out</span><span class="nm">text_features <em>float32 [32,1,256]</em></span> | |
| <span class="dir">out</span><span class="nm">text_padding_mask <em>bool [1,32], true where token id = 0</em></span> | |
| </div> | |
| </div> | |
| <div class="tcard"> | |
| <div class="hd"><span class="f">sam3_decoder (stage 2)</span><span class="chip retired">not in production</span></div> | |
| <div class="io"> | |
| <span class="dir">in</span><span class="nm">the 3 feature maps</span> | |
| <span class="dir">in</span><span class="nm">tokenized_text <em>int64 [1,32]</em></span> | |
| <span class="dir">out</span><span class="nm">same 4 outputs as sam3_decoder_notext</span> | |
| </div> | |
| </div> | |
| </div> | |
| </section> | |
| <section id="method"> | |
| <div class="sec-head"> | |
| <h2>How the text split was made</h2> | |
| </div> | |
| <ol class="steps"> | |
| <li><div><p class="t"><b>Found the text-only part.</b></p><p>Every node in <code>sam3_decoder.onnx</code> that is reachable from <code>tokenized_text</code> but not from the image features is text encoder: 353.4M of the decoder's 379.4M parameters (93%).</p></div></li> | |
| <li><div><p class="t"><b>Found the narrowest cut.</b></p><p>A minimum vertex cut between the token ids and the image-dependent nodes has exactly two tensors: the projected text features and the padding mask. They were renamed <code>text_features</code> and <code>text_padding_mask</code>.</p></div></li> | |
| <li><div><p class="t"><b>Split the decoder and checked it.</b></p><p>In FP32 ONNX Runtime, text encoder → text-free decoder matches <code>sam3_decoder.onnx</code> bit for bit (largest difference 0.0 on all 4 outputs, 4 prompts).</p></div></li> | |
| <li><div><p class="t"><b>Built the new decoder engine.</b></p><p>Same <code>--fp16 --int8</code> recipe: 44 MB, against 465 MB for the stage-2 decoder engine.</p></div></li> | |
| <li><div><p class="t"><b>Kept the text encoder off TensorRT.</b></p><p>As an INT8/FP16 engine its output was about 30% off FP32. It runs only once, so the prompt bank is computed in FP32 with ONNX Runtime on CPU.</p></div></li> | |
| <li><div><p class="t"><b>Verified end to end.</b></p><p>1 of 72 detections differs from FP32 (stage 2: 8), and the running process dropped from 1,872 to 1,474 MiB.</p></div></li> | |
| </ol> | |
| </section> | |
| <section id="run"> | |
| <div class="sec-head"> | |
| <h2>How to run it</h2> | |
| <p>Full code, including an ONNX Runtime reference pipeline, is on the <a href="https://huggingface.co/sanskar003/SAM3-Split-TextFree" target="_blank" rel="noopener">model card</a>.</p> | |
| </div> | |
| <div class="runs"> | |
| <div class="run"> | |
| <div class="when"><h3>Whenever the prompt list changes</h3><span>CPU only, a few seconds. No engine rebuild.</span></div> | |
| <div class="codebox"><pre>tok = AutoTokenizer.from_pretrained("facebook/sam3"); tok.pad_token_id = 0 | |
| ids = tok(prompts, padding="max_length", max_length=32, truncation=True, return_tensors="np")["input_ids"] | |
| text_encoder = ort.InferenceSession("sam3_text_encoder.onnx") | |
| <span class="c"># run each prompt's ids through text_encoder and save the outputs to prompt_bank.npz</span></pre></div> | |
| </div> | |
| <div class="run"> | |
| <div class="when"><h3>Once per GPU model</h3><span>Engines only run on the GPU model they were built on.</span></div> | |
| <div class="codebox"><pre>trtexec --onnx=sam3_encoder.onnx --fp16 --int8 --builderOptimizationLevel=5 --memPoolSize=workspace:4G --saveEngine=sam3_encoder.engine | |
| trtexec --onnx=sam3_decoder_notext.onnx --fp16 --int8 --builderOptimizationLevel=5 --memPoolSize=workspace:4G --saveEngine=sam3_decoder_notext.engine</pre></div> | |
| </div> | |
| <div class="run"> | |
| <div class="when"><h3>In production</h3><span>Load the two engines and the prompt bank once, then for each image:</span></div> | |
| <div class="codebox"><pre>features = encoder.run({"image": x}) <span class="c"># once per image</span> | |
| for prompt in prompts: <span class="c"># once per prompt</span> | |
| outputs = decoder.run({**features, **prompt_bank[prompt]})</pre></div> | |
| </div> | |
| </div> | |
| </section> | |
| <footer>All three stages measured in October 2026 on an RTX 4070 (16 GB): stage 1 in PyTorch 2.6 FP16, stages 2 and 3 with TensorRT 10.3.</footer> | |
| </main> | |
| <div class="tip" id="tip" role="tooltip"></div> | |
| <script> | |
| (function () { | |
| var tip = document.getElementById('tip'); | |
| function place(x, y) { | |
| var w = tip.offsetWidth, h = tip.offsetHeight; | |
| var left = Math.min(Math.max(8, x + 12), window.innerWidth - w - 8); | |
| var top = y - h - 12 < 8 ? y + 16 : y - h - 12; | |
| tip.style.left = left + 'px'; tip.style.top = top + 'px'; | |
| } | |
| function show(el, x, y) { tip.textContent = el.getAttribute('data-tip'); tip.classList.add('on'); place(x, y); } | |
| function hide() { tip.classList.remove('on'); } | |
| document.querySelectorAll('[data-tip]').forEach(function (el) { | |
| el.addEventListener('mouseenter', function (e) { show(el, e.clientX, e.clientY); }); | |
| el.addEventListener('mousemove', function (e) { place(e.clientX, e.clientY); }); | |
| el.addEventListener('mouseleave', hide); | |
| el.addEventListener('focus', function () { var r = el.getBoundingClientRect(); show(el, r.left + r.width / 2, r.top); }); | |
| el.addEventListener('blur', hide); | |
| }); | |
| document.querySelectorAll('.codebox').forEach(function (box) { | |
| var pre = box.querySelector('pre'); | |
| var btn = document.createElement('button'); | |
| btn.type = 'button'; btn.className = 'copy'; btn.textContent = 'Copy'; | |
| btn.addEventListener('click', function () { | |
| var text = pre.innerText.split('\n').map(function (l) { return l.replace(/\s+#.*$/, ''); }).join('\n').trim(); | |
| try { | |
| navigator.clipboard.writeText(text).then(function () { btn.textContent = 'Copied'; }, | |
| function () { btn.textContent = 'Copy failed'; }); | |
| } catch (e) { btn.textContent = 'Copy failed'; } | |
| setTimeout(function () { btn.textContent = 'Copy'; }, 1600); | |
| }); | |
| box.appendChild(btn); | |
| }); | |
| })(); | |
| </script> | |
| </body> | |
| </html> | |