// MiniCPM5 (GGUF) WebGPU chat bundle. Import { MiniCPM5ForCausalLM } from this file.
var b1=Object.defineProperty;var Cc=(e,t,r)=>()=>{if(r)throw r[0];try{return e&&(t=e(e=0)),t}catch(n){throw r=[n],n}};var qc=(e,t)=>{for(var r in t)b1(e,r,{get:t[r],enumerable:!0})};var il={};qc(il,{default:()=>s_,fileURLToPath:()=>n_,join:()=>r_,open:()=>e_,readFile:()=>J0,stat:()=>t_});var ms,J0,e_,t_,r_,n_,s_,ul=Cc(()=>{ms=e=>()=>{throw new Error("Node built-in node:fs/promises."+e+" is unavailable in this browser bundle (node-only code path).")},J0=ms("readFile"),e_=ms("open"),t_=ms("stat"),r_=ms("join"),n_=ms("fileURLToPath"),s_={}});var zp={};qc(zp,{default:()=>c_,fileURLToPath:()=>l_,join:()=>o_,open:()=>i_,readFile:()=>a_,stat:()=>u_});var hs,a_,i_,u_,o_,l_,c_,Kp=Cc(()=>{hs=e=>()=>{throw new Error("Node built-in node:url."+e+" is unavailable in this browser bundle (node-only code path).")},a_=hs("readFile"),i_=hs("open"),u_=hs("stat"),o_=hs("join"),l_=hs("fileURLToPath"),c_={}});function Xs(e){let t=[],r="",n=0,s=0,a=i=>{if(!i.startsWith(r))return"";let u=i.slice(r.length);return r=i,u};return{push(i){t.push(i);let u=e(t),o=u.length;for(;o>0&&u[o-1]==="\uFFFD";)o--;let l=o!==u.length;return n=!l||o>s?0:n+1,s=o,a(l?n>3?u:u.slice(0,o):u)},flush(){if(t.length===0)return"";let i=e(t);if(n=0,i.startsWith(r))return a(i);let u=0;for(;u<r.length&&u<i.length&&r[u]===i[u];)u+=1;let o=i.slice(u);return r=i,o},get text(){return r}}}function Ic(e,t){let r=Math.min(e.length,t.length),n=0;for(;n<r&&e[n]===t[n];)n+=1;return n}var _1=["a","b"];function Mn(e){let t=0;for(;t<e.length&&e[t].role==="system";)t+=1;return e.slice(0,t)}function Ys(e,t){return e.rewindableLengthAtMost?.(t)??(t>0&&(e.canTruncateTo?.(t)??!0)?t:0)}function Js(e,t,r){if(r.length===0)return[];try{let[n,s]=_1.map(a=>e.encode(t([...r,{role:"user",content:a}]),{add_special_tokens:!1}).ids);return n.slice(0,Ic(n,s))}catch{return[]}}function Jt(e,t,r){return t.length===0?"":e.decode(t,r)}function Bc(e,t,r,n=[]){for(let a=n.length;a>=0;--a){let i=[...r,...n.slice(0,a)];if(i.length===0)continue;let u=Jt(e,i,{skip_special_tokens:!1});if(!t.startsWith(u))continue;let o=e.encode(t.slice(u.length),{add_special_tokens:!1}).ids,l=[...i,...o];if(Jt(e,l,{skip_special_tokens:!1})===t)return{fullyEncodedIds:l,inputIds:l,prefixLength:r.length}}let s=e.encode(t,{add_special_tokens:!1}).ids;return{fullyEncodedIds:s,inputIds:s,prefixLength:Ic(r,s)}}var an=class{#e;#t=[];#r=[];#n=[];#s=null;constructor(t){this.#e=t}get cacheLength(){return this.#t.length}prepare(t,r){if(r===void 0&&this.#n.length===0)return{...Bc(this.#e,t,this.#t,this.#r),imageSpans:[]};let n=r?.imageTokenId??this.#s;if(this.#s!==null&&this.#s!==n)throw new Error(`chat token ledger holds image token ${this.#s}, not ${n}`);let s=r?.images??[],{ids:a,placeholders:i}=k1(this.#t,this.#n),u=Bc(this.#e,t,a,this.#r),o=u.prefixLength;for(let p=0;p<this.#n.length;++p){let f=this.#n[p],g=s[p];if(g===void 0||g.digest!==f.digest||g.tokens!==f.tokens){o=Math.min(o,i[p]);break}}let l=p=>A1(p,s,n),c=l(u.inputIds);return{fullyEncodedIds:(u.inputIds===u.fullyEncodedIds?c:l(u.fullyEncodedIds)).ids,inputIds:c.ids,prefixLength:y1(a,o,this.#n,i),imageSpans:c.spans}}commit(t,r,n,s){let a=[...t,...r];if(!Number.isInteger(n)||n<t.length||n>a.length)throw new Error(`chat token ledger cannot commit cache length ${n} for ${t.length} input + ${r.length} generated tokens`);this.#t=a.slice(0,n),this.#r=a.slice(n),s!==void 0&&(this.#s=s.imageTokenId),this.#n=(s?.imageSpans??[]).filter(i=>i.start+i.tokens<=n)}rewind(t){if(!Number.isInteger(t)||t<0||t>this.#t.length)throw new Error(`chat token ledger cannot rewind to ${t} of ${this.#t.length} cached tokens`);let r=this.#n.find(n=>n.start<t&&t<n.start+n.tokens);if(r)throw new Error(`chat token ledger cannot rewind to ${t}: it splits the image at rows ${r.start}..${r.start+r.tokens-1}`);this.#t=this.#t.slice(0,t),this.#r=[],this.#n=this.#n.filter(n=>n.start<t)}clear(){this.#t=[],this.#r=[],this.#n=[]}};function k1(e,t){let r=[],n=[],s=0;for(let a of t){for(;s<a.start;++s)r.push(e[s]);n.push(r.length),r.push(e[a.start]),s=a.start+a.tokens}for(;s<e.length;++s)r.push(e[s]);return{ids:r,placeholders:n}}function A1(e,t,r){let n=[],s=[];for(let a of e){if(a!==r){n.push(a);continue}let i=t[s.length];if(i===void 0)throw new Error(`prompt holds more image placeholders than its ${t.length} image(s)`);s.push({start:n.length,tokens:i.tokens,digest:i.digest,image:s.length});for(let u=0;u<i.tokens;++u)n.push(r)}if(s.length!==t.length)throw new Error(`prompt holds ${s.length} image placeholder(s) but ${t.length} image(s) were given`);return{ids:n,spans:s}}function y1(e,t,r,n){let s=Math.min(t,e.length);for(let a=0;a<r.length&&n[a]<t;++a)s+=r[a].tokens-1;return s}function S1(e,t){return e?.assistantContent?.(t)??t.visibleText}function bu(e,t){return{...e?.assistantMessageFields?.(t)??{},role:"assistant",content:S1(e,t)}}function Pr(e){return typeof e=="number"&&Number.isFinite(e)}var Ec=/^<function=([^>\n]+)>([\s\S]*)<\/function>$/,_u=/<parameter=([^>\n]+)>\n?/y,Pc="</parameter>",x1=/^<function\s+name="([^"]*)"\s*>([\s\S]*?)(?:<\/function>)?$/,ku=/<param\s+name="([^"]*)"\s*>/y,Fn="</param>",Qn="<![CDATA[";var T1=[{open:"<tool_call>",close:"</tool_call>",inMarkup:!1},{open:"<function",close:"</function>",inMarkup:!0}],Gc=Object.freeze([{type:"function",function:{name:"webgpu_delimiter_probe",description:"probe",parameters:{type:"object",properties:{}}}}]);function Dc(e,t){let r=T1.flatMap(({open:s,close:a,inMarkup:i})=>{let u=t(s),o=t(a);return typeof u=="number"&&typeof o=="number"?[{text:s,open:u,close:o,inMarkup:i}]:[]}),n=(e===null?void 0:r.find(s=>e.includes(s.text)))??r.find(s=>!s.inMarkup);return n?{open:n.open,close:n.close,inMarkup:n.inMarkup}:null}function C1(e){let t=e.trim();if(t.startsWith("{"))return I1(t);if(/^<function\s/.test(t))return q1(t);let r=Ec.exec(t)??(t.startsWith("<function=")?Ec.exec(`${t}
</function>`):null);if(!r)return{error:'the call is neither <function=...> nor <function name="..."> markup, nor a JSON object'};let n=r[2],s=[],a=0;for(;;){for(;a<n.length&&/\s/.test(n[a]);)a+=1;if(a===n.length)break;_u.lastIndex=a;let i=_u.exec(n);if(!i)return{error:`unexpected text in the call: ${JSON.stringify(n.slice(a,a+40))}`};let u=_u.lastIndex,o=n.indexOf(Pc,u),l=n.indexOf("<parameter=",u);if(o>=0&&(l<0||o<l)){s.push([i[1].trim(),{text:n.slice(u,o).replace(/\n$/,"")}]),a=o+Pc.length;continue}let c=l>=0?l:n.length;s.push([i[1].trim(),{text:n.slice(u,c).replace(/\s+$/,"")}]),a=c}return{name:r[1].trim(),arguments:s}}function q1(e){let t=x1.exec(e);if(!t)return{error:`the call's <function> tag does not parse: ${JSON.stringify(e.slice(0,60))}`};let r=t[2],n=[],s=0;for(;;){for(;s<r.length&&/\s/.test(r[s]);)s+=1;if(s===r.length)break;ku.lastIndex=s;let a=ku.exec(r);if(!a)return{error:`unexpected text in the call: ${JSON.stringify(r.slice(s,s+40))}`};let i=ku.lastIndex;if(r.startsWith(Qn,i)){let c=r.indexOf("]]>",i+Qn.length);if(c>=0){for(n.push([a[1],{text:r.slice(i+Qn.length,c)}]),s=c+3;s<r.length&&/\s/.test(r[s]);)s+=1;r.startsWith(Fn,s)&&(s+=Fn.length);continue}n.push([a[1],{text:r.slice(i+Qn.length).replace(/(?:\]\]?)?\s*$/,"")}]);break}let u=r.indexOf(Fn,i),o=r.indexOf("<param",i);if(u>=0&&(o<0||u<o)){n.push([a[1],{text:r.slice(i,u)}]),s=u+Fn.length;continue}let l=o>=0?o:r.length;n.push([a[1],{text:r.slice(i,l).replace(/\s+$/,"")}]),s=l}return{name:t[1].trim(),arguments:n}}function B1(e){let t=Object.entries(e.function.arguments??{}).map(([r,n])=>{let s=typeof n!="string"?JSON.stringify(n):/[<&\n]/.test(n)?`${Qn}${n}]]>`:n;return`<param name="${r}">${s}${Fn}`});return`<function name="${e.function.name}">${t.join("")}</function>`}function Lc(e){let t=e.tool_calls;if(e.role!=="assistant"||typeof e.content!="string"||!Array.isArray(t)||t.length===0)return e;let{tool_calls:r,...n}=e,s=e.content,a=t.map((i,u)=>(u>0||s?`
`:"")+B1(i)).join("");return{...n,content:s+a}}function I1(e){let t;try{t=JSON.parse(e)}catch{return{error:"the call's JSON does not parse"}}let r=t;if(typeof r!="object"||r===null||typeof r.name!="string")return{error:`the call's JSON has no string "name"`};let n=r.arguments??{};return typeof n!="object"||n===null||Array.isArray(n)?{error:`the call's "arguments" is not an object`}:{name:r.name,arguments:Object.entries(n).map(([s,a])=>[s,{value:a}])}}var E1=/^-?(?:0|[1-9]\d*)$/,P1=/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/;function Rc(e,t){switch(e){case"string":return{value:t};case"integer":return E1.test(t)&&Number.isSafeInteger(Number(t))?{value:Number(t)}:void 0;case"number":return P1.test(t)?{value:Number(t)}:void 0;case"boolean":return t==="true"?{value:!0}:t==="false"?{value:!1}:void 0;case"null":return t==="null"?{value:null}:void 0;case"object":case"array":try{let r=JSON.parse(t);return Oc(e,r)?{value:r}:void 0}catch{return}default:return}}function Oc(e,t){switch(e){case"string":return typeof t=="string";case"integer":return typeof t=="number"&&Number.isSafeInteger(t);case"number":return typeof t=="number"&&Number.isFinite(t);case"boolean":return typeof t=="boolean";case"null":return t===null;case"object":return typeof t=="object"&&t!==null&&!Array.isArray(t);case"array":return Array.isArray(t);default:return!1}}function R1(e){return e.type===void 0?null:typeof e.type=="string"?[e.type]:e.type}function G1(e,t,r){let n=R1(r),s;if("text"in t)if(n===null)s=t.text;else{let a=n.map(i=>Rc(i,t.text)).filter(i=>i!==void 0);if(a.length===0)return{error:`argument "${e}": ${JSON.stringify(t.text)} is not ${n.join(" or ")}`};if(a.length>1&&n.includes("string"))return{error:`argument "${e}": ${JSON.stringify(t.text)} could be ${n.filter(i=>Rc(i,t.text)).join(" or ")}`};s=a[0].value}else if(s=t.value,n!==null&&!n.some(a=>Oc(a,s)))return{error:`argument "${e}": ${JSON.stringify(s)} is not ${n.join(" or ")}`};return r.enum&&!r.enum.some(a=>D1(a,s))?{error:`argument "${e}": ${JSON.stringify(s)} is not one of ${r.enum.map(a=>JSON.stringify(a)).join(", ")}`}:{value:s}}function D1(e,t){return JSON.stringify(e)===JSON.stringify(t)}var Au=0;function L1(){return Au=(Au+1)%16777216,`call_${Au.toString(36)}${Math.floor(Math.random()*4294967296).toString(36)}`}function Mc(e,t){let r=L1(),n=C1(e);if("error"in n)return{id:r,type:"function",function:{name:"",arguments:{}},errors:[n.error]};let s=[],a=t.find(l=>l.function.name===n.name);a||s.push(`no tool named "${n.name}" was offered`);let i=a?.function.parameters??{},u=i.properties??{},o={};for(let[l,c]of n.arguments){if(Object.hasOwn(o,l)){s.push(`argument "${l}" is given twice`);continue}let d=Object.hasOwn(u,l)?u[l]:void 0,p=i.additionalProperties,f=d??(typeof p=="object"?p:p===!0?{}:void 0);if(a&&f===void 0){s.push(`argument "${l}" is not a parameter of "${n.name}"`),o[l]="text"in c?c.text:c.value;continue}let g=G1(l,c,f??{});"error"in g?(s.push(g.error),o[l]="text"in c?c.text:c.value):o[l]=g.value}for(let l of i.required??[])Object.hasOwn(o,l)||s.push(`required argument "${l}" is missing`);return{id:r,type:"function",function:{name:n.name,arguments:o},...s.length>0?{errors:s}:{}}}function yu(e){if(!Array.isArray(e))throw new TypeError('tools must be an array of { type: "function", function: { name, parameters } }');let t=new Set;for(let r of e){let n=r?.function;if(r?.type!=="function"||typeof n?.name!="string"||n.name==="")throw new TypeError('each tool must be { type: "function", function: { name, description?, parameters? } }');if(t.has(n.name))throw new TypeError(`tool "${n.name}" is offered twice`);t.add(n.name)}}function Fc(e,t){let r=Qc(e.minimumCapacity??0,"minimumCapacity"),n=e.maxLength??e.max_length;if(n!==void 0){if(n<r)throw new Error(`maxLength ${n} is smaller than minimumCapacity ${r}`);return n}return Math.max(t,r)}function Qc(e,t){if(typeof e!="number"||!Number.isSafeInteger(e)||e<0)throw new Error(`${t} must be a non-negative safe integer, got ${String(e)}`);return e}function O1(e){return Qc(e??0,"max_new_tokens")}var jn=64;function un(e){if(!e.do_sample)return null;let t=e.temperature??1,r=e.top_k??50,n=e.top_p??1,s=e.min_p??0;if(typeof t!="number"||!Number.isFinite(t)||t<0)throw new RangeError(`temperature must be a finite number >= 0, got ${String(t)}`);if(!Number.isInteger(r)||r<1||r>jn)throw new RangeError(`top_k must be an integer from 1 to ${jn} (sampling draws among at most ${jn} candidates), got ${String(r)}`);if(typeof n!="number"||!(n>0&&n<=1))throw new RangeError(`top_p must be in (0, 1], got ${String(n)}`);if(typeof s!="number"||!(s>=0&&s<=1))throw new RangeError(`min_p must be in [0, 1], got ${String(s)}`);if(e.seed!==void 0&&!Number.isSafeInteger(e.seed))throw new RangeError(`seed must be an integer, got ${String(e.seed)}`);if(t===0||r===1)return null;let a=e.seed===void 0?Math.floor(Math.random()*4294967296):e.seed;return{top_k:r,temperature:t,top_p:n,min_p:s,seed:Number(BigInt.asUintN(32,BigInt(a)))}}function ea(e,t={}){if(e.images!==void 0&&e.images.length>0)throw new Error("this model does not accept images; load its vision-language wrapper to generate with them");return{maxNewTokens:O1(e.max_new_tokens),eosTokenId:e.eos_token_id??t.eosTokenId??null,stopOnEos:e.stop_on_eos!==!1,onPrefillDone:e.on_prefill_done??null,onPrefillProgress:e.on_prefill_progress??null,signal:e.signal??null}}function ta(e,t){return t==null?!1:Array.isArray(t)?t.includes(e):e===t}var M1=3,F1=256,ra=class extends Error{required;capacity;constructor(t,r){super(`the conversation needs ${t} tokens (prompt plus one reply token) but the context holds ${r}`),this.name="ContextOverflowError",this.required=t,this.capacity=r}},Rr=class extends Error{constructor(t){super(t),this.name="BusyError"}},Gr=class extends Error{reason;constructor(t,r,n){super(`the GPU device was lost (${t})${r?`: ${r}`:""}`,n),this.name="DeviceLostError",this.reason=t}};function jc(e,t){if(typeof e!="number"||!Number.isSafeInteger(e)||e<1)throw new RangeError(`${t} must be a positive integer, got ${String(e)}`);return e}var na=class{#e=!1;#t="";text="";push(t){let r=t;if(!this.#e){if(r=r.replace(/^\s+/,""),r==="")return"";this.#e=!0}let n=this.#t+r,s=/\s+$/.exec(n),a=s?n.slice(0,s.index):n;return this.#t=s?s[0]:"",this.text+=a,a}},sa=class{#e;#t="";matched=!1;constructor(t){this.#e=t}push(t){if(this.matched)return"";let r=this.#t+t,n=-1;for(let a of this.#e){let i=r.indexOf(a);i!==-1&&(n===-1||i<n)&&(n=i)}if(n!==-1)return this.matched=!0,this.#t="",r.slice(0,n);let s=0;for(let a of this.#e)for(let i=Math.min(a.length-1,r.length);i>s;--i)if(r.endsWith(a.slice(0,i))){s=i;break}return this.#t=r.slice(r.length-s),r.slice(0,r.length-s)}flush(){let t=this.#t;return this.#t="",t}},xu=class{#e;#t;#r;#n;#s;#i;#o;#a;#u=null;calls=[];#l=this.#p();#c=this.#p();reasoning=new na;answer=new na;#f;#d;reasoningTokens=0;thought;constructor(t){this.#i=t.decode,this.#o=t.decodeRaw,this.#e=t.thinkOpen,this.#t=t.thinkClose,this.#r=t.calls?.open??null,this.#n=t.calls?.close??null,this.#s=t.calls?.inMarkup??!1,this.#a=t.startInReasoning?"reasoning":"text",this.thought=t.startInReasoning,this.#f=new sa(t.stopStrings),this.#d=new sa(t.stopStrings)}closeOpenCall(){this.#u!==null&&(this.calls.push(this.#u.length>0?this.#o(this.#u):""),this.#u=null)}get stopped(){return this.#f.matched||this.#d.matched}#p(){return Xs(t=>this.#i(t))}push(t){if(this.#u!==null)return t===this.#n?(this.#s&&this.#u.push(t),this.calls.push(this.#u.length>0?this.#o(this.#u):""),this.#u=null):this.#u.push(t),[];if(t===this.#r&&this.#a==="text")return this.#u=this.#s?[t]:[],this.#g("text");if(this.#a==="reasoning"&&(this.reasoningTokens+=1),t===this.#t&&this.#a==="reasoning"){let r=this.#g("reasoning");return this.#a="text",r}if(t===this.#e&&this.#a==="text"){let r=this.#g("text");return this.#a="reasoning",this.thought=!0,this.reasoningTokens+=1,r}return this.#m(this.#a,this.#w(this.#a).push(t))}finish(){return[...this.#g("reasoning"),...this.#g("text")]}#w(t){return t==="reasoning"?this.#l:this.#c}#g(t){let r=t==="reasoning"?this.#f:this.#d,n=r.push(this.#w(t).flush()),s=this.#h(t,r.matched?n:n+r.flush());return t==="reasoning"?this.#l=this.#p():this.#c=this.#p(),s}#m(t,r){return r===""?[]:this.#h(t,(t==="reasoning"?this.#f:this.#d).push(r))}#h(t,r){if(r==="")return[];let n=(t==="reasoning"?this.reasoning:this.answer).push(r);return n===""?[]:[{type:t,text:n}]}},Tu=class{result;#e=new AbortController;#t=[];#r;#n=null;#s=!1;#i=null;#o=!1;constructor(t,r){let n=()=>{};if(r)if(r.aborted)this.#e.abort(r.reason);else{let u=()=>this.#e.abort(r.reason);r.addEventListener("abort",u,{once:!0}),n=()=>r.removeEventListener("abort",u)}let s,a=new Promise((u,o)=>{s=o});this.#r=s;let i=t(this);i.catch(()=>{}),this.result=Promise.race([i,a]).then(u=>(this.#a(null),u),u=>{throw this.#a({error:u}),u}).finally(n),this.result.catch(()=>{})}cancel(){this.#e.abort()}get signal(){return this.#e.signal}emit(t){this.#s||(this.#t.push(t),this.#n?.())}abandon(t){this.#r(t),this.#e.abort(t)}#a(t){this.#s=!0,this.#i=t,this.#n?.()}[Symbol.asyncIterator](){if(this.#o)throw new TypeError("a ChatTurn's events can be iterated once");return this.#o=!0,{next:async()=>{for(;;){let t=this.#t.shift();if(t)return{value:t,done:!1};if(this.#s){if(this.#i)throw this.#i.error;return{value:void 0,done:!0}}await new Promise(r=>{this.#n=r}),this.#n=null}},return:async()=>(this.#s||this.cancel(),{value:void 0,done:!0})}}},aa=class{#e;#t;#r;#n;#s=new Map;#i=null;#o=null;constructor(t){this.#e=t.host,this.#t=t.state,this.#r=t.ledger,this.#n=t.release;let r=jc(t.options?.max_checkpoints??M1,"max_checkpoints");t.state.cache.setRewindPointCapacity?.(r)}get_seq_length(){return this.#t.cache.get_seq_length()}get max_length(){return this.#t.cache.maxLength}generate(t,r={}){return new Tu(n=>this.#w(t,r,n),r.signal)}count_tokens(t,r={}){this.#a(),r.tools!==void 0&&yu(r.tools);let{inputIds:n,imageTokens:s}=this.#p(t,this.#e.templateArgs(this.#c(r)),r.tools??null);return{prompt_tokens:n.length,image_tokens:s}}reset(){if(this.#a(),this.#i)throw new Rr("a turn is running on this session; cancel it before resetting");this.#l()}#a(){if(this.#o)throw new Error("this ChatSession has been disposed")}dispose(){return this.#o??=this.#u()}async#u(){let t=this.#i;t&&(t.cancel(),await t.result.catch(()=>{})),this.#n()}deviceLost(t){this.#i?.abandon(t)}#l(){this.#t.cache.truncate(0),this.#r.clear()}#c(t){return{...t.chat_template_kwargs,...t.enable_thinking!==void 0?{enable_thinking:t.enable_thinking}:{}}}#f(t){if(this.#s.size===0)return t;let r=t.findLastIndex(n=>n.role==="user");return t.map((n,s)=>{if(n.role!=="assistant"||typeof n.content!="string")return n;let a=this.#s.get(Su(n));return a?this.#e.historyMessage({...a,position:s>r?"after_last_user":"before_last_user"}):n})}#d(t,r,n){let s=new Set(n.filter(a=>a.role==="assistant"&&typeof a.content=="string").map(a=>Su(a)));for(let a of this.#s.keys())s.has(a)||this.#s.delete(a);this.#s.set(Su(t),r)}#p(t,r,n){let s=this.#e,a=s.renderPrompt(this.#f(t),!0,r,n),i=s.planImages(t),u=this.#r.prepare(a,i),o=u.inputIds,l=u.imageSpans.reduce((c,d)=>c+d.tokens,0);return{prompt:a,images:i,reconciled:u,inputIds:o,imageTokens:l}}async#w(t,r,n){this.#a();let s=this.#e.acquireLease();this.#i=n;try{return await this.#g(t,r,n)}finally{this.#i=null,s()}}async#g(t,r,n){let s=performance.now(),a=this.#e,i=this.#t,u=i.cache,o=this.#r,l=n.signal,c=jc(r.max_new_tokens??a.defaultMaxNewTokens,"max_new_tokens"),d=r.eos_token_id??a.eosTokenIds,p=this.#m(r),f=j1(r.stop_strings),g=r.tools??null;g!==null&&yu(g),u.get_seq_length()!==o.cacheLength&&this.#l();let m=a.templateArgs(this.#c(r)),{prompt:h,images:v,reconciled:b,inputIds:_,imageTokens:C}=this.#p(t,m,g),y=u.maxLength;if(_.length+1>y)throw new ra(_.length+1,y);let E=Math.min(c,y-_.length),k=b.imageSpans,S=v&&{imageTokenId:v.imageTokenId,imageSpans:k},A=(W,P)=>v?k.filter(R=>R.start>=W&&R.start<P).map(R=>v.images[R.image]):[],w=[],I=!1,z=0,O=0,j=0,F=0,K=!1,Z=({done:W})=>{K=!0,n.emit({type:"prefill",done:j+W,total:F})},Q=r.decode_pipeline_depth,N=(W,P,R,Y,ne)=>{let re=A(Y,Y+W.length);return a.streamTokens(i,{suffixIds:W,maxNewTokens:P,eosTokenId:d,stopOnEos:R,signal:l,onPrefillProgress:Z,...re.length>0?{images:re}:{},...Q!==void 0?{decodePipelineDepth:Q}:{},...ne?{sampling:ne}:{}})},q=async W=>{let P=u.get_seq_length();for await(let R of N(_.slice(P,W),1,!1,P,null));if(j+=W-P,u.get_seq_length()!==W)throw new Error(`prefill left the cache at ${u.get_seq_length()}, not ${W}`)},x=this.#h(h),B=new xu({decode:W=>Jt(a.tokenizer,W,{skip_special_tokens:!0}),decodeRaw:W=>Jt(a.tokenizer,W,{skip_special_tokens:!1}),thinkOpen:a.thinkOpenTokenId,thinkClose:a.thinkCloseTokenId,calls:a.toolCallDelimiters,startInReasoning:x,stopStrings:f});try{try{let P=Math.min(b.prefixLength,_.length-1),R=Ys(u,P);if(R===0?this.#l():R!==u.get_seq_length()&&(u.truncate(R),o.rewind(R)),O=u.get_seq_length(),F=_.length-O,O===0){let D=await this.#y(t,m,_,q,g);D>0&&(O=D,F=_.length-D)}u.get_seq_length()>0&&await u.captureRewindPoint?.();let Y=u.get_seq_length(),ne=_.length-Y;u.captureRewindPoint&&ne>=2&&(ne>=F1||A(Y,_.length).length>0)&&(await q(_.length-1),await u.captureRewindPoint());let re=u.get_seq_length();for await(let D of N(_.slice(re),E,!0,re,p)){w.length===0&&(z=performance.now(),K&&n.emit({type:"prefill",done:F,total:F})),w.push(D);for(let se of B.push(D))n.emit(se);if(l.aborted||B.stopped)break}l.aborted&&(I=!0)}catch(P){if(!(l.aborted&&P===l.reason))throw P;I=!0}for(let P of B.finish())n.emit(P);let W=u.get_seq_length();W>=_.length?o.commit(_,w,W,S):o.commit(_.slice(0,W),[],W,S)}catch(W){throw this.#l(),W}let U=performance.now(),M=!I&&!B.stopped&&w.length<E;M&&B.closeOpenCall();let G=I?"cancelled":!B.stopped&&w.length>=E?"length":M&&B.calls.length>0?"tool_calls":"stop",$=G==="tool_calls"?B.calls.map(W=>Mc(W,g??[])):null,X={role:"assistant",content:B.answer.text,...B.thought?{reasoning_content:B.reasoning.text}:{},...$?{tool_calls:$}:{}};for(let W of $??[])n.emit({type:"tool_call",tool_call:W});this.#d(X,{visibleText:Jt(a.tokenizer,w,{skip_special_tokens:!0}),rawText:Jt(a.tokenizer,w,{skip_special_tokens:!1}),templateArgs:m},t);let H=z?U-z:0;return{finish_reason:G,...G==="length"?{limit:E<c?"context":"max_new_tokens"}:{},...p?{seed:p.seed}:{},message:X,usage:{prompt_tokens:_.length,cached_tokens:O,image_tokens:C,completion_tokens:w.length,reasoning_tokens:B.reasoningTokens},timings:{ttft_ms:(z||U)-s,decode_ms:H,tokens_per_second:w.length>1&&H>0?(w.length-1)/(H/1e3):0}}}#m(t){let r=this.#e.generationConfig;return un({do_sample:t.do_sample??r.do_sample,...Q1({temperature:t.temperature??r.temperature,top_k:t.top_k??r.top_k,top_p:t.top_p??r.top_p,min_p:t.min_p??r.min_p,seed:t.seed})})}#h(t){let{thinkOpenTokenId:r,thinkCloseTokenId:n,tokenizer:s}=this.#e;if(r===null||n===null)return!1;let a=s.decode([r],{skip_special_tokens:!1}),i=s.decode([n],{skip_special_tokens:!1});return t.lastIndexOf(a)>t.lastIndexOf(i)}async#y(t,r,n,s,a){let i=this.#e,u=this.#t.cache;if(!u.captureRewindPoint)return 0;let o=Mn(t);if(o.length===0)return 0;let l=Js(i.tokenizer,c=>i.renderPrompt(c,!1,r,a),o);if(l.length===0||l.length>=n.length)return 0;for(let c=0;c<l.length;++c)if(l[c]!==n[c])return 0;return await i.restoreSystemPrefix?.(this.#t,l)?(this.#r.commit(l,[],l.length),l.length):(await s(l.length),this.#r.commit(l,[],l.length),await u.captureRewindPoint(),await i.persistSystemPrefix?.(this.#t,l),0)}};function Q1(e){return Object.fromEntries(Object.entries(e).filter(([,t])=>t!==void 0))}function j1(e){if(e===void 0)return[];if(!Array.isArray(e)||e.some(t=>typeof t!="string"||t===""))throw new TypeError("stop_strings must be an array of non-empty strings");return e}function Su(e){let t=e.tool_calls?.map(r=>[r.function?.name,r.function?.arguments])??null;return JSON.stringify([e.content,e.reasoning_content??null,t])}var W1=Object.freeze({vendor:"",architecture:"",device:"",description:""}),Wn=Object.freeze(["maxTextureDimension1D","maxTextureDimension2D","maxTextureDimension3D","maxTextureArrayLayers","maxBindGroups","maxBindGroupsPlusVertexBuffers","maxBindingsPerBindGroup","maxDynamicUniformBuffersPerPipelineLayout","maxDynamicStorageBuffersPerPipelineLayout","maxSampledTexturesPerShaderStage","maxSamplersPerShaderStage","maxStorageBuffersPerShaderStage","maxStorageTexturesPerShaderStage","maxUniformBuffersPerShaderStage","maxUniformBufferBindingSize","maxStorageBufferBindingSize","minUniformBufferOffsetAlignment","minStorageBufferOffsetAlignment","maxVertexBuffers","maxBufferSize","maxVertexAttributes","maxVertexBufferArrayStride","maxInterStageShaderVariables","maxColorAttachments","maxColorAttachmentBytesPerSample","maxComputeWorkgroupStorageSize","maxComputeInvocationsPerWorkgroup","maxComputeWorkgroupSizeX","maxComputeWorkgroupSizeY","maxComputeWorkgroupSizeZ","maxComputeWorkgroupsPerDimension","maxImmediateSize","maxStorageBuffersInVertexStage","maxStorageBuffersInFragmentStage","maxStorageTexturesInVertexStage","maxStorageTexturesInFragmentStage"]),pr=Object.freeze(["shader-f16","subgroups","subgroup-size-control","chromium-experimental-subgroup-matrix","atomic-vec2u-min-max","timestamp-query","texture-formats-tier1","texture-formats-tier2"]),Wt=65535,z1=Object.freeze({maxTextureDimension1D:8192,maxTextureDimension2D:8192,maxTextureDimension3D:2048,maxTextureArrayLayers:256,maxBindGroups:4,maxBindGroupsPlusVertexBuffers:24,maxBindingsPerBindGroup:1e3,maxDynamicUniformBuffersPerPipelineLayout:8,maxDynamicStorageBuffersPerPipelineLayout:4,maxSampledTexturesPerShaderStage:16,maxSamplersPerShaderStage:16,maxStorageBuffersPerShaderStage:10,maxStorageTexturesPerShaderStage:4,maxUniformBuffersPerShaderStage:12,maxUniformBufferBindingSize:64*1024,maxStorageBufferBindingSize:128*1024*1024,minUniformBufferOffsetAlignment:256,minStorageBufferOffsetAlignment:256,maxVertexBuffers:8,maxBufferSize:256*1024*1024,maxVertexAttributes:16,maxVertexBufferArrayStride:2048,maxInterStageShaderVariables:16,maxColorAttachments:8,maxColorAttachmentBytesPerSample:32,maxComputeInvocationsPerWorkgroup:256,maxComputeWorkgroupSizeX:256,maxComputeWorkgroupSizeY:256,maxComputeWorkgroupSizeZ:64,maxComputeWorkgroupStorageSize:16*1024,maxComputeWorkgroupsPerDimension:Wt,maxImmediateSize:0}),Wc=new WeakSet,zc=new WeakMap;function ze(e=null){if(e!==null&&typeof e=="object"&&Wc.has(e))return e;let t=e!==null&&typeof e=="object"?e:void 0;if(t!==void 0){let a=zc.get(t);if(a!==void 0&&a.adapterInfo===t.adapterInfo&&a.features===t.features&&a.wgslLanguageFeatures===t.wgslLanguageFeatures&&a.wgslCompileFailures===t.wgslCompileFailures&&a.limits===t.limits)return a.normalized}let r=oa(e)?e:{},n=oa(r.adapterInfo)?r.adapterInfo:{},s={adapterInfo:N1(n),features:on(r.features),wgslLanguageFeatures:zn(r.wgslLanguageFeatures),wgslCompileFailures:Iu(r.wgslCompileFailures),limits:la(r.limits)};return Wc.add(s),t!==void 0&&zc.set(t,{normalized:s,adapterInfo:t.adapterInfo,features:t.features,wgslLanguageFeatures:t.wgslLanguageFeatures,wgslCompileFailures:t.wgslCompileFailures,limits:t.limits}),s}function Bu(e){let t=new Set(e),r={size:t.size,has(n){return t.has(n)},entries(){return t.entries()},forEach(n,s){t.forEach(a=>n.call(s,a,a,r))},keys(){return t.keys()},values(){return t.values()},[Symbol.iterator](){return t[Symbol.iterator]()}};return r}function on(e=[]){return Bu(Ru(e))}function zn(e=[]){return Bu(Ru(e))}function Iu(e=[]){return Bu(Ru(e))}function yt(e=null){let t=ze(e);return{adapterInfo:{...t.adapterInfo},features:Array.from(t.features.values()).sort(),wgslLanguageFeatures:Array.from(t.wgslLanguageFeatures.values()).sort(),wgslCompileFailures:Array.from(t.wgslCompileFailures.values()).sort(),limits:{...t.limits}}}function Eu(e,t){if(!e)return null;for(let n of e.features??[])if(!t.features.has(n))return`requires device.features.has("${n}")`;for(let[n,s]of Object.entries(e.limits??{})){if(s===void 0)continue;let a=t.limits[n];if(typeof a!="number"||a<s)return`requires device.limits.${n} >= ${s}`}if(e.subgroupMinSize!==void 0){let n=t.adapterInfo.subgroupMinSize;if(typeof n!="number")return`requires adapterInfo.subgroupMinSize >= ${e.subgroupMinSize} (adapter does not report subgroup sizes)`;if(n<e.subgroupMinSize)return`requires adapterInfo.subgroupMinSize >= ${e.subgroupMinSize}`}let r=e.subgroupMatrixConfigs;if(r!==void 0&&r.length>0){let n=t.adapterInfo.subgroupMatrixConfigs;if(Array.isArray(n)&&!r.some(s=>n.some(a=>K1(s,a))))return`requires adapterInfo.subgroupMatrixConfigs to match one of [${r.map(ia).join(", ")}] (device reports [${n.map(ia).join(", ")}])`}return null}function K1(e,t){return(e.componentType===void 0||e.componentType===t.componentType)&&(e.resultComponentType===void 0||e.resultComponentType===t.resultComponentType)&&(e.M===void 0||e.M===t.M)&&(e.N===void 0||e.N===t.N)&&(e.K===void 0||e.K===t.K)}function ia(e){return`${e.componentType??"*"}-${e.resultComponentType??"*"}-${e.M??"*"}x${e.N??"*"}x${e.K??"*"}`}var ua=Object.freeze(["f32","f16","u32","i32","u8","i8"]);function U1(e){if(!oa(e))return null;let t=e.componentType,r=e.resultComponentType,{M:n,N:s,K:a}=e;return!ua.includes(t)||!ua.includes(r)||!Cu(n)||!Cu(s)||!Cu(a)?null:Object.freeze({componentType:t,resultComponentType:r,M:n,N:s,K:a})}function ln(e){if(e==null||typeof e=="string"||typeof e[Symbol.iterator]!="function")return null;let t=[];for(let r of e){let n=U1(r);n!==null&&t.push(n)}return t.sort((r,n)=>ia(r).localeCompare(ia(n))),Object.freeze(t)}function Cu(e){return typeof e=="number"&&Number.isInteger(e)&&e>0}function Pu(e){let t=e.adapterInfo;return{vendor:typeof t.vendor=="string"?t.vendor.toLowerCase():"",architecture:typeof t.architecture=="string"?t.architecture.toLowerCase():"",...typeof t.subgroupMinSize=="number"?{subgroupMinSize:t.subgroupMinSize}:{},...typeof t.subgroupMaxSize=="number"?{subgroupMaxSize:t.subgroupMaxSize}:{},isFallbackAdapter:t.isFallbackAdapter===!0,...Array.isArray(t.subgroupMatrixConfigs)?{subgroupMatrixConfigs:t.subgroupMatrixConfigs}:{}}}function N1(e){let{subgroupMatrixConfigs:t,...r}=e,n=ln(t);return{...W1,...r,...n!==null?{subgroupMatrixConfigs:n}:{}}}function la(e){let t=oa(e)?e:{},r={...z1};for(let s of Wn){let a=H1(t,s);a!==void 0&&(r[s]=a)}for(let[s,a]of Object.entries(t))typeof a=="number"&&Number.isFinite(a)&&(r[s]=a);let n=r.maxComputeWorkgroupsPerDimension;return typeof n=="number"&&(r.maxComputeWorkgroupsPerDimension=Math.min(n,Wt)),r}function Ru(e){return e==null?[]:Array.isArray(e)?e.filter(qu):typeof e[Symbol.iterator]=="function"?Array.from(e).filter(qu):typeof e.values=="function"?Array.from(e.values()).filter(qu):[]}function H1(e,t){let r=e[t];return typeof r=="number"&&Number.isFinite(r)?r:void 0}function qu(e){return typeof e=="string"}function oa(e){return e!==null&&typeof e=="object"&&!Array.isArray(e)}var $1=Object.freeze({float16:"float16",float32:"float32",int8:"int8",int16:"int16",int32:"int32",int64:"int64",uint8:"uint8",uint32:"uint32",bool:"bool"}),Dr=Object.freeze({...$1,int64:"int64"});function Lr(e,t){return e===t||e==="int64"&&(t==="int32"||t==="uint32")}var Ge=Object.freeze({float16:{storageByteSize:2,onDiskByteSize:2,wgslScalar:"f16",arrayCtor:Uint16Array},float32:{storageByteSize:4,onDiskByteSize:4,wgslScalar:"f32",arrayCtor:Float32Array},int8:{storageByteSize:4,onDiskByteSize:1,wgslScalar:"i32",arrayCtor:Int32Array},int16:{storageByteSize:4,onDiskByteSize:2,wgslScalar:"i32",arrayCtor:Int32Array},int32:{storageByteSize:4,onDiskByteSize:4,wgslScalar:"i32",arrayCtor:Int32Array},int64:{storageByteSize:8,onDiskByteSize:8,wgslScalar:"vec2<u32>",arrayCtor:BigInt64Array},uint8:{storageByteSize:4,onDiskByteSize:1,wgslScalar:"u32",arrayCtor:Uint32Array},uint32:{storageByteSize:4,onDiskByteSize:4,wgslScalar:"u32",arrayCtor:Uint32Array},bool:{storageByteSize:4,onDiskByteSize:1,wgslScalar:"u32",arrayCtor:Uint32Array}});function je(e){if(Object.hasOwn(Ge,e))return e;throw new Error(`Unsupported dtype: ${e}`)}function ve(e){return Ge[e].storageByteSize}function Gu(e){return e==="uint8"}function ca(e,t){if(!Gu(e))throw new Error(`${t}: the packed storage layout is defined for uint8 tensors, not ${e}`)}function et(e,t,r){if(!Number.isSafeInteger(t)||t<0)throw new Error(`tensorStorageByteLength: element count must be a nonnegative safe integer, got ${t}`);let n;if(r==="packed"?(ca(e,"tensorStorageByteLength"),n=Ze(t,4)):n=t*ve(e),!Number.isSafeInteger(n))throw new Error("tensorStorageByteLength: byte length exceeds safe integer storage");return n}function Kc(e){let t=new Uint32Array(Math.ceil(e.length/4));for(let r=0;r<e.length;r+=1){let n=e[r];if(!Number.isInteger(n)||n<0||n>255)throw new Error(`packUint8Elements: element ${r} (${n}) is not a byte`);t[r>>>2]|=n<<(r&3)*8}return t}function Uc(e,t){if(e.length*4<t)throw new Error(`unpackUint8Elements: ${e.length} words cannot hold ${t} elements`);let r=new Uint32Array(t);for(let n=0;n<t;n+=1)r[n]=e[n>>>2]>>>(n&3)*8&255;return r}function we(e){if(!Array.isArray(e))throw new Error("shape must be an array");let t=1;for(let r of e){if(!Number.isInteger(r)||r<0)throw new Error(`invalid shape dimension: ${r}`);t*=r}return t}function Ze(e,t){return Math.ceil(e/t)*t}function nt(e){return Math.max(4,Ze(e,4))}function da(e,t){return e.length===t.length&&e.every((r,n)=>r===t[n])}function st(e){return e!==null&&typeof e=="object"&&typeof e.dtype=="string"&&Array.isArray(e.shape)&&typeof e.size=="number"&&typeof e.byteLength=="number"&&"buffer"in e&&"runtime"in e}function Or(){}function fe(e,t){let r=we([...t]);return{dtype:e,shape:[...t],size:r,byteLength:r*ve(e),byteOffset:0,buffer:{destroy(){}},runtime:null,destroy(){}}}function Kn(e){if(e instanceof BigInt64Array)throw new Error("Opaque int64 data cannot be used for numeric tensor arithmetic");return e}function V1(e){return Object.fromEntries(Object.entries(e).filter(([,t])=>t!==void 0))}var Z1=/^vec([234])<(u32|i32|f32)>$/;function Nc(e){if(e==="u32"||e==="i32"||e==="f32")return{align:4,size:4,scalar:e,components:1};let t=Z1.exec(e);if(!t)throw new Error(`Unsupported uniform field type: ${e}`);let r=Number(t[1]);return{align:r===2?8:16,size:r===3?12:r*4,scalar:t[2],components:r}}function X1(e){return e==="u32"||e==="i32"||e==="f32"}function Du(e,t={}){return Un(e,"u32",t)}function Lu(e,t={}){return Un(e,"i32",t)}function Ou(e,t={}){return Un(e,"f32",t)}function Un(e,t,r={}){return Nn(e,"uniform field"),Nc(t),Object.freeze({kind:"uniform-field",name:e,type:t,semantic:r.semantic,required:r.required??!r.internal,internal:!!r.internal,default:r.default,description:r.description})}function fa(e,t,r={}){if(Nn(e,"uniform struct"),!Array.isArray(t)||t.length===0)throw new Error(`uniform struct ${e} requires at least one field`);let n=t.map(s=>J1(s,e));return Object.freeze({kind:"uniform-struct",name:e,fields:Object.freeze(e3(n,r))})}function Mu(e){Fu(e);let t=e.fields.map(r=>`${r.name}: ${r.type}`).join(", ");return`struct ${e.name} { ${t} };`}function pa(e,t={}){if(Fu(e),!e.fields.every(u=>X1(u.type)))return Y1(e,t);let r=Math.max(16,Math.ceil(e.fields.length*4/16)*16),n=new ArrayBuffer(r),s=new Uint32Array(n),a=new Int32Array(n),i=new Float32Array(n);return e.fields.forEach((u,o)=>{let l=t[u.name];if(l===void 0&&u.required)throw new Error(`Missing uniform field ${e.name}.${u.name}`);if(l===void 0&&(l=u.default??0),typeof l!="number")throw new Error(`Uniform ${e.name}.${u.name} expects a scalar number`);Hc(u.type,o,l,`${e.name}.${u.name}`,s,a,i)}),new Uint32Array(n)}function Y1(e,t){let r=[],n=0;for(let o of e.fields){let l=Nc(o.type);n=Ze(n,l.align),r.push({field:o,layout:l,offset:n}),n+=l.size}let s=new ArrayBuffer(Math.max(16,Ze(n,16))),a=new Uint32Array(s),i=new Int32Array(s),u=new Float32Array(s);for(let{field:o,layout:l,offset:c}of r){let d=t[o.name];if(d===void 0&&o.required)throw new Error(`Missing uniform field ${e.name}.${o.name}`);d===void 0&&(d=o.default??(l.components===1?0:new Array(l.components).fill(0)));let p=typeof d=="number"?[d]:d;if(p.length!==l.components)throw new Error(`Uniform ${e.name}.${o.name} expects ${l.components} component(s), got ${p.length}`);for(let f=0;f<l.components;++f)Hc(l.scalar,(c+f*4)/4,p[f],`${e.name}.${o.name}[${f}]`,a,i,u)}return new Uint32Array(s)}function Hc(e,t,r,n,s,a,i){if(e==="u32"){if(!Number.isInteger(r)||r<0||r>4294967295)throw new Error(`Uniform ${n} must be an integer u32 in [0, 4294967295]`);s[t]=r;return}if(e==="i32"){if(!Number.isInteger(r)||r<-2147483648||r>2147483647)throw new Error(`Uniform ${n} must be an integer i32 in [-2147483648, 2147483647]`);a[t]=r;return}if(Number.isNaN(r))throw new Error(`Uniform ${n} must not be NaN (f32)`);i[t]=r}function $c(e,t,r,n){return e.createUniformU32(pa(t,r),n)}function Vc(e){return Fu(e),{kind:e.kind,name:e.name,fields:e.fields.map(t=>V1({name:t.name,type:t.type,required:t.required,internal:t.internal,semantic:t.semantic,default:t.default,description:t.description}))}}function J1(e,t){if(!e||e.kind!=="uniform-field")throw new Error(`uniform struct ${t} fields must be created with u32(), i32(), or f32()`);return e}function e3(e,t){let r=t.alignFieldsTo??4;if(!Number.isInteger(r)||r<=0)throw new Error(`alignFieldsTo must be a positive integer, got ${r}`);let n=[...e],s=n.length%r;if(s===0)return Object.freeze(n);let a=r-s,i=0,u=new Set(n.map(o=>o.name));for(let o=0;o<a;++o){for(;u.has(`_pad${i}`);)i++;let l=`_pad${i++}`;u.add(l),n.push(Un(l,"u32",{internal:!0,required:!1,default:0}))}return Object.freeze(n)}function Fu(e){if(!e||e.kind!=="uniform-struct")throw new Error("Expected a uniform struct schema")}function Nn(e,t){if(typeof e!="string"||!/^[A-Za-z_][A-Za-z0-9_]*$/.test(e))throw new Error(`${t} name must be a WGSL-compatible identifier, got ${e}`)}var Xc=Object.freeze({read:"read-only-storage",read_write:"storage"});function Qu(e,t={}){Nn(e,"storage binding");let r=t.access??"read";if(!(r in Xc))throw new Error(`storage binding ${e} has unsupported access ${r}`);return Object.freeze({kind:"storage",name:e,arg:t.arg,access:r,elementType:t.elementType??"f32",...t.length!==void 0?{length:t.length}:{},semantic:t.semantic})}function ju(e,t,r={}){Nn(e,"uniform binding");let n;if(a3(t)?n=fa(r.structName??s3(e),t):n=t,!n||n.kind!=="uniform-struct")throw new Error(`uniform binding ${e} requires a uniformStruct schema or field array`);return Object.freeze({kind:"uniform",name:e,struct:n,semantic:r.semantic})}function cn(e){if(!Array.isArray(e)||e.length===0)throw new Error("bindGroup requires at least one binding");let t=new Set;return Object.freeze(e.map((r,n)=>{if(!r||r.kind!=="storage"&&r.kind!=="uniform")throw new Error("bindGroup entries must be storage() or uniform() bindings");if(t.has(r.name))throw new Error(`duplicate bindGroup binding name: ${r.name}`);return t.add(r.name),Object.freeze({...r,binding:n})}))}function Wu(e,t={}){return cn(e).map(r=>t3(r,t)).join(`
`)}function Yc(e){let t=[],r=new Map;for(let n of cn(e)){if(n.kind!=="uniform")continue;let s=JSON.stringify(Vc(n.struct)),a=r.get(n.struct.name);if(a!==void 0){if(a!==s)throw new Error(`uniform struct ${n.struct.name} is declared with conflicting schemas`);continue}r.set(n.struct.name,s),t.push(Mu(n.struct))}return t.join(`
`)}function Jc(e,t,r,n={}){let s=cn(t),a=n.labelPrefix??"kernel",i=[],u=()=>{let l=[];for(let c of i.splice(0))try{c()}catch(d){l.push(d)}if(l.length===1)throw l[0];if(l.length>1)throw new AggregateError(l,"WebGPU binding cleanup failed")},o=[];try{for(let l of s){let c=r?.[l.name];if(c==null)throw new Error(`Missing resource for binding ${l.name}`);if(l.kind==="uniform"){let g;n3(c)?g=c:(g=$c(e,l.struct,c,`${a}-${l.name}`),typeof g.destroy=="function"&&i.push(()=>g.destroy?.())),o.push({buffer:g,type:"uniform",binding:l.binding});continue}let d=c,p=typeof d.byteOffset=="number"?d.byteOffset:0,f=typeof d.byteLength=="number"?nt(d.byteLength):void 0;if(l.minByteSize!==void 0&&f!==void 0&&f<l.minByteSize)throw new Error(`binding ${l.name}: bound tensor provides ${f} bytes but the declared fixed length requires ${l.minByteSize}`);o.push({tensor:c,type:Xc[l.access],binding:l.binding,...p?{offset:p}:{},...f!==void 0?{size:f}:{}})}return{bindings:o,cleanup:u}}catch(l){try{u()}catch(c){throw new AggregateError([l,c],"WebGPU binding materialization and cleanup both failed")}throw l}}function t3(e,t){if(e.kind==="storage"){let r=e.access==="read_write"?"read_write":"read",n=zu(e.elementType,t,`binding ${e.name} elementType`);return e.length!==void 0?`@group(0) @binding(${e.binding}) var<storage, ${r}> ${e.name}: array<${n}, ${td(e.length,t,e.name)}u>;`:`@group(0) @binding(${e.binding}) var<storage, ${r}> ${e.name}: array<${n}>;`}return`@group(0) @binding(${e.binding}) var<uniform> ${e.name}: ${e.struct.name};`}var Zc={f32:4,u32:4,i32:4,f16:2};function r3(e){if(/^atomic<\s*[ui]32\s*>$/.test(e))return 4;if(/^atomic<\s*(?:vec2u|vec2<\s*u32\s*>)\s*>$/.test(e))return 8;let t=/^vec([234])<\s*([a-z0-9]+)\s*>$/.exec(e);if(t){let n=Zc[t[2]];if(n!==void 0){let s=Number(t[1]);return(s===3?4:s)*n}}let r=Zc[e];if(r!==void 0)return r;throw new Error(`unsupported storage element type for a sized binding: ${e}`)}function ed(e,t){return e.map(r=>{if(r.kind!=="storage"||r.length===void 0)return r;let n=zu(r.elementType,t,`binding ${r.name} elementType`),s=td(r.length,t,r.name);return{...r,minByteSize:s*r3(String(n))}})}function td(e,t,r){let n=typeof e=="number"?e:zu(e,t,`binding ${r} length`),s=typeof n=="number"?n:Number(n);if(!Number.isSafeInteger(s)||s<=0)throw new Error(`binding ${r} length must resolve to a positive integer, got ${String(n)}`);return s}function zu(e,t,r){if(typeof e!="string"||!e.startsWith("$"))return e;let n=e.slice(1),s=t[n];if(s==null)throw new Error(`Missing template value ${n} for ${r}`);return s}function n3(e){return e!==null&&typeof e=="object"&&typeof e.destroy=="function"&&!("shape"in e)}function s3(e){return e.length===0?e:e[0].toUpperCase()+e.slice(1)}function a3(e){return Array.isArray(e)}var i3=Object.defineProperty,u3=(e,t,r)=>t in e?i3(e,t,{enumerable:!0,configurable:!0,writable:!0,value:r}):e[t]=r,o3=(e,t,r)=>(u3(e,typeof t!="symbol"?t+"":t,r),r),L=Object.freeze({Text:"Text",NumericLiteral:"NumericLiteral",StringLiteral:"StringLiteral",Identifier:"Identifier",Equals:"Equals",OpenParen:"OpenParen",CloseParen:"CloseParen",OpenStatement:"OpenStatement",CloseStatement:"CloseStatement",OpenExpression:"OpenExpression",CloseExpression:"CloseExpression",OpenSquareBracket:"OpenSquareBracket",CloseSquareBracket:"CloseSquareBracket",OpenCurlyBracket:"OpenCurlyBracket",CloseCurlyBracket:"CloseCurlyBracket",Comma:"Comma",Dot:"Dot",Colon:"Colon",Pipe:"Pipe",CallOperator:"CallOperator",AdditiveBinaryOperator:"AdditiveBinaryOperator",MultiplicativeBinaryOperator:"MultiplicativeBinaryOperator",ExponentiationBinaryOperator:"ExponentiationBinaryOperator",ComparisonBinaryOperator:"ComparisonBinaryOperator",UnaryOperator:"UnaryOperator",Comment:"Comment"}),gt=class{constructor(e,t){this.value=e,this.type=t}};function rd(e){return/\w/.test(e)}function Mr(e){return/[0-9]/.test(e)}function nd(e){return/\s/.test(e)}var l3=[["{%",L.OpenStatement],["%}",L.CloseStatement],["{{",L.OpenExpression],["}}",L.CloseExpression],["(",L.OpenParen],[")",L.CloseParen],["{",L.OpenCurlyBracket],["}",L.CloseCurlyBracket],["[",L.OpenSquareBracket],["]",L.CloseSquareBracket],[",",L.Comma],[".",L.Dot],[":",L.Colon],["|",L.Pipe],["<=",L.ComparisonBinaryOperator],[">=",L.ComparisonBinaryOperator],["==",L.ComparisonBinaryOperator],["!=",L.ComparisonBinaryOperator],["<",L.ComparisonBinaryOperator],[">",L.ComparisonBinaryOperator],["+",L.AdditiveBinaryOperator],["-",L.AdditiveBinaryOperator],["~",L.AdditiveBinaryOperator],["**",L.ExponentiationBinaryOperator],["*",L.MultiplicativeBinaryOperator],["//",L.MultiplicativeBinaryOperator],["/",L.MultiplicativeBinaryOperator],["%",L.MultiplicativeBinaryOperator],["=",L.Equals]],c3=new Map([["n",`
`],["t","	"],["r","\r"],["b","\b"],["f","\f"],["v","\v"],["'","'"],['"','"'],["\\","\\"]]);function d3(e,t={}){return e.endsWith(`
`)&&(e=e.slice(0,-1)),t.lstrip_blocks&&(e=e.replace(/^[ \t]*({[#%-])/gm,"$1")),t.trim_blocks&&(e=e.replace(/([#%-]})\n/g,"$1")),e.replace(/(\s*){%(-?)\s*(?:end)?generation\s*(-?)%}(\s*)/gs,(r,n,s,a,i)=>(s?"":n)+(a?"":i))}function ba(e,t={}){let r=[],n=d3(e,t),s=0,a=0,i=l=>{let c="";for(;l(n[s]);){if(n[s]==="\\"){if(++s,s>=n.length)throw new SyntaxError("Unexpected end of input");let d=n[s++],p=c3.get(d);if(p===void 0)throw new SyntaxError(`Unexpected escaped character: ${d}`);c+=p;continue}if(c+=n[s++],s>=n.length)throw new SyntaxError("Unexpected end of input")}return c},u=()=>{let l=r.at(-1);l&&l.type===L.Text&&(l.value=l.value.trimEnd(),l.value===""&&r.pop())},o=()=>{for(;s<n.length&&nd(n[s]);)++s};e:for(;s<n.length;){let l=r.at(-1)?.type;if(l===void 0||l===L.CloseStatement||l===L.CloseExpression||l===L.Comment){let d="";for(;s<n.length&&!(n[s]==="{"&&(n[s+1]==="%"||n[s+1]==="{"||n[s+1]==="#"));)d+=n[s++];if(d.length>0){r.push(new gt(d,L.Text));continue}}if(n[s]==="{"&&n[s+1]==="#"){s+=2;let d=n[s]==="-";d&&++s;let p="";for(;n[s]!=="#"||n[s+1]!=="}";){if(s+2>=n.length)throw new SyntaxError("Missing end of comment tag");p+=n[s++]}let f=p.endsWith("-");f&&(p=p.slice(0,-1)),d&&u(),r.push(new gt(p,L.Comment)),s+=2,f&&o();continue}if(n.slice(s,s+3)==="{%-"){u(),r.push(new gt("{%",L.OpenStatement)),s+=3;continue}if(n.slice(s,s+3)==="{{-"){u(),r.push(new gt("{{",L.OpenExpression)),a=0,s+=3;continue}if(i(nd),n.slice(s,s+3)==="-%}"){r.push(new gt("%}",L.CloseStatement)),s+=3,o();continue}if(n.slice(s,s+3)==="-}}"){r.push(new gt("}}",L.CloseExpression)),s+=3,o();continue}let c=n[s];if(c==="-"||c==="+"){let d=r.at(-1)?.type;if(d===L.Text||d===void 0)throw new SyntaxError(`Unexpected character: ${c}`);switch(d){case L.Identifier:case L.NumericLiteral:case L.StringLiteral:case L.CloseParen:case L.CloseSquareBracket:break;default:{++s;let p=i(Mr);if(p.length>0&&n[s]==="."&&Mr(n[s+1])){++s;let f=i(Mr);p=`${p}.${f}`}r.push(new gt(`${c}${p}`,p.length>0?L.NumericLiteral:L.UnaryOperator));continue}}}for(let[d,p]of l3){if(d==="}}"&&a>0)continue;if(n.slice(s,s+d.length)===d){r.push(new gt(d,p)),p===L.OpenExpression?a=0:p===L.OpenCurlyBracket?++a:p===L.CloseCurlyBracket&&--a,s+=d.length;continue e}}if(c==="'"||c==='"'){++s;let d=i(p=>p!==c);r.push(new gt(d,L.StringLiteral)),++s;continue}if(Mr(c)){let d=i(Mr);if(r.at(-1)?.type!==L.Dot&&n[s]==="."&&Mr(n[s+1])){++s;let p=i(Mr);d=`${d}.${p}`}r.push(new gt(d,L.NumericLiteral));continue}if(rd(c)){let d=i(rd);r.push(new gt(d,L.Identifier));continue}throw new SyntaxError(`Unexpected character: ${c}`)}return r}var ht=class{type="Statement"},f3=class extends ht{constructor(e){super(),this.body=e}type="Program"},p3=class extends ht{constructor(e,t,r){super(),this.test=e,this.body=t,this.alternate=r}type="If"},g3=class extends ht{constructor(e,t,r,n){super(),this.loopvar=e,this.iterable=t,this.body=r,this.defaultBlock=n}type="For"},m3=class extends ht{type="Break"},h3=class extends ht{type="Continue"},v3=class extends ht{constructor(e,t,r){super(),this.assignee=e,this.value=t,this.body=r}type="Set"},w3=class extends ht{constructor(e,t,r){super(),this.name=e,this.args=t,this.body=r}type="Macro"},b3=class extends ht{constructor(e){super(),this.value=e}type="Comment"},at=class extends ht{type="Expression"},_3=class extends at{constructor(e,t,r){super(),this.object=e,this.property=t,this.computed=r}type="MemberExpression"},sd=class extends at{constructor(e,t){super(),this.callee=e,this.args=t}type="CallExpression"},gr=class extends at{constructor(e){super(),this.value=e}type="Identifier"},fn=class extends at{constructor(e){super(),this.value=e}type="Literal"},k3=class extends fn{type="IntegerLiteral"},A3=class extends fn{type="FloatLiteral"},ad=class extends fn{type="StringLiteral"},y3=class extends fn{type="ArrayLiteral"},id=class extends fn{type="TupleLiteral"},S3=class extends fn{type="ObjectLiteral"},dn=class extends at{constructor(e,t,r){super(),this.operator=e,this.left=t,this.right=r}type="BinaryExpression"},x3=class extends at{constructor(e,t){super(),this.operand=e,this.filter=t}type="FilterExpression"},T3=class extends ht{constructor(e,t){super(),this.filter=e,this.body=t}type="FilterStatement"},C3=class extends at{constructor(e,t){super(),this.lhs=e,this.test=t}type="SelectExpression"},q3=class extends at{constructor(e,t,r){super(),this.operand=e,this.negate=t,this.test=r}type="TestExpression"},ud=class extends at{constructor(e,t){super(),this.operator=e,this.argument=t}type="UnaryExpression"},B3=class extends at{constructor(e=void 0,t=void 0,r=void 0){super(),this.start=e,this.stop=t,this.step=r}type="SliceExpression"},Ku=class extends at{constructor(e,t){super(),this.key=e,this.value=t}type="KeywordArgumentExpression"},I3=class extends at{constructor(e){super(),this.argument=e}type="SpreadExpression"},E3=class extends at{constructor(e){super(),this.argument=e}type="KeywordSpreadExpression"},P3=class extends ht{constructor(e,t,r){super(),this.call=e,this.callerArgs=t,this.body=r}type="CallStatement"},R3=class extends at{constructor(e,t,r){super(),this.condition=e,this.trueExpr=t,this.falseExpr=r}type="Ternary"};function _a(e){let t=new f3([]),r=0;function n(q,x){let B=e[r++];if(!B||B.type!==q)throw new Error(`Parser Error: ${x}. ${B.type} !== ${q}.`);return B}function s(q){if(!o(q))throw new SyntaxError(`Expected ${q}`);++r}function a(){switch(e[r].type){case L.Comment:return new b3(e[r++].value);case L.Text:return l();case L.OpenStatement:return c();case L.OpenExpression:return d();default:throw new SyntaxError(`Unexpected token type: ${e[r].type}`)}}function i(...q){return r+q.length<=e.length&&q.every((x,B)=>x===e[r+B].type)}function u(...q){return e[r]?.type===L.OpenStatement&&e[r+1]?.type===L.Identifier&&q.includes(e[r+1]?.value)}function o(...q){return r+q.length<=e.length&&q.every((x,B)=>e[r+B].type==="Identifier"&&x===e[r+B].value)}function l(){return new ad(n(L.Text,"Expected text token").value)}function c(){if(n(L.OpenStatement,"Expected opening statement token"),e[r].type!==L.Identifier)throw new SyntaxError(`Unknown statement, got ${e[r].type}`);let q=e[r].value,x;switch(q){case"set":++r,x=p();break;case"if":++r,x=f(),n(L.OpenStatement,"Expected {% token"),s("endif"),n(L.CloseStatement,"Expected %} token");break;case"macro":++r,x=g(),n(L.OpenStatement,"Expected {% token"),s("endmacro"),n(L.CloseStatement,"Expected %} token");break;case"for":++r,x=h(),n(L.OpenStatement,"Expected {% token"),s("endfor"),n(L.CloseStatement,"Expected %} token");break;case"call":{++r;let B=null;i(L.OpenParen)&&(B=w("parameters"));let U=N();if(U.type!=="Identifier")throw new SyntaxError("Expected identifier following call statement");let M=w("call");n(L.CloseStatement,"Expected closing statement token");let G=[];for(;!u("endcall");)G.push(a());n(L.OpenStatement,"Expected '{%'"),s("endcall"),n(L.CloseStatement,"Expected closing statement token");let $=new sd(U,M);x=new P3($,B,G);break}case"break":++r,n(L.CloseStatement,"Expected closing statement token"),x=new m3;break;case"continue":++r,n(L.CloseStatement,"Expected closing statement token"),x=new h3;break;case"filter":{++r;let B=N();B instanceof gr&&i(L.OpenParen)&&(B=A(B)),n(L.CloseStatement,"Expected closing statement token");let U=[];for(;!u("endfilter");)U.push(a());n(L.OpenStatement,"Expected '{%'"),s("endfilter"),n(L.CloseStatement,"Expected '%}'"),x=new T3(B,U);break}default:throw new SyntaxError(`Unknown statement type: ${q}`)}return x}function d(){n(L.OpenExpression,"Expected opening expression token");let q=v();return n(L.CloseExpression,"Expected closing expression token"),q}function p(){let q=m(),x=null,B=[];if(i(L.Equals))++r,x=m();else{for(n(L.CloseStatement,"Expected %} token");!u("endset");)B.push(a());n(L.OpenStatement,"Expected {% token"),s("endset")}return n(L.CloseStatement,"Expected closing statement token"),new v3(q,x,B)}function f(){let q=v();n(L.CloseStatement,"Expected closing statement token");let x=[],B=[];for(;!u("elif","else","endif");)x.push(a());if(u("elif")){++r,++r;let U=f();B.push(U)}else if(u("else"))for(++r,++r,n(L.CloseStatement,"Expected closing statement token");!u("endif");)B.push(a());return new p3(q,x,B)}function g(){let q=N();if(q.type!=="Identifier")throw new SyntaxError("Expected identifier following macro statement");let x=w("parameters");n(L.CloseStatement,"Expected closing statement token");let B=[];for(;!u("endmacro");)B.push(a());return new w3(q,x,B)}function m(q=!1){let x=q?N:v,B=[x()],U=i(L.Comma);for(;U&&(++r,B.push(x()),!!i(L.Comma)););return U?new id(B):B[0]}function h(){let q=m(!0);if(!(q instanceof gr||q instanceof id))throw new SyntaxError(`Expected identifier/tuple for the loop variable, got ${q.type} instead`);if(!o("in"))throw new SyntaxError("Expected `in` keyword following loop variable");++r;let x=v();n(L.CloseStatement,"Expected closing statement token");let B=[];for(;!u("endfor","else");)B.push(a());let U=[];if(u("else"))for(++r,++r,n(L.CloseStatement,"Expected closing statement token");!u("endfor");)U.push(a());return new g3(q,x,B,U)}function v(){return b()}function b(){let q=_();if(o("if")){++r;let x=_();if(o("else")){++r;let B=b();return new R3(x,q,B)}else return new C3(q,x)}return q}function _(){let q=C();for(;o("or");){let x=e[r];++r;let B=C();q=new dn(x,q,B)}return q}function C(){let q=y();for(;o("and");){let x=e[r];++r;let B=y();q=new dn(x,q,B)}return q}function y(){let q;for(;o("not");){let x=e[r];++r;let B=y();q=new ud(x,B)}return q??E()}function E(){let q=k();for(;;){let x;if(o("not","in"))x=new gt("not in",L.Identifier),r+=2;else if(o("in"))x=e[r++];else if(i(L.ComparisonBinaryOperator))x=e[r++];else break;let B=k();q=new dn(x,q,B)}return q}function k(){let q=j();for(;i(L.AdditiveBinaryOperator);){let x=e[r];++r;let B=j();q=new dn(x,q,B)}return q}function S(){let q=O(N());return i(L.OpenParen)?A(q):q}function A(q){let x=new sd(q,w("call"));return x=O(x),i(L.OpenParen)&&(x=A(x)),x}function w(q){n(L.OpenParen,"Expected opening parenthesis for arguments list");let x=I(q);return n(L.CloseParen,"Expected closing parenthesis for arguments list"),x}function I(q){let x=[],B=new Set,U=!1,M=!1;for(;!i(L.CloseParen);){let G=i(L.ExponentiationBinaryOperator),$=i(L.MultiplicativeBinaryOperator)&&e[r].value==="*";if(q==="parameters"&&(G||$))throw new SyntaxError("Argument unpacking is not allowed in parameter declarations");if(G){if(++r,x.push(new E3(v())),i(L.Comma)&&++r,!i(L.CloseParen))throw new SyntaxError("Expected closing parenthesis: `**` must be applied to the final argument");break}let X;if($){if(M)throw new SyntaxError("Only one `*` argument unpacking is allowed");M=!0,++r;let H=v();X=new I3(H)}else{if(X=v(),i(L.Equals)){if(++r,!(X instanceof gr))throw new SyntaxError(q==="parameters"?"Expected identifier for parameter declaration":"Expected identifier for keyword argument");let H=v();X=new Ku(X,H)}if(q==="parameters"){if(!(X instanceof gr||X instanceof Ku))throw new SyntaxError("Expected identifier for parameter declaration");let H=X instanceof gr?X.value:X.key.value;if(B.has(H))throw new SyntaxError(`Duplicate parameter name: ${H}`);B.add(H)}if(X instanceof Ku)U=!0;else{if(U)throw new SyntaxError(q==="call"?"Positional arguments must come before keyword arguments":"Non-default argument follows default argument");if(M)throw new SyntaxError("Positional arguments must not follow `*` argument unpacking")}}x.push(X),i(L.Comma)&&++r}return x}function z(){let q=[],x=!1;for(;!i(L.CloseSquareBracket);)i(L.Colon)?(q.push(void 0),++r,x=!0):(q.push(v()),i(L.Colon)&&(++r,x=!0));if(q.length===0)throw new SyntaxError("Expected at least one argument for member/slice expression");if(x){if(q.length>3)throw new SyntaxError("Expected 0-3 arguments for slice expression");return new B3(...q)}return q[0]}function O(q){for(;i(L.Dot)||i(L.OpenSquareBracket);){let x=e[r];++r;let B,U=x.type===L.OpenSquareBracket;if(U)B=z(),n(L.CloseSquareBracket,"Expected closing square bracket");else if(B=N(),B.type!=="Identifier"&&B.type!=="IntegerLiteral")throw new SyntaxError("Expected identifier or integer following dot operator");q=new _3(q,B,U)}return q}function j(){let q=F();for(;i(L.MultiplicativeBinaryOperator);){let x=e[r++],B=F();q=new dn(x,q,B)}return q}function F(){let q=K();for(;i(L.ExponentiationBinaryOperator);){let x=e[r++],B=K();q=new dn(x,q,B)}return q}function K(){let q=Z();for(;o("is");){++r;let x=o("not");x&&++r;let B=N();if(!(B instanceof gr))throw new SyntaxError("Expected identifier for the test");q=new q3(q,x,B)}return q}function Z(){let q=Q();for(;i(L.Pipe);){++r;let x=N();if(!(x instanceof gr))throw new SyntaxError("Expected identifier for the filter");i(L.OpenParen)&&(x=A(x)),q=new x3(q,x)}return q}function Q(){let q=e[r];if(q&&(q.type===L.UnaryOperator||q.type===L.AdditiveBinaryOperator)&&(q.value==="-"||q.value==="+")){let x=e[r++],B=Q();return new ud(x,B)}return S()}function N(){let q=e[r++];switch(q.type){case L.NumericLiteral:{let x=q.value;return x.includes(".")?new A3(Number(x)):new k3(Number(x))}case L.StringLiteral:{let x=q.value;for(;i(L.StringLiteral);)x+=e[r++].value;return new ad(x)}case L.Identifier:return new gr(q.value);case L.OpenParen:{let x=m();return n(L.CloseParen,"Expected closing parenthesis, got ${tokens[current].type} instead."),x}case L.OpenSquareBracket:{let x=[];for(;!i(L.CloseSquareBracket);)x.push(v()),i(L.Comma)&&++r;return++r,new y3(x)}case L.OpenCurlyBracket:{let x=new Map;for(;!i(L.CloseCurlyBracket);){let B=v();n(L.Colon,"Expected colon between key and value in object literal");let U=v();x.set(B,U),i(L.Comma)&&++r}return++r,new S3(x)}default:throw new SyntaxError(`Unexpected token: ${q.type}`)}}for(;r<e.length;)t.body.push(a());return t}function G3(e,t,r=1){if(t===void 0&&(t=e,e=0),r===0)throw new Error("range() step must not be zero");let n=[];if(r>0)for(let s=e;s<t;s+=r)n.push(s);else for(let s=e;s>t;s+=r)n.push(s);return n}function od(e,t,r,n=1){let s=Math.sign(n);s>=0?(t=(t??=0)<0?Math.max(e.length+t,0):Math.min(t,e.length),r=(r??=e.length)<0?Math.max(e.length+r,0):Math.min(r,e.length)):(t=(t??=e.length-1)<0?Math.max(e.length+t,-1):Math.min(t,e.length-1),r=(r??=-1)<-1?Math.max(e.length+r,-1):Math.min(r,e.length-1));let a=[];for(let i=t;s*i<s*r;i+=n)a.push(e[i]);return a}function D3(e){return e.replace(/\b\w/g,t=>t.toUpperCase())}function L3(e){return O3(new Date,e)}function O3(e,t){let r=new Intl.DateTimeFormat(void 0,{month:"long"}),n=new Intl.DateTimeFormat(void 0,{month:"short"}),s=a=>a<10?"0"+a:a.toString();return t.replace(/%[YmdbBHM%]/g,a=>{switch(a){case"%Y":return e.getFullYear().toString();case"%m":return s(e.getMonth()+1);case"%d":return s(e.getDate());case"%b":return n.format(e);case"%B":return r.format(e);case"%H":return s(e.getHours());case"%M":return s(e.getMinutes());case"%%":return"%";default:return a}})}function M3(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function F3(e,t,r,n){if(n===0)return e;let s=n==null||n<0?1/0:n,a=t.length===0?new RegExp("(?=)","gu"):new RegExp(M3(t),"gu");return e.replaceAll(a,i=>s>0?(--s,r):i)}var ld=class extends Error{},cd=class extends Error{},Q3=new Map,Tt=class{type="RuntimeValue";value;get builtins(){return Q3}constructor(e=void 0){this.value=e}__bool__(){return new ue(!!this.value)}toString(){return String(this.value)}},oe=class extends Tt{type="IntegerValue"},ke=class extends Tt{type="FloatValue";toString(){return Object.is(this.value,-0)?"-0.0":this.value%1===0?this.value.toFixed(1):this.value.toString()}},J=class extends Tt{type="StringValue";_builtins;get builtins(){return this._builtins??=new Map([["upper",new Ee(()=>new J(this.value.toUpperCase()))],["lower",new Ee(()=>new J(this.value.toLowerCase()))],["strip",new Ee(()=>new J(this.value.trim()))],["title",new Ee(()=>new J(D3(this.value)))],["capitalize",new Ee(()=>new J(this.value.charAt(0).toUpperCase()+this.value.slice(1)))],["length",new oe(this.value.length)],["rstrip",new Ee(()=>new J(this.value.trimEnd()))],["lstrip",new Ee(()=>new J(this.value.trimStart()))],["startswith",new Ee(e=>{if(e.length===0)throw new Error("startswith() requires at least one argument");let t=e[0];if(t instanceof J)return new ue(this.value.startsWith(t.value));if(t instanceof pe){for(let r of t.value){if(!(r instanceof J))throw new Error("startswith() tuple elements must be strings");if(this.value.startsWith(r.value))return new ue(!0)}return new ue(!1)}throw new Error("startswith() argument must be a string or tuple of strings")})],["endswith",new Ee(e=>{if(e.length===0)throw new Error("endswith() requires at least one argument");let t=e[0];if(t instanceof J)return new ue(this.value.endsWith(t.value));if(t instanceof pe){for(let r of t.value){if(!(r instanceof J))throw new Error("endswith() tuple elements must be strings");if(this.value.endsWith(r.value))return new ue(!0)}return new ue(!1)}throw new Error("endswith() argument must be a string or tuple of strings")})],["split",new Ee(e=>{let t=e[0]??new De;if(!(t instanceof J||t instanceof De))throw new Error("sep argument must be a string or null");let r=e[1]??new oe(-1);if(!(r instanceof oe))throw new Error("maxsplit argument must be a number");let n=[];if(t instanceof De){let s=this.value.trimStart();for(let{0:a,index:i}of s.matchAll(/\S+/g)){if(r.value!==-1&&n.length>=r.value&&i!==void 0){n.push(a+s.slice(i+a.length));break}n.push(a)}}else{if(t.value==="")throw new Error("empty separator");n=this.value.split(t.value),r.value!==-1&&n.length>r.value&&n.push(n.splice(r.value).join(t.value))}return new pe(n.map(s=>new J(s)))})],["replace",new Ee(e=>{if(e.length<2)throw new Error("replace() requires at least two arguments");let t=e[0],r=e[1];if(!(t instanceof J&&r instanceof J))throw new Error("replace() arguments must be strings");let n;if(e.length>2?e[2].type==="KeywordArgumentsValue"?n=e[2].value.get("count")??new De:n=e[2]:n=new De,!(n instanceof oe||n instanceof De))throw new Error("replace() count argument must be a number or null");return new J(F3(this.value,t.value,r.value,n.value))})]])}},ue=class extends Tt{type="BooleanValue"},j3=/[\x7f-\uffff]/g;function dd(e){return e.replace(j3,t=>"\\u"+t.charCodeAt(0).toString(16).padStart(4,"0"))}function mr(e,t={},r=0,n=!0){let{indent:s=null,ensureAscii:a=!1,separators:i=null,sortKeys:u=!1}=t,o,l;switch(i?[o,l]=i:s?(o=",",l=": "):(o=", ",l=": "),e.type){case"NullValue":return"null";case"UndefinedValue":return n?"null":"undefined";case"IntegerValue":case"FloatValue":case"BooleanValue":return JSON.stringify(e.value);case"StringValue":{let c=JSON.stringify(e.value);return a&&(c=dd(c)),c}case"ArrayValue":case"NamespaceValue":case"ObjectValue":{let c=s?" ".repeat(s):"",d=`
`+c.repeat(r),p=d+c;if(e.type==="ArrayValue"){let f=e.value.map(g=>mr(g,t,r+1,n));return s?`[${p}${f.join(`${o}${p}`)}${d}]`:`[${f.join(o)}]`}else{let f=Array.from(e.value.entries());u&&(f=f.sort(([m],[h])=>m.localeCompare(h)));let g=f.map(([m,h])=>{let v=JSON.stringify(m);a&&(v=dd(v));let b=`${v}${l}${mr(h,t,r+1,n)}`;return s?`${p}${b}`:b});return s?`{${g.join(o)}${d}}`:`{${g.join(o)}}`}}default:throw new Error(`Cannot convert to JSON: ${e.type}`)}}var Xe=class extends Tt{type="ObjectValue";_builtins;__bool__(){return new ue(this.value.size>0)}get builtins(){return this._builtins??=new Map([["get",new Ee(([e,t])=>{if(!(e instanceof J))throw new Error(`Object key must be a string: got ${e.type}`);return this.value.get(e.value)??t??new De})],["items",new Ee(()=>this.items())],["keys",new Ee(()=>this.keys())],["values",new Ee(()=>this.values())],["dictsort",new Ee(e=>{let t=new Map,r=e.filter(u=>u instanceof Fr?(t=u.value,!1):!0),n=r.at(0)??t.get("case_sensitive")??new ue(!1);if(!(n instanceof ue))throw new Error("case_sensitive must be a boolean");let s=r.at(1)??t.get("by")??new J("key");if(!(s instanceof J))throw new Error("by must be a string");if(!["key","value"].includes(s.value))throw new Error("by must be either 'key' or 'value'");let a=r.at(2)??t.get("reverse")??new ue(!1);if(!(a instanceof ue))throw new Error("reverse must be a boolean");let i=Array.from(this.value.entries()).map(([u,o])=>new pe([new J(u),o])).sort((u,o)=>{let l=s.value==="key"?0:1,c=u.value[l],d=o.value[l],p=Uu(c,d,n.value);return a.value?-p:p});return new pe(i)})]])}items(){return new pe(Array.from(this.value.entries()).map(([e,t])=>new pe([new J(e),t])))}keys(){return new pe(Array.from(this.value.keys()).map(e=>new J(e)))}values(){return new pe(Array.from(this.value.values()))}toString(){return mr(this,{},0,!1)}},Fr=class extends Xe{type="KeywordArgumentsValue"},Hu=class extends Tt{type="NamespaceValue";toString(){return mr(this,{},0,!1)}},pe=class extends Tt{type="ArrayValue";_builtins;get builtins(){return this._builtins??=new Map([["length",new oe(this.value.length)]])}__bool__(){return new ue(this.value.length>0)}toString(){return mr(this,{},0,!1)}},fd=class extends pe{type="TupleValue"},Ee=class extends Tt{type="FunctionValue"},De=class extends Tt{type="NullValue"},Ie=class extends Tt{type="UndefinedValue"};function W3(e){let t;if(e instanceof pe?t=e.value:e instanceof J&&(t=Array.from(e.value,s=>new J(s))),!t||t.length!==2)throw new Error("namespace expected an object or an iterable of [key, value] pairs");let[r,n]=t;if(!(r instanceof J))throw new Error("namespace keys must be strings");return[r,n]}var hd=class{constructor(e){this.parent=e}variables=new Map([["namespace",new Ee(e=>{let t=e.slice(),r;if(t.at(-1)instanceof Fr&&(r=t.pop()),t.length>1)throw new Error(`namespace expected at most 1 argument, got ${t.length}`);let n=new Map;if(t.length===1){let s=t[0];if(s instanceof Xe)for(let[a,i]of s.value)n.set(a,i);else if(s instanceof pe)for(let a of s.value){let[i,u]=W3(a);n.set(i.value,u)}else throw new Error(`'${s.type}' object is not iterable`)}if(r)for(let[s,a]of r.value)n.set(s,a);return new Hu(n)})]]);tests=hd.TESTS;set(e,t){return this.declareVariable(e,ma(t))}declareVariable(e,t){if(this.variables.has(e))throw new SyntaxError(`Variable already declared: ${e}`);return this.variables.set(e,t),t}setVariable(e,t){return this.variables.set(e,t),t}resolve(e){if(this.variables.has(e))return this;if(this.parent)return this.parent.resolve(e);throw new Error(`Unknown variable: ${e}`)}lookupVariable(e){try{return this.resolve(e).variables.get(e)??new Ie}catch{return new Ie}}},mt=hd;o3(mt,"TESTS",new Map([["boolean",e=>e.type==="BooleanValue"],["callable",e=>e instanceof Ee],["odd",e=>{if(!(e instanceof oe))throw new Error(`cannot odd on ${e.type}`);return e.value%2!==0}],["even",e=>{if(!(e instanceof oe))throw new Error(`cannot even on ${e.type}`);return e.value%2===0}],["false",e=>e.type==="BooleanValue"&&!e.value],["true",e=>e.type==="BooleanValue"&&e.value],["none",e=>e.type==="NullValue"],["string",e=>e.type==="StringValue"],["number",e=>e instanceof oe||e instanceof ke],["integer",e=>e instanceof oe],["iterable",e=>e.type==="ArrayValue"||e.type==="StringValue"],["mapping",e=>e instanceof Xe],["sequence",e=>e instanceof pe||e instanceof Xe||e instanceof J],["lower",e=>{let t=e.value;return e.type==="StringValue"&&t===t.toLowerCase()}],["upper",e=>{let t=e.value;return e.type==="StringValue"&&t===t.toUpperCase()}],["none",e=>e.type==="NullValue"],["defined",e=>e.type!=="UndefinedValue"],["undefined",e=>e.type==="UndefinedValue"],["equalto",(e,t)=>e.value===t.value],["eq",(e,t)=>e.value===t.value]]));function z3(e){e.set("false",!1),e.set("true",!0),e.set("none",null),e.set("raise_exception",t=>{throw new Error(t)}),e.set("range",G3),e.set("strftime_now",L3),e.set("True",!0),e.set("False",!1),e.set("None",null)}function ga(e){return e instanceof Xe||e instanceof Hu}function ha(e){return e instanceof oe||e instanceof ke||e instanceof ue}function va(e){return e instanceof ue?Number(e.value):e.value}function pd(e,t){let r=t.split("."),n=e;for(let s of r)if(ga(n))n=n.value.get(s)??new Ie;else if(n instanceof pe){let a=parseInt(s,10);if(!isNaN(a)&&a>=0&&a<n.value.length)n=n.value[a];else return new Ie}else return new Ie;return n}function Uu(e,t,r=!1){if(e instanceof De&&t instanceof De)return 0;if(e instanceof De||t instanceof De)throw new Error(`Cannot compare ${e.type} with ${t.type}`);if(e instanceof Ie&&t instanceof Ie)return 0;if(e instanceof Ie||t instanceof Ie)throw new Error(`Cannot compare ${e.type} with ${t.type}`);if(ha(e)&&ha(t)){let n=va(e),s=va(t);return n<s?-1:n>s?1:0}if(e.type!==t.type)throw new Error(`Cannot compare different types: ${e.type} and ${t.type}`);if(e.type==="StringValue"){let n=e.value,s=t.value;return r||(n=n.toLowerCase(),s=s.toLowerCase()),n<s?-1:n>s?1:0}else throw new Error(`Cannot compare type: ${e.type}`)}function Nu(e){if(e.type==="Identifier")return{name:e.value};let t=e;return{name:t.key.value,defaultValue:t.value}}function gd(e){return e==="kwargs"||e==="varargs"}function md(e,t){let r=new Set,n=new Set,s=c=>{gd(c)&&r.add(c)},a=c=>{if(c.type==="Identifier")s(c.value);else if(c.type==="TupleLiteral")for(let d of c.value)a(d)},i=c=>{s(Nu(c).name)},u=c=>{for(let d of c){let p=Nu(d);p.defaultValue&&l(p.defaultValue)}},o=c=>{c.type==="CallExpression"&&l(c.args)},l=c=>{if(Array.isArray(c)){for(let d of c)l(d);return}if(c instanceof Map){for(let[d,p]of c)l(d),l(p);return}if(c instanceof ht)switch(c.type){case"For":{let d=c;if(a(d.loopvar),d.iterable.type==="SelectExpression"){let p=d.iterable;l(p.lhs),l(d.body),l(d.defaultBlock),l(p.test);return}l(d.iterable),l(d.body),l(d.defaultBlock);return}case"Set":{let d=c;a(d.assignee),l(d.value),l(d.body);return}case"Macro":{let d=c;for(let p of d.args)i(p);u(d.args),l(d.body);return}case"CallStatement":{let d=c;l(d.call);for(let p of d.callerArgs??[])i(p);u(d.callerArgs??[]),l(d.body);return}case"FilterStatement":{let d=c;l(d.body),o(d.filter);return}case"Identifier":{let d=c.value;gd(d)&&!r.has(d)&&n.add(d);return}case"MemberExpression":{let d=c;l(d.object),d.computed&&l(d.property);return}case"FilterExpression":{let d=c;l(d.operand),o(d.filter);return}case"TestExpression":l(c.operand);return;case"SelectExpression":{let d=c;l(d.test),l(d.lhs);return}case"KeywordArgumentExpression":l(c.value);return;default:for(let d of Object.values(c))l(d)}};for(let c of e)i(c);return l(t),n}var Hn=class{global;constructor(e){this.global=e??new mt}run(e){return this.evaluate(e,this.global)}evaluateBinaryExpression(e,t){let r=this.evaluate(e.left,t);switch(e.operator.value){case"and":return r.__bool__().value?this.evaluate(e.right,t):r;case"or":return r.__bool__().value?r:this.evaluate(e.right,t)}let n=this.evaluate(e.right,t);switch(e.operator.value){case"==":return new ue(r.value==n.value);case"!=":return new ue(r.value!=n.value)}if(r instanceof Ie||n instanceof Ie){if(n instanceof Ie&&["in","not in"].includes(e.operator.value))return new ue(e.operator.value==="not in");throw new Error(`Cannot perform operation ${e.operator.value} on undefined values`)}else{if(r instanceof De||n instanceof De)throw new Error("Cannot perform operation on null values");if(e.operator.value==="~")return new J(r.value.toString()+n.value.toString());if(e.operator.value==="**"&&ha(r)&&ha(n)){let s=va(r),a=va(n);if(s===0&&a<0)throw new Error("0.0 cannot be raised to a negative power");let i=s**a;if(!Number.isFinite(i))throw new Error("Exponentiation result is not a finite real number");return r instanceof ke||n instanceof ke||a<0?new ke(i):new oe(i)}else if((r instanceof oe||r instanceof ke)&&(n instanceof oe||n instanceof ke)){let s=r.value,a=n.value;switch(e.operator.value){case"+":case"-":case"*":{let i=e.operator.value==="+"?s+a:e.operator.value==="-"?s-a:s*a;return r instanceof ke||n instanceof ke?new ke(i):new oe(i)}case"/":return new ke(s/a);case"//":{let i=Math.floor(s/a);return r instanceof ke||n instanceof ke?new ke(i):new oe(i)}case"%":{let i=s%a;return r instanceof ke||n instanceof ke?new ke(i):new oe(i)}case"<":return new ue(s<a);case">":return new ue(s>a);case">=":return new ue(s>=a);case"<=":return new ue(s<=a)}}else if(r instanceof pe&&n instanceof pe){if(e.operator.value==="+")return new pe(r.value.concat(n.value))}else if(n instanceof pe){let s=n.value.find(a=>a.value===r.value)!==void 0;switch(e.operator.value){case"in":return new ue(s);case"not in":return new ue(!s)}}}if((r instanceof J||n instanceof J)&&e.operator.value==="+")return new J(r.value.toString()+n.value.toString());if(r instanceof J&&n instanceof J)switch(e.operator.value){case"in":return new ue(n.value.includes(r.value));case"not in":return new ue(!n.value.includes(r.value))}if(r instanceof J&&n instanceof Xe)switch(e.operator.value){case"in":return new ue(n.value.has(r.value));case"not in":return new ue(!n.value.has(r.value))}throw new SyntaxError(`Unknown operator "${e.operator.value}" between ${r.type} and ${n.type}`)}evaluateArguments(e,t){let r=[],n=new Map,s=(a,i)=>{if(n.has(a))throw new Error(`Got multiple values for keyword argument '${a}'`);n.set(a,i)};for(let a of e)if(a.type==="SpreadExpression"){let i=a,u=this.evaluate(i.argument,t);if(!(u instanceof pe))throw new Error(`Cannot unpack non-iterable type: ${u.type}`);for(let o of u.value)r.push(o)}else a.type!=="KeywordArgumentExpression"&&a.type!=="KeywordSpreadExpression"&&r.push(this.evaluate(a,t));for(let a of e)if(a.type==="KeywordArgumentExpression"){let i=a;s(i.key.value,this.evaluate(i.value,t))}else if(a.type==="KeywordSpreadExpression"){let i=a,u=this.evaluate(i.argument,t);if(!(u instanceof Xe))throw new Error(`Argument after ** must be a mapping, not ${u.type}`);for(let[o,l]of u.value)s(o,l)}return[r,n]}applyFilter(e,t,r){if(t.type==="Identifier"){let n=t;if(n.value==="safe")return e;if(n.value==="tojson")return new J(mr(e,{}));if(e instanceof pe)switch(n.value){case"list":return e;case"first":return e.value[0];case"last":return e.value[e.value.length-1];case"length":return new oe(e.value.length);case"reverse":return new pe(e.value.slice().reverse());case"sort":return new pe(e.value.slice().sort((s,a)=>Uu(s,a,!1)));case"join":return new J(e.value.map(s=>s.value).join(""));case"string":return new J(mr(e,{},0,!1));case"unique":{let s=new Set,a=[];for(let i of e.value)s.has(i.value)||(s.add(i.value),a.push(i));return new pe(a)}default:throw new Error(`Unknown ArrayValue filter: ${n.value}`)}else if(e instanceof J)switch(n.value){case"length":case"upper":case"lower":case"title":case"capitalize":{let s=e.builtins.get(n.value);if(s instanceof Ee)return s.value([],r);if(s instanceof oe)return s;throw new Error(`Unknown StringValue filter: ${n.value}`)}case"trim":return new J(e.value.trim());case"indent":return new J(e.value.split(`
`).map((s,a)=>a===0||s.length===0?s:"    "+s).join(`
`));case"join":case"string":return e;case"int":{let s=parseInt(e.value,10);return new oe(isNaN(s)?0:s)}case"float":{let s=parseFloat(e.value);return new ke(isNaN(s)?0:s)}default:throw new Error(`Unknown StringValue filter: ${n.value}`)}else if(e instanceof oe||e instanceof ke)switch(n.value){case"abs":return e instanceof oe?new oe(Math.abs(e.value)):new ke(Math.abs(e.value));case"int":return new oe(Math.floor(e.value));case"float":return new ke(e.value);case"string":return new J(e.toString());default:throw new Error(`Unknown NumericValue filter: ${n.value}`)}else if(e instanceof Xe)switch(n.value){case"items":return new pe(Array.from(e.value.entries()).map(([s,a])=>new pe([new J(s),a])));case"length":return new oe(e.value.size);default:{let s=e.builtins.get(n.value);if(s)return s instanceof Ee?s.value([],r):s;throw new Error(`Unknown ObjectValue filter: ${n.value}`)}}else if(e instanceof ue)switch(n.value){case"bool":return new ue(e.value);case"int":return new oe(e.value?1:0);case"float":return new ke(e.value?1:0);case"string":return new J(e.value?"true":"false");default:throw new Error(`Unknown BooleanValue filter: ${n.value}`)}throw new Error(`Cannot apply filter "${n.value}" to type: ${e.type}`)}else if(t.type==="CallExpression"){let n=t;if(n.callee.type!=="Identifier")throw new Error(`Unknown filter: ${n.callee.type}`);let s=n.callee.value;if(s==="tojson"){let[,a]=this.evaluateArguments(n.args,r),i=a.get("indent")??new De;if(!(i instanceof oe||i instanceof De))throw new Error("If set, indent must be a number");let u=a.get("ensure_ascii")??new ue(!1);if(!(u instanceof ue))throw new Error("If set, ensure_ascii must be a boolean");let o=a.get("sort_keys")??new ue(!1);if(!(o instanceof ue))throw new Error("If set, sort_keys must be a boolean");let l=a.get("separators")??new De,c=null;if(l instanceof pe||l instanceof fd){if(l.value.length!==2)throw new Error("separators must be a tuple of two strings");let[d,p]=l.value;if(!(d instanceof J)||!(p instanceof J))throw new Error("separators must be a tuple of two strings");c=[d.value,p.value]}else if(!(l instanceof De))throw new Error("If set, separators must be a tuple of two strings");return new J(mr(e,{indent:i.value,ensureAscii:u.value,sortKeys:o.value,separators:c}))}else if(s==="join"){let a;if(e instanceof J)a=Array.from(e.value);else if(e instanceof pe)a=e.value.map(l=>l.value);else throw new Error(`Cannot apply filter "${s}" to type: ${e.type}`);let[i,u]=this.evaluateArguments(n.args,r),o=i.at(0)??u.get("separator")??new J("");if(!(o instanceof J))throw new Error("separator must be a string");return new J(a.join(o.value))}else if(s==="int"||s==="float"){let[a,i]=this.evaluateArguments(n.args,r),u=a.at(0)??i.get("default")??(s==="int"?new oe(0):new ke(0));if(e instanceof J){let o=s==="int"?parseInt(e.value,10):parseFloat(e.value);return isNaN(o)?u:s==="int"?new oe(o):new ke(o)}else{if(e instanceof oe||e instanceof ke)return e;if(e instanceof ue)return s==="int"?new oe(e.value?1:0):new ke(e.value?1:0);throw new Error(`Cannot apply filter "${s}" to type: ${e.type}`)}}else if(s==="default"){let[a,i]=this.evaluateArguments(n.args,r),u=a[0]??new J(""),o=a[1]??i.get("boolean")??new ue(!1);if(!(o instanceof ue))throw new Error("`default` filter flag must be a boolean");return e instanceof Ie||o.value&&!e.__bool__().value?u:e}if(e instanceof pe){switch(s){case"sort":{let[a,i]=this.evaluateArguments(n.args,r),u=a.at(0)??i.get("reverse")??new ue(!1);if(!(u instanceof ue))throw new Error("reverse must be a boolean");let o=a.at(1)??i.get("case_sensitive")??new ue(!1);if(!(o instanceof ue))throw new Error("case_sensitive must be a boolean");let l=a.at(2)??i.get("attribute")??new De;if(!(l instanceof J||l instanceof oe||l instanceof De))throw new Error("attribute must be a string, integer, or null");let c=d=>{if(l instanceof De)return d;let p=l instanceof oe?String(l.value):l.value;return pd(d,p)};return new pe(e.value.slice().sort((d,p)=>{let f=c(d),g=c(p),m=Uu(f,g,o.value);return u.value?-m:m}))}case"selectattr":case"rejectattr":{let a=s==="selectattr";if(e.value.some(d=>!ga(d)))throw new Error(`\`${s}\` can only be applied to array of objects`);if(n.args.some(d=>d.type!=="StringLiteral"))throw new Error(`arguments of \`${s}\` must be strings`);let[i,u,o]=n.args.map(d=>this.evaluate(d,r)),l;if(u){let d=r.tests.get(u.value);if(!d)throw new Error(`Unknown test: ${u.value}`);l=d}else l=(...d)=>d[0].__bool__().value;let c=e.value.filter(d=>{let p=d.value.get(i.value),f=p?l(p,o):!1;return a?f:!f});return new pe(c)}case"map":{let[,a]=this.evaluateArguments(n.args,r);if(a.has("attribute")){let i=a.get("attribute");if(!(i instanceof J))throw new Error("attribute must be a string");let u=a.get("default"),o=e.value.map(l=>{if(!ga(l))throw new Error("items in map must be an object");let c=pd(l,i.value);return c instanceof Ie?u??new Ie:c});return new pe(o)}else throw new Error("`map` expressions without `attribute` set are not currently supported.")}}throw new Error(`Unknown ArrayValue filter: ${s}`)}else if(e instanceof J){switch(s){case"indent":{let[a,i]=this.evaluateArguments(n.args,r),u=a.at(0)??i.get("width")??new oe(4);if(!(u instanceof oe))throw new Error("width must be a number");let o=a.at(1)??i.get("first")??new ue(!1),l=a.at(2)??i.get("blank")??new ue(!1),c=e.value.split(`
`),d=" ".repeat(u.value),p=c.map((f,g)=>!o.value&&g===0||!l.value&&f.length===0?f:d+f);return new J(p.join(`
`))}case"replace":{let a=e.builtins.get("replace");if(!(a instanceof Ee))throw new Error("replace filter not available");let[i,u]=this.evaluateArguments(n.args,r);return a.value([...i,new Fr(u)],r)}}throw new Error(`Unknown StringValue filter: ${s}`)}else if(e instanceof Xe){let a=e.builtins.get(s);if(a&&a instanceof Ee){let[i,u]=this.evaluateArguments(n.args,r);return u.size>0&&i.push(new Fr(u)),a.value(i,r)}throw new Error(`Unknown ObjectValue filter: ${s}`)}else throw new Error(`Cannot apply filter "${s}" to type: ${e.type}`)}throw new Error(`Unknown filter: ${t.type}`)}evaluateFilterExpression(e,t){let r=this.evaluate(e.operand,t);return this.applyFilter(r,e.filter,t)}evaluateTestExpression(e,t){let r=this.evaluate(e.operand,t),n=t.tests.get(e.test.value);if(!n)throw new Error(`Unknown test: ${e.test.value}`);let s=n(r);return new ue(e.negate?!s:s)}evaluateSelectExpression(e,t){return this.evaluate(e.test,t).__bool__().value?this.evaluate(e.lhs,t):new Ie}evaluateUnaryExpression(e,t){let r=this.evaluate(e.argument,t);switch(e.operator.value){case"not":return new ue(!r.value);case"+":case"-":{let n=e.operator.value==="-"?-1:1;if(r instanceof oe||r instanceof ke||r instanceof ue){let s=r instanceof ue?r.value?1:0:r.value,a=n*s;return r instanceof ke?new ke(a):new oe(a)}throw new SyntaxError(`Unknown operator "${e.operator.value}" for ${r.type}`)}default:throw new SyntaxError(`Unknown operator: ${e.operator.value}`)}}evaluateTernaryExpression(e,t){return this.evaluate(e.condition,t).__bool__().value?this.evaluate(e.trueExpr,t):this.evaluate(e.falseExpr,t)}evalProgram(e,t){return this.evaluateBlock(e.body,t)}evaluateBlock(e,t){let r="";for(let n of e){let s=this.evaluate(n,t);s.type!=="NullValue"&&s.type!=="UndefinedValue"&&(r+=s.toString())}return new J(r)}evaluateIdentifier(e,t){return t.lookupVariable(e.value)}evaluateCallExpression(e,t){let[r,n]=this.evaluateArguments(e.args,t);n.size>0&&r.push(new Fr(n));let s=this.evaluate(e.callee,t);if(s.type!=="FunctionValue")throw new Error(`Cannot call something that is not a function: got ${s.type}`);return s.value(r,t)}evaluateSliceExpression(e,t,r){if(!(e instanceof pe||e instanceof J))throw new Error("Slice object must be an array or string");let n=this.evaluate(t.start,r),s=this.evaluate(t.stop,r),a=this.evaluate(t.step,r);if(!(n instanceof oe||n instanceof Ie))throw new Error("Slice start must be numeric or undefined");if(!(s instanceof oe||s instanceof Ie))throw new Error("Slice stop must be numeric or undefined");if(!(a instanceof oe||a instanceof Ie))throw new Error("Slice step must be numeric or undefined");return e instanceof pe?new pe(od(e.value,n.value,s.value,a.value)):new J(od(Array.from(e.value),n.value,s.value,a.value).join(""))}evaluateMemberExpression(e,t){let r=this.evaluate(e.object,t),n;if(e.computed){if(e.property.type==="SliceExpression")return this.evaluateSliceExpression(r,e.property,t);n=this.evaluate(e.property,t)}else e.property.type==="IntegerLiteral"?n=new oe(e.property.value):n=new J(e.property.value);let s;if(ga(r)){if(!(n instanceof J))throw new Error(`Cannot access property with non-string: got ${n.type}`);s=r.value.get(n.value)??(r instanceof Xe?r.builtins.get(n.value):void 0)}else if(r instanceof pe||r instanceof J)if(n instanceof oe)s=r.value.at(n.value),r instanceof J&&(s=new J(r.value.at(n.value)));else if(n instanceof J)s=r.builtins.get(n.value);else throw new Error(`Cannot access property with non-string/non-number: got ${n.type}`);else{if(!(n instanceof J))throw new Error(`Cannot access property with non-string: got ${n.type}`);s=r.builtins.get(n.value)}return s instanceof Tt?s:new Ie}evaluateSet(e,t){let r=e.value?this.evaluate(e.value,t):this.evaluateBlock(e.body,t);if(e.assignee.type==="Identifier"){let n=e.assignee.value;t.setVariable(n,r)}else if(e.assignee.type==="TupleLiteral"){let n=e.assignee;if(!(r instanceof pe))throw new Error(`Cannot unpack non-iterable type in set: ${r.type}`);let s=r.value;if(s.length!==n.value.length)throw new Error(`Too ${n.value.length>s.length?"few":"many"} items to unpack in set`);for(let a=0;a<n.value.length;++a){let i=n.value[a];if(i.type!=="Identifier")throw new Error(`Cannot unpack to non-identifier in set: ${i.type}`);t.setVariable(i.value,s[a])}}else if(e.assignee.type==="MemberExpression"){let n=e.assignee,s=this.evaluate(n.object,t);if(!(s instanceof Hu))throw new Error("cannot assign attribute on non-namespace object");if(n.property.type!=="Identifier")throw new Error("Cannot assign to member with non-identifier property");s.value.set(n.property.value,r)}else throw new Error(`Invalid LHS inside assignment expression: ${JSON.stringify(e.assignee)}`);return new De}evaluateIf(e,t){let r=this.evaluate(e.test,t);return this.evaluateBlock(r.__bool__().value?e.body:e.alternate,t)}evaluateFor(e,t){let r=new mt(t),n,s;if(e.iterable.type==="SelectExpression"){let l=e.iterable;s=this.evaluate(l.lhs,r),n=l.test}else s=this.evaluate(e.iterable,r);if(!(s instanceof pe||s instanceof Xe))throw new Error(`Expected iterable or object type in for loop: got ${s.type}`);s instanceof Xe&&(s=s.keys());let a=[],i=[];for(let l=0;l<s.value.length;++l){let c=new mt(r),d=s.value[l],p;if(e.loopvar.type==="Identifier")p=f=>f.setVariable(e.loopvar.value,d);else if(e.loopvar.type==="TupleLiteral"){let f=e.loopvar;if(d.type!=="ArrayValue")throw new Error(`Cannot unpack non-iterable type: ${d.type}`);let g=d;if(f.value.length!==g.value.length)throw new Error(`Too ${f.value.length>g.value.length?"few":"many"} items to unpack`);p=m=>{for(let h=0;h<f.value.length;++h){if(f.value[h].type!=="Identifier")throw new Error(`Cannot unpack non-identifier type: ${f.value[h].type}`);m.setVariable(f.value[h].value,g.value[h])}}}else throw new Error(`Invalid loop variable(s): ${e.loopvar.type}`);n&&(p(c),!this.evaluate(n,c).__bool__().value)||(a.push(d),i.push(p))}let u="",o=!0;for(let l=0;l<a.length;++l){let c=new Map([["index",new oe(l+1)],["index0",new oe(l)],["revindex",new oe(a.length-l)],["revindex0",new oe(a.length-l-1)],["first",new ue(l===0)],["last",new ue(l===a.length-1)],["length",new oe(a.length)],["previtem",l>0?a[l-1]:new Ie],["nextitem",l<a.length-1?a[l+1]:new Ie]]);r.setVariable("loop",new Xe(c)),i[l](r);try{let d=this.evaluateBlock(e.body,r);u+=d.value}catch(d){if(d instanceof cd)continue;if(d instanceof ld)break;throw d}o=!1}if(o){let l=this.evaluateBlock(e.defaultBlock,r);u+=l.value}return new J(u)}bindMacroArguments(e,t,r,n,s){let a=n.slice(),i=new Map;a.at(-1)instanceof Fr&&(i=new Map(a.pop().value));let u=[];for(let o=0;o<t.length;++o){let{name:l,defaultValue:c}=Nu(t[o]),d=a[o];d===void 0&&i.has(l)&&(d=i.get(l),i.delete(l)),d===void 0&&c!==void 0&&u.push([l,c]),s.setVariable(l,d??new Ie)}if(r.has("kwargs"))s.setVariable("kwargs",new Xe(i));else if(i.size>0)throw new Error(`macro ${e} takes no keyword argument '${i.keys().next().value}'`);if(r.has("varargs"))s.setVariable("varargs",new pe(a.slice(t.length)));else if(a.length>t.length)throw new Error(`macro ${e} takes not more than ${t.length} argument(s)`);for(let[o,l]of u)s.setVariable(o,this.evaluate(l,s))}evaluateMacro(e,t){let r=md(e.args,e.body);return t.setVariable(e.name.value,new Ee((n,s)=>{let a=new mt(s);return this.bindMacroArguments(`'${e.name.value}'`,e.args,r,n,a),this.evaluateBlock(e.body,a)})),new De}evaluateCallStatement(e,t){let r=e.callerArgs??[],n=md(r,e.body),s=new Ee(l=>{let c=new mt(t);return this.bindMacroArguments("None",r,n,l,c),this.evaluateBlock(e.body,c)}),[a,i]=this.evaluateArguments(e.call.args,t);a.push(new Fr(i));let u=this.evaluate(e.call.callee,t);if(u.type!=="FunctionValue")throw new Error(`Cannot call something that is not a function: got ${u.type}`);let o=new mt(t);return o.setVariable("caller",s),u.value(a,o)}evaluateFilterStatement(e,t){let r=this.evaluateBlock(e.body,t);return this.applyFilter(r,e.filter,t)}evaluate(e,t){if(!e)return new Ie;switch(e.type){case"Program":return this.evalProgram(e,t);case"Set":return this.evaluateSet(e,t);case"If":return this.evaluateIf(e,t);case"For":return this.evaluateFor(e,t);case"Macro":return this.evaluateMacro(e,t);case"CallStatement":return this.evaluateCallStatement(e,t);case"Break":throw new ld;case"Continue":throw new cd;case"IntegerLiteral":return new oe(e.value);case"FloatLiteral":return new ke(e.value);case"StringLiteral":return new J(e.value);case"ArrayLiteral":return new pe(e.value.map(r=>this.evaluate(r,t)));case"TupleLiteral":return new fd(e.value.map(r=>this.evaluate(r,t)));case"ObjectLiteral":{let r=new Map;for(let[n,s]of e.value){let a=this.evaluate(n,t);if(!(a instanceof J))throw new Error(`Object keys must be strings: got ${a.type}`);r.set(a.value,this.evaluate(s,t))}return new Xe(r)}case"Identifier":return this.evaluateIdentifier(e,t);case"CallExpression":return this.evaluateCallExpression(e,t);case"MemberExpression":return this.evaluateMemberExpression(e,t);case"UnaryExpression":return this.evaluateUnaryExpression(e,t);case"BinaryExpression":return this.evaluateBinaryExpression(e,t);case"FilterExpression":return this.evaluateFilterExpression(e,t);case"FilterStatement":return this.evaluateFilterStatement(e,t);case"TestExpression":return this.evaluateTestExpression(e,t);case"SelectExpression":return this.evaluateSelectExpression(e,t);case"Ternary":return this.evaluateTernaryExpression(e,t);case"Comment":return new De;default:throw new SyntaxError(`Unknown node type: ${e.type}`)}}};function ma(e){switch(typeof e){case"number":return Number.isInteger(e)?new oe(e):new ke(e);case"string":return new J(e);case"boolean":return new ue(e);case"undefined":return new Ie;case"object":return e===null?new De:Array.isArray(e)?new pe(e.map(ma)):new Xe(new Map(Object.entries(e).map(([t,r])=>[t,ma(r)])));case"function":return new Ee((t,r)=>{let n=e(...t.map(s=>s.value))??null;return ma(n)});default:throw new Error(`Cannot convert to runtime value: ${e}`)}}var Ke=`
`,K3="{%- ",U3=" -%}",Me=Object.freeze({CONDITIONAL:0,LOGICAL_OR:1,LOGICAL_AND:2,LOGICAL_NOT:3,COMPARISON:4,ADDITIVE:5,MULTIPLICATIVE:6,EXPONENTIATION:7,TEST:8,FILTER:9,UNARY_SIGN:10,ATOM:11});function vd(e){switch(e.operator.type){case"ExponentiationBinaryOperator":return Me.EXPONENTIATION;case"MultiplicativeBinaryOperator":return Me.MULTIPLICATIVE;case"AdditiveBinaryOperator":return Me.ADDITIVE;case"ComparisonBinaryOperator":return Me.COMPARISON;case"Identifier":return e.operator.value==="and"?Me.LOGICAL_AND:e.operator.value==="in"||e.operator.value==="not in"?Me.COMPARISON:Me.LOGICAL_OR}return Me.LOGICAL_OR}function wd(e){switch(e.type){case"SelectExpression":case"Ternary":return Me.CONDITIONAL;case"BinaryExpression":return vd(e);case"UnaryExpression":return e.operator.value==="not"?Me.LOGICAL_NOT:Me.UNARY_SIGN;case"TestExpression":return Me.TEST;case"FilterExpression":return Me.FILTER;default:return Me.ATOM}}function St(e,t){let r=Le(e);return wd(e)<t?`(${r})`:r}function N3(e,t="	"){let r=typeof t=="number"?" ".repeat(t):t;return xt(e.body,0,r).replace(/\n$/,"")}function Ue(...e){return K3+e.join(" ")+U3}function xt(e,t,r){return e.map(n=>H3(n,t,r)).join(Ke)}function wa(e){return e.map(t=>Le(t)).join(", ")}function H3(e,t,r){let n=r.repeat(t);switch(e.type){case"Program":return xt(e.body,t,r);case"If":return $3(e,t,r);case"For":return V3(e,t,r);case"Set":return Z3(e,t,r);case"Macro":return X3(e,t,r);case"Break":return n+Ue("break");case"Continue":return n+Ue("continue");case"CallStatement":return Y3(e,t,r);case"FilterStatement":return J3(e,t,r);case"Comment":return n+"{# "+e.value+" #}";default:return n+"{{- "+Le(e)+" -}}"}}function $3(e,t,r){let n=r.repeat(t),s=[],a=e;for(;a&&(s.push({test:a.test,body:a.body}),a.alternate.length===1&&a.alternate[0].type==="If");)a=a.alternate[0];let i=n+Ue("if",Le(s[0].test))+Ke+xt(s[0].body,t+1,r);for(let u=1;u<s.length;++u)i+=Ke+n+Ue("elif",Le(s[u].test))+Ke+xt(s[u].body,t+1,r);return a&&a.alternate.length>0&&(i+=Ke+n+Ue("else")+Ke+xt(a.alternate,t+1,r)),i+=Ke+n+Ue("endif"),i}function V3(e,t,r){let n=r.repeat(t),s="";if(e.iterable.type==="SelectExpression"){let i=e.iterable;s=`${Le(i.lhs)} if ${Le(i.test)}`}else s=Le(e.iterable);let a=n+Ue("for",Le(e.loopvar),"in",s)+Ke+xt(e.body,t+1,r);return e.defaultBlock.length>0&&(a+=Ke+n+Ue("else")+Ke+xt(e.defaultBlock,t+1,r)),a+=Ke+n+Ue("endfor"),a}function Z3(e,t,r){let n=r.repeat(t),s=Le(e.assignee),a=e.value?Le(e.value):"",i=n+Ue("set",`${s}${e.value?" = "+a:""}`);return e.body.length===0?i:i+Ke+xt(e.body,t+1,r)+Ke+n+Ue("endset")}function X3(e,t,r){let n=r.repeat(t),s=wa(e.args);return n+Ue("macro",`${e.name.value}(${s})`)+Ke+xt(e.body,t+1,r)+Ke+n+Ue("endmacro")}function Y3(e,t,r){let n=r.repeat(t),s=e.callerArgs&&e.callerArgs.length>0?`(${wa(e.callerArgs)})`:"",a=Le(e.call),i=n+Ue(`call${s}`,a)+Ke;return i+=xt(e.body,t+1,r)+Ke,i+=n+Ue("endcall"),i}function J3(e,t,r){let n=r.repeat(t),s=e.filter.type==="Identifier"?e.filter.value:Le(e.filter),a=n+Ue("filter",s)+Ke;return a+=xt(e.body,t+1,r)+Ke,a+=n+Ue("endfilter"),a}function Le(e){switch(e.type){case"SpreadExpression":return`*${Le(e.argument)}`;case"KeywordSpreadExpression":return`**${Le(e.argument)}`;case"Identifier":return e.value;case"IntegerLiteral":return`${e.value}`;case"FloatLiteral":{let t=e.value;return Object.is(t,-0)?"-0.0":t%1===0?t.toFixed(1):t.toString()}case"StringLiteral":return JSON.stringify(e.value);case"BinaryExpression":{let t=e,r=vd(t),n=St(t.left,r),s=St(t.right,r+1);return`${n} ${t.operator.value} ${s}`}case"UnaryExpression":{let t=e,r=t.argument.type==="UnaryExpression"?wd(t):Me.ATOM;return t.operator.value+(t.operator.value==="not"?" ":"")+St(t.argument,r)}case"CallExpression":{let t=e,r=wa(t.args);return`${Le(t.callee)}(${r})`}case"MemberExpression":{let t=e,r=St(t.object,Me.ATOM),n=Le(t.property);return!t.computed&&t.property.type!=="Identifier"&&t.property.type!=="IntegerLiteral"&&(n=`(${n})`),t.computed?`${r}[${n}]`:`${r}.${n}`}case"FilterExpression":{let t=e,r=St(t.operand,Me.FILTER);return t.filter.type==="CallExpression"?`${r} | ${Le(t.filter)}`:`${r} | ${t.filter.value}`}case"SelectExpression":{let t=e;return`${St(t.lhs,Me.LOGICAL_OR)} if ${St(t.test,Me.LOGICAL_OR)}`}case"TestExpression":{let t=e;return`${St(t.operand,Me.TEST)} is${t.negate?" not":""} ${t.test.value}`}case"ArrayLiteral":case"TupleLiteral":{let t=wa(e.value),r=e.type==="ArrayLiteral"?"[]":"()";return`${r[0]}${t}${r[1]}`}case"ObjectLiteral":return`{${Array.from(e.value.entries()).map(([r,n])=>`${Le(r)}: ${Le(n)}`).join(", ")}}`;case"SliceExpression":{let t=e,r=t.start?Le(t.start):"",n=t.stop?Le(t.stop):"",s=t.step?`:${Le(t.step)}`:"";return`${r}:${n}${s}`}case"KeywordArgumentExpression":{let t=e;return`${t.key.value}=${Le(t.value)}`}case"Ternary":{let t=e;return`${St(t.trueExpr,Me.LOGICAL_OR)} if ${St(t.condition,Me.LOGICAL_OR)} else ${St(t.falseExpr,Me.CONDITIONAL)}`}default:throw new Error(`Unknown expression type: ${e.type}`)}}var bd=class{parsed;constructor(e){let t=ba(e,{lstrip_blocks:!0,trim_blocks:!0});this.parsed=_a(t)}render(e){let t=new mt;if(z3(t),e)for(let[s,a]of Object.entries(e))t.set(s,a);return new Hn(t).run(this.parsed).value}format(e){return N3(this.parsed,e?.indent||"	")}};var ka=new Set(["repeat","tunables","dtypes","logicalDtypes","tensorDtypes","ranks","shapes","present","args","attrs","device","env","constants","derive","source","DISPATCH_FOLD_WIDTH","f16Ok","subgroupMatrixConfigAvailable","ceil","floor","min","max","pow","ceilDiv","pow2ceil","log2ceil","numel","dim","rows","outer","inner","prefix","suffix","fill","moveAxis","permuteShape","reversedAxes","broadcastShape","matmulShape","broadcastable","sameShape","hasAxis","axisValue","isUniqueIntList","has","dtypeBytes","onnxDtypeCode","pick","true","false","none","True","False","None","range","loop","namespace","dict"]);function er(e){if(typeof e=="number"){if(Number.isNaN(e))return"NaN";if(e===Number.POSITIVE_INFINITY)return"Infinity";if(e===Number.NEGATIVE_INFINITY)return"-Infinity";if(Object.is(e,-0))return"-0"}if(Array.isArray(e))return`[${e.map(er).join(",")}]`;if(e&&typeof e=="object"){let t=e;return`{${Object.keys(t).sort().map(r=>`${JSON.stringify(r)}:${er(t[r])}`).join(",")}}`}return JSON.stringify(e)}var e4=Object.freeze(new Set(Object.getOwnPropertyNames(Object.prototype)));function Aa(e){return!e4.has(e)}function $u(e,t=""){if(Array.isArray(e)){for(let r=0;r<e.length;r+=1){let n=e[r];if(n===null||typeof n!="object")continue;let s=$u(n,`${t}[${r}]`);if(s!==void 0)return s}return}if(!(e===null||typeof e!="object"))for(let[r,n]of Object.entries(e)){let s=t===""?r:`${t}.${r}`;if(!Aa(r))return s;if(n===null||typeof n!="object")continue;let a=$u(n,s);if(a!==void 0)return a}}function _d(e,t){let r=$u(e);if(r!==void 0)throw new Error(`${t}: key "${r}" shadows an Object.prototype member; rename it`)}var kd=[{code:1,label:"F32",logical:"float32",tensor:"tensor(float)"},{code:2,label:"U8",logical:"uint8",tensor:"tensor(uint8)"},{code:3,label:"I8",logical:"int8",tensor:"tensor(int8)"},{code:5,label:"I16",logical:"int16",tensor:"tensor(int16)"},{code:6,label:"I32",logical:"int32",tensor:"tensor(int32)"},{code:7,label:"I64",logical:"int64",tensor:"tensor(int64)"},{code:9,label:"BOOL",logical:"bool",tensor:"tensor(bool)"},{code:10,label:"F16",logical:"float16",tensor:"tensor(float16)"},{code:12,label:"U32",logical:"uint32",tensor:"tensor(uint32)"}],t4=new Map(kd.map(e=>[e.code,e])),Ad=new Map,r4=new Map;for(let e of kd)Ad.set(e.logical,e),r4.set(e.tensor,e.logical);function yd(e){return t4.get(e)}function Sd(e){let t=Ad.get(e);if(t===void 0)throw new Error(`Unsupported ONNX logical dtype: ${String(e)}`);return t.code}function ya(e,t=0){return`${(e/(1024*1024)).toFixed(t)} MiB`}function zt(e,t,r){return e===1?t:r!==void 0?r:/(s|x|z|ch|sh)$/.test(t)?`${t}es`:`${t}s`}var P6=Object.freeze([]);function n4(e){return e!==null&&typeof e=="object"&&typeof e.type=="string"}function Qr(e){if(!n4(e))throw new Error("expected a typed Jinja AST node");return e}function ge(e,t){try{return Qr(e[t])}catch{throw new Error(`Jinja ${e.type??"node"}.${t} is not an AST node`)}}function Ne(e,t){let r=e[t];if(!Array.isArray(r))throw new Error(`Jinja ${e.type??"node"}.${t} is not an AST node array`);for(let n of r)Qr(n);return r}function it(e){if(e.type!=="Identifier"||typeof e.value!="string")throw new Error(`Jinja ${e.type??"node"} is not an Identifier`);return e.value}function xd(e){return e.type==="KeywordArgumentExpression"?it(ge(e,"key")):it(e)}function Vu(e,t){if(e.type==="Identifier"){t(it(e));return}if(e.type==="TupleLiteral"){for(let r of Ne(e,"value"))Vu(r,t);return}throw new Error(`Jinja assignment target ${e.type??"node"} is not a lexical binding`)}function pn(e){if(e.type==="Identifier")return[it(e)];if(e.type!=="MemberExpression")return;let t=pn(ge(e,"object"));if(t===void 0)return;let r=ge(e,"property");return e.computed===!0?r.type==="StringLiteral"&&typeof r.value=="string"?[...t,r.value]:void 0:r.type==="Identifier"?[...t,it(r)]:void 0}function gn(e){return e.type==="Identifier"?it(e):e.type==="MemberExpression"?gn(ge(e,"object")):void 0}function vt(e,t){if(!e||typeof e!="object")return;if(Array.isArray(e)){for(let s of e)vt(s,t);return}if(e instanceof Map){for(let[s,a]of e)vt(s,t),vt(a,t);return}let r=e;if(t(r))return;let n=r.type==="MemberExpression"&&r.computed!==!0;for(let s of Object.keys(r))s==="type"||n&&s==="property"||vt(r[s],t)}function xa(e,t){let r=!1;return vt(e,n=>r?!0:(r=t(n),r)),r}function Td(e){return xa(e,t=>t.type==="Break"||t.type==="Continue")}function Cd(e){if(e.type==="ArrayLiteral"||e.type==="TupleLiteral")return Ne(e,"value").length===0;if(e.type==="StringLiteral")return e.value==="";if(e.type==="ObjectLiteral"){if(!(e.value instanceof Map))throw new Error("Jinja ObjectLiteral.value is not a Map");return e.value.size===0}return!1}var Sa=class e{bindings;parent;constructor(t,r){this.parent=t,this.bindings=new Map(r)}child(){return new e(this)}fork(){return new e(this.parent,this.bindings)}define(t,r){this.bindings.set(t,r)}resolve(t){return this.bindings.get(t)??this.parent?.resolve(t)}replaceResolved(t,r){return this.bindings.has(t)?(this.bindings.set(t,r),!0):this.parent?.replaceResolved(t,r)??!1}localEntries(){return this.bindings}};var qd=Object.freeze({}),$n=Object.freeze({dynamicPossible:!0});function Ca(e,t){let r=ge(e,"object");r.type==="MemberExpression"&&Ca(r,t),e.computed===!0&&t(ge(e,"property"))}function Ta(e,t,r=$n){Vu(e,n=>t.define(n,r))}function Bd(e,t){if(e.type!=="Identifier")return $n;let r=t.resolve(it(e));if((r?.macros?.length??0)===0)return $n;let n=r?.externalPossible===!0||r?.dynamicPossible===!0;return Object.freeze({...n?{dynamicPossible:!0}:{},macros:r.macros})}function Id(e,t){return xa(e,r=>r.type==="Identifier"&&(t.resolve(it(r))?.macros?.length??0)>0)}function Ed(e,t,r){let n=new Set([...t.localEntries().keys(),...r.localEntries().keys()]);for(let s of n){let a=e.resolve(s),i=t.localEntries().get(s)??a,u=r.localEntries().get(s)??a;if(i===u&&i!==void 0){e.define(s,i);continue}let o=[...new Set([...i?.macros??[],...u?.macros??[]])],l=i===void 0||u===void 0||i.externalPossible===!0||u.externalPossible===!0,c=i?.dynamicPossible===!0||u?.dynamicPossible===!0;e.define(s,Object.freeze({...l?{externalPossible:!0}:{},...c?{dynamicPossible:!0}:{},...o.length>0?{macros:Object.freeze(o)}:{}}))}}function Pd(e){let t=Qr(e);if(t.type!=="Program")throw new Error("Jinja template dependency analysis expected a Program AST");let r=new Set,n=new Map,s=new Set,a=(p,f)=>{if(r.add(p),f===void 0){s.add(p),n.delete(p);return}if(s.has(p))return;let g=n.get(p);g===void 0&&(g=new Set,n.set(p,g)),g.add(f)},i=new Set,u=!1,o,l=(p,f,g,m)=>{if(i.has(p)){u=!0;return}i.add(p);try{let h=f.child();g!==void 0&&h.define("caller",Object.freeze({macros:Object.freeze([g])}));let v=[],b=new Map,_=m===void 0;for(let C of m??[])C.type==="KeywordArgumentExpression"?b.set(it(ge(C,"key")),ge(C,"value")):C.type==="SpreadExpression"?_=!0:v.push(C);for(let C=0;C<p.args.length;C+=1){let y=p.args[C],E=xd(y),k=v[C],S=f;if(y.type==="KeywordArgumentExpression"&&(k??=b.get(E),k===void 0)){let A=ge(y,"value");c(A,h),k=_?void 0:A,S=h}h.define(E,k===void 0?$n:Bd(k,S))}o(p.body,h)}finally{i.delete(p)}},c=(p,f,g,m)=>{switch(p.type){case"Identifier":{let h=it(p),v=f.resolve(h);if((v===void 0||v.externalPossible===!0)&&a(h),m!==void 0)for(let b of v?.macros??[])l(b,f,g,m);return}case"IntegerLiteral":case"FloatLiteral":case"StringLiteral":return;case"MemberExpression":{let h=pn(p),v=h?.[0]??gn(p),b=v===void 0?void 0:f.resolve(v);if(v!==void 0&&(b===void 0||b.externalPossible===!0)){h!==void 0&&h.length>=2?a(v,h[1]):a(v),Ca(p,_=>c(_,f));return}c(ge(p,"object"),f),p.computed===!0&&c(ge(p,"property"),f);return}case"CallExpression":{let h=ge(p,"callee"),v=Ne(p,"args"),b=h.type==="MemberExpression"?gn(h):void 0,_=b===void 0?void 0:f.resolve(b),C=h.type==="Identifier"?f.resolve(it(h)):void 0,y=h.type==="Identifier"?C!==void 0&&(C.dynamicPossible===!0||(C.macros?.length??0)===0):b!==void 0&&_!==void 0,E=h.type==="Identifier"?C===void 0||C.externalPossible===!0:b!==void 0&&(_===void 0||_.externalPossible===!0),k=h.type!=="Identifier"&&!(h.type==="MemberExpression"&&b!==void 0);(y||k||E&&v.some(S=>Id(S,f)))&&(u=!0),b!==void 0&&(_===void 0||_.externalPossible===!0)?(a(b),Ca(h,S=>c(S,f))):c(h,f,g,v);for(let S of v)c(S,f);return}case"ArrayLiteral":case"TupleLiteral":for(let h of Ne(p,"value"))c(h,f);return;case"ObjectLiteral":if(!(p.value instanceof Map))throw new Error("Jinja ObjectLiteral.value is not a Map");for(let[h,v]of p.value)c(Qr(h),f),c(Qr(v),f);return;case"BinaryExpression":c(ge(p,"left"),f),c(ge(p,"right"),f);return;case"FilterExpression":{c(ge(p,"operand"),f);let h=ge(p,"filter");if(h.type==="CallExpression")for(let v of Ne(h,"args"))c(v,f);return}case"TestExpression":{let h=ge(p,"operand");c(h,f);return}case"SelectExpression":c(ge(p,"test"),f),c(ge(p,"lhs"),f);return;case"UnaryExpression":case"SpreadExpression":c(ge(p,"argument"),f);return;case"SliceExpression":for(let h of["start","stop","step"])p[h]!==void 0&&c(ge(p,h),f);return;case"KeywordArgumentExpression":c(ge(p,"value"),f);return;case"Ternary":c(ge(p,"condition"),f),c(ge(p,"trueExpr"),f),c(ge(p,"falseExpr"),f);return;default:{let h=!0;vt(p,v=>{if(h)return h=!1,!1;if(v.type==="Identifier"&&typeof v.value=="string"){let b=f.resolve(v.value);(b===void 0||b.externalPossible===!0)&&a(v.value);for(let _ of b?.macros??[])l(_,f);return!0}return!1})}}};if(o=(p,f)=>{for(let g of p)switch(g.type){case"Comment":case"Break":case"Continue":break;case"If":{c(ge(g,"test"),f);let m=f.fork(),h=f.fork();o(Ne(g,"body"),m),o(Ne(g,"alternate"),h),Ed(f,m,h);break}case"For":{let m=ge(g,"iterable"),h=ge(g,"loopvar");if(m.type==="SelectExpression"){c(ge(m,"lhs"),f);let C=f.child();Ta(h,C),c(ge(m,"test"),C)}else c(m,f);let v=f.child();Ta(h,v),v.define("loop",qd);let b=Ne(g,"body");o(b,v),b.some(C=>xa(C,y=>y.type==="Macro"))&&(u=!0);let _=f.child();b.some(Td)&&!Cd(m)&&Ed(_,v,f.child()),o(Ne(g,"defaultBlock"),_);break}case"Set":{let m=ge(g,"assignee"),h=g.value===null?void 0:ge(g,"value");h===void 0?o(Ne(g,"body"),f):c(h,f),m.type==="MemberExpression"?(c(ge(m,"object"),f),m.computed===!0&&c(ge(m,"property"),f),h!==void 0&&Id(h,f)&&(u=!0)):Ta(m,f,h===void 0?$n:Bd(h,f));break}case"Macro":{let m=it(ge(g,"name")),h=Object.freeze({args:Ne(g,"args"),body:Ne(g,"body")});f.define(m,Object.freeze({macros:Object.freeze([h])}));break}case"CallStatement":{let m=ge(g,"call"),h=Array.isArray(g.callerArgs)?g.callerArgs.map(Qr):[],v=Object.freeze({args:Object.freeze(h),body:Ne(g,"body")});c(m,f,v);let b=ge(m,"callee"),_=b.type==="Identifier"?f.resolve(it(b)):void 0;if(_===void 0||_.externalPossible===!0||_.dynamicPossible===!0||(_.macros?.length??0)===0){u=!0;let y=f.child();y.define("caller",qd);for(let E of h)Ta(E,y);o(v.body,y)}break}case"FilterStatement":{o(Ne(g,"body"),f);let m=ge(g,"filter");if(m.type==="CallExpression")for(let h of Ne(m,"args"))c(h,f);break}default:c(g,f)}},o(Ne(t,"body"),new Sa),u){let p=f=>{if(f.type==="CallExpression"){let g=ge(f,"callee"),m=g.type==="MemberExpression"?gn(g):void 0;return m!==void 0&&a(m),!1}if(f.type==="MemberExpression"){let g=pn(f),m=g?.[0]??gn(f);return m!==void 0&&(g!==void 0&&g.length>=2?a(m,g[1]):a(m)),Ca(f,h=>vt(h,p)),!0}return f.type==="Identifier"&&typeof f.value=="string"?(a(f.value),!0):!1};vt(t,p)}let d=new Map;for(let p of r)d.set(p,s.has(p)?null:Object.freeze([...n.get(p)??[]].sort()));return Object.freeze({roots:r,namespaceReads:d})}var s4;function qa(){return s4}var Zu=new Map;function mn(e){return e==null?e:Array.isArray(e)?e.map(mn):e instanceof Map?Object.fromEntries([...e].map(([t,r])=>[t,mn(r)])):typeof e=="object"&&"value"in e?e.type==="NullValue"?null:mn(e.value):e}function Ba(e,t){if(typeof e=="function")return!0;if(e===null||typeof e!="object")return!1;if(e.type==="FunctionValue")return!0;if(Array.isArray(e)){let r=t??new Set;return r.has(e)?!1:(r.add(e),e.some(n=>Ba(n,r)))}if(e instanceof Map){let r=t??new Set;if(r.has(e))return!1;r.add(e);for(let n of e.values())if(Ba(n,r))return!0;return!1}if("value"in e){let r=e.value;if(r===null||typeof r!="object"&&typeof r!="function")return!1;let n=t??new Set;return n.has(e)?!1:(n.add(e),Ba(r,n))}return!1}function _e(e){return((...t)=>e(...t.map(mn)))}function Ld(e,t){let r=Math.max(e.length,t.length),n=new Array(r);for(let s=0;s<r;++s){let a=s-(r-e.length),i=s-(r-t.length),u=a<0?1:e[a],o=i<0?1:t[i];if(u!==o&&u!==1&&o!==1)throw new Error(`Shapes [${e}] and [${t}] are not broadcast-compatible`);n[s]=u===1?o:u}return n}function a4(e,t){if(e.length===0||t.length===0)throw new Error("MatMul inputs must each have rank >= 1");let r=e.length===1,n=t.length===1,s=e.at(-1),a=n?t[0]:t.at(-2);if(s!==a)throw new Error(`MatMul inner dimensions do not match (${s} != ${a})`);let i=Ld(r?[]:e.slice(0,-2),n?[]:t.slice(0,-2));return r&&n?i:r?[...i,t.at(-1)]:n?[...i,e.at(-2)]:[...i,e.at(-2),t.at(-1)]}function i4(e,t,r){if(!Array.isArray(e)||e.length===0)throw new Error("moveAxis() expects a non-empty shape");if(!Number.isInteger(t)||!Number.isInteger(r))throw new Error("moveAxis() axes must be integers");let n=e.length;if(t<-n||t>=n||r<-n||r>=n)throw new Error(`moveAxis() axes must be in [-${n}, ${n-1}]`);let s=t<0?t+n:t,a=r<0?r+n:r,i=[...e],[u]=i.splice(s,1);return i.splice(a,0,u),i}function u4(e,t){if(!Array.isArray(e))throw new Error("permuteShape() expects a shape");let r=e.length;if(!Array.isArray(t)||t.length!==r)throw new Error(`permuteShape() expects one axis per shape entry, got ${Array.isArray(t)?t.length:"a non-list"} for rank ${r}`);let n=new Set,s=new Array(r);for(let a=0;a<r;++a){let i=t[a];if(!Number.isInteger(i)||i<-r||i>=r)throw new Error(`permuteShape() axes must be in [-${r}, ${r-1}]`);let u=i<0?i+r:i;if(n.has(u))throw new Error(`permuteShape() axes must be unique, saw ${u} twice`);n.add(u),s[a]=e[u]}return s}var Ea=Object.freeze({ceil:_e(e=>Math.ceil(e)),floor:_e(e=>Math.floor(e)),min:_e((...e)=>Math.min(...e)),max:_e((...e)=>Math.max(...e)),pow:_e((e,t)=>Math.pow(e,t)),ceilDiv:_e((e,t)=>Math.ceil(e/t)),pow2ceil:_e(e=>e<=1?1:2**Math.ceil(Math.log2(e))),log2ceil:_e(e=>e<=1?0:Math.ceil(Math.log2(e))),numel:_e(e=>e.reduce((t,r)=>t*r,1)),dim:_e((e,t)=>{let r=(t%e.length+e.length)%e.length;return e[r]}),rows:_e((e,t)=>{let r=(t%e.length+e.length)%e.length;return e.reduce((n,s,a)=>a===r?n:n*s,1)}),outer:_e((e,t)=>{let r=(t%e.length+e.length)%e.length;return e.slice(0,r).reduce((n,s)=>n*s,1)}),inner:_e((e,t)=>{let r=(t%e.length+e.length)%e.length;return e.slice(r+1).reduce((n,s)=>n*s,1)}),prefix:_e((e,t)=>e.slice(0,Math.max(0,t))),suffix:_e((e,t)=>e.slice(Math.max(0,t))),fill:_e((e,t)=>{if(!Number.isInteger(t)||t<0)throw new Error("fill() count must be a non-negative integer");return Array.from({length:t},()=>e)}),moveAxis:_e(i4),permuteShape:_e(u4),reversedAxes:_e(e=>{if(!Number.isInteger(e)||e<0)throw new Error("reversedAxes() expects a non-negative integer rank");return Array.from({length:e},(t,r)=>e-1-r)}),broadcastShape:_e(Ld),matmulShape:_e(a4),broadcastable:_e((e,t)=>{if(e.length>t.length)return!1;let r=t.length-e.length;return e.every((n,s)=>n===1||n===t[s+r])}),sameShape:_e((e,t)=>e.length===t.length&&e.every((r,n)=>r===t[n])),hasAxis:_e((e,t,r)=>{let n=(t%r+r)%r;return e.some(s=>(s%r+r)%r===n)}),axisValue:_e((e,t,r,n,s)=>{let a=(r%n+n)%n,i=t.findIndex(u=>(u%n+n)%n===a);return i>=0&&i<e.length?e[i]:s}),isUniqueIntList:_e((e,t,r,n=0)=>{if(!Array.isArray(e)||!Number.isInteger(t)||!Number.isInteger(r)||r<t||!Number.isInteger(n)||n<0)return!1;let s=new Set;for(let a of e){if(!Number.isInteger(a)||a<t||a>=r)return!1;let i=n===0?a:(a%n+n)%n;if(s.has(i))return!1;s.add(i)}return!0}),has:_e((e,t)=>e instanceof Map?e.has(t):!!(e&&Object.prototype.hasOwnProperty.call(e,t))),dtypeBytes:_e(e=>{let t=Ge[Pa(e)];if(!t)throw new Error(`Unsupported dtype for dtypeBytes(): ${e}`);return t.storageByteSize}),onnxDtypeCode:_e(e=>Sd(e==="int64"?e:je(e))),pick:_e((e,t)=>{if(!Array.isArray(e))throw new Error("pick() expects a list of [condition, value] pairs as its first argument");for(let r of e){if(!Array.isArray(r)||r.length<2)throw new Error("pick() entries must be [condition, value] pairs");if(r[0])return r[1]}return t})});function Od(e){let t=Ge[Pa(e)];if(!t)throw new Error(`Unsupported WebGPU dtype: ${e}`);return t.wgslScalar}function Pa(e){return e==="f32"?"float32":e==="f16"?"float16":e==="u32"?"uint32":e==="i32"?"int32":e==="vec2<u32>"?"int64":je(e)}var Md=["f16Ok","subgroupMatrixConfigAvailable"],Fd=["DISPATCH_FOLD_WIDTH"],Qd=["repeat","tunables","derive","dtypes","logicalDtypes","tensorDtypes","ranks","shapes","present","args","attrs","device"];for(let e of[...Qd,...Md,...Fd,...Object.keys(Ea),"env"])if(!ka.has(e))throw new Error(`reserved scope name "${e}" is missing from the reserved list`);var Rd=new WeakMap,o4=new WeakSet;function Vn(e){let t=ze(e),r=Rd.get(t);return r===void 0&&(r={features:t.features,wgslLanguageFeatures:t.wgslLanguageFeatures,limits:t.limits,adapterInfo:Pu(t)},Rd.set(t,r),o4.add(r)),r}var Ju=[["true",!0],["false",!1],["none",null],["True",!0],["False",!1],["None",null]],l4=[...Ju,["range",c4],["raise_exception",e=>{throw new Error(e)}]];function c4(e,t,r=1){if(t===void 0&&(t=e,e=0),r===0)throw new Error("range() step must not be zero");let n=[];if(r>0)for(let s=e;s<t;s+=r)n.push(s);else for(let s=e;s>t;s+=r)n.push(s);return n}var Xu=class{environment;interpreter;cachedBindings=new Map;activeNames=[];nextNames=[];initialNames;evaluating=!1;constructor(){this.environment=new mt;for(let[t,r]of Ju)this.environment.set(t,r);this.interpreter=new Hn(this.environment),this.initialNames=new Set(this.environment.variables.keys())}get isEvaluating(){return this.evaluating}evaluate(t,r,n){let s=!1;this.evaluating=!0;try{let a=Wd(t,r,n);if(r.statement===void 0)throw r.parseError??new Error(`Empty WebGPU expression: ${t}`);s=y4(r,a);let i=r.lookupIdentifiers,u=r.freeIdentifiers.length,o=this.nextNames;o.length=0;for(let d=0;d<i.length;d+=1){let p=i[d],f=a[d];d<u||f.present?(this.bind(p,f.value),o.push(p)):this.remove(p,!0)}let l=this.activeNames;for(let d=0;d<l.length;d+=1){let p=l[d];o.includes(p)||this.remove(p,!1)}l.length=0;for(let d=0;d<o.length;d+=1)l.push(o[d]);let c=this.interpreter.evaluate(r.statement,this.environment);return Ba(c)&&(s=!1),mn(c)}catch(a){throw this.reset(),a}finally{this.evaluating=!1,s||this.reset()}}bind(t,r){if(this.initialNames.has(t)){this.environment.set(t,r);return}let n=this.cachedBindings.get(t);if(n!==void 0&&Object.is(n.raw,r)){this.environment.setVariable(t,n.runtime);return}this.environment.variables.delete(t),this.cachedBindings.delete(t);let s=this.environment.set(t,r);this.cachedBindings.set(t,{raw:r,runtime:s})}remove(t,r){this.initialNames.has(t)||this.environment.variables.delete(t),r&&this.cachedBindings.delete(t)}reset(){for(let t of this.environment.variables.keys())this.initialNames.has(t)||this.environment.variables.delete(t);this.cachedBindings.clear(),this.activeNames.length=0}},Ia;function jd(e){let t=Ia;Ia=new Xu;try{return e()}finally{Ia=t}}function lt(e,t={}){if(typeof e!="string")return e;let r=Nd(e),n=Ia;if(n!==void 0&&!n.isEvaluating)return n.evaluate(e,r,t);let s=Wd(e,r,t);if(r.statement===void 0)throw r.parseError??new Error(`Empty WebGPU expression: ${e}`);let a=new mt;for(let[o,l]of Ju)a.set(o,l);let i=r.lookupIdentifiers,u=r.freeIdentifiers.length;for(let o=0;o<i.length;o+=1){let l=s[o];(o<u||l.present)&&a.set(i[o],l.value)}return mn(new Hn(a).evaluate(r.statement,a))}function Wd(e,t,r){let n=t.lookupIdentifiers,s=new Array(n.length),a=t.freeIdentifiers.length,i;for(let u=0;u<n.length;u+=1){let o=zd(r,n[u]);s[u]=o,u<a&&!o.present&&(i??=[]).push(n[u])}if(i!==void 0)throw new Error(`Unknown ${zt(i.length,"identifier")} ${i.map(u=>`"${u}"`).join(", ")} in WebGPU expression: ${e}
Identifiers resolve against the expression scope. Namespaces: ${[...Qd].sort().join(", ")} (plus the bare names declared in \`derive\` and the constants ${Fd.join(", ")}).
device sub-fields: features, wgslLanguageFeatures, limits, adapterInfo.
Helper functions: ${[...Object.keys(Ea),...Md].join(", ")}.
If a string literal was intended, quote it (e.g. '"${i[0]}"' not '${i[0]}').`);return s}function d4(e,t,r){let n=e.lookupIdentifiers.indexOf(r);return n<0?void 0:t[n]}var f4=Object.freeze({present:!1});function Yu(e,t){return e!==void 0&&Object.hasOwn(e,t)&&Object.prototype.propertyIsEnumerable.call(e,t)?{present:!0,value:e[t]}:null}function zd(e,t,r=!1){let n=e;if(t==="env"&&n.env!==void 0)return{present:!0,value:n.env};let s=Yu(e.derived,t);if(s)return s;switch(t){case"repeat":return{present:!0,value:e.repeat??{}};case"tunables":return{present:!0,value:e.tunables??{}};case"derive":return{present:!0,value:e.derived??{}};case"DISPATCH_FOLD_WIDTH":return{present:!0,value:Wt}}switch(t){case"dtypes":return{present:!0,value:e.dtypes??{}};case"logicalDtypes":return{present:!0,value:e.logicalDtypes??{}};case"tensorDtypes":return{present:!0,value:e.tensorDtypes??{}};case"ranks":return{present:!0,value:e.ranks??{}};case"shapes":return{present:!0,value:e.shapes??{}};case"present":return{present:!0,value:e.present??{}};case"args":return{present:!0,value:e.args??{}};case"attrs":return{present:!0,value:e.attrs??{}};case"device":return{present:!0,value:Vn(e.device),sessionCallRoot:"device-view"};case"f16Ok":{let u=Vn(e.device).features.has("shader-f16");return{present:!0,sessionCallRoot:"pure-function",value:_e(o=>o!=="f16"&&o!=="float16"||u)}}case"subgroupMatrixConfigAvailable":{let i=Pu(ze(e.device)).subgroupMatrixConfigs;return{present:!0,sessionCallRoot:"pure-function",value:_e((u,o,l,c,d)=>i===void 0||i.some(p=>p.componentType===u&&(o==null||p.resultComponentType===o)&&p.M===l&&p.N===c&&p.K===d))}}}let a=Yu(Ea,t);return a?{...a,sessionCallRoot:"pure-function"}:f4}function eo(e,t={},r){let n=to(e),s=n.template,a={};for(let o of g4(n)){let l=zd(t,o,!0);l.present&&(a[o]=l.value)}let i=b4(n,a,h4(n),t),u=qa();return u===void 0?_4(s,a,i):u(Object.freeze({source:e,parsed:s.parsed,context:a,...i===void 0?{}:{memoKey:i},...r===void 0?{}:{provenance:r},render:()=>s.render(a)}))}function to(e){let t=Zu.get(e);return t===void 0&&(t={id:Zu.size,template:p4(e)},Zu.set(e,t)),t}function p4(e){let t=_a(ba(e,{lstrip_blocks:!0,trim_blocks:!0}));return{parsed:t,render(r){let n=new mt;for(let[s,a]of l4)n.set(s,a);for(let[s,a]of Object.entries(r))n.set(s,a);return new Hn(n).run(t).value}}}function g4(e){if(e.referenced!==void 0)return e.referenced;let t=new Set;return vt(e.template.parsed,r=>(r.type==="Identifier"&&typeof r.value=="string"&&t.add(r.value),!1)),e.referenced=t,t}function Kd(e){let t=e.externalNames;return t===void 0&&(t=Pd(e.template.parsed),e.externalNames=t),t}function m4(e){return Kd(e).roots}function Ud(e){return m4(to(e))}function Ra(e,t){let r=to(e),n=Kd(r).namespaceReads;return n?.has(t)?n.get(t):Object.freeze([])}function h4(e){if(e.deviceReads!==void 0)return e.deviceReads;let t=new Set,r=!1;vt(e.template.parsed,s=>{if(r)return!0;if(s.type==="MemberExpression"){let a=pn(s);if(a===void 0||a[0]!=="device"&&a[0]!=="env")return!1;let i=a[0]==="device"?a.slice(1):a[1]==="device"?a.slice(2):void 0;return i!==void 0&&(i.length===0?r=!0:t.add(i.join("."))),!0}return s.type==="Identifier"&&(s.value==="device"||s.value==="env")?(r=!0,!0):!1});let n=r?"opaque":Object.freeze([...t].sort());return e.deviceReads=n,n}var v4=2048,jr=new Map;function w4(e,t){let r=t.split("."),n=e,s=e;for(let a of r){if(n===null||typeof n!="object"&&typeof n!="function")return n;s=n,n=n[a]}return typeof n=="function"?s:n}function tr(e,t){if(e===null)return"null";let r=typeof e;if(r==="string")return JSON.stringify(e);if(r==="number")return er(e);if(r==="boolean"||r==="bigint")return String(e);if(r==="undefined")return"undef";if(r!=="object")return;let n=e;if(!t.has(n)){t.add(n);try{if(Array.isArray(n)){let u=[];for(let o of n){let l=tr(o,t);if(l===void 0)return;u.push(l)}return`[${u.join(",")}]`}if(n instanceof Map||ArrayBuffer.isView(n))return;let s=n;if(typeof s.has=="function"&&typeof s[Symbol.iterator]=="function"){let u=[];for(let l of n){let c=tr(l,t);if(c===void 0)return;u.push(c)}let o=[];for(let l of Object.keys(n).sort()){let c=n[l];if(typeof c=="function")continue;let d=tr(c,t);if(d===void 0)return;o.push(`${JSON.stringify(l)}:${d}`)}return`S[${u.sort().join(",")}]{${o.join(",")}}`}let a=n,i=[];for(let u of Object.keys(a).sort()){let o=tr(a[u],t);if(o===void 0)return;i.push(`${JSON.stringify(u)}:${o}`)}return`{${i.join(",")}}`}finally{t.delete(n)}}}function Gd(e,t,r){if(t==="opaque")return tr(e,r);let n=[];for(let s of t){let a=tr(w4(e,s),r);if(a===void 0)return;n.push(`${JSON.stringify(s)}:${a}`)}return`D{${n.join(",")}}`}function b4(e,t,r,n){let s=!1;for(let o of Object.keys(t)){let l=t[o];if(typeof l=="function"&&Yu(Ea,o)?.value!==l){s=!0;break}}let a=s?"opaque":r,i=new Set,u=[];for(let o of Object.keys(t).sort()){let l=t[o],c;if(typeof l=="function")c=`fn:${o}`;else if(o==="device")c=Gd(l,a,i);else if(o==="env"&&l!==null&&typeof l=="object"){let d=l,p=[];for(let f of Object.keys(d).sort()){let g=f==="device"?Gd(d[f],a,i):tr(d[f],i);if(g===void 0)return;p.push(`${JSON.stringify(f)}:${g}`)}c=`{${p.join(",")}}`}else c=tr(l,i);if(c===void 0)return;u.push(`${JSON.stringify(o)}:${c}`)}if(s){let o=tr(Vn(n.device),i);if(o===void 0)return;u.push(`@device:${o}`)}return`${e.id} ${u.join(",")}`}function _4(e,t,r){if(r===void 0)return e.render(t);let n=jr.get(r);if(n!==void 0)return jr.delete(r),jr.set(r,n),n;let s=e.render(t);if(jr.set(r,s),jr.size>v4){let a=jr.keys().next().value;a!==void 0&&jr.delete(a)}return s}var Dd=new Map,k4=Object.freeze({lstrip_blocks:!0,trim_blocks:!0}),A4=new Set(["true","false","none","True","False","None"]);function Nd(e){let t=Dd.get(e);if(t!==void 0)return t;let r=new Set,n=new Set,s=[],a=o=>{if(!o||typeof o!="object")return;if(Array.isArray(o)){for(let c of o)a(c);return}if(o instanceof Map){for(let[c,d]of o)a(c),a(d);return}let l=o;switch(l.type){case"Identifier":A4.has(l.value)||r.add(l.value);return;case"MemberExpression":a(l.object),l.computed&&a(l.property);return;case"BinaryExpression":a(l.left),a(l.right);return;case"UnaryExpression":a(l.argument);return;case"FilterExpression":{a(l.operand);let c=l.filter;c?.type==="CallExpression"&&a(c.args);return}case"TestExpression":{let c=l.test,d=l.operand;(c?.value==="defined"||c?.value==="undefined")&&d?.type==="Identifier"?n.add(d.value):a(l.operand);return}case"CallExpression":{s.push([...Hd(l.callee)??[]]),a(l.callee),a(l.args);return}case"KeywordArgumentExpression":a(l.value);return;default:{for(let[c,d]of Object.entries(l))c!=="type"&&a(d);return}}},i,u;try{let o=_a(ba(`{{ ${e} }}`,k4));i=o.body[0],a(o)}catch(o){u=o}for(let o of r)n.delete(o);return t=Object.freeze({statement:i,...u!==void 0?{parseError:u}:{},freeIdentifiers:Object.freeze([...r]),probedIdentifiers:Object.freeze([...n]),lookupIdentifiers:Object.freeze([...r,...n]),callPaths:Object.freeze(s.map(o=>Object.freeze(o)))}),Dd.set(e,t),t}function y4(e,t){for(let r of e.callPaths){let n=r[0],s=n===void 0?void 0:d4(e,t,n);if(!(r.length===1&&s?.sessionCallRoot==="pure-function")&&!(s?.sessionCallRoot==="device-view"&&r.length===3&&r[0]==="device"&&(r[1]==="features"||r[1]==="wgslLanguageFeatures")&&r[2]==="has"))return!1}return!0}function Hd(e){if(!e||typeof e!="object")return null;let t=e;if(t.type==="Identifier"&&typeof t.value=="string")return[t.value];if(t.type==="MemberExpression"&&t.computed!==!0){let r=Hd(t.object);if(!r)return null;let n=t.property;if(n?.type==="Identifier"&&typeof n.value=="string")return[...r,n.value]}return null}function $d(e){let t=Nd(e);return{free:t.freeIdentifiers,probed:t.probedIdentifiers,parseError:t.parseError!==void 0}}function Vd(e,t){return Object.fromEntries(Object.entries(e??{}).map(([r,n])=>[r,Ct(n,t)]))}function Ct(e,t){return typeof e=="string"?lt(e,t):Array.isArray(e)?e.map(r=>Ct(r,t)):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).map(([r,n])=>[r,Ct(n,t)])):e}function Ga(e){let t=new WeakMap,r=(n,s)=>{if(n===null||typeof n!="object")return n;if(t.has(n))return t.get(n);if(t.set(n,n),Array.isArray(n)){let o=n.map((c,d)=>r(c,`${s}[${d}]`)),l=o.every((c,d)=>c===n[d])?n:o;return t.set(n,l),l}let a=[],i=!1;for(let[o,l]of Object.entries(n)){if(o==="$comment"){if(typeof l!="string")throw new Error(`${s}.$comment must be a string`);i=!0;continue}let c=r(l,`${s}.${o}`);i||=c!==l,a.push([o,c])}let u=i?Object.fromEntries(a):n;return t.set(n,u),u};return r(e,"manifest")}var S4=new Set(["read-only-storage","storage","uniform"]),qt=/^[A-Za-z_][A-Za-z0-9_]*$/,s2=/^[A-Za-z0-9][A-Za-z0-9_]*$/,x4=new Set(["u32","i32","f32","bool","string"]);function zr(e){let t=e instanceof Map?new Map(e):new Map(Object.entries(e));return Object.freeze({readText(r){let n=t.get(r);if(n===void 0)throw new Error(`WebGPU template asset is missing: ${r}`);return n},has(r){return t.has(r)}})}var Zd=new WeakSet;function io(e){e=Ga(e),Ye(e,"WebGPU manifest"),Zd.has(e)||(_d(e,"WebGPU manifest"),Zd.add(e));let t=e.domain??"ai.onnx";ih(t,"WebGPU manifest domain");let r=e.name;if(typeof r!="string"||r.length===0)throw new Error("WebGPU manifest requires a non-empty name");let n=`${t}.${r}`;if(e.conformance!==void 0&&e.conformance!=="internal-lowering"&&e.conformance!=="legacy-default-domain")throw new Error(`WebGPU manifest ${n} has invalid conformance ${JSON.stringify(e.conformance)}`);for(let _ of Object.keys(e))if(!T4.has(_))throw new Error(`WebGPU manifest ${n} has unknown top-level key "${_}"${Xd[_]?` (${Xd[_]})`:""}`);if(!wt(e.inputs))throw new Error(`WebGPU manifest ${n} requires an inputs map`);if(!wt(e.outputs))throw new Error(`WebGPU manifest ${n} requires an outputs map`);let s=q4(e.typeConstraints,n),a=Yd(e.inputs,"input",n,s),i=Yd(e.outputs,"output",n,s),u=B4(e.args,n),o=I4(a,i,u,n);if(e.description!==void 0&&typeof e.description!="string")throw new Error(`WebGPU manifest ${n} description must be a string`);let{defaults:l,names:c}=E4(e.attributes,n),d=P4(e.attributeConstraints,l,new Set(c),n),p=D4(a,i,s),f=O4(e.bindings,n,p);g2(e.derive,`WebGPU manifest ${n} derive`);let g=new Set([...ka,...Object.keys(s)]),m=oo(e.derive,`WebGPU manifest ${n} derive`,g),h=c2(e.when,`manifest ${n}`),v=e.requires===void 0?null:d2(e.requires,`WebGPU manifest ${n}`),b=Q4(e,n,{entries:f,ports:p,reservedDeriveNames:g},h,v);return Object.freeze({domain:t,name:r,id:n,...e.conformance===void 0?{}:{conformance:e.conformance},sinceVersion:e.sinceVersion,inputs:Object.freeze(a),outputs:Object.freeze(i),args:o,attributes:l,attributeNames:c,attributeConstraints:d,derive:m,typeConstraints:s,tunables:a2(e.tunables,`WebGPU manifest ${n} tunables`),variants:Object.freeze(b)})}var T4=new Set(["$schema","domain","name","description","conformance","sinceVersion","inputs","outputs","args","attributes","attributeConstraints","typeConstraints","tunables","derive","when","requires","bindings","variants","onnx","aten"]),Xd={attributeDescriptions:"an attribute's prose is `attributes.<name>.description`",tunableDescriptions:"a tunable's prose is `tunables.<name>.description`",attributeNames:"a default-less attribute is an `attributes` entry without `default`",constants:"`derive` is the one value channel; every derive is template-visible and a pass is keyed by the derives its template reads",variantFamilies:"a family is a `variants` member: { cases, variants }",copy:"a copy is expanded where the manifest is read, by expandSharedLibraries given a copy source",bindingSets:"spec 2.0 declares binding ENTRIES under `bindings` and passes list entry handles",bindingEntries:"spec 2.0 declares binding entries under `bindings`"},uo=new Set(Object.values(Dr)),C4=new Set(["description","onnx","dtype","rank","optional","layout","shape","storage","narrowing"]);function q4(e,t){let r={};for(let[n,s]of Object.entries(e??{})){if(!Array.isArray(s)||s.length===0)throw new Error(`${t}.typeConstraints.${n} must be a non-empty array`);for(let a of s)if(!uo.has(a))throw new Error(`${t}.typeConstraints.${n} has unsupported logical dtype ${String(a)}`);r[n]=Object.freeze([...s])}return Object.freeze(r)}function Yd(e,t,r,n){let s=[];for(let[a,i]of Object.entries(e)){let u=`${r}.${t}s.${a}`;if(!qt.test(a))throw new Error(`${u} is not a WGSL-compatible tensor name`);Ye(i,u);for(let l of Object.keys(i))if(!C4.has(l))throw new Error(`${u} has unknown key "${l}"`);if(i.onnx!==void 0&&(typeof i.onnx!="string"||i.onnx.length===0))throw new Error(`${u}.onnx must be a non-empty string`);if(i.description!==void 0&&typeof i.description!="string")throw new Error(`${u}.description must be a string`);if(i.dtype!==void 0&&typeof i.dtype!="string")throw new Error(`${u}.dtype must be a string`);if(i.optional!==void 0&&typeof i.optional!="boolean")throw new Error(`${u}.optional must be a boolean`);let o=i.dtype===void 0?void 0:uo.has(i.dtype)?[i.dtype]:n[i.dtype];if(i.storage!==void 0){if(!Object.hasOwn(Ge,i.storage))throw new Error(`${u}.storage ${String(i.storage)} is not a WebGPU storage dtype`);if(o===void 0)throw new Error(`${u} declares a storage dtype but its dtype ${JSON.stringify(i.dtype)} is not a known logical dtype or type variable`);if(o.length===1&&o[0]==="int64"){if(i.storage!=="int32"&&i.storage!=="uint32"&&i.storage!=="int64")throw new Error(`${u}: an int64 port requires int64 storage or an int32/uint32 projection`)}else if(o.length!==1||!Lr(o[0],i.storage))throw new Error(`${u}.storage ${i.storage} does not exactly match logical dtype [${o.join(", ")}]; only int64 -> int32/uint32 projection is supported`)}if(i.narrowing!==void 0){if(i.narrowing!=="checked"&&i.narrowing!=="saturating")throw new Error(`${u}.narrowing must be checked or saturating`);if(t!=="input"||o?.length!==1||o[0]!=="int64"||i.storage!=="int32"&&i.storage!=="uint32")throw new Error(`${u}.narrowing is valid only on a projected logical int64 input`)}else if(t==="input"&&o?.length===1&&o[0]==="int64"&&(i.storage==="int32"||i.storage==="uint32"))throw new Error(`${u}: a projected int64 input must declare narrowing as checked or saturating`);if(i.layout!==void 0){if(t==="output")throw new Error(`${u} cannot declare a storage layout`);if(i.layout!=="packed")throw new Error(`${u} has unsupported storage layout ${JSON.stringify(i.layout)}`);if(!(o!==void 0&&o.length>0&&o.every(c=>Object.hasOwn(Ge,c)&&Gu(c))))throw new Error(`${u} declares the packed storage layout but its dtype ${JSON.stringify(i.dtype)} does not resolve to uint8 only`)}s.push(Object.freeze({...i,role:a}))}return s}function B4(e,t){let r={};for(let[n,s]of Object.entries(e??{})){let a=`${t}.args.${n}`;if(!qt.test(n))throw new Error(`${a} is not a valid arg name`);if(Ye(s,a),!x4.has(s.kind))throw new Error(`${a} has unsupported kind ${String(s.kind)}${s.kind==="tensor"?" (tensors are inputs/outputs ports in spec 2.0)":""}`);for(let i of Object.keys(s))if(!["kind","description","onnx","required","oneOf"].includes(i))throw new Error(`${a} has unknown key "${i}"`);if(s.description!==void 0&&typeof s.description!="string")throw new Error(`${a}.description must be a string`);if(s.oneOf!==void 0&&(s.kind!=="string"||!Array.isArray(s.oneOf)||s.oneOf.length===0))throw new Error(`${a}.oneOf is valid only on a non-empty string arg`);r[n]=s}return r}function I4(e,t,r,n){let s=[],a=new Map,i=new Set(t.map(u=>u.role));for(let[u,o]of[["input",e],["output",t]])for(let l of o){if(a.has(l.role)){if(u==="output"){let c=e[a.get(l.role)];if(c.dtype!==l.dtype||c.storage!==l.storage)throw new Error(`${n}: inout tensor "${l.role}" must declare the same dtype and storage under inputs and outputs`);continue}throw new Error(`${n}: tensor name "${l.role}" is declared twice`)}a.set(l.role,s.length),s.push([l.role,Object.freeze({kind:"tensor",semantic:l.role,role:u==="input"&&i.has(l.role)?"inout":u,...l.onnx!==void 0?{onnx:l.onnx}:{},...l.storage!==void 0?{dtype:l.storage}:{},...l.narrowing!==void 0?{narrowing:l.narrowing}:{},...l.optional?{required:!1}:{}})])}for(let[u,o]of Object.entries(r)){if(a.has(u))throw new Error(`${n}: arg "${u}" collides with a tensor of the same name`);a.set(u,s.length),s.push([u,Object.freeze({...o})])}return Object.freeze(Object.fromEntries(s))}function E4(e,t){if(e===void 0)return{defaults:Object.freeze({}),names:Object.freeze([])};Ye(e,`WebGPU manifest ${t} attributes`);let r={},n=[];for(let[s,a]of Object.entries(e)){let i=`WebGPU manifest ${t} attributes.${s}`;if(s.length===0)throw new Error(`${i}: attribute names must be non-empty`);if(!wt(a))throw new Error(`${i} must be an object { default?, description? } (a bare value is the spec 2.0 draft spelling; write { "default": ${JSON.stringify(a)} })`);for(let u of Object.keys(a))if(u!=="default"&&u!=="description")throw new Error(`${i} has unknown key "${u}"`);if(a.description!==void 0&&typeof a.description!="string")throw new Error(`${i}.description must be a string`);if(Object.hasOwn(a,"default")){if(a.default===void 0)throw new Error(`${i}.default must be a JSON value`);r[s]=a.default}n.push(s)}return{defaults:Object.freeze(r),names:Object.freeze(n.sort())}}function a2(e,t){if(e===void 0)return Object.freeze({});Ye(e,t);let r={};for(let[n,s]of Object.entries(e)){let a=`${t}.${n}`;if(!qt.test(n))throw new Error(`${a}: tunable names must be identifiers`);if(!wt(s))throw new Error(`${a} must be an object { default, description? }; got ${JSON.stringify(s)}`);for(let u of Object.keys(s))if(u!=="default"&&u!=="description")throw new Error(`${a} has unknown key "${u}"`);let i=s.default;if(typeof i!="number"&&typeof i!="string"&&typeof i!="boolean")throw new Error(`${a}.default must be a number, string or boolean`);if(typeof i=="number"&&!Number.isFinite(i))throw new Error(`${a}.default must be finite`);if(s.description!==void 0&&typeof s.description!="string")throw new Error(`${a}.description must be a string`);r[n]=i}return Object.freeze(r)}function oo(e,t,r){let n=uh(e,t);for(let s of Object.keys(n)){if(!qt.test(s))throw new Error(`${t}.${s}: derive names must be identifiers`);if(r.has(s))throw new Error(`${t}.${s} would shadow a scope name templates and expressions already bind; rename the derive`)}return Object.freeze(n)}function P4(e,t,r,n){if(e===void 0)return Object.freeze({});Ye(e,`WebGPU manifest ${n} attributeConstraints`);let s=[];for(let[a,i]of Object.entries(e)){let u=`WebGPU manifest ${n} attributeConstraints.${a}`;if(!r.has(a))throw new Error(`${u} does not name a declared attribute`);Ye(i,u);let o=Object.keys(i).filter(d=>d!=="required"&&d!=="values");if(o.length>0)throw new Error(`${u} has unknown ${zt(o.length,"field")}: ${o.join(", ")}`);let l=i.required??!1;if(typeof l!="boolean")throw new Error(`${u}.required must be a boolean`);if(l&&Object.hasOwn(t,a))throw new Error(`${u}.required is redundant because the attribute has a default`);let c=null;if(i.values!==void 0){if(!Array.isArray(i.values)||i.values.length===0)throw new Error(`${u}.values must be a non-empty array`);let d=[];for(let p of i.values){if(p!==null&&typeof p!="string"&&typeof p!="boolean"&&!(typeof p=="number"&&Number.isFinite(p)))throw new Error(`${u}.values must contain only finite JSON scalars`);if(d.some(f=>Ma(f,p)))throw new Error(`${u}.values contains duplicate ${JSON.stringify(p)}`);d.push(p)}c=Object.freeze(d)}if(!l&&c===null)throw new Error(`${u} must declare required: true and/or values`);if(Object.hasOwn(t,a)&&c!==null&&!c.some(d=>Ma(d,t[a])))throw new Error(`WebGPU manifest ${n} default attribute ${a}=${JSON.stringify(t[a])} is not in attributeConstraints.${a}.values`);s.push([a,Object.freeze({required:l,values:c})])}return Object.freeze(Object.fromEntries(s))}function Ma(e,t){return typeof e=="number"&&typeof t=="number"?Number.isFinite(e)&&Number.isFinite(t)&&Math.fround(e)===Math.fround(t):e===t}function i2(e){let t=new Map;for(let[r,n]of Object.entries(e))n.kind==="tensor"&&t.set(r,[r]);return t}function u2(e){let t=new Map;for(let[r,n]of Object.entries(e))n.kind==="tensor"&&n.dtype!==void 0&&t.set(r,n.dtype);return t}function lo(e,t){let r=new Map;for(let n of t)n.layout!==void 0&&r.set(n.role,n.layout);return r}function o2(e){return t=>t}function l2(e){return e.map(t=>Xn(t)?ju(t.name,fa(t.struct.name,t.struct.fields.map(ah))):Qu(t.name,{arg:t.arg,access:t.buffer.type==="storage"?"read_write":"read",elementType:t.elementType,...t.length!==void 0?{length:t.length}:{}}))}var R4="main",G4=new Set(["buffer","name","arg","scratch","elementType","length","structName","struct"]);function D4(e,t,r){let n=i=>i.storage==="int64"||i.storage===void 0&&(i.dtype==="int64"||i.dtype!==void 0&&r[i.dtype]?.length===1&&r[i.dtype][0]==="int64"),s=new Map,a=new Set(t.map(i=>i.role));for(let i of e)s.set(i.role,{port:i,opaqueInt64:n(i),direction:a.has(i.role)?"inout":"input"});for(let i of t)s.has(i.role)||s.set(i.role,{port:i,opaqueInt64:n(i),direction:"output"});return s}function L4(e,t){let{port:r}=e;if(r.storage!==void 0)return Ge[r.storage].wgslScalar;if(r.dtype===void 0)throw new Error(`${t} binds port "${r.role}", which declares no dtype; write the entry's elementType`);if(uo.has(r.dtype)){let n=Object.hasOwn(Ge,r.dtype)?Ge[r.dtype]:void 0;if(n===void 0)throw new Error(`${t} binds port "${r.role}" of logical dtype ${r.dtype}, which has no storage scalar; write the entry's elementType`);return n.wgslScalar}return`$${r.dtype}`}function O4(e,t,r){let n=new Map;if(e===void 0)return n;Ye(e,`WebGPU manifest ${t} bindings`);for(let[s,a]of Object.entries(e)){if(!qt.test(s))throw new Error(`WebGPU manifest ${t} binding handle "${s}" must be an identifier`);n.set(s,La(a,s,`WebGPU manifest ${t} bindings.${s}`,r,null))}return n}function La(e,t,r,n,s){Ye(e,r);for(let p of Object.keys(e))if(!G4.has(p))throw new Error(`${r} has unknown key "${p}"${p==="semantic"?" (spec 2.0 bindings resolve by arg/name and name scratch explicitly)":""}`);for(let p of["name","arg","scratch","elementType","structName"])if(e[p]!==void 0&&(typeof e[p]!="string"||e[p].length===0))throw new Error(`${r}.${p} must be a non-empty string`);let a=e.name??t??e.arg??e.scratch;if(a==null)throw new Error(`${r}: an inline binding entry needs a name (name, arg or scratch)`);if(!qt.test(a))throw new Error(`${r}.name must be a WGSL identifier`);for(let[p,f]of[["name",a],["arg",e.arg],["scratch",e.scratch]])if(typeof f=="string"&&!Aa(f))throw new Error(`${r}.${p} "${f}" is a JavaScript prototype property name`);let i={name:a,...e.arg!==void 0&&e.arg!==a?{arg:e.arg}:{},...e.scratch!==void 0&&e.scratch!==a?{scratch:e.scratch}:{}},u=e.scratch??(s?.has(a)?a:void 0),o=u!==void 0?s?.get(u):void 0,l=u===void 0?n.get(e.arg??a):void 0,c=e.buffer??(e.struct!==void 0?"uniform":u!==void 0?"storage":l!==void 0?l.direction==="input"?"read-only-storage":"storage":void 0);if(c===void 0)throw new Error(`${r} binds "${e.arg??a}", which is neither a declared tensor nor an intermediate, so its buffer type cannot default; write buffer`);if(!S4.has(c))throw new Error(`${r}.buffer must be read-only-storage, storage or uniform`);if(c==="uniform"){if(e.elementType!==void 0||e.length!==void 0)throw new Error(`${r}: a uniform binding takes struct fields, not elementType/length`);if(!Array.isArray(e.struct)||e.struct.length===0)throw new Error(`${r}: a uniform binding requires a non-empty struct field list`);return F4(e.struct,r),Object.freeze({...i,buffer:{type:c},struct:Object.freeze({name:e.structName??"Params",fields:Object.freeze([...e.struct])})})}if(e.struct!==void 0||e.structName!==void 0)throw new Error(`${r}: a storage binding takes elementType, not struct fields`);let d=e.elementType;if(d===void 0)if(o!==void 0)d=Ge[o.dtype].wgslScalar;else if(l!==void 0)d=L4(l,r);else throw u!==void 0?new Error(`${r} names scratch "${u}" outside a variant, so its elementType cannot default; write elementType`):new Error(`${r}: a storage binding requires elementType`);if(l?.opaqueInt64&&!d.startsWith("$")&&d.replace(/\s+/g,"")!=="vec2<u32>")throw new Error(`${r}: an int64 tensor binding must use vec2<u32>`);if(e.length!==void 0){let p=e.length,f=typeof p=="number"&&Number.isSafeInteger(p)&&p>0,g=typeof p=="string"&&p.startsWith("$")&&p.length>1;if(!f&&!g)throw new Error(`${r}.length must be a positive integer or a $-prefixed derive reference`)}return Object.freeze({...i,buffer:{type:c},elementType:d,...e.length!==void 0?{length:e.length}:{}})}function M4(e,t,r,n,s){let a=`WebGPU pass ${r} bindings[${t}]`;if(typeof e=="string"){let i=n.entries.get(e);if(i!==void 0)return i;if(n.ports.has(e))return La({},e,a,n.ports,s);if(s.has(e))return La({scratch:e},e,a,n.ports,s);throw new Error(`${a} "${e}" is neither a binding entry handle, a declared tensor nor an intermediate of the variant`)}if(wt(e))return La(e,null,a,n.ports,s);throw new Error(`${a} must be a handle, a tensor or intermediate name, or an inline binding entry`)}function F4(e,t){let r=new Set;for(let[n,s]of e.entries()){if(Ye(s,`${t} field ${n}`),typeof s.name!="string"||!qt.test(s.name))throw new Error(`${t} field ${n} name must be an identifier`);if(!Aa(s.name))throw new Error(`${t} field ${n} name "${s.name}" is a JavaScript prototype property name`);if(r.has(s.name))throw new Error(`${t} repeats field ${s.name}`);if(r.add(s.name),!["u32","i32","f32"].includes(s.type))throw new Error(`${t} field ${s.name} has invalid type ${String(s.type)}`);for(let a of Object.keys(s))if(!["name","type","value","required","internal","default"].includes(a))throw new Error(`${t} field ${s.name} has unknown key "${a}"`);if(s.internal!==void 0&&typeof s.internal!="boolean")throw new Error(`${t} field ${s.name} internal must be a boolean`)}}function Xn(e){return e.buffer.type==="uniform"}function Q4(e,t,r,n,s){let a=fo(e,t);if(a.length===0)throw new Error(`WebGPU manifest ${t} requires a non-empty variants list`);let i=a.map((o,l)=>W4(o,l,r,n,s)),u=new Set;for(let o of i){if(u.has(o.id))throw new Error(`WebGPU manifest ${t} has duplicate expanded variant id "${o.id}"`);u.add(o.id)}return K4(i,t),i}function co(e,t){return t.priority-e.priority}var j4=new Set(["id","description","priority","when","demoteWhen","supersededBy","requires","tunables","derive","intermediates","passes"]);function W4(e,t,r,n,s){Ye(e,"WebGPU variant");let a=e.id??`variant_${t}`;if(typeof a!="string"||!s2.test(a))throw new Error(`WebGPU variant id must be stable, got ${String(a)}`);for(let p of Object.keys(e))if(!j4.has(p))throw new Error(`WebGPU variant ${a} has unknown key "${p}"${p==="bindings"?" (spec 2.0 lists bindings on each pass)":p==="constants"?" (`derive` is the one value channel)":""}`);if(e.description!==void 0&&typeof e.description!="string")throw new Error(`WebGPU variant ${a} description must be a string`);let i=e.priority??0;if(typeof i!="number"||!Number.isFinite(i))throw new Error(`WebGPU variant ${a} priority must be a finite number`);if(!Array.isArray(e.passes)||e.passes.length===0)throw new Error(`WebGPU variant ${a} requires passes`);g2(e.derive,`WebGPU variant ${a} derive`);let u=V4(e.intermediates,a),o=new Map(u.map(p=>[p.id,p]));for(let p of o.keys())if(r.ports.has(p))throw new Error(`WebGPU variant ${a} intermediate "${p}" is also a tensor name`);let l=f2(e.passes,r,o,a),c=p2(l);z4(c,o,r.ports,a);let d=e.requires===void 0?null:d2(e.requires,`WebGPU variant ${a}`);return Object.freeze({id:a,priority:i,when:Object.freeze([...n,...c2(e.when,`variant ${a}`)]),demoteWhen:Jd(e.demoteWhen,`WebGPU variant ${a} demoteWhen`),supersededBy:Jd(e.supersededBy,`WebGPU variant ${a} supersededBy`),requires:H4(s,d,a),tunables:a2(e.tunables,`WebGPU variant ${a} tunables`),derive:oo(e.derive,`WebGPU variant ${a} derive`,r.reservedDeriveNames),intermediates:Object.freeze(u),passes:Object.freeze(c),passSequence:Object.freeze(l)})}function z4(e,t,r,n){let s=new Set(t.keys()),a=new Set(r.keys());for(let i of e)for(let u of i.bindings){if(u.buffer.type==="uniform")continue;if(u.scratch!==void 0){if(!s.has(u.scratch))throw new Error(`WebGPU variant ${n} pass ${i.id} binding "${u.name}" names scratch "${u.scratch}", which the variant does not declare as an intermediate`);continue}if(s.has(u.name))continue;let o=u.arg??u.name;if(!a.has(o))throw new Error(`WebGPU variant ${n} pass ${i.id} binding "${u.name}" names request tensor "${o}", which is neither an input, an output nor an intermediate of the variant`)}}function c2(e,t){if(e===void 0)return Object.freeze([]);if(!Array.isArray(e))throw new Error(`WebGPU ${t} when must be a list of expression strings${typeof e=="string"?" (spec 2.0 has no single-string form: wrap it in a list)":""}`);if(e.length===0)throw new Error(`WebGPU ${t} when must not be an empty list (omit the key for an always-eligible variant)`);for(let r of e)if(typeof r!="string"||r.length===0)throw new Error(`WebGPU ${t} when[] entries must be non-empty expression strings, got ${JSON.stringify(r)}`);return Object.freeze([...e])}function Jd(e,t){if(e===void 0)return null;if(!Array.isArray(e)||e.length===0)throw new Error(`${t} must be a non-empty string[]${typeof e=="string"?" (spec 2.0 has no single-string form: wrap it in a list)":""}`);for(let r of e)if(typeof r!="string"||r.length===0)throw new Error(`${t} entries must be non-empty strings, got ${JSON.stringify(r)}`);return Object.freeze([...e])}function K4(e,t){let r=new Map(e.map(s=>[s.id,s])),n=new Map(e.map((s,a)=>[s.id,a]));for(let s of e)for(let a of s.supersededBy??[]){if(a===s.id)throw new Error(`WebGPU manifest ${t} variant "${s.id}" lists itself in supersededBy`);let i=r.get(a);if(!i)throw new Error(`WebGPU manifest ${t} variant "${s.id}" supersededBy names unknown variant "${a}"`);if((co(i,s)||(n.get(a)??0)-(n.get(s.id)??0))>=0)throw new Error(`WebGPU manifest ${t} variant "${s.id}" (priority ${s.priority}) is supersededBy "${a}" (priority ${i.priority}), which does not outrank it. supersededBy removes a variant where a HIGHER-ranked sibling already covers the shape; to express the reverse, put the guard on the other variant.`)}}var e2=new Set(["features","limits","subgroupMinSize","subgroupMatrixConfigs"]),U4=new Set(pr),N4=new Set(Wn);function d2(e,t){if(!wt(e))throw new Error(`${t} requires must be an object`);for(let l of Object.keys(e))if(!e2.has(l))throw new Error(`${t} requires has unknown key "${l}"; expected one of ${[...e2].join(", ")}`);let{features:r,limits:n,subgroupMinSize:s,subgroupMatrixConfigs:a}=e,i=Z4(r,`${t} requires.features`);for(let l of i)if(!U4.has(l))throw new Error(`${t} requires.features has unknown feature "${l}"; expected one of ${pr.join(", ")}`);if(n!==void 0&&!wt(n))throw new Error(`${t} requires.limits must be an object`);for(let[l,c]of Object.entries(n??{})){if(!N4.has(l))throw new Error(`${t} requires.limits has unknown limit "${l}"; not a canonical WebGPU limit name`);if(typeof c!="number"||!Number.isFinite(c))throw new Error(`${t} requires.limits.${l} must be a finite number`)}let u=$4(a,t);if(s!==void 0&&(!Number.isInteger(s)||s<1))throw new Error(`${t} requires.subgroupMinSize must be a positive integer`);let o={...i.length>0?{features:i}:{},...n&&Object.keys(n).length>0?{limits:n}:{},...s!==void 0?{subgroupMinSize:s}:{},...u!==null?{subgroupMatrixConfigs:u}:{}};return Object.keys(o).length===0?null:o}function H4(e,t,r){if(e===null)return t;if(t===null)return e;let n=[...new Set([...e.features??[],...t.features??[]])],s={...e.limits??{}};for(let[u,o]of Object.entries(t.limits??{}))o!==void 0&&(s[u]=Math.max(s[u]??-1/0,o));let a=t.subgroupMinSize!==void 0||e.subgroupMinSize!==void 0?Math.max(t.subgroupMinSize??0,e.subgroupMinSize??0):void 0;if(e.subgroupMatrixConfigs!==void 0&&t.subgroupMatrixConfigs!==void 0)throw new Error(`WebGPU variant ${r} and the manifest both declare requires.subgroupMatrixConfigs; declare the pattern list once`);let i=t.subgroupMatrixConfigs??e.subgroupMatrixConfigs;return{...n.length>0?{features:n}:{},...Object.keys(s).length>0?{limits:s}:{},...a!==void 0?{subgroupMinSize:a}:{},...i!==void 0?{subgroupMatrixConfigs:i}:{}}}function $4(e,t){if(e===void 0)return null;if(!Array.isArray(e)||e.length===0)throw new Error(`${t} requires.subgroupMatrixConfigs must be a non-empty array of config patterns`);let r=ua,n=e.map((s,a)=>{let i=`${t} requires.subgroupMatrixConfigs[${a}]`;if(!wt(s))throw new Error(`${i} must be an object with optional componentType/resultComponentType/M/N/K`);let u=Object.keys(s).filter(l=>!["componentType","resultComponentType","M","N","K"].includes(l));if(u.length>0)throw new Error(`${i} has unknown ${u.length===1?"key":"keys"} ${u.join(", ")}`);for(let l of["componentType","resultComponentType"]){let c=s[l];if(c!==void 0&&(typeof c!="string"||!r.includes(c)))throw new Error(`${i}.${l} must be one of ${r.join(", ")}, got ${JSON.stringify(c)}`)}for(let l of["M","N","K"]){let c=s[l];if(c!==void 0&&(!Number.isInteger(c)||c<1))throw new Error(`${i}.${l} must be a positive integer, got ${JSON.stringify(c)}`)}let o={...s.componentType!==void 0?{componentType:s.componentType}:{},...s.resultComponentType!==void 0?{resultComponentType:s.resultComponentType}:{},...s.M!==void 0?{M:s.M}:{},...s.N!==void 0?{N:s.N}:{},...s.K!==void 0?{K:s.K}:{}};if(Object.keys(o).length===0)throw new Error(`${i} must constrain at least one field (an empty pattern matches everything)`);return Object.freeze(o)});return Object.freeze(n)}function V4(e,t){if(e===void 0)return[];if(!Array.isArray(e))throw new Error(`WebGPU variant ${t} intermediates must be an array`);return e.map((r,n)=>{if(Ye(r,`WebGPU variant ${t} intermediate ${n}`),typeof r.id!="string"||r.id.length===0)throw new Error(`WebGPU variant ${t} intermediate ${n} id must be a non-empty string`);if(typeof r.dtype!="string"||!Object.hasOwn(Ge,r.dtype))throw new Error(`WebGPU variant ${t} intermediate ${r.id} dtype must be a concrete WebGPU storage dtype`);if(r.dtype==="int64")throw new Error(`WebGPU variant ${t}: opaque int64 intermediates are unsupported`);if(r.zeroInit!==void 0&&r.zeroInit!==!0)throw new Error(`WebGPU variant ${t} intermediate ${r.id} zeroInit must be true when present (omit the key when the kernel does not require a zeroed buffer)`);return Object.freeze({id:r.id,dtype:r.dtype,shape:r.shape,...r.zeroInit===!0?{zeroInit:!0}:{}})})}function Z4(e,t){if(e===void 0)return[];if(!Array.isArray(e)||![...e].every(r=>typeof r=="string"))throw new Error(`${t} must be an array of strings`);return[...e]}var X4=new Set(["id","name","shader","derive","bindings","dispatch","profile","subgroupCollectivesWidth","submitAfter","viewAlias"]);function f2(e,t,r,n,s=new Set){return e.map((a,i)=>{if(!Y4(a))return J4(a,i,t,r);let u=a.id??`repeat_${i}`;if(typeof u!="string"||!s2.test(u))throw new Error(`WebGPU variant ${n} repeat group id must be stable, got ${String(u)}`);Ye(a.repeat,`WebGPU repeat group ${u}.repeat`);let o=a.repeat.index;if(typeof o!="string"||!qt.test(o))throw new Error(`WebGPU repeat group ${u} index must be an identifier, got ${String(o)}`);if(o==="index"||o==="count")throw new Error(`WebGPU repeat group ${u} index cannot use reserved repeat member ${o}`);if(s.has(o))throw new Error(`WebGPU repeat group ${u} index ${o} shadows an outer repeat index`);if(a.repeat.count===void 0)throw new Error(`WebGPU repeat group ${u} requires repeat.count`);if(!Array.isArray(a.passes)||a.passes.length===0)throw new Error(`WebGPU repeat group ${u} requires passes`);let l=new Set(s);return l.add(o),Object.freeze({kind:"repeat",id:u,count:a.repeat.count,index:o,passes:Object.freeze(f2(a.passes,t,r,n,l))})})}function Y4(e){return wt(e)&&"repeat"in e}function p2(e){let t=[];for(let r of e)"kind"in r&&r.kind==="repeat"?t.push(...p2(r.passes)):t.push(r);return t}function J4(e,t,r,n){Ye(e,`WebGPU pass ${t}`);let s=e.id??`pass_${t}`;if(typeof s!="string"||s.length===0)throw new Error(`WebGPU pass ${t} id must be a non-empty string`);for(let d of Object.keys(e))if(!X4.has(d))throw new Error(`WebGPU pass ${s} has unknown key "${d}"${d==="source"||d==="constants"?" (a pass spells `shader` and pass-level `derive`)":""}`);if(e.name!==void 0&&(typeof e.name!="string"||e.name.length===0))throw new Error(`WebGPU pass ${s} name must be a non-empty string`);if(typeof e.shader!="string"||e.shader.length===0)throw new Error(`WebGPU pass ${s} requires shader`);if(e.submitAfter!==void 0&&typeof e.submitAfter!="boolean")throw new Error(`WebGPU pass ${s} submitAfter must be a boolean`);let a=eh(e.subgroupCollectivesWidth,s);if(!e.dispatch)throw new Error(`WebGPU pass ${s} requires dispatch`);Ye(e.dispatch,`WebGPU pass ${s} dispatch`);let i="threads"in e.dispatch?Object.freeze({divisor:e.dispatch.workgroupSize}):void 0,u=rh(e.dispatch,s);if(!Array.isArray(e.bindings)||e.bindings.length===0)throw new Error(`WebGPU pass ${s} requires a non-empty bindings list`);let o=Object.freeze(e.bindings.map((d,p)=>M4(d,p,s,r,n))),l=new Set;for(let d of o){if(l.has(d.name))throw new Error(`WebGPU pass ${s} binds the WGSL name "${d.name}" twice`);l.add(d.name)}let c=oo(e.derive,`WebGPU pass ${s} derive`,r.reservedDeriveNames);return Object.freeze({id:s,...e.name?{name:e.name}:{},shader:e.shader,entryPoint:R4,bindings:o,derive:c,dispatch:u,...i!==void 0?{dispatchWorkgroupContract:i}:{},profile:Object.freeze({...e.profile??{}}),subgroupCollectivesWidth:a,submitAfter:e.submitAfter===!0,...e.viewAlias?{viewAlias:th(e.viewAlias,o,`pass ${s}`)}:{}})}function eh(e,t){if(e===void 0)return null;if(e==="portable"||typeof e=="number"&&Number.isInteger(e)&&e>=4&&e<=128&&(e&e-1)===0)return e;throw new Error(`WebGPU pass ${t} subgroupCollectivesWidth must be "portable" or a power of two in [4, 128]`)}function th(e,t,r){if(!Array.isArray(e)||e.length===0)throw new Error(`WebGPU ${r} viewAlias must be a non-empty array of { input, output } binding-name pairs`);let n=s=>t.find(a=>a.name===s)?.buffer.type;return Object.freeze(e.map(s=>{if(typeof s?.input!="string"||typeof s?.output!="string")throw new Error(`WebGPU ${r} viewAlias pair must be { "input": <binding>, "output": <binding> }`);if(n(s.input)!=="read-only-storage")throw new Error(`WebGPU ${r} viewAlias.input "${s.input}" must name a read-only-storage binding of the pass`);if(n(s.output)!=="storage")throw new Error(`WebGPU ${r} viewAlias.output "${s.output}" must name a storage (read_write) binding of the pass`);return Object.freeze({input:s.input,output:s.output})}))}var Wr=Wt;function rh(e,t){let r=e,n="threads"in r,s="workgroups"in r;if("gridStride"in r)throw new Error(`WebGPU pass ${t} dispatch uses gridStride, which spec 2.0 removed: write the clamp explicitly, x = min(<count>, device.limits.maxComputeWorkgroupsPerDimension)`);if(!n&&!s){for(let i of Object.keys(r))if(i!=="x"&&i!=="y"&&i!=="z")throw new Error(`WebGPU pass ${t} dispatch has unknown key "${i}"`);if(r.x===void 0)throw new Error(`WebGPU pass ${t} dispatch requires x`);for(let i of["x","y","z"])r[i]!==void 0&&Da(r[i],t);return e}if(n&&s)throw new Error(`WebGPU pass ${t} dispatch must use exactly one of 'threads'+'workgroupSize' or 'workgroups'`);let a=n?["threads","workgroupSize"]:["workgroups"];for(let i of Object.keys(r))if(!a.includes(i))throw new Error(`WebGPU pass ${t} dispatch shorthand has unknown key "${i}"${["x","y","z"].includes(i)?" (spec 2.0 has no pinned axes; write the explicit x/y/z grid)":""}`);if(n){if(r.workgroupSize===void 0)throw new Error(`WebGPU pass ${t} dispatch shorthand requires both 'threads' and 'workgroupSize'`);return t2(`ceilDiv((${Da(r.threads,t)}), (${Da(r.workgroupSize,t)}))`)}return t2(Da(r.workgroups,t))}function t2(e){return Object.freeze({x:`min(${e}, ${Wr})`,y:`ceilDiv(${e}, ${Wr})`,z:1})}function Da(e,t){if(typeof e=="string")return e;if(typeof e=="number"&&Number.isSafeInteger(e)&&e>=0)return String(e);throw new Error(`WebGPU pass ${t} dispatch sizes must be expression strings or non-negative safe integers, got ${JSON.stringify(e)}`)}function fo(e,t){if(e=Ga(e),e.variantFamilies!==void 0)throw new Error(`WebGPU manifest ${t} has a top-level variantFamilies list; a family is a \`variants\` member { cases, variants }`);let r=[];for(let[n,s]of(e.variants??[]).entries()){if(!wt(s))throw new Error(`WebGPU manifest ${t}: variants[${n}] must be an object`);if("use"in s)throw new Error(`WebGPU manifest ${t} variants[${n}] is an unexpanded shared fragment reference ("use": ${JSON.stringify(s.use)}); expand the shared libraries before normalizing`);if(!("cases"in s)&&!("axes"in s)&&!("exclude"in s)){r.push(s);continue}let a=s;for(let u of Object.keys(a))if(u!=="cases"&&u!=="variants"&&u!=="description"&&u!=="axes"&&!(u==="exclude"&&"axes"in a))throw new Error(`WebGPU manifest ${t}: variants[${n}] has unknown key "${u}"`);if(a.description!==void 0&&typeof a.description!="string")throw new Error(`WebGPU manifest ${t}: variants[${n}] description must be a string`);if(!Array.isArray(a.variants)||a.variants.length===0)throw new Error(`WebGPU manifest ${t}: a family requires at least one variant template`);let{cases:i}=nh(a,`WebGPU manifest ${t}: variants[${n}]`);for(let[u,{values:o}]of i.entries())for(let[l,c]of a.variants.entries()){let d=`WebGPU manifest ${t}: variants[${n}] case ${u} template ${l}`,p=sh(c,o,d);ao(p,d,`variants[${n}].variants[${l}]`),r.push(p)}}return r}function r2(e,t){if(!Zn(e)||Object.keys(e).length===0)throw new Error(`${t} must be a non-empty record`);for(let[r,n]of Object.entries(e)){if(!qt.test(r))throw new Error(`${t} key "${r}" must be an identifier`);if(!no(n))throw new Error(`${t}.${r} must be a finite JSON value`)}}function nh(e,t){e=Ga(e);let r=[],n=(d,p,f)=>{Object.keys(d).length&&r2(d,`${t} ${f===void 0?"case":`axis ${f}`} ${p}`);let g=new Map;for(let m of Object.keys(d)){let h={case:p,...f===void 0?{}:{axis:f},key:m};r.push(h),g.set(m,h)}return{values:d,fields:g}};if(!("axes"in e)){if(!Array.isArray(e.cases)||e.cases.length===0)throw new Error(`${t}: family cases must be a non-empty array of case records`);return{cases:e.cases.map((d,p)=>(r2(d,`${t} case ${p}`),n(d,p))),fields:r}}if(!Zn(e.axes)||Object.keys(e.axes).length!==2)throw new Error(`${t}: axes must have exactly two named axes`);let s=Object.keys(e.axes);for(let d of s)if(!qt.test(d))throw new Error(`${t}: axis name "${d}" must be an identifier`);let a=new Map,i=Object.entries(e.axes).map(([d,p])=>{if(!Array.isArray(p)||p.length===0)throw new Error(`${t}: axis ${d} must be a non-empty array`);let f=new Set;return p.map((g,m)=>{if(!Zn(g)||typeof g.$key!="string"||!qt.test(g.$key))throw new Error(`${t}: axis ${d}[${m}] requires an identifier $key`);if(f.has(g.$key))throw new Error(`${t}: duplicate $key "${g.$key}" in axis ${d}`);f.add(g.$key);let h=Object.fromEntries(Object.entries(g).filter(([v])=>v!=="$key"));for(let v of Object.keys(h)){if(s.includes(v))throw new Error(`${t}: value "${v}" collides with an axis selector`);let b=a.get(v);if(b!==void 0&&b!==d)throw new Error(`${t}: value "${v}" belongs to both axes`);a.set(v,d)}return{key:g.$key,...n(h,m,d)}})}),u=(d,p,f)=>{if(!Zn(d))throw new Error(`${p} must be a selector record`);for(let[g,m]of s.entries())if(typeof d[m]!="string"||!i[g].some(h=>h.key===d[m]))throw new Error(`${p} must select an existing ${m} key`);if(f&&Object.keys(d).some(g=>!s.includes(g)))throw new Error(`${p} must name exactly both axes`);return JSON.stringify(s.map(g=>d[g]))},o=new Set;if(e.exclude!==void 0&&!Array.isArray(e.exclude))throw new Error(`${t}: exclude must be an array`);for(let[d,p]of(e.exclude??[]).entries()){let f=u(p,`${t} exclude[${d}]`,!0);if(o.has(f))throw new Error(`${t}: duplicate excluded cell ${f}`);o.add(f)}if(e.cases!==void 0&&!Array.isArray(e.cases))throw new Error(`${t}: cases overrides must be an array`);let l=new Map;for(let[d,p]of(e.cases??[]).entries()){let f=u(p,`${t} case ${d}`,!1);if(l.has(f))throw new Error(`${t}: duplicate cell override ${f}`);if(o.has(f))throw new Error(`${t}: override targets excluded cell ${f}`);let g=Object.fromEntries(Object.entries(p).filter(([m])=>!s.includes(m)));l.set(f,n(g,d))}let c=[];for(let d of i[0])for(let p of i[1]){let f=JSON.stringify([d.key,p.key]);if(o.has(f))continue;let g=l.get(f);c.push({values:{...d.values,...p.values,...g?.values},fields:new Map([...d.fields,...p.fields,...g?.fields??[]])})}if(c.length===0)throw new Error(`${t}: axes must retain at least one cell`);return{cases:c,fields:r}}function no(e,t=new Set){if(e===null||typeof e=="string"||typeof e=="boolean")return!0;if(typeof e=="number")return Number.isFinite(e);if(typeof e!="object"||t.has(e))return!1;t.add(e);let r;if(Array.isArray(e)){r=!0;for(let n=0;n<e.length;n+=1)if(!Object.hasOwn(e,n)||!no(e[n],t)){r=!1;break}}else Zn(e)?r=Object.values(e).every(n=>no(n,t)):r=!1;return t.delete(e),r}function Zn(e){if(e===null||typeof e!="object"||Array.isArray(e))return!1;let t=Object.getPrototypeOf(e);return t===Object.prototype||t===null}var n2=/^\{([A-Za-z_]\w*)\}$/,so=Symbol("variantFamily.omit");function sh(e,t,r){let n=Oa(e,t,[],r);if(n===so)throw new Error(`${r}: a case cannot delete the whole variant template`);return n}function Oa(e,t,r,n){if(typeof e=="string"){let s=n2.exec(e);if(s&&Object.hasOwn(t,s[1])){let a=t[s[1]];if(a===null){let i=r.at(-1);if(typeof i=="number")throw new Error(`${n}: null substitution "{${s[1]}}" cannot delete the array element at ${ro(r)} (substitute an empty array to splice nothing)`);if(r.length===0||i==="id"||r.length===1&&r[0]==="passes")throw new Error(`${n}: null substitution "{${s[1]}}" cannot remove ${ro(r)}, which is required`);return so}return Fa(a)}return e.replace(/\{([A-Za-z_]\w*)\}/g,(a,i)=>{if(!Object.hasOwn(t,i))return a;let u=t[i];if(u===null||typeof u=="object")throw new Error(`${n}: substitution "{${i}}" at ${ro(r)} must occupy the whole value to take a ${u===null?"null":"structured"} case value`);return String(u)})}if(Array.isArray(e)){let s=[];for(let a=0;a<e.length;a+=1){let i=e[a],u=typeof i=="string"?n2.exec(i):null,o=u!==null&&Object.hasOwn(t,u[1])?t[u[1]]:void 0;if(Array.isArray(o)){for(let l of o)s.push(Fa(l));continue}s.push(Oa(i,t,[...r,a],n))}return s}if(e!==null&&typeof e=="object"){let s=[],a=new Set;for(let[i,u]of Object.entries(e)){let o=String(Oa(i,t,[...r,i],n));if(a.has(o))throw new Error(`${n}: substitution produced duplicate object key "${o}"`);a.add(o);let l=Oa(u,t,[...r,i],n);l!==so&&s.push([o,l])}return Object.fromEntries(s)}return e}function ro(e){return e.length===0?"variant":`variant${e.map(t=>typeof t=="number"?`[${t}]`:`.${t}`).join("")}`}function Fa(e){return Array.isArray(e)?e.map(Fa):e!==null&&typeof e=="object"?Object.fromEntries(Object.entries(e).map(([t,r])=>[t,Fa(r)])):e}function ao(e,t,r){if(typeof e=="string"){let n=/\{[A-Za-z_]\w*\}/.exec(e);if(n!==null)throw new Error(`${t} leaves unresolved placeholder "${n[0]}" at ${r}`);return}if(Array.isArray(e)){for(let n=0;n<e.length;n+=1)ao(e[n],t,`${r}[${n}]`);return}if(e!==null&&typeof e=="object")for(let[n,s]of Object.entries(e)){let a=/\{[A-Za-z_]\w*\}/.exec(n);if(a!==null)throw new Error(`${t} leaves unresolved placeholder "${a[0]}" in object key at ${r}`);ao(s,t,`${r}.${n}`)}}function g2(e,t){if(wt(e)&&Object.hasOwn(e,"$use"))throw new Error(`${t} still carries an unexpanded "$use" shared-derive splice \u2014 expand the shared libraries before normalizing`)}function ah(e){if(e.type==="u32")return Du(e.name,e);if(e.type==="i32")return Lu(e.name,e);if(e.type==="f32")return Ou(e.name,e);throw new Error(`Unsupported WebGPU uniform field type: ${e.type}`)}function ih(e,t){if(!/^[a-z][a-z0-9]*(?:\.[A-Za-z][A-Za-z0-9_]*)*$/.test(e))throw new Error(`${t} must be a dotted canonical domain, got ${e}`)}function Ye(e,t){if(!wt(e))throw new Error(`${t} must be an object`)}function uh(e,t){return e===void 0?{}:(Ye(e,t),{...e})}function wt(e){return e!==null&&typeof e=="object"&&!Array.isArray(e)}var m2="@group(0) @binding(0) var<storage, read_write> slot: array<atomic<u32>>;",v2=Object.freeze([{construct:"atomicCompareExchangeWeak",specifies:"WGSL 17.11.3 makes atomicCompareExchangeWeak a mandatory built-in for every atomic type in both the storage and workgroup address spaces, returning __atomic_compare_exchange_result<T> { old_value: T, exchanged: bool }. It is core WGSL: no feature flag guards it and no conformant implementation may reject it.",defect:`WebKit's WGSL-to-Metal translation emits
    template<typename T, typename U = bool>
    struct __atomic_compare_exchange_result { T old_value; U exchanged; };
instantiated with an address-space-qualified T, which Metal refuses: "field may not be qualified with an address space". Every pipeline containing the built-in fails to create, whatever the kernel does with the result.`,measured:["Safari on iOS, Apple A-series GPU"],unaffected:["Chrome on Apple M-series \u2014 the same GPU family, which is what makes this an engine fact rather than a hardware one","Chrome on Intel Gen9","Chrome on NVIDIA Pascal"],tracking:null,detection:{mode:"build",probe:`${m2}
@compute @workgroup_size(1)
fn main() {
  let swapped = atomicCompareExchangeWeak(&slot[0], 0u, 1u);
  if (swapped.exchanged) {
    atomicStore(&slot[0], swapped.old_value);
  }
}
`,control:`${m2}
@compute @workgroup_size(1)
fn main() {
  let previous = atomicLoad(&slot[0]);
  if (previous == 0u) {
    atomicStore(&slot[0], previous);
  }
}
`}},{construct:"subgroupExclusiveAdd",specifies:"WGSL 17.12 defines the subgroup arithmetic built-ins over the active invocations of a subgroup, and says nothing that would make their results unusable as operands. A value returned by subgroupAdd or subgroupExclusiveAdd is an ordinary u32.",defect:"Feeding the result of subgroupAdd(1u) or subgroupExclusiveAdd(1u) into an atomicAdd that hands out per-subgroup claims makes every subgroup read a claim of zero, so most destinations are never written and kernels that compact or scan through such claims return wrong results. Each collective is correct on its own, which makes this a miscompile of the composition rather than a broken built-in.",measured:["Chrome, NVIDIA Maxwell GPU"],unaffected:["every other tested device, across Apple, Intel and NVIDIA Pascal GPUs"],tracking:null,detection:{mode:"behavior",avoidedBy:"Kernels take the subgroup width and the electing lane from the subgroup_size / subgroup_invocation_id BUILTINS rather than from the collectives' results. That is correct on the affected device and on every other, so no kernel is withheld anywhere."}},{construct:"num_subgroups",specifies:"WGSL's subgroups extension defines the num_subgroups built-in input as the number of subgroups in the current workgroup, and subgroup_id as each subgroup's index in [0, num_subgroups).",defect:"A workgroup that spans several subgroups reports num_subgroups = 1, so a cross-subgroup fold sized by it reads only the first subgroup's partial and returns a fraction of the reduction.",measured:["Chrome on Android, Qualcomm Adreno 6xx to 8xx GPUs"],unaffected:["Chrome on Apple M-series, Intel and NVIDIA GPUs"],tracking:null,detection:{mode:"behavior",avoidedBy:"Cross-subgroup folds bound their loop over the subgroup partials by max(num_subgroups, ceil(workgroup_size / subgroup_size)), so an under-reported count can neither skip the fold nor shorten it, and keep indexing the partials by subgroup_id. On a conformant device the reported count always wins, so the fold is exact for any subgroup layout."}},{construct:"workgroupBarrier",specifies:"WGSL's memory model keeps one invocation's accesses to a workgroup location coherent in program order, and workgroupBarrier orders every invocation's workgroup-memory writes before the loads that follow it. A load after a barrier returns the latest value stored to that location, whether the loading invocation stored it or a peer did.",defect:"A load from a workgroup array at a compile-time-constant index returns the value an earlier load of the same element returned, although the array was stored to in between through an invocation-dependent index and the loads are separated by workgroupBarrier: a second inlined tree reduction over the same bank returns the first reduction's slot-0 total, and a serial top-k that marks its pick with a dynamic-index store selects the same element on every pick.",measured:["Chrome on Android, Qualcomm Adreno 810 and 830 GPUs"],unaffected:["Chrome on Android, Qualcomm Adreno 710 GPU","Chrome on AMD GPUs with 64-wide subgroups"],tracking:null,detection:{mode:"behavior",avoidedBy:"A kernel never reads back, at a constant index, a workgroup element it re-stored through an invocation-dependent index after an earlier read of the same element: each tree reduction in one kernel folds through its own bank, and selections rank from values that are not written again after the barrier that publishes them."}},{construct:"local_invocation_index",specifies:"WGSL's memory model keeps one invocation's accesses to a location coherent in program order, and storageBarrier / workgroupBarrier make an invocation's storage writes visible to the rest of its workgroup. Once the loading invocation, or a peer ordered before it by a barrier, has stored to a location, no later load in the dispatch can return the value it held before the dispatch. local_invocation_index is the linearized local_invocation_id.",defect:"A loop whose induction variable starts at local_invocation_index and whose constant stride is at least its constant bound, so that no invocation runs it twice, loads a storage location at a compile-time-constant address (after unrolling and bounds clamping) after a barrier, and receives the value the location held before the dispatch, even when the loading invocation stored to it itself. atomicLoad is affected the same way; read-modify-write atomics, workgroup memory and invocation-dependent addresses are not. The same kernel with the index taken from local_invocation_id, or with the loop written as an if, reads the stored value.",measured:["Dawn (Chrome's WebGPU engine) on macOS, Apple M-series GPU"],unaffected:["Safari on iOS, Apple A-series GPU","Dawn on Windows (D3D12), Intel Xe3-LPG GPU"],tracking:null,detection:{mode:"behavior",avoidedBy:"A kernel whose loop starts at the flat local index and re-reads storage the same dispatch wrote computes that index from local_invocation_id (x + y * width + z * width * height) instead of reading local_invocation_index. The value is the same by definition, and the loads then read the stored value on every implementation measured."}},{construct:"tanh",specifies:"WGSL bounds the f32 tanh built-in only through its definition: its accuracy is inherited from sinh(x) / cosh(x), which inherit from exp (3 + 2|x| ULP) and division (2.5 ULP). At x = -1.25 that allows about 8.7e-7 absolute error.",defect:"tanh is off by about 7.5e-6 relative (6.3e-6 absolute at x = -1.25, seven times the bound) while exp on the same device is within one ULP of the other engines. A recurrence that applies tanh at every step carries the error into its state.",measured:["Chrome on Windows (D3D12), NVIDIA Ampere, Ada Lovelace and Blackwell GPUs"],unaffected:["Chrome on Linux (Vulkan), NVIDIA Blackwell GPU, which makes this an engine fact rather than a hardware one","Chrome on Windows (D3D12), NVIDIA Pascal GPU","Chrome and Safari on Apple, AMD, Intel, Qualcomm and Arm GPUs"],tracking:null,detection:{mode:"behavior",avoidedBy:"The RNN, GRU and LSTM cells compute tanh from exp: an odd polynomial below |x| = 0.625 and (1 - e) / (1 + e) with e = exp(-2|x|) above it, both evaluated and one selected, which saturates to exactly +/-1 for large and infinite input. Adapters that report vendor apple keep the built-in, which measured exact there and is faster; an adapter that reports no vendor takes the exp form. Elementwise Tanh, Mish, Gelu and the fused activations still call the built-in."}}]);function oh(e){return v2.find(t=>t.construct===e)}async function w2(e,t={}){let r=[],n=async s=>{try{return await e(s)}catch(a){return a instanceof Error?a.message:String(a)}};for(let s of t.bugs??v2){if(s.detection.mode!=="build"||await n(s.detection.probe)===null)continue;let i=await n(s.detection.control);if(i!==null){t.onInconclusive?.(s.construct,`the control shader failed too (${i}), so the probe says nothing about ${s.construct}`);continue}r.push(s.construct)}return r}function lh(e){let t="",r=0,n=0;for(;n<e.length;){if(r===0&&e.startsWith("//",n)){let s=e.indexOf(`
`,n);if(s===-1)break;t+=`
`,n=s+1;continue}if(e.startsWith("/*",n)){r+=1,n+=2;continue}if(r>0&&e.startsWith("*/",n)){r-=1,n+=2,t+=" ";continue}if(r>0){e[n]===`
`&&(t+=`
`),n+=1;continue}t+=e[n],n+=1}return t}function ch(e){return new RegExp(`(?<![A-Za-z0-9_])${e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}(?![A-Za-z0-9_])`)}function b2(e,t){let r=[...t];if(r.length===0)return[];let n=e.map(lh);return r.filter(s=>{let a=ch(s);return n.some(i=>a.test(i))})}function dh(e){let t=e.constructs.map(n=>{let s=oh(n);if(s===void 0)return n;let a=s.tracking===null?"unfiled upstream":s.tracking;return`${n} (${s.measured.join("; ")} \u2014 ${a})`}),r=e.used===null?"and no other variant can serve this call":`so ${e.used} is used instead`;return`${e.opId}: this device cannot build ${t.join(", ")}, which ${e.skipped} needs, ${r}. This is a WebGPU implementation bug, not a missing feature \u2014 the construct is mandatory WGSL.`}var h2=new Set;function _2(e){let t=`${e.opId}|${e.skipped}|${e.used??""}|${e.constructs.join(",")}`;h2.has(t)||(h2.add(t),console.warn(dh(e)))}var po=class extends Map{limit;constructor(t){super(),this.limit=t}get(t){let r=super.get(t);return r!==void 0&&(super.delete(t),super.set(t,r)),r}set(t,r){if(super.delete(t),super.set(t,r),this.size>this.limit){let n=this.keys().next();n.done!==!0&&super.delete(n.value)}return this}};function hr(e){return e===void 0?new Map:new po(e)}var Qa=class{entries;hits=0;misses=0;constructor(t){this.entries=hr(t)}get(t){let r=this.entries.get(t);if(r===void 0){this.misses+=1;return}return this.hits+=1,r}getOrCreate(t,r){let n=this.get(t);if(n!==void 0)return n;let s=r();return this.entries.set(t,s),s}clear(){this.entries.clear(),this.hits=0,this.misses=0}},ja=class extends Qa{},Wa=class extends Qa{};function go(e){return{attrs:e.attrs,dtypes:e.dtypes,logicalDtypes:e.logicalDtypes,tensorDtypes:e.tensorDtypes,ranks:e.ranks,tunables:e.tunables}}function y2(e,t,r,n=null){return`${Bt({op:e.id,sinceVersion:e.sinceVersion,variant:t.id,...go(r),specialization:n})}|device=${ph(r.device)}|bindings=${fh(t)}`}var k2=new WeakMap;function fh(e){let t=k2.get(e);if(t!==void 0)return t;let r=e.passes.map(s=>s.bindings.map(a=>({name:a.name,binding:void 0,group:void 0,type:a.buffer.type,elementType:a.elementType,...a.length!==void 0?{length:a.length}:{},struct:gh(a.struct)}))),n=Bt(r);return k2.set(e,n),n}var A2=new WeakMap;function ph(e){if(e===null||typeof e!="object")return Bt(yt(e));let t=e,r=A2.get(e);if(r!==void 0&&r.features===t.features&&r.wgslLanguageFeatures===t.wgslLanguageFeatures&&r.limits===t.limits&&r.adapterInfo===t.adapterInfo&&r.wgslCompileFailures===t.wgslCompileFailures)return r.text;let n=Bt(yt(e));return A2.set(e,{text:n,features:t.features,wgslLanguageFeatures:t.wgslLanguageFeatures,limits:t.limits,adapterInfo:t.adapterInfo,wgslCompileFailures:t.wgslCompileFailures}),n}function gh(e){return e&&{name:e.name,fields:e.fields.map(t=>({name:t.name,type:t.type}))}}function S2(e){return Bt({shapes:e.shapes,args:e.args})}function x2(e,t,r){return`${e}|shapes=${t}|plan=${er(r)}`}var Bt=er;function mh(e){let t=2166136261;for(let r=0;r<e.length;++r)t^=e.charCodeAt(r),t=Math.imul(t,16777619);return(t>>>0).toString(16).padStart(8,"0")}function za(e){return`${mh(e)}:${e.length}`}function T2(e,t,r){let n=t.map((s,a)=>{let i=s.offset??0,u=s.size!==void 0&&s.size>0?s.size:"";return`${s.binding??a}:${r(s)}:${i}:${u}`});return`${e}|${n.join("|")}`}var hh=/(?:^|;)[ \t]*enable\s+[^;]*\bf16\b[^;]*;/m,vh=/\bf16\b/,wh=/(?:^|;)[ \t]*enable\s+[^;]*\bsubgroups\b[^;]*;/m,bh=/(?:^|;)[ \t]*enable\s+[^;]*\bsubgroup_size_control\b[^;]*;/m,_h=new RegExp("\\b(?:subgroup(?:Add|ExclusiveAdd|InclusiveAdd|Mul|ExclusiveMul|InclusiveMul|And|Or|Xor|Min|Max|All|Any|Ballot|Elect|Broadcast|BroadcastFirst|Shuffle|ShuffleXor|ShuffleUp|ShuffleDown)|quad(?:Broadcast|SwapX|SwapY|SwapDiagonal))\\s*\\(|@builtin\\s*\\(\\s*(?:subgroup_invocation_id|subgroup_size|subgroup_id|num_subgroups)\\s*\\)"),kh=/@subgroup_size\s*\(/,Ah=/(?:^|[^A-Za-z0-9_.])(?:0[xX][0-9a-fA-F]*\.?[0-9a-fA-F]*[pP][+-]?[0-9]+|[0-9]+\.?[0-9]*(?:[eE][+-]?[0-9]+)?|\.[0-9]+(?:[eE][+-]?[0-9]+)?)h\b/;function mo(e){let t="",r=0,n=e.indexOf("/"),s=e.length;for(;n!==-1&&n<s;){let a=e[n+1];if(a==="/"){t+=e.slice(r,n);let i=e.indexOf(`
`,n+2);n=i===-1?s:i,r=n}else if(a==="*"){t+=e.slice(r,n);let i=1;for(n+=2;n<s&&i>0;){if(e[n]==="/"&&e[n+1]==="*"){i+=1,n+=2;continue}if(e[n]==="*"&&e[n+1]==="/"){i-=1,n+=2;continue}e[n]===`
`&&(t+=`
`),n+=1}r=n}else n+=1;n=e.indexOf("/",n)}return t+e.slice(r)}var Ka=new Map,yh=4096;function Sh(e){let t=[];return(vh.test(e)||Ah.test(e))&&!hh.test(e)&&t.push("f16"),_h.test(e)&&!wh.test(e)&&t.push("subgroups"),kh.test(e)&&!bh.test(e)&&t.push("subgroup_size_control"),t}var xh=!0;function C2(e){let t=Ka.get(e);if(t===void 0){let r=Sh(mo(e));t={inferred:r.length===0?e:`${r.map(n=>`enable ${n};
`).join("")}${e}`,f16Only:r.includes("f16")?`enable f16;
${e}`:e},Ka.size>=yh&&Ka.clear(),Ka.set(e,t)}return xh?t.inferred:t.f16Only}function q2(e,t){if(t.length===0)return;let r=mo(e),n=new Set(t),s="__opaque_int64_value__",a="";for(let u=0;u<r.length;){let o=/^([A-Za-z_]\w*)\s*\[/.exec(r.slice(u));if(!o){a+=r[u++];continue}let l=u+o[0].length,c=1;for(;l<r.length&&c>0;)r[l]==="["&&++c,r[l]==="]"&&--c,++l;a+=n.has(o[1])?s:`${o[1]}[index]`,u=l}r=a;for(let u=0;u<8;++u){let o=[...r.matchAll(new RegExp(`\\b(?:let|var)\\s+([A-Za-z_]\\w*)(?:\\s*:[^=;]+)?\\s*=\\s*${s}\\s*;`,"g"))].map(l=>l[1]).filter(l=>l!==s);if(o.length===0)break;for(let l of o)r=r.replace(new RegExp(`\\b${l}\\b`,"g"),s)}for(let u="";u!==r;)u=r,r=r.replace(new RegExp(`\\(\\s*${s}\\s*\\)`,"g"),s);let i="(?:==|!=|<=|>=|<<|>>|[+*/%<>&|^~!-])";if(new RegExp(`${s}\\s*(?:\\.|\\[|${i})|${i}\\s*${s}|\\b(?:min|max|abs|clamp|bitcast|f32|i32|u32)\\s*(?:<[^>]+>)?\\s*\\(\\s*${s}`).test(r))throw new Error("Opaque int64 storage permits whole-element copies and selection only; value arithmetic, comparisons, and word extraction are unsupported")}function vr(e,t){return e.scratch!==void 0&&t.has(e.scratch)?e.scratch:t.has(e.name)?e.name:void 0}var B2=new WeakMap;function Th(e,t){let r=B2.get(t);return r===void 0&&(r=za(t.source),B2.set(t,r)),`webgpu:${e}:pass=${t.pipelineCacheId}:source=${r}`}function Yn(e,t){let r=new Set(e.plan.scratches.map(s=>s.id)),n=new Set;for(let s of e.program.passes)if(t.has(s.id))for(let a of s.bindingSpecs){let i=vr(a,r);i!==void 0&&n.add(i)}return n}function Ch(e){let t=0;for(let r of e){if(r.type==="uniform")continue;let n=r.tensor,s=typeof r.size=="number"?r.size:typeof n?.byteLength=="number"?n.byteLength:void 0;typeof s=="number"&&Number.isFinite(s)&&s>0&&(t+=s)}return t}function qh(e,t){return typeof e?.bytesMoved=="number"?{profile:e}:t<=0?e?{profile:e}:{}:{profile:{...e??{},bytesMoved:t}}}async function E2(e,t,r={}){let n=Jn(e,t);try{if(n.programs.length===0)return;if(typeof e.runProgramSequence=="function")await e.runProgramSequence(n.programs,r);else for(let s of n.programs)await e.runProgram(s,r)}finally{n.cleanup()}}function Jn(e,t,r={}){let n=Rh(t);if(!n.some(Boolean))return{programs:[],zeroInitClears:[],constantPrograms:[],cleanup(){}};let s=Gh(t,n),a=r.scratchResources,i=a===void 0,u=a===void 0?Ih(e,t,s):Ph(t,a,s),o=[],l=[],c=[],d=r.constantPassIds,p=i?[]:Eh(t,u,s),f=!1,g=()=>{if(f)return;f=!0;let m;for(let h of o)try{h()}catch(v){m??=v}if(i)try{P2(u)}catch(h){m??=h}if(m!==void 0)throw m};try{for(let m=0;m<t.program.passes.length;++m){let h=t.program.passes[m],v=t.plan.passes[m];if(!n[m])continue;let b=`${t.program.op} (variant ${t.program.variant}, pass ${h.id})`,_=Lh(t,h.bindingSpecs,v.uniforms,u),C=[];for(let E of h.bindingSpecs){let k=_[E.name];if(!st(k)||k.dtype!=="int64")continue;C.push(E.name);let S=E.elementType,A=S?.startsWith("$")?h.derived[S.slice(1)]:S;if(typeof A!="string"||A.replace(/\s+/g,"")!=="vec2<u32>")throw new Error(`${b}: int64 binding ${E.name} must use vec2<u32>`)}if(q2(h.source,C),Fh(h,_))continue;Oh(b,h.bindings,E=>_[E.name]);let y=Jc(e,h.bindings,_,{labelPrefix:h.name});o.push(y.cleanup),(d?.has(h.id)?c:l).push({name:h.name,op:t.program.op,kernelId:h.kernelId,source:h.source,entryPoint:h.entryPoint,cacheKey:Th(t.program.key,h),bindings:y.bindings,dispatchWorkgroups:v.dispatchWorkgroups,...h.submitAfter?{submitAfter:!0}:{},...qh({...v.profile??{},variant:t.program.variant},Ch(y.bindings))})}return{programs:l,zeroInitClears:p,constantPrograms:c,cleanup:g}}catch(m){try{g()}catch(h){throw new AggregateError([m,h],"WebGPU plan materialization and cleanup both failed")}throw m}}function Bh(e,t){return t[e.arg??""]??t[e.name]}function Ih(e,t,r){let n={};if(r.size===0)return n;if(typeof e.empty!="function")throw new Error("WebGPU multi-pass plan requires a runtime with empty(dtype, shape)");try{for(let s of t.plan.scratches)r.has(s.id)&&(n[s.id]=e.empty(s.dtype,s.shape,`webgpu-scratch-${s.id}`))}catch(s){try{P2(n)}catch(a){throw new AggregateError([s,a],"WebGPU scratch allocation and cleanup both failed")}throw s}return n}function Eh(e,t,r){let n=[];for(let s of e.plan.scratches){if(s.zeroInit!==!0||!r.has(s.id))continue;let a=t[s.id];if(a===void 0)continue;let i=typeof a.byteOffset=="number"?a.byteOffset:0,u=typeof a.byteLength=="number"?a.byteLength:s.byteLength;if(u!==0){if(i%4!==0||u%4!==0)throw new Error(`WebGPU scratch ${s.id} declares zeroInit but its buffer range (offset ${i}, length ${u}) is not 4-byte aligned; a zeroInit scratch must be 4-byte aligned in both offset and length (use a 4-byte dtype, or pad the declared shape)`);n.push({kind:"clear",dst:a,dstOffset:i,byteLength:u,name:`${e.program.op}:${s.id}-zero-init`})}}return n}function Ph(e,t,r){let n={};for(let s of e.plan.scratches){if(!r.has(s.id))continue;let a=t[s.id];if(a==null)throw new Error(`Missing caller-owned WebGPU scratch resource ${s.id}`);let i=a;if(i.dtype!==s.dtype||!Array.isArray(i.shape)||!da(i.shape,s.shape)){let u=Array.isArray(i.shape)?`[${i.shape.join(",")}]`:String(i.shape);throw new Error(`Caller-owned WebGPU scratch ${s.id} must be ${s.dtype}[${s.shape.join(",")}], got ${String(i.dtype)}${u}`)}n[s.id]=a}return n}function Rh(e){if(e.program.passes.length!==e.plan.passes.length)throw new Error(`WebGPU program/plan pass count mismatch: ${e.program.passes.length} program passes, ${e.plan.passes.length} plan passes`);return e.plan.passes.map((t,r)=>{let n=e.program.passes[r];if(n.id!==t.id)throw new Error(`WebGPU program/plan pass ${r} identity mismatch: program ${n.id}, plan ${t.id}`);return!Dh(t.dispatchWorkgroups)})}function Gh(e,t){let r=new Set(e.plan.scratches.map(s=>s.id)),n=new Set;for(let s=0;s<e.program.passes.length;s+=1)if(t[s])for(let a of e.program.passes[s].bindingSpecs){let i=vr(a,r);i!==void 0&&n.add(i)}return n}function Dh(e){return e.some(t=>t===0)}function P2(e){let t;for(let r of Object.values(e))if(typeof r.destroy=="function")try{r.destroy()}catch(n){t??=n}if(t!==void 0)throw t}function Lh(e,t,r,n){let s={},a=new Set(Object.keys(n));for(let i of t){if(i.buffer.type==="uniform"){let u=I2(e,i,n,a);s[i.name]=u??r[i.name];continue}s[i.name]=I2(e,i,n,a)}return s}function I2(e,t,r,n){let s=vr(t,n);return(s===void 0?void 0:r[s])??Bh(t,e.request.resources)}function Oh(e,t,r){let n=null;for(let s of t){let a=Mh(s);if(a===null)continue;let i=ho(r(s));if(i===null)continue;n===null&&(n=new Map);let u=n.get(i.buffer);if(u===void 0){n.set(i.buffer,{name:s.name,access:a,view:i});continue}if(u.access==="read"&&a==="read")continue;if(u.access==="read_write"&&a==="read_write"){if(!Qh(u.view.offset,u.view.end,i.offset,i.end))continue;throw new Error(`${e}: storage bindings "${u.name}" and "${s.name}" alias overlapping writable ranges of the same GPU buffer; overlapping storage aliases are unsafe when either binding is writable`)}let[o,l]=a==="read_write"?[u.name,s.name]:[s.name,u.name];throw new Error(`${e}: storage aliasing hazard \u2014 binding "${o}" (read-only-storage) and binding "${l}" (storage, read_write) resolve to the same GPU buffer. WebGPU usage scopes cover the whole buffer, so this poisons the entire command buffer and corrupts downstream results. Bind it once as storage (read_write) for intentional in-place, or use distinct buffers.`)}}function Mh(e){if("buffer"in e&&e.buffer!==void 0)switch(e.buffer.type){case"read-only-storage":return"read";case"storage":return"read_write";default:return null}let t=e;return t.kind==="storage"?t.access:null}function Fh(e,t){let r=e.viewAlias;if(r===void 0)return!1;for(let{input:n,output:s}of r){let a=ho(t[n]),i=ho(t[s]);if(a===null||i===null||!(a.buffer===i.buffer&&a.offset===i.offset&&a.end===i.end)||a.dtype!==i.dtype)return!1}return!0}function ho(e){if(e===null||typeof e!="object")return null;let t=e,r=t.buffer??e;if(r===null||typeof r!="object"&&typeof r!="function")return null;let n=typeof t.byteOffset=="number"?t.byteOffset:0,s=typeof t.byteLength=="number"?t.byteLength:typeof t.size=="number"?t.size:Number.POSITIVE_INFINITY,a=typeof t.dtype=="string"?t.dtype:void 0;return{buffer:r,offset:n,end:n+s,dtype:a}}function Qh(e,t,r,n){return e<n&&r<t}var R2;function vo(e,t){return R2===void 0?void 0:R2(e,t)}function G2(e){let t=e,r=new Set(Object.keys(t.args??{}));for(let s of["inputs","outputs"]){let a=t[s];if(Array.isArray(a))for(let i of a)typeof i?.role=="string"&&r.add(i.role);else if(a&&typeof a=="object")for(let i of Object.keys(a))r.add(i)}let n=t.bindings;if(n&&typeof n=="object"&&!Array.isArray(n))for(let[s,a]of Object.entries(n))r.add(typeof a?.name=="string"?a.name:s),typeof a?.arg=="string"&&r.add(a.arg);for(let s of t.variants??[]){let a=i=>{for(let u of i){if(u?.repeat!==void 0){a(u.passes??[]);continue}for(let o of Array.isArray(u?.bindings)?u.bindings:[])o&&typeof o=="object"&&(typeof o.name=="string"&&r.add(o.name),typeof o.arg=="string"&&r.add(o.arg))}};a(s?.passes??[])}return r}function D2(e,t,r=!1){return e.map(n=>{let s=r?[0]:jh(Ct(n.shape,t),n.id),a=Pa(n.dtype);return Object.freeze({id:n.id,dtype:a,shape:s,byteLength:et(a,we(s)),...n.zeroInit===!0?{zeroInit:!0}:{}})})}function jh(e,t){let r=Array.isArray(e)?e:[e];if(r.length===0)throw new Error(`WebGPU scratch ${t} shape must not be empty`);return Object.freeze(r.map(n=>{if(!Number.isInteger(n)||Number(n)<0)throw new Error(`WebGPU scratch ${t} shape dimension must be a nonnegative integer, got ${String(n)}`);return Number(n)}))}var L2=65535,Ua=131072;function O2(e,t){let r=[];if(F2(e.passSequence,t,t.repeat,"","",!1,r),r.length===0)throw new Error(`WebGPU variant ${e.id} expands to zero passes; every eligible request must retain at least one pass (all-empty outputs use retained zero-dispatch passes)`);if(r.length>Ua)throw new Error(`WebGPU variant ${e.id} expands to ${r.length} passes; maximum is ${Ua}`);let n=new Set;for(let s of r){if(n.has(s.id))throw new Error(`WebGPU variant ${e.id} expands duplicate pass id ${s.id}`);n.add(s.id)}return Object.freeze(r)}function M2(e){return e.some(Q2)}function F2(e,t,r,n,s,a,i){for(let u of e){if(!Q2(u)){if(i.push(Object.freeze({pass:u,id:`${n}${u.id}`,pipelineBaseId:`${s}${u.id}`,repeat:r,repeated:a})),i.length>Ua)return;continue}let o=lt(u.count,{...t,repeat:r});if(!Number.isInteger(o)||Number(o)<0||Number(o)>L2)throw new Error(`WebGPU repeat group ${u.id} count must resolve to an integer in [0, ${L2}], got ${String(o)}`);let l=Number(o);for(let c=0;c<l;++c){let d=Object.freeze({...r,index:c,count:l,[u.index]:c,[`${u.index}_count`]:l});if(F2(u.passes,t,d,`${n}${u.id}[${c}].`,`${s}${u.id}.`,!0,i),i.length>Ua)return}}}function Q2(e){return"kind"in e&&e.kind==="repeat"}function Wh(e){return"kind"in e&&e.kind==="repeat"}function j2(e,t,r){let n=new Set,s=new Map,a=!1,i=o=>{try{let l=s.get(o.shader);if(l===void 0&&(l=Ra(r(o.shader),"tunables"),s.set(o.shader,l)),l===null)a=!0;else for(let c of l)n.add(c)}catch{a=!0}},u=o=>{for(let l of o)Wh(l)?u(l.passes):i(l)};return u(t.passSequence),Object.freeze({tunables:a?null:Object.freeze([...n].sort())})}function W2(e,t){if(t===null)return e;let r={};for(let n of t)Object.hasOwn(e,n)&&(r[n]=e[n]);return r}var bo=Object.freeze([]);function zh(e,t){return t.size===0?bo:b2(e.passes.map(r=>r.source),t)}function X2(e){return typeof e=="string"?e:String(e)}function Kh(e,t){let r=X2(e);return{kind:"whenFailed",clause:r,value:String(t),message:`when clause failed: \`${r}\` \u2192 ${String(t)}`}}function Uh(e){let t=0,r=0;for(let n of e.passes){let s=0,a=0;for(let i of n.bindings){let u=i.buffer?.type;u&&(u==="uniform"?a+=1:s+=1)}t=Math.max(t,s),r=Math.max(r,a)}return{storage:t,uniform:r}}function Nh(e,t){let{storage:r,uniform:n}=Uh(e),s=t.limits.maxStorageBuffersPerShaderStage;if(typeof s=="number"&&r>s)return`needs ${r} storage buffers per shader stage, device allows ${s} (maxStorageBuffersPerShaderStage)`;let a=t.limits.maxUniformBuffersPerShaderStage;return typeof a=="number"&&n>a?`needs ${n} uniform buffers per shader stage, device allows ${a} (maxUniformBuffersPerShaderStage)`:null}var z2=new WeakMap;function Hh(e,t){let r=z2.get(e);r===void 0&&(r=new WeakMap,z2.set(e,r));let n=r.get(t);if(n!==void 0&&n.features===t.features&&n.limits===t.limits&&n.adapterInfo===t.adapterInfo)return n;let s={features:t.features,limits:t.limits,adapterInfo:t.adapterInfo,requirementReason:Eu(e.requires,t),budgetReason:Nh(e,t)};return r.set(t,s),s}function $h(e,t){let r={};for(let n of Object.keys(e)){let s=e[n];s!==void 0&&(r[n]=es(s)?{dtype:s.dtype,shape:s.shape,...s.logicalDtype===void 0?{}:{logicalDtype:s.logicalDtype}}:s)}return Bt({inputs:r,args:t.args??{},attrs:t.attrs??{}})}var K2=new WeakMap;function Vh(e){let t=K2.get(e);if(t!==void 0)return t;let r=Object.entries(e.derive),n=new Set(r.map(([u])=>u)),s=new Set,a=u=>{if(typeof u=="string"){let o=$d(u);for(let l of[...o.free,...o.probed])!n.has(l)||s.has(l)||(s.add(l),a(e.derive[l]));return}if(Array.isArray(u)){for(let o of u)a(o);return}if(u&&typeof u=="object")for(let o of Object.values(u))a(o)};for(let u of e.outputs)u.shape!==void 0&&a(u.shape);let i=Object.freeze(r.filter(([u])=>s.has(u)).map(([u,o])=>Object.freeze([u,o])));return K2.set(e,i),i}function U2(e,t){let r=Object.freeze(ed(cn(l2(e.bindings)),t).map(s=>Object.freeze(s))),n=[Yc(r),Wu(r,t)].filter(Boolean).join(`
`);return Object.freeze({kernelBindings:r,resourceDeclarations:n})}function Zh(e,t){return e.derived===t.derived&&e.repeat===t.repeat&&e.tunables===t.tunables&&e.device===t.device&&e.dtypes===t.dtypes&&e.logicalDtypes===t.logicalDtypes&&e.tensorDtypes===t.tensorDtypes&&e.ranks===t.ranks&&e.shapes===t.shapes&&e.present===t.present&&e.args===t.args&&e.attrs===t.attrs&&e.env===t.env}var ts=class{manifest;assets;programCache;planCache;prepareCache;explainCache;selectedVariants;manifestDerivedCache;expandedTemplateCache=new Map;passKeyedReadsCache=new WeakMap;passDerivedBase=new WeakMap;passBindingsMemo=new WeakMap;passDerivedMemo=new WeakMap;variantProgramDependencies=new WeakMap;inferCache;scopeNamesByArg;storageDtypesByPortRole;layoutsByArg;acceptedArgKeys;acceptedResourceKeys;acceptedAttributeKeys;variantsById;renderedSourceAudit;requestCacheLimit;constructor(t,r={}){this.manifest=io(t),this.assets=Jh(r.assets??{}),this.renderedSourceAudit=r.renderedSourceAudit;let n=r.requestCacheLimit;this.requestCacheLimit=n,this.programCache=r.programCache??new ja(n),this.planCache=r.planCache??new Wa(n),this.prepareCache=hr(n),this.explainCache=hr(n),this.selectedVariants=hr(n),this.manifestDerivedCache=hr(n),this.inferCache=hr(n),this.scopeNamesByArg=i2(this.manifest.args),this.storageDtypesByPortRole=u2(this.manifest.args),this.layoutsByArg=lo(this.manifest.args,this.manifest.inputs),this.variantsById=new Map(this.manifest.variants.map(s=>[s.id,s])),this.acceptedArgKeys=new Set(Object.keys(this.manifest.args)),this.acceptedAttributeKeys=new Set(this.manifest.attributeNames),this.acceptedResourceKeys=G2(this.manifest)}get prepareCacheSize(){return this.prepareCache.size}explain(t,r,n={}){let s=vo(this,this.manifest),a=this.requestScope(t.device,r,s),{baseScope:i,scopeKey:u}=a,o=this.pinVariant(u,n.variant),l=this.explainCache.get(o);if(l)return l;let c=s?.selectionRequest(a),d=this.applyManifestDeriveCached(u,i,c),p=new Map,f=this.variantCandidates(n.variant).map(h=>{let v=this.checkVariant(h,d,p,r,c);return v.ok?{id:h.id,ok:!0,demoted:v.demotions.length>0?Object.freeze([...v.demotions]):null}:{id:h.id,ok:!1,rejection:v.rejection}}),g=null;for(let h of f)if(h.ok){if(h.demoted===null){g=h.id;break}g??=h.id}let m={selected:g,candidates:f};return this.explainCache.set(o,m),m}selectVariantId(t,r){this.validateRequest(r);let{baseScope:n,scopeKey:s}=this.requestScope(t.device,r,void 0),a=this.prepareCache.get(s);if(a!==void 0)return a.program.variant;let i=this.selectedVariants.get(s);return i===void 0&&(i=this.selectVariant(this.applyManifestDeriveCached(s,n,void 0),void 0,r),this.selectedVariants.set(s,i)),i.id}prepare(t,r,n={}){let s=vo(this,this.manifest),a=s?.rememberedRequestScope(r,t.device);a?.validated!==!0&&this.validateRequest(r);let i=a??this.requestScope(t.device,r,s);i.validated=!0;let{baseScope:u,scopeKey:o}=i,l=this.pinVariant(o,n.variant),c=this.prepareCache.get(l);if(c)return{program:c.program,plan:c.plan,request:r};let d=s?.selectionRequest(i),p=this.applyManifestDeriveCached(o,u,d),f=n.variant===void 0?this.selectedVariants.get(l):void 0,{variant:g,scope:m}=f===void 0?this.selectVariantAndScope(p,r,n.variant,d):{variant:f,scope:this.applyVariantDerive(p,r,f,d)},{passInstances:h,programKey:v,program:b}=this.buildProgramFor(g,m),_=this.allOutputsEmpty(m);this.validateDispatchWorkgroupContracts(g,m,h,b,_);let C=this.buildPlanPayload(g,m,h,_),y=S2(m),E=x2(v,y,C),k=this.planCache.getOrCreate(E,()=>this.buildPlan(E,v,C));return this.prepareCache.set(l,{program:b,plan:k}),this.selectedVariants.delete(l),{program:b,plan:k,request:r}}clearPreparedState(){this.prepareCache.clear(),this.explainCache.clear(),this.selectedVariants.clear(),this.manifestDerivedCache.clear(),this.programCache.clear(),this.planCache.clear()}inferOutputs(t,r={}){let n=$h(t,r),s=this.inferCache.get(n);if(s)return s;this.rejectUnknownKeys(this.manifest.id,"binding",t,r.args);let a=o2(this.manifest.args);for(let c of this.manifest.outputs){if(c.shape!==void 0||c.optional)continue;let d=a(c.role),p=t[d];if(p===void 0||!es(p))throw new Error(`${this.manifest.id}: output "${c.role}" has no shape expression \u2014 pass "${d}" as an explicit destination, or declare an outputs[].shape for it in the manifest.`)}let i={};for(let[c,d]of Object.entries(t))d!==void 0&&(i[c]=es(d)?tv(d):d);let u={resources:i,...r.args?{args:r.args}:{},...r.attrs?{attrs:r.attrs}:{}},o=this.applyManifestDeriveForInference(this.buildBaseScope(ze(void 0),u)),l={};for(let c of this.manifest.outputs){let d=a(c.role),p=t[d];if(c.optional&&(p===void 0||!es(p)))continue;if(c.shape===void 0){if(p!==void 0&&es(p)){l[d]=this.inferredOutput(c,p.shape,o);continue}if(c.optional)continue;throw new Error(`${this.manifest.id}: output "${c.role}" has no shape expression \u2014 pass "${d}" as an explicit destination, or declare an outputs[].shape for it in the manifest.`)}let f=typeof c.shape=="string"?lt(c.shape,o):c.shape.map(g=>Ct(g,o));if(!Array.isArray(f)||!f.every(g=>Number.isInteger(g)&&g>=0))throw new Error(`${this.manifest.id}: output "${c.role}" shape evaluated to ${JSON.stringify(f)} (expected non-negative integers)`);l[d]=this.inferredOutput(c,f,o)}return this.inferCache.set(n,l),l}inferredOutput(t,r,n){let s=this.resolveOutputLogicalDtype(t,n),a=this.storageDtypesByPortRole.get(t.role)??(s!==void 0&&Object.hasOwn(Ge,s)?s:void 0);if(a===void 0)throw new Error(`${this.manifest.id}: output "${t.role}" resolves to logical dtype ${String(s)}, but has no concrete WebGPU storage dtype`);return{role:t.role,shape:[...r],dtype:a,...s===void 0||s===a?{}:{logicalDtype:s}}}resolveOutputLogicalDtype(t,r){if(t.dtype===void 0)return"float32";if(Object.hasOwn(Dr,t.dtype))return t.dtype;let n=r.logicalDtypes[t.dtype];if(n!==void 0)return n;let s=this.manifest.typeConstraints[t.dtype];if(this.manifest.domain==="ai.onnx"&&this.manifest.attributeConstraints.to?.required){let i=r.attrs.to,u=typeof i=="number"?yd(i)?.logical:void 0;if(u===void 0)throw new Error(`${this.manifest.id}: required attribute to=${JSON.stringify(i)} is not a supported ONNX TensorProto dtype code`);if(s===void 0||!s.includes(u))throw new Error(`${this.manifest.id}: required attribute to resolves to ${u}, not allowed by output type constraint ${t.dtype} = [${s?.join(", ")??""}]`);return u}if(s?.length===1)return s[0];let a=r.tensorDtypes;for(let i of this.manifest.inputs){let u=a[i.role];if(typeof u=="string"&&(!Array.isArray(s)||s.some(o=>o===u))&&Object.hasOwn(Dr,u))return u}throw new Error(`${this.manifest.id}: output "${t.role}" dtype "${t.dtype}" is a constraint var with no bound input \u2014 pass a dtype override or bind an input sharing the var.`)}async run(t,r,n={}){let s=this.prepare(t,r,n);return await E2(t,s,n.runOptions??{}),s}buildProgramFor(t,r){let n=O2(t,r),s=this.programDependencies(t),a=this.programSpecialization(t,r,n),i={...r,tunables:W2(r.tunables,s.tunables)},u=y2(this.manifest,t,i,a),o=this.programCache.getOrCreate(u,()=>this.buildProgram(u,t,r,n));return{passInstances:n,programKey:u,program:o}}wgslBuildBlockers(t,r,n,s){if(s.size===0)return bo;try{let a=this.applyVariantDerive(r,n,t);return zh(this.buildProgramFor(t,a).program,s)}catch{return bo}}buildProgram(t,r,n,s){let a=new Map,i=s.map(u=>this.buildProgramPass(r,u,n,a));return Object.freeze({key:t,op:this.manifest.id,variant:r.id,passes:Object.freeze(i)})}buildProgramPass(t,r,n,s){let a=r.pass,i={...n,repeat:r.repeat},u=this.passDerived(a,i);for(let f of a.bindings){if(f.buffer.type==="uniform"||n.tensorDtypes[f.arg??f.name]!=="int64")continue;let g=f.elementType,m=g?.startsWith("$")?u[g.slice(1)]:g;if(typeof m!="string"||m.replace(/\s+/g,"")!=="vec2<u32>")throw new Error(`${this.manifest.id} variant ${t.id} pass ${r.id}: int64 binding ${f.name} must use vec2<u32>`)}let o={...i,derived:u},{kernelBindings:l,resourceDeclarations:c}=this.passBindings(a,u),d=this.buildTemplateSource(t,a,o,c),p=r.pipelineBaseId;if(r.repeated){let f=s.get(r.pipelineBaseId);f===void 0&&(f=new Map,s.set(r.pipelineBaseId,f));let g=f.get(d);g!==void 0?(d=g.source,p=g.pipelineCacheId):(p=`${r.pipelineBaseId}|source=${f.size}`,f.set(d,{source:d,pipelineCacheId:p}))}return Object.freeze({id:r.id,pipelineCacheId:p,...r.repeated?{repeatBaseId:r.pipelineBaseId}:{},kernelId:`${this.manifest.id}.${t.id}.${a.id}`,name:a.name??`${this.manifest.id}.${t.id}.${a.id}`,source:d,derived:u,entryPoint:a.entryPoint,bindings:l,bindingSpecs:a.bindings,...a.submitAfter?{submitAfter:!0}:{},...a.viewAlias?{viewAlias:a.viewAlias}:{}})}passBindings(t,r){let n=this.passBindingsMemo.get(t);if(n===void 0){let u=[];for(let o of t.bindings)if(!Xn(o))for(let l of[o.elementType,o.length])typeof l=="string"&&l.startsWith("$")&&u.push(l.slice(1));n={refs:u,built:hr(this.requestCacheLimit)},this.passBindingsMemo.set(t,n)}let s=[];for(let u of n.refs){let o=r[u];if(o==null)return U2(t,r);s.push(o)}let a=Bt(s),i=n.built.get(a);return i===void 0&&(i=U2(t,r),n.built.set(a,i)),i}expandTemplate(t,r){let n=`${r?"p":"s"}\0${t}`,s=this.expandedTemplateCache.get(n);if(s!==void 0)return s;let a=u=>this.assets.readText(u),i=r?Xh(t,a(t),a,new Set):{source:J2(t,a(t),a,new Set)};return this.expandedTemplateCache.set(n,i),i}programDependencies(t){let r=this.variantProgramDependencies.get(t);return r===void 0&&(r=j2(this.manifest,t,n=>this.expandTemplate(n,!1).source),this.variantProgramDependencies.set(t,r)),r}buildTemplateSource(t,r,n,s){let{source:a,provenance:i}=this.expandTemplate(r.shader,qa()!==void 0),u=C2(eo(a,{...n,env:{device:Vn(n.device),wgsl:{resourceDeclarations:s}}},i));if(this.renderedSourceAudit!==void 0)for(let o of this.renderedSourceAudit.unsupported(u,r.entryPoint,n.device))throw new Error(`WebGPU op ${this.manifest.id} variant ${t.id} pass ${r.id} rendered WGSL ${o}`);return u}programSpecialization(t,r,n){return M2(t.passSequence)?{passes:n.map(s=>{let a=s.pass,i={...r,repeat:s.repeat};return{id:s.id,pipelineBaseId:s.pipelineBaseId,shader:a.shader,...this.keyedPassSpecialization(a,i),...a.submitAfter?{submitAfter:!0}:{}}})}:{passes:t.passes.map(s=>({id:s.id,shader:s.shader,...this.keyedPassSpecialization(s,r),...s.submitAfter?{submitAfter:!0}:{}}))}}keyedPassSpecialization(t,r){let n=this.passDerived(t,r),s=this.passKeyedReads(t),a=(u,o)=>{if(o==="all")return u;let l={};for(let c of Object.keys(u))o.has(c)&&(l[c]=u[c]);return l},i={derived:a(n,s.derived)};return s.args!==null&&(i.args=a(r.args,s.args)),s.shapes!==null&&(i.shapes=a(r.shapes,s.shapes)),s.present!==null&&(i.present=a(r.present,s.present)),i}passKeyedReads(t){let r=this.passKeyedReadsCache.get(t);if(r!==void 0)return r;let n=this.expandTemplate(t.shader,!1).source,s=Ud(n),a=s.has("derive")?"all":new Set(s);if(a!=="all")for(let o of t.bindings)for(let l of["elementType"in o?o.elementType:void 0,"length"in o?o.length:void 0])typeof l=="string"&&l.startsWith("$")&&a.add(l.slice(1));let i=o=>{if(!s.has(o))return null;let l=Ra(n,o);return l===null?"all":new Set(l)},u=Object.freeze({derived:a,args:i("args"),shapes:i("shapes"),present:i("present")});return this.passKeyedReadsCache.set(t,u),u}buildPlanPayload(t,r,n,s){let a=D2(t.intermediates,r,s),i=n.map(u=>this.buildPlanPass(t,u,r,s));return Object.freeze({scratches:a,passes:Object.freeze(i)})}buildPlan(t,r,n){return Object.freeze({key:t,programKey:r,...n})}allOutputsEmpty(t){let r=0;for(let[n,s]of Object.entries(this.manifest.args)){if(s.role!=="output"&&s.role!=="inout")continue;let a=t.resources[n];if(!(a==null&&s.required===!1)&&(!st(a)||(r+=1,a.shape.reduce((i,u)=>i*u,1)!==0)))return!1}return r>0}validateDispatchWorkgroupContracts(t,r,n,s,a){if(a)return;let i=this.renderedSourceAudit;if(i!==void 0)for(let u=0;u<n.length;++u){let o=n[u],l=o.pass.dispatchWorkgroupContract;if(l===void 0)continue;let c=s.passes[u];if(c===void 0)throw new Error(`WebGPU op ${this.manifest.id} variant ${t.id} dispatch/workgroup validation could not find program pass ${o.id}`);let d={...r,repeat:o.repeat},p={...d,derived:this.passDerived(o.pass,d)},f=`WebGPU op ${this.manifest.id} variant ${t.id} pass ${o.id} (base ${o.pass.id}, shader ${o.pass.shader}) dispatch threads workgroupSize divisor`,g=JSON.stringify(l.divisor)??String(l.divisor),m;try{m=lt(l.divisor,p)}catch(b){throw new Error(`${f} ${g} could not be evaluated: ${b instanceof Error?b.message:String(b)}`)}if(typeof m!="number"||!Number.isSafeInteger(m)||m<=0)throw new Error(`${f} must resolve to a positive safe integer, got ${String(m)} from ${g}`);let h=i.workgroupSize(c.source,c.entryPoint);if(h===null)throw new Error(`${f} is ${m} (from ${g}), but selected entry ${JSON.stringify(c.entryPoint)} does not have a statically resolved @workgroup_size. Make the selected entry size resolvable, or express cooperative logical tiling with an explicit workgroups dispatch.`);if(h.some(b=>!Number.isSafeInteger(b)||b<=0))throw new Error(`${f} cannot compare against invalid resolved @workgroup_size(${h.join(", ")})`);let v=BigInt(h[0])*BigInt(h[1])*BigInt(h[2]);if(v>BigInt(Number.MAX_SAFE_INTEGER))throw new Error(`${f} cannot compare against invalid resolved @workgroup_size(${h.join(", ")})`);if(BigInt(m)!==v)throw new Error(`${f} is ${m} (from ${g}), but selected entry ${JSON.stringify(c.entryPoint)} renders @workgroup_size(${h.join(", ")}) with product ${v}. Count physical invocations in threads and use the same physical product as workgroupSize, or express cooperative logical tiling with an explicit workgroups dispatch.`)}}buildPlanPass(t,r,n,s){let a=r.pass,i={...n,repeat:r.repeat};if(s)return Object.freeze({id:r.id,dispatchWorkgroups:[0,0,0],uniforms:this.zeroFilledUniforms(a)});let u={...i,derived:this.passDerived(a,i)},o=a.bindings.filter(Xn).reduce((c,d)=>(c[d.name]=this.uniformValues(d,u),c),{}),l=Vd(a.profile,u);return Object.freeze({id:r.id,dispatchWorkgroups:this.passDispatchWorkgroups(a,u),uniforms:o,...Object.keys(l).length>0?{profile:l}:{}})}zeroFilledUniforms(t){let r={};for(let n of t.bindings)Xn(n)&&(r[n.name]=Object.fromEntries(n.struct.fields.map(s=>[s.name,0])));return r}passDispatchWorkgroups(t,r){let n=r.device.limits.maxComputeWorkgroupsPerDimension;return[lt(t.dispatch.x,r),lt(t.dispatch.y??1,r),lt(t.dispatch.z??1,r)].map((s,a)=>{if(!Number.isInteger(s)||Number(s)<0)throw new Error(`WebGPU pass ${t.id} dispatch axis ${a} must resolve to a nonnegative integer, got ${String(s)}`);if(Number(s)>n){let i=Number(s)===Wr&&n<Wr;throw new Error(`WebGPU pass ${t.id} dispatch axis ${a} = ${s} exceeds maxComputeWorkgroupsPerDimension (${n}); `+(i?`this is the 2D fold width, which every WebGPU device must support \u2014 this adapter reports less than the required ${Wr}.`:`clamp the dispatch (min(..., ${n})) and grid-stride over the remainder, or fold into 2D/3D.`))}return Number(s)})}uniformValues(t,r){let n=t.struct,s={};for(let a of n.fields){let i=a.value??r.args[a.name]??a.default;if(i===void 0)continue;let u=lt(i,r),o=a.type==="f32";if(typeof u!="number"||Number.isNaN(u)||!Number.isFinite(u)&&!o)throw new Error(`WebGPU uniform ${t.name}.${a.name} must resolve to a finite number`);s[a.name]=u}return s}passDerived(t,r){let n=this.passDerivedBase.get(r.derived);n===void 0&&(n=new WeakMap,this.passDerivedBase.set(r.derived,n));let s=n.get(r.dtypes);s===void 0&&(s=Object.freeze({...r.dtypes,...r.derived}),n.set(r.dtypes,s));let a=Object.entries(t.derive);if(a.length===0)return s;let i=this.passDerivedMemo.get(r.derived);i===void 0&&(i=new Map,this.passDerivedMemo.set(r.derived,i));let u=i.get(t);u===void 0&&(u=[],i.set(t,u));for(let l of u)if(Zh(l.scope,r))return l.derived;let o={...s};for(let[l,c]of a)o[l]=Ct(c,{...r,derived:o});return u.push({scope:r,derived:o}),o}selectVariantAndScope(t,r,n,s){let a=this.selectVariant(t,n,r,s),i=this.applyVariantDerive(t,r,a,s);return{variant:a,scope:i}}selectVariant(t,r,n,s){let a=[],i=[],u=null,o=new Map,l=c=>{for(let d of i)_2({opId:this.manifest.id,skipped:d.id,used:c?.id??null,constructs:d.constructs})};for(let c of this.variantCandidates(r)){let d=this.checkVariant(c,t,o,n,s);if(!d.ok){a.push(`${c.id}: ${d.rejection.message}`),d.rejection.kind==="wgslBuildFailure"&&i.push({id:c.id,constructs:d.rejection.constructs});continue}if(d.demotions.length===0)return l(c),c;u??=c}if(u!==null)return l(u),u;throw l(null),new Error(`No supported WebGPU variant for ${this.manifest.id}; rejected ${a.join("; ")}`)}variantCandidates(t){if(t!==void 0){let r=this.manifest.variants.find(n=>n.id===t);if(!r)throw new Error(`WebGPU op ${this.manifest.id} has no variant ${t}`);return[r]}return[...this.manifest.variants].sort(co)}checkVariant(t,r,n=new Map,s,a){let i=n.get(t.id);if(i)return i;let u=this.computeVariantCheck(t,r,n,s,a);return n.set(t.id,u),u}computeVariantCheck(t,r,n,s,a){let i=Hh(t,r.device),u=i.requirementReason;if(u)return{ok:!1,rejection:{kind:"requires",message:u}};let o=i.budgetReason;if(o)return{ok:!1,rejection:{kind:"bufferBudget",message:o}};let l=r,c=a?.clauseReads(t.id),d=Array.isArray(t.when)?t.when:[t.when];for(let m=0;m<d.length;m+=1){let h=d[m],v;try{v=a===void 0?lt(h,l):a.evalClause(h,l,c?.when[m])}catch(b){let _=X2(h);return{ok:!1,rejection:{kind:"whenError",clause:_,message:`when eval error in \`${_}\`: ${b.message}`}}}if(v!==!0)return{ok:!1,rejection:Kh(h,v)}}for(let m of t.supersededBy??[]){let h=this.variantsById.get(m);if(!h)continue;let v=this.checkVariant(h,r,n,s,a);if(v.ok&&v.demotions.length===0)return{ok:!1,rejection:{kind:"superseded",variant:m,message:`superseded by \`${m}\`, which is eligible here and takes precedence`}}}let p=ze(r.device).wgslCompileFailures;if(p.size>0&&s!==void 0){let m=this.wgslBuildBlockers(t,r,s,p);if(m.length>0)return{ok:!1,rejection:{kind:"wgslBuildFailure",constructs:m,message:`renders ${m.join(", ")}, which this device rejects at pipeline creation`}}}let f=[],g=t.demoteWhen??[];for(let m=0;m<g.length;m+=1){let h=g[m],v;try{v=a===void 0?lt(h,l):a.evalClause(h,l,c?.demoteWhen[m])}catch(b){return{ok:!1,rejection:{kind:"demoteError",clause:h,message:`demoteWhen eval error in \`${h}\`: ${b.message}`}}}v===!0&&f.push(h)}return{ok:!0,demotions:f}}buildBaseScope(t,r){let n={...r.resources},s={...this.manifest.attributes,...r.attrs??{}};this.validateAttributes(s);let a={...r.args??{}};for(let[m,h]of Object.entries(n))!st(h)&&typeof h!="object"&&(a[m]=h);let i=Object.fromEntries(Object.keys({...this.manifest.args,...n}).map(m=>[m,n[m]!==void 0&&n[m]!==null])),u={},o={},l={},c={},d={},p={};for(let m of this.scopeNamesByArg.keys()){let h=n[m];if(st(h)){for(let v of this.scopeNamesByArg.get(m)){u[v]=h.shape,o[v]=h.shape.length,l[v]=h.dtype;let b=rv(h);b!==void 0&&(c[v]=b)}this.validateInputLayout(m,h)}}let f=this.validateInputDtypes(l,c);for(let[m,h]of f)d[m]=h.logicalDtype,p[m]=Od(h.storageDtype);return{device:t,attrs:s,args:a,resources:n,present:i,shapes:u,ranks:o,tensorDtypes:l,logicalDtypes:d,dtypes:p,derived:{},tunables:{...this.manifest.tunables,...r.tunables??{}},repeat:{}}}validateAttributes(t){for(let r of Object.keys(t))if(!this.acceptedAttributeKeys.has(r))throw new Error(`WebGPU op ${this.manifest.id}: unknown attribute "${r}" \u2014 not declared by the manifest (known: ${this.manifest.attributeNames.join(", ")||"none"})`);for(let[r,n]of Object.entries(this.manifest.attributeConstraints)){if(n.required&&(!Object.hasOwn(t,r)||t[r]===void 0))throw new Error(`WebGPU op ${this.manifest.id} missing required attribute ${r}`);if(!Object.hasOwn(t,r)||n.values===null)continue;let s=t[r];if(!n.values.some(a=>Ma(a,s)))throw new Error(`WebGPU op ${this.manifest.id} attribute ${r} must be one of [${n.values.map(a=>JSON.stringify(a)).join(", ")}], got ${JSON.stringify(s)}`)}}validateInputLayout(t,r){if(r.runtime===null||r.runtime===void 0)return;let n=this.layoutsByArg.get(t);if(r.layout!==n)throw new Error(`${this.manifest.id}: input "${t}" is bound in the ${r.layout??"widened"} storage layout, but its port declares ${n??"widened"}; upload it in the declared layout`)}validateInputDtypes(t,r){let n=new Map;for(let s of[...this.manifest.inputs,...this.manifest.outputs]){let a=s.dtype;if(!a)continue;let i=t[s.role];if(i===void 0)continue;let u=this.manifest.inputs.includes(s)?"input":"output",o=this.storageDtypesByPortRole.get(s.role);if(o!==void 0&&i!==o)throw new Error(`${this.manifest.id}: ${u} "${s.role}" has WebGPU storage dtype ${i}, not the declared ${o} storage ABI`);let l=r[s.role],c;if(Object.hasOwn(Dr,a)){c=a;let p=o??a;if(i!==p)throw new Error(`${this.manifest.id}: ${u} "${s.role}" has dtype ${i}, not the declared concrete dtype ${p}`)}else{let p=this.manifest.typeConstraints[a];if(p===void 0){if(l!==void 0&&l!==i)throw new Error(`${this.manifest.id}: ${u} "${s.role}" carries logical dtype ${l} over ${i} storage, but unconstrained type variable ${a} resolves to its physical dtype`);n.set(a,{logicalDtype:i,storageDtype:i,role:s.role});continue}if(p.length===1&&p[0]==="int64"){if(i!==(o??"int64"))throw new Error(`${this.manifest.id}: ${u} "${s.role}" requires ${o??"int64"} storage for logical int64, got ${i}`);c="int64"}else{if(!p.some(f=>f===i))throw new Error(`${this.manifest.id}: ${u} "${s.role}" has dtype ${i}, not allowed by type constraint ${a} = [${p.join(", ")}]`);c=i}}if(l!==void 0&&l!==c)throw new Error(`${this.manifest.id}: ${u} "${s.role}" carries logical dtype ${l} over ${i} storage, but this port resolves to logical dtype ${c} and does not declare the same storage projection`);let d=n.get(a);if(d!==void 0){if(d.logicalDtype!==c)throw new Error(`${this.manifest.id}: ${u} "${s.role}" resolves to logical dtype ${c}, but shared type variable ${a} already resolved to ${d.logicalDtype} from "${d.role}"`);if(d.storageDtype!==i)throw new Error(`${this.manifest.id}: ${u} "${s.role}" has storage dtype ${i}, but shared type variable ${a} already resolved to ${d.storageDtype} from "${d.role}"`)}n.set(a,{logicalDtype:c,storageDtype:i,role:s.role})}return n}applyManifestDerive(t,r){if(r!==void 0)return{...t,derived:r.manifestDerive(t)};let n={};for(let[s,a]of Object.entries(this.manifest.derive))n[s]=Ct(a,{...t,derived:n});return{...t,derived:n}}prepareScopeKey(t,r){return Bt(this.prepareScopeKeyFields(t,r))}prepareScopeKeyFields(t,r){return{d:ev(t,r.device),...go(r),args:r.args,shapes:r.shapes,present:r.present}}pinVariant(t,r){return r===void 0?t:`${t}\0${r}`}applyManifestDeriveCached(t,r,n){let s=this.manifestDerivedCache.get(t);if(s!==void 0)return{...r,derived:s};let a=this.applyManifestDerive(r,n);return this.manifestDerivedCache.set(t,a.derived),a}requestScope(t,r,n){let s=n?.rememberedRequestScope(r,t);if(s!==void 0)return s;let a=this.buildBaseScope(ze(t),r);return n===void 0?{rawDevice:t,baseScope:a,scopeKey:this.prepareScopeKey(t,a),validated:!1}:n.rememberRequestScope(r,t,a,this.prepareScopeKeyFields(t,a))}applyManifestDeriveForInference(t){let r={};for(let[n,s]of Vh(this.manifest))try{r[n]=Ct(s,{...t,derived:r})}catch{}return{...t,derived:r}}applyVariantDerive(t,r,n,s){let a={...t,tunables:{...this.manifest.tunables,...n.tunables,...r.tunables??{}}},i={...t.derived};if(s!==void 0)s.variantDerive(n,a,i);else for(let[u,o]of Object.entries(n.derive))i[u]=Ct(o,{...a,derived:i});return{...a,derived:i}}rejectUnknownKeys(t,r,n,s){for(let a of Object.keys(n))if(!this.acceptedResourceKeys.has(a))throw new Error(`${t}: unknown ${r} "${a}" \u2014 not a declared arg or binding name (known: ${[...this.acceptedResourceKeys].sort().join(", ")})`);for(let a of Object.keys(s??{}))if(!this.acceptedArgKeys.has(a))throw new Error(`${t}: unknown arg "${a}" \u2014 not declared in manifest.args (known: ${[...this.acceptedArgKeys].sort().join(", ")})`)}validateRequest(t){this.rejectUnknownKeys(`WebGPU op ${this.manifest.id}`,"resource",t.resources,t.args);for(let[r,n]of Object.entries(this.manifest.args)){let s=t.resources[r]??t.args?.[r];if(s==null){if(n.required!==!1)throw new Error(`WebGPU op ${this.manifest.id} missing required arg ${r}`);continue}Yh(r,n,s)}}},N2=/([ \t]*){%\s*include\s+["']([^"']+)["']\s*%}([ \t]*\r?\n)?/g;function H2(e,t,r){if(r===void 0)return!1;let n=t===0?`
`:e[t-1];return n===`
`||n==="\r"}var Y2=Object.freeze([]);function J2(e,t,r,n){return Na(e,t,r,n)}function Xh(e,t,r,n){let s={expandedOffset:0,spans:[]},a=rs(e),i=Na(a,t,r,n,s,Y2);return Object.freeze({source:i,provenance:Object.freeze({rootKey:a,spans:Object.freeze(s.spans)})})}function Na(e,t,r,n,s,a=Y2){let i=rs(e);if(n.has(i))throw new Error(`Circular WebGPU template include: ${[...n,i].join(" -> ")}`);if(n.add(i),s===void 0){let g=t.replace(N2,(m,h,v,b,_)=>{let C=$2(i,v),y=Na(C,r(C),r,n);return H2(t,_,b)?y:`${h}${y}${b??""}`});return n.delete(i),g}let u=0,o=0,l=1,c=1,d=(g,m)=>{for(let h=g;h<m;h+=1){let v=t.charCodeAt(h);v===10||v===11||v===12||v===13||v===133||v===8232||v===8233?(l+=1,c=1,v===13&&h+1<m&&t.charCodeAt(h+1)===10&&(h+=1)):c+=1}},p=(g,m)=>{if(g===m)return;let h=m-g;s.spans.push(Object.freeze({expandedStart:s.expandedOffset,expandedEnd:s.expandedOffset+h,sourceKey:i,sourceStart:g,sourceEnd:m,sourceLine:l,sourceColumn:c,includeInstancePath:a})),s.expandedOffset+=h},f=t.replace(N2,(g,m,h,v,b)=>{let _=H2(t,b,v),C=b+m.length,y=b+g.length,E=y-(v?.length??0);p(o,_?b:C);let k=_?y:E;d(o,k),o=k;let S=$2(i,h),A=Object.freeze([...a,u]);u+=1;let w=Na(S,r(S),r,n,s,A);return _?w:`${m}${w}${v??""}`});return p(o,t.length),n.delete(i),f}function $2(e,t){if(t.startsWith(".")){let r=e.slice(0,Math.max(0,e.lastIndexOf("/")));return rs(`${r}/${t}`)}return rs(t)}function rs(e){let t=[];for(let r of e.replaceAll("\\","/").split("/"))if(!(r===""||r===".")){if(r===".."){if(t.length===0)throw new Error(`WebGPU template path escapes package root: ${e}`);t.pop();continue}t.push(r)}return t.join("/")}function Yh(e,t,r){if(t.kind==="tensor"){if(!st(r))throw new Error(`WebGPU arg ${e} must be a GPU tensor`);if(t.dtype&&r.dtype!==t.dtype)throw new Error(`WebGPU arg ${e} dtype ${r.dtype} does not match ${t.dtype}`);return}if(t.kind==="string"){if(typeof r!="string")throw new Error(`WebGPU arg ${e} must be a string`);if(t.oneOf&&!t.oneOf.includes(r))throw new Error(`WebGPU arg ${e} must be one of ${t.oneOf.join(", ")}`);return}if(t.kind==="bool"){if(typeof r!="boolean")throw new Error(`WebGPU arg ${e} must be a boolean`);return}if(typeof r!="number"||!Number.isFinite(r))throw new Error(`WebGPU arg ${e} must be a finite number`);if(t.kind==="u32"&&(!Number.isInteger(r)||r<0||r>4294967295))throw new Error(`WebGPU arg ${e} must be a u32`);if(t.kind==="i32"&&(!Number.isInteger(r)||r<-2147483648||r>2147483647))throw new Error(`WebGPU arg ${e} must be an i32`)}function Jh(e){return typeof e.readText=="function"?e:zr(e)}var V2=new WeakMap;function ev(e,t){if(e&&typeof e=="object"){let r=e,n=V2.get(e);if(n!==void 0&&n.features===r.features&&n.wgslLanguageFeatures===r.wgslLanguageFeatures&&n.limits===r.limits&&n.adapterInfo===r.adapterInfo&&n.wgslCompileFailures===r.wgslCompileFailures)return n.serial;let s=Z2(Bt(yt(t)));return V2.set(e,{serial:s,features:r.features,wgslLanguageFeatures:r.wgslLanguageFeatures,limits:r.limits,adapterInfo:r.adapterInfo,wgslCompileFailures:r.wgslCompileFailures}),s}return Z2(Bt(yt(t)))}var wo=new Map;function Z2(e){let t=wo.get(e);if(t!==void 0)return t;let r=`d${wo.size}`;return wo.set(e,r),r}function es(e){return e!==null&&typeof e=="object"&&typeof e.dtype=="string"&&Array.isArray(e.shape)}function tv(e){let t=e.shape.reduce((r,n)=>r*n,1);return{dtype:e.dtype,...e.logicalDtype===void 0?{}:{logicalDtype:e.logicalDtype},shape:e.shape,size:t,byteLength:t*4,buffer:{},runtime:null}}function rv(e){if(e===null||typeof e!="object")return;let t=e.logicalDtype;if(t!==void 0){if(typeof t=="string"&&Object.hasOwn(Dr,t))return t;throw new Error(`tensor has unsupported logical dtype ${String(t)}`)}}var ef=new Map([["ops/_shared/gelu-tanh.wgsl.jinja",'{% include "ops/_shared/tanh-safe.wgsl.jinja" %}fn gelu_tanh(v:f32)->f32{return 0.5*v*(1.0+tanh_safe(0.7978845608028654*(v+0.044715*v*v*v)));}'],["ops/_shared/gemma4-gelu-grid.wgsl.jinja","fn gelu_grid(g:f32,s:f32)->f32{if(s==0.0){return gelu_tanh(g);}return gelu_lut[u32(clamp(round(g/s),-128.0,127.0)+128.0)];}"],["ops/_shared/swiglu-value.wgsl.jinja",`{% macro sigmoid_denominator(value, one) %}{{ one }} + exp(-{{ value }}){% endmacro %}{% macro sigmoid_value(value, one) %}{{ one }} / ({{ sigmoid_denominator(value, one) }}){% endmacro %}{% macro swiglu_gate(gate, alpha, one) %}{{ gate }} / ({{ sigmoid_denominator(gate ~ " * " ~ alpha, one) }}){% endmacro %}{% macro onnx_swiglu_stages(prefix, gate, linear, alpha, tensorType, wideType, one) %}{% if "f16" in tensorType %}{% set roundStage = "half_stage4" if "vec4" in tensorType else "half_stage" %}let {{ prefix }}_alpha={{ roundStage }}({{ alpha }});let {{ prefix }}_alpha_gate={{ roundStage }}({{ prefix }}_alpha*{{ gate }});let {{ prefix }}_sigmoid={{ roundStage }}({{ sigmoid_value(prefix ~ "_alpha_gate", one) }});let {{ prefix }}_swish={{ roundStage }}({{ gate }}*{{ prefix }}_sigmoid);let {{ prefix }}_result={{ tensorType }}({{ roundStage }}({{ prefix }}_swish*{{ linear }}));
{% else %}let {{ prefix }}_gate={{ tensorType }}({{ gate }});let {{ prefix }}_linear={{ tensorType }}({{ linear }});let {{ prefix }}_alpha={{ tensorType }}({{ alpha }});let {{ prefix }}_alpha_gate={{ tensorType }}({{ wideType }}({{ prefix }}_alpha)*{{ wideType }}({{ prefix }}_gate));let {{ prefix }}_sigmoid={{ tensorType }}({{ sigmoid_value(wideType ~ "(" ~ prefix ~ "_alpha_gate)", one) }});let {{ prefix }}_swish={{ tensorType }}({{ wideType }}({{ prefix }}_gate)*{{ wideType }}({{ prefix }}_sigmoid));let {{ prefix }}_result={{ tensorType }}({{ wideType }}({{ prefix }}_swish)*{{ wideType }}({{ prefix }}_linear));
{% endif %}{% endmacro %}`],["ops/_shared/tanh-safe.wgsl.jinja","fn tanh_safe(x:f32)->f32{if(x>10.0){return 1.0;}if(x<-10.0){return-1.0;}if(x>-1.0e-4&&x<1.0e-4){return x;}return tanh(x);}"],["ops/_shared/broadcast-offset.wgsl.jinja",`{% macro offset_fn(fn_name, opShape, opRank, op_same, op_numel, outShape, outRank, out_numel) %}fn {{ fn_name }}({% if out_numel != 0 and op_numel != 1 %}out_index: u32{% endif %}) -> u32 {
{% if out_numel == 0 %}return 0u;
{% elif op_numel == 1 %}return 0u;
{% elif op_same %}return out_index;
{% else %}var offset=0u;
{% for axis in range(outRank) %}{% set op_axis = axis - (outRank - opRank) %}{% if op_axis >= 0 and opShape[op_axis] != 1 %}{% set c_stride = namespace(value=1) %}{% for j in range(axis + 1, outRank) %}{% set c_stride.value = c_stride.value * outShape[j] %}{% endfor %}{% set op_stride = namespace(value=1) %}{% for j in range(op_axis + 1, opRank) %}{% set op_stride.value = op_stride.value * opShape[j] %}{% endfor %}{% if c_stride.value == 1 %}let coord{{ axis }}=out_index% {{ outShape[axis] }}u;
{% else %}let coord{{ axis }}=(out_index/{{ c_stride.value }}u)% {{ outShape[axis] }}u;
{% endif %}{% if op_stride.value == 1 %}offset=offset+coord{{ axis }};
{% else %}offset=offset+coord{{ axis }}*{{ op_stride.value }}u;
{% endif %}{% endif %}{% endfor %}return offset;
{% endif %}}{% endmacro %}{% macro broadcast_offset_call(fn_name, opShape, outShape, out_index) %}{% set op_numel = namespace(value=1) %}{% for d in opShape %}{% set op_numel.value = op_numel.value * d %}{% endfor %}{% set out_numel = namespace(value=1) %}{% for d in outShape %}{% set out_numel.value = out_numel.value * d %}{% endfor %}{{ fn_name }}({% if out_numel.value != 0 and op_numel.value != 1 %}{{ out_index }}{% endif %}){% endmacro %}{% macro broadcast_offset_fn(fn_name, opShape, opRank, outShape, outRank) %}{% set op_numel = namespace(value=1) %}{% for d in opShape %}{% set op_numel.value = op_numel.value * d %}{% endfor %}{% set out_numel = namespace(value=1) %}{% for d in outShape %}{% set out_numel.value = out_numel.value * d %}{% endfor %}{% set op_same = namespace(value=(opRank == outRank)) %}{% if op_same.value %}{% for axis in range(outRank) %}{% if opShape[axis] != outShape[axis] %}{% set op_same.value = false %}{% endif %}{% endfor %}{% endif %}{{ offset_fn(fn_name, opShape, opRank, op_same.value, op_numel.value, outShape, outRank, out_numel.value) }}{% endmacro %}{% macro binary_broadcast_offsets() %}{% set aShape = aShape | default([]) %}{% set aRank = aRank | default(0) %}{% set bShape = bShape | default([]) %}{% set bRank = bRank | default(0) %}{% set cShape = cShape | default([]) %}{% set cRank = cRank | default(0) %}{{ broadcast_offset_fn("a_offset", aShape, aRank, cShape, cRank) }}
{{ broadcast_offset_fn("b_offset", bShape, bRank, cShape, cRank) }}{% endmacro %}`],["ops/_shared/div-exact.wgsl.jinja","fn div_exact(x:f32,s:f32,t:f32)->f32{let q0=x*t;let r=fma(-s,q0,x);return fma(r,t,q0);}"],["ops/_shared/float-reciprocal-exact.wgsl.jinja","fn recip_exact(s:f32)->f32{let t0=1.0/s;let t1=fma(fma(-s,t0,1.0),t0,t0);return fma(fma(-s,t1,1.0),t1,t1);}"],["ops/_shared/float16-rne-bits.wgsl.jinja","fn round_f16_bits_rte(value:f32)->u32{let bits=bitcast<u32>(value);let sign=(bits>>16u)&0x8000u;let exponent_f32=(bits>>23u)&0xffu;let mantissa_f32=bits&0x7fffffu;if(exponent_f32==0xffu){if(mantissa_f32!=0u){return 0x7e00u;}return sign|0x7c00u;}var exponent_f16=i32(exponent_f32)-127+15;if(exponent_f16>=0x1f){return sign|0x7c00u;}if(exponent_f16<=0){if(exponent_f16<-10){return sign;}let significand=mantissa_f32|0x800000u;let shift=u32(14-exponent_f16);let halfway=1u<<(shift-1u);let discarded=significand&((1u<<shift)-1u);var fraction=significand>>shift;if(discarded>halfway||(discarded==halfway&&(fraction&1u)==1u)){fraction=fraction+1u;}return sign|fraction;}let halfway=1u<<12u;let discarded=mantissa_f32&0x1fffu;var mantissa_f16=mantissa_f32>>13u;if(discarded>halfway||(discarded==halfway&&(mantissa_f16&1u)==1u)){mantissa_f16=mantissa_f16+1u;if(mantissa_f16==0x400u){mantissa_f16=0u;exponent_f16=exponent_f16+1;} }if(exponent_f16>=0x1f){return sign|0x7c00u;}return sign|(u32(exponent_f16)<<10u)|mantissa_f16;}fn widen_f16_bits(value:u32)->f32{return unpack2x16float(value&0xffffu).x;}"],["ops/_shared/half-stage-rounding.wgsl.jinja",`{% include "ops/_shared/float16-rne-bits.wgsl.jinja" %}fn half_stage(value:f32)->f32{return widen_f16_bits(round_f16_bits_rte(value));}
{% if halfStageVector | default(false) %}fn half_stage4(value:vec4<f32>)->vec4<f32>{return vec4<f32>(half_stage(value.x),half_stage(value.y),half_stage(value.z),half_stage(value.w));}
{% endif %}`],["ops/_shared/kimi-bf16-unpack-pair.wgsl.jinja","fn unpack_bf16_pair(word:u32)->vec2<f32>{let bits=vec2<u32>(word&0xffffu,word>>16u);return bitcast<vec2<f32>>(bits<<vec2<u32>(16u));}"],["ops/_shared/norm-half-row.wgsl.jinja",`{% set halfOutputScalar = halfOutputScalar | default("f16") %}{% set hasBias = hasBias | default(false) %}{% if halfOutputScalar == "f16" %}{% set halfStageVector = vec4 %}{% include "ops/_shared/half-stage-rounding.wgsl.jinja" %}{% else %}{% include "ops/_shared/float16-rne-bits.wgsl.jinja" %}{% endif %}{% if not halfConstantsProvided | default(false) %}const HIDDEN:u32={{ hidden }}u;const WG:u32={{ wg }}u;const EPSILON:f32={{ epsilon }};
{% endif %}{% set halfWriteMean = halfWriteMean if halfWriteMean is defined else (writeStats and modeSpec == "layer") %}{% set halfWriteInv = halfWriteInv if halfWriteInv is defined else writeStats %}fn pair_add(a:vec2<f32>,b:vec2<f32>)->vec2<f32>{let s=fma(a.x,1.0,b.x);let bv=fma(-1.0,a.x,s);let av=fma(-1.0,bv,s);let a_error=fma(-1.0,av,a.x);let b_error=fma(-1.0,bv,b.x);let error=fma(a_error,1.0,b_error);let e=fma(fma(error,1.0,a.y),1.0,b.y);let hi=fma(s,1.0,e);return vec2<f32>(hi,fma(-1.0,fma(-1.0,s,hi),e));}fn pair_mul(a:vec2<f32>,b:vec2<f32>)->vec2<f32>{let p=fma(a.x,b.x,0.0);let error=fma(a.x,b.x,-p);return pair_add(vec2<f32>(p,0.0),vec2<f32>(error+(a.x*b.y+b.x*a.y),0.0));}fn pair_div(a:vec2<f32>,b:f32)->vec2<f32>{let q=a.x/b;let residual=pair_add(a,-pair_mul(vec2<f32>(q,0.0),vec2<f32>(b,0.0)));return pair_add(vec2<f32>(q,0.0),vec2<f32>((residual.x+residual.y)/b,0.0));}fn pair_inverse_sqrt(a:vec2<f32>)->vec2<f32>{let r=inverseSqrt(a.x);let rr=pair_mul(vec2<f32>(r,0.0),vec2<f32>(r,0.0));let residual=pair_add(vec2<f32>(1.0,0.0),-pair_mul(a,rr));return pair_add(vec2<f32>(r,0.0),vec2<f32>((0.5*r)*(residual.x+residual.y),0.0));}fn half_normalized(value:vec2<f32>)->f32{return widen_f16_bits(round_f16_bits_rte(fma(value.x,1.0,value.y)));}
{% if combineSubgroups %}var<workgroup>sg_partials:array<vec2<f32>,WG>;fn reduce_pair(value:vec2<f32>,sg_lane:u32,sg_id:u32,num_sg:u32,sg_size:u32)->vec2<f32>{var s=value;let active_lanes=subgroupBallot(true);let counts=countOneBits(active_lanes);if(counts.x+counts.y+counts.z+counts.w==sg_size){for(var step=sg_size/2u;step>0u;step/=2u){let other=subgroupShuffleDown(s,step);if(sg_lane+step<sg_size){s=pair_add(s,other);} }s=subgroupBroadcastFirst(s);}else{s=vec2<f32>(0.0);for(var word=0u;word<4u;word++){var mask=active_lanes[word];while(mask!=0u){let lane=word*32u+firstTrailingBit(mask);s=pair_add(s,subgroupShuffle(value,lane));mask&=mask-1u;} } }let fold_count=max(num_sg,(WG+sg_size-1u)>>firstTrailingBit(sg_size));if(fold_count==1u){return s;}if(subgroupElect()){sg_partials[sg_id]=s;}workgroupBarrier();var total=vec2<f32>(0.0);for(var i=0u;i<fold_count;i++){total=pair_add(total,sg_partials[i]);}workgroupBarrier();return total;}
{% else %}var<workgroup>partial:array<vec2<f32>,WG>;fn reduce_pair(value:vec2<f32>,tid:u32)->vec2<f32>{partial[tid]=value;workgroupBarrier();for(var step=WG/2u;step>0u;step/=2u){if(tid<step){partial[tid]=pair_add(partial[tid],partial[tid+step]);}workgroupBarrier();}let total=partial[0];workgroupBarrier();return total;}
{% endif %}{% set reduceArgs = "sg_lane, sg_id, num_sg, sg_size" if combineSubgroups else "tid" %}{% if not vec4 %}fn load_value(index:u32)->f32{return f32(x[index]);}
{% endif %}fn normalize_half_row(row:u32,tid:u32
{% if combineSubgroups %},sg_lane:u32,sg_id:u32,num_sg:u32,sg_size:u32
{% endif %}){if(row>=params.rows){return;}let base=row*HIDDEN;
{% if modeSpec == "layer" %}var local_sum=vec2<f32>(0.0);
{% if vec4 %}for(var i=tid;i<HIDDEN/4u;i+=WG){let v=vec4<f32>(x[base/4u+i]);
{% for component in ["x", "y", "z", "w"] %}local_sum=pair_add(local_sum,vec2<f32>(v.{{ component }},0.0));
{% endfor %}}
{% else %}for(var i=tid;i<HIDDEN;i+=WG){local_sum=pair_add(local_sum,vec2<f32>(load_value(base+i),0.0));}
{% endif %}let mean=pair_div(reduce_pair(local_sum,{{ reduceArgs }}),f32(HIDDEN));
{% else %}let mean=vec2<f32>(0.0);
{% endif %}var local_square=vec2<f32>(0.0);
{% if vec4 %}for(var i=tid;i<HIDDEN/4u;i+=WG){let v=vec4<f32>(x[base/4u+i]);
{% for component in ["x", "y", "z", "w"] %}let centered_{{ component }}=pair_add(vec2<f32>(v.{{ component }},0.0),-mean);local_square=pair_add(local_square,pair_mul(centered_{{ component }},centered_{{ component }}));
{% endfor %}}
{% else %}for(var i=tid;i<HIDDEN;i+=WG){let centered=pair_add(vec2<f32>(load_value(base+i),0.0),-mean);local_square=pair_add(local_square,pair_mul(centered,centered));}
{% endif %}let variance=pair_div(reduce_pair(local_square,{{ reduceArgs }}),f32(HIDDEN));let inv=pair_inverse_sqrt(pair_add(variance,vec2<f32>(EPSILON,0.0)));
{% if halfWriteMean %}if(tid==0u){mean_out[row]=mean.x;}
{% endif %}{% if halfWriteInv %}if(tid==0u){inv_std_out[row]=inv.x;}
{% endif %}{% if vec4 %}for(var i=tid;i<HIDDEN/4u;i+=WG){let v=vec4<f32>(x[base/4u+i]);var normalized:vec4<f32>;for(var component=0u;component<4u;component++){normalized[component]=half_normalized(pair_mul(pair_add(vec2<f32>(v[component],0.0),-mean),inv));}var value=half_stage4(normalized*vec4<f32>(scale[i]));
{% if modeSpec == "layer" and hasBias %}value=half_stage4(value+vec4<f32>(bias[i]));
{% endif %}y[base/4u+i]=vec4<f16>(value);}
{% else %}for(var i=tid;i<HIDDEN;i+=WG){let normalized=half_normalized(pair_mul(pair_add(vec2<f32>(load_value(base+i),0.0),-mean),inv));
{% if halfOutputScalar == "f16" %}var value=half_stage(normalized*f32(scale[{{ halfScaleOffset | default("i") }}]));
{% else %}var value=normalized*f32(scale[{{ halfScaleOffset | default("i") }}]);
{% endif %}{% if modeSpec == "layer" and hasBias %}value=half_stage(value+f32(bias[{{ halfBiasOffset | default("i") }}]));
{% endif %}y[base+i]={{ halfOutputScalar }}(value);}
{% endif %}}`],["ops/_shared/norm-row-stats.wgsl.jinja",`{% set combineSubgroups = combineSubgroups and not (scalar == "f16" and compensateHalfStats is defined and compensateHalfStats and (device.adapterInfo.subgroupMinSize | default(0)) > 32) %}{% set scalarIo = scalarIo if scalarIo is defined else false %}{% set packedWidth = packedWidth | default(4) %}{% set packedF32 = "vec" ~ packedWidth ~ "<f32>" %}{% set packedBf16Embedding = packedBf16Embedding if packedBf16Embedding is defined else false %}{% set writeStats = writeStats if writeStats is defined else false %}{% set lpOrder = p if p is defined else 0 %}{% set rmsWeightOffset = rmsWeightOffset if rmsWeightOffset is defined else false %}{% set rmsScaleAfterCast = rmsScaleAfterCast if rmsScaleAfterCast is defined else false %}{% set rmsResidualAdd = rmsResidualAdd if rmsResidualAdd is defined else false %}{% set rmsChainNorm = rmsChainNorm if rmsChainNorm is defined else false %}{% set hiddenPairs = hiddenPairs | default(0) %}{% set numRows = numRows | default(0) %}{% set epsilon = epsilon | default("0.0") %}{% set epsilon2 = epsilon2 | default("0.0") %}{% set numGroupsSpec = numGroupsSpec | default(0) %}{% set cpg = cpg | default(0) %}{% set spatialVec = spatialVec | default(0) %}{% set spatial = spatial | default(0) %}{% set channels = channels | default(0) %}{% set hasBias = hasBias is defined and hasBias %}{% if rmsWeightOffset %}{% set rmsScaleVec = "(vec4<f32>(1.0) + vec4<f32>(scale[i]))" %}{% set rmsScaleScalar = "(1.0 + f32(scale[i]))" %}{% else %}{% set rmsScaleVec = "vec4<f32>(scale[i])" %}{% set rmsScaleScalar = "f32(scale[i])" %}{% endif %}{% set reduceThreadParameters = ", sg_id: u32, num_sg: u32, sg_size: u32"
    if combineSubgroups else ", tid: u32" %}{% set reduceThreadArguments = ", sg_id, num_sg, sg_size"
    if combineSubgroups else ", tid" %}{% if combineSubgroups %}enable subgroups;
{% endif %}{{ env.wgsl.resourceDeclarations }}
{% set affineRowBroadcast = affineRowBroadcast | default(false) %}{% set batchRows = batchRows | default(1) %}{% set batchLanes = batchLanes | default(0) %}{% set xRowShape = xRowShape | default([]) %}{% set scaleRowShape = scaleRowShape | default([]) %}{% set biasRowShape = biasRowShape | default([]) %}{% if affineRowBroadcast %}{% include "ops/_shared/broadcast-offset.wgsl.jinja" %}{{ broadcast_offset_fn("scale_row_offset", scaleRowShape, scaleRowShape | length, xRowShape, xRowShape | length) }}
{% if hasBias %}{{ broadcast_offset_fn("bias_row_offset", biasRowShape, biasRowShape | length, xRowShape, xRowShape | length) }}
{% endif %}{% endif %}{% if scalar == "f16" and compensateHalfStats is defined and compensateHalfStats %}{% include "ops/_shared/norm-half-row.wgsl.jinja" %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg_id:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>
{% if combineSubgroups %},@builtin(subgroup_invocation_id)sg_lane:u32,@builtin(subgroup_id)sg_id:u32,@builtin(num_subgroups)num_sg:u32,@builtin(subgroup_size)sg_size:u32
{% endif %}){let row=wg_id.x+wg_id.y*params.rowStride;
  normalize_half_row(row, lid.x{% if combineSubgroups %}, sg_lane, sg_id, num_sg, sg_size{% endif %});
}
{% else %}{% if modeSpec == "layer" %}{% elif modeSpec == "group" %}{% elif modeSpec == "instance" %}{% elif modeSpec == "rms" %}{% else %}{% endif %}{% if modeSpec != "lp" or not vec4 %}const HIDDEN:u32={{ hidden }}u;
{% endif %}{% if vec4 %}const HIDDEN_V:u32={{ hiddenVec }}u;
{% endif %}{% if packedBf16Embedding %}const HIDDEN_PAIRS:u32={{ hiddenPairs }}u;const NUM_ROWS:u32={{ numRows }}u;
{% endif %}const WG:u32={{ wg }}u;
{% if batchRows > 1 %}const ROW_LANES:u32={{ batchLanes }}u;const ROW_BATCH:u32={{ batchRows }}u;
{% endif %}{% if modeSpec != "lp" %}const EPSILON:f32={{ epsilon }};
{% endif %}{% if rmsChainNorm %}const EPSILON2:f32={{ epsilon2 }};
{% endif %}{% if modeSpec == "group" %}const NUM_GROUPS:u32={{ numGroupsSpec }}u;const CPG:u32={{ cpg }}u;
{% if vec4 %}const SPATIAL_V:u32={{ spatialVec }}u;
{% else %}const SPATIAL:u32={{ spatial }}u;
{% endif %}{% endif %}{% if modeSpec == "instance" %}const CHANNELS:u32={{ channels }}u;
{% endif %}{% if packedBf16Embedding %}{% if vec4 %}{% include "ops/_shared/kimi-bf16-unpack-pair.wgsl.jinja" %}{% endif %}{% if not vec4 %}fn embedding_scalar(source_row:u32,hidden:u32)->f32{if(source_row>=NUM_ROWS){return 0.0;}let word=x[source_row*HIDDEN_PAIRS+(hidden>>1u)];let bits=select(word&0xffffu,word>>16u,(hidden&1u)!=0u);return bitcast<f32>(bits<<16u);}
{% endif %}{% if vec4 %}fn embedding_vec4(source_row:u32,hidden_vec:u32)->vec4<f32>{if(source_row>=NUM_ROWS){return vec4<f32>(0.0);}let base=source_row*HIDDEN_PAIRS+hidden_vec*2u;let low=unpack_bf16_pair(x[base]);let high=unpack_bf16_pair(x[base+1u]);return vec4<f32>(low,high);}
{% endif %}{% endif %}{% if vec4 and scalarIo %}fn load_vec4(index:u32)->vec4<f32>{return vec4<f32>(x[index],x[index+1u],x[index+2u],x[index+3u]);}
{% endif %}{% if combineSubgroups %}{% if modeSpec == "rms" or modeSpec == "lp" %}var<workgroup>sg_partials:array<f32,WG>;
{% else %}var<workgroup>sg_partials:array<vec2<f32>,WG>;
{% endif %}{% if modeSpec == "rms" or modeSpec == "lp" %}fn reduce_scalar(value:f32{{ reduceThreadParameters }})->f32{let s=subgroupAdd(value);let fold_count=max(num_sg,(WG+sg_size-1u)>>firstTrailingBit(sg_size));if(fold_count==1u){return s;}if(subgroupElect()){sg_partials[sg_id]=s;}workgroupBarrier();var total=0.0;for(var i=0u;i<fold_count;i++){total+=sg_partials[i];}workgroupBarrier();return total;}
{% else %}fn reduce_pair(value:vec2<f32>{{ reduceThreadParameters }})->vec2<f32>{let s=vec2<f32>(subgroupAdd(value.x),subgroupAdd(value.y));let fold_count=max(num_sg,(WG+sg_size-1u)>>firstTrailingBit(sg_size));if(fold_count==1u){return s;}if(subgroupElect()){sg_partials[sg_id]=s;}workgroupBarrier();var total=vec2<f32>(0.0);for(var i=0u;i<fold_count;i++){total+=sg_partials[i];}workgroupBarrier();return total;}
{% endif %}{% else %}{% if modeSpec == "rms" or modeSpec == "lp" %}var<workgroup>tr0:array<f32,WG>;fn reduce_scalar(value:f32,tid:u32)->f32{tr0[tid]=value;workgroupBarrier();var stride:u32=WG/2u;loop{if(stride==0u){break;}if(tid<stride){tr0[tid]=tr0[tid]+tr0[tid+stride];}stride=stride/2u;workgroupBarrier();}let reduced=tr0[0];workgroupBarrier();return reduced;}
{% else %}var<workgroup>tr0:array<f32,WG>;var<workgroup>tr1:array<f32,WG>;fn reduce_pair(value:vec2<f32>,tid:u32)->vec2<f32>{tr0[tid]=value.x;tr1[tid]=value.y;workgroupBarrier();
  var stride: u32 = {% if batchRows > 1 %}ROW_LANES{% else %}WG{% endif %} / 2u;
loop{if(stride==0u){break;}
    if (tid{% if batchRows > 1 %} % ROW_LANES{% endif %} < stride) {
tr0[tid]=tr0[tid]+tr0[tid+stride];tr1[tid]=tr1[tid]+tr1[tid+stride];}stride=stride/2u;workgroupBarrier();}
{% if batchRows > 1 %}let row_base=tid-tid%ROW_LANES;let reduced=vec2<f32>(tr0[row_base],tr1[row_base]);
{% else %}let reduced=vec2<f32>(tr0[0],tr1[0]);
{% endif %}workgroupBarrier();return reduced;}
{% endif %}{% endif %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg_id:vec3<u32>,
  @builtin(local_invocation_id) lid: vec3<u32>{% if combineSubgroups %},
@builtin(subgroup_id)sg_id:u32,@builtin(num_subgroups)num_sg:u32,
  @builtin(subgroup_size) sg_size: u32{% endif %}){
{% if batchRows > 1 %}let row=(wg_id.x+wg_id.y*params.rowStride)*ROW_BATCH+lid.x/ROW_LANES;let row_active=row<params.rows;
{% else %}let row=wg_id.x+wg_id.y*params.rowStride;if(row>=params.rows){return;}
{% endif %}let tid=lid.x;
{% if packedBf16Embedding %}let source_row=indices[row];
{% if vec4 %}let base=row*HIDDEN_V;
{% else %}let base=row*HIDDEN;
{% endif %}{% elif vec4 and not scalarIo %}let base=row*HIDDEN_V;
{% else %}let base=row*HIDDEN;
{% endif %}{% if modeSpec == "layer" or modeSpec == "group" or modeSpec == "instance" %}{% endif %}{% if affineRowBroadcast %}let scale_base={{ broadcast_offset_call("scale_row_offset", scaleRowShape, xRowShape, "row") }}*HIDDEN_V;
{% if hasBias %}let bias_base={{ broadcast_offset_call("bias_row_offset", biasRowShape, xRowShape, "row") }}*HIDDEN_V;
{% endif %}{% endif %}{% if modeSpec == "layer" or modeSpec == "group" or modeSpec == "instance" %}{% if vec4 %}{% if scalarIo %}let shift=f32(x[base]);
{% else %}{% if batchRows > 1 %}var shift=0.0;if(row_active){shift=f32(x[base].x);}
{% else %}let shift=f32(x[base].x);
{% endif %}{% endif %}{% else %}let shift=f32(x[base]);
{% endif %}{% endif %}{% if modeSpec == "rms" or modeSpec == "lp" %}var acc=0.0;
{% else %}var acc=vec2<f32>(0.0,0.0);
{% endif %}{% if vec4 %}  for (var i = tid{% if batchRows > 1 %} % ROW_LANES{% endif %}; {% if batchRows > 1 %}row_active && {% endif %}i < HIDDEN_V; i = i + {% if batchRows > 1 %}ROW_LANES{% else %}WG{% endif %}) {
{% if packedBf16Embedding %}let v=embedding_vec4(source_row,i);embedding_out[base+i]=v;
{% elif scalarIo %}let v=load_vec4(base+i*4u);
{% else %}let v={{ packedF32 }}(x[base+i]);
{% endif %}{% if modeSpec == "layer" or modeSpec == "group" or modeSpec == "instance" %}let d=v-{{ packedF32 }}(shift);
    acc.x = acc.x + d.x + d.y{% if packedWidth == 4 %} + d.z + d.w{% endif %};
acc.y=acc.y+dot(d,d);
{% elif modeSpec == "rms" %}acc=acc+dot(v,v);
{% else %}{% if lpOrder == 1 %}let a=abs(v);acc=acc+a.x+a.y+a.z+a.w;
{% else %}acc=acc+dot(v,v);
{% endif %}{% endif %}}
{% else %}for(var i=tid;i<HIDDEN;i=i+WG){
{% if packedBf16Embedding %}let v=embedding_scalar(source_row,i);embedding_out[base+i]=v;
{% else %}let v=f32(x[base+i]);
{% endif %}{% if modeSpec == "layer" or modeSpec == "group" or modeSpec == "instance" %}let d=v-shift;acc.x=acc.x+d;acc.y=acc.y+d*d;
{% elif modeSpec == "rms" %}acc=acc+v*v;
{% else %}{% if lpOrder == 1 %}acc=acc+abs(v);
{% else %}acc=acc+v*v;
{% endif %}{% endif %}}
{% endif %}{% if modeSpec == "rms" or modeSpec == "lp" %}let total=reduce_scalar(acc{{ reduceThreadArguments }});
{% else %}let totals=reduce_pair(acc{{ reduceThreadArguments }});
{% endif %}{% if modeSpec == "layer" %}let mean_d=totals.x/f32(HIDDEN);let variance=max(totals.y/f32(HIDDEN)-mean_d*mean_d,0.0);let inv=inverseSqrt(variance+EPSILON);let row_mean=shift+mean_d;
{% if writeStats %}if(tid==0u){mean_out[row]=row_mean;inv_std_out[row]=inv;}
{% endif %}{% elif modeSpec == "group" %}let mean_d=totals.x/f32(HIDDEN);let variance=max(totals.y/f32(HIDDEN)-mean_d*mean_d,0.0);let denom=sqrt(variance+EPSILON);let row_mean=shift+mean_d;let g_ch_base=(row%NUM_GROUPS)*CPG;
{% elif modeSpec == "instance" %}let mean_d=totals.x/f32(HIDDEN);let variance=max(totals.y/f32(HIDDEN)-mean_d*mean_d,0.0);let inv=inverseSqrt(variance+EPSILON);let row_mean=shift+mean_d;let c=row%CHANNELS;let ch_scale=f32(scale[c]);let ch_bias=f32(bias[c]);
{% elif modeSpec == "rms" %}let inv=inverseSqrt(total/f32(HIDDEN)+EPSILON);
{% if writeStats %}if(tid==0u){inv_std_out[row]=inv;}
{% endif %}{% else %}{% if lpOrder == 2 %}let norm=sqrt(total);
{% else %}let norm=total;
{% endif %}{% endif %}{% if rmsChainNorm %}var acc2=0.0;
{% endif %}{% if vec4 %}  for (var i = tid{% if batchRows > 1 %} % ROW_LANES{% endif %}; {% if batchRows > 1 %}row_active && {% endif %}i < HIDDEN_V; i = i + {% if batchRows > 1 %}ROW_LANES{% else %}WG{% endif %}) {
{% if packedBf16Embedding %}let idx=base+i;let v=embedding_vec4(source_row,i);
{% elif scalarIo %}let idx=base+i*4u;let v=load_vec4(idx);
{% else %}let idx=base+i;let v={{ packedF32 }}(x[idx]);
{% endif %}{% if modeSpec == "layer" %}    var value = (v - {{ packedF32 }}(row_mean)) * inv * {{ packedF32 }}(scale[{% if affineRowBroadcast %}scale_base + {% endif %}i]);
{% if hasBias %}    value = value + {{ packedF32 }}(bias[{% if affineRowBroadcast %}bias_base + {% endif %}i]);
{% endif %}y[idx]={{ vecType }}(value);
{% elif modeSpec == "group" %}let ch=g_ch_base+i/SPATIAL_V;let normed=(v-vec4<f32>(row_mean))/vec4<f32>(denom);y[idx]={{ vecType }}(normed*vec4<f32>(f32(scale[ch]))+vec4<f32>(f32(bias[ch])));
{% elif modeSpec == "instance" %}{% if scalarIo %}let value=(v-vec4<f32>(row_mean))*inv*vec4<f32>(ch_scale)+vec4<f32>(ch_bias);y[idx]=value.x;y[idx+1u]=value.y;y[idx+2u]=value.z;y[idx+3u]=value.w;
{% else %}y[idx]={{ vecType }}((v-vec4<f32>(row_mean))*inv*vec4<f32>(ch_scale)+vec4<f32>(ch_bias));
{% endif %}{% elif modeSpec == "rms" %}{% if rmsScaleAfterCast %}y[idx]={{ vecType }}(v*inv)*{{ vecType }}({{ rmsScaleVec }});
{% elif rmsChainNorm %}let hv=y[idx]+fma(v*inv,{{ rmsScaleVec }},vec4<f32>(0.0));y[idx]=hv;acc2=acc2+dot(hv,hv);
{% elif rmsResidualAdd %}y[idx]=y[idx]+fma(v*inv,{{ rmsScaleVec }},vec4<f32>(0.0));
{% else %}y[idx]={{ vecType }}(v*inv*{{ rmsScaleVec }});
{% endif %}{% else %}let normalized=select(v/vec4<f32>(norm),vec4<f32>(0.0),vec4<bool>(norm==0.0));y[idx]={{ vecType }}(normalized);
{% endif %}}
{% if rmsChainNorm %}workgroupBarrier();let total2=reduce_scalar(acc2{{ reduceThreadArguments }});let inv2=inverseSqrt(total2/f32(HIDDEN)+EPSILON2);for(var i=tid;i<HIDDEN_V;i=i+WG){let idx=base+i;let hv=vec4<f32>(y[idx]);normed2[idx]={{ vecType }}(hv*inv2*vec4<f32>(scale2[i]));}
{% endif %}{% else %}for(var i=tid;i<HIDDEN;i=i+WG){let idx=base+i;
{% if packedBf16Embedding %}let v=embedding_scalar(source_row,i);
{% else %}let v=f32(x[idx]);
{% endif %}{% if modeSpec == "layer" %}var value=(v-row_mean)*inv*f32(scale[i]);
{% if hasBias %}value=value+f32(bias[i]);
{% endif %}y[idx]={{ scalar }}(value);
{% elif modeSpec == "group" %}let ch=g_ch_base+i/SPATIAL;let normed=(v-row_mean)/denom;y[idx]={{ scalar }}(normed*f32(scale[ch])+f32(bias[ch]));
{% elif modeSpec == "instance" %}y[idx]={{ scalar }}((v-row_mean)*inv*ch_scale+ch_bias);
{% elif modeSpec == "rms" %}{% if rmsScaleAfterCast %}y[idx]={{ scalar }}(v*inv)*{{ scalar }}({{ rmsScaleScalar }});
{% else %}y[idx]={{ scalar }}(v*inv*{{ rmsScaleScalar }});
{% endif %}{% else %}let normalized=select(v/norm,0.0,norm==0.0);y[idx]={{ scalar }}(normalized);
{% endif %}}
{% endif %}}
{% endif %}`],["ops/_shared/reduce-sum.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{% set reduceSumName = reduceSumName if reduceSumName is defined else "reduce_sum" %}{% set reduceSumBank = reduceSumBank if reduceSumBank is defined else "partial" %}{% set trailingBarrier = trailingBarrier is defined and trailingBarrier %}fn {{ reduceSumName }}(value:f32,tid:u32)->f32{ {{ reduceSumBank }}[tid]=value;workgroupBarrier();{{ wgsl_tree_fold([reduceSumBank], idx="tid", wg="WG", form="head") }}
{% if trailingBarrier %}let total={{ reduceSumBank }}[0];workgroupBarrier();return total;
{% else %}return {{ reduceSumBank }}[0];
{% endif %}}`],["ops/_shared/tree-fold-inplace.wgsl.jinja",`{% macro wgsl_tree_fold_stmt(a, op, idx, svar) %}{% if op == "max" or op == "min" %}{{ a }}[{{ idx }}] = {{ op }}({{ a }}[{{ idx }}], {{ a }}[{{ idx }} + {{ svar }}]);{% else %}{{ a }}[{{ idx }}] = {{ a }}[{{ idx }}] {{ "*" if op == "prod" else "+" }} {{ a }}[{{ idx }} + {{ svar }}];{% endif %}{% endmacro %}{% macro wgsl_tree_fold(arrays, op="add", idx="lid", wg="WORKGROUP_SIZE", svar="stride", typed=false, form="tail", barrierFirst=false, reuse=false) %}{% if reuse %}{{ svar }}={{ wg }}/2u;
{% else %}var {{ svar }}{{ ": u32 " if typed else " " }}={{ wg }}/2u;
{% endif %}loop{
{% if form == "head" %}if({{ svar }}==0u){break;}
{% endif %}if({{ idx }}<{{ svar }}){
{% for a in arrays %}{{ wgsl_tree_fold_stmt(a, op, idx, svar) }}
{% endfor %}}
{% if form == "head" %}{% if barrierFirst %}workgroupBarrier();{{ svar }}={{ svar }}/2u;
{% else %}{{ svar }}={{ svar }}/2u;workgroupBarrier();
{% endif %}{% else %}workgroupBarrier();if({{ svar }}==1u){break;} {{ svar }}={{ svar }}/2u;
{% endif %}  }{% endmacro %}`],["ops/_shared/attn-flash-decode-splitk-merge.wgsl.jinja",`{% if splitQueries is not defined %}{% set splitQueries = false %}{% endif %}{% if hasGate is not defined %}{% set hasGate = false %}{% endif %}{% set qSeq = qSeq | default(0) %}{% set qHiddenV4 = qHiddenV4 | default(0) %}{% set qHidden = qHidden | default(0) %}{% set hasBias = hasBias is defined and hasBias %}{{ env.wgsl.resourceDeclarations }}
{% if safeSigmoidGate is defined and safeSigmoidGate %}{% include "ops/_shared/sigmoid-safe.wgsl.jinja" %}{% endif %}{% if layout == "bhsd" %}{% elif layout == "layer_cache" %}{% else %}{% endif %}const HEAD_DIM_V4:u32={{ headDimV4 }}u;const Q_HEADS:u32={{ qNumHeads }}u;const NUM_SPLITS:u32={{ numSplits }}u;
{% if splitQueries %}const Q_SEQ:u32={{ qSeq }}u;
{% endif %}{% if layout == "bsh" %}const Q_HIDDEN_V4:u32={{ qHiddenV4 }}u;
{% endif %}{% if hasBias %}const HEAD_DIM:u32={{ headDim }}u;const Q_HIDDEN:u32={{ qHidden }}u;
{% endif %}{% include "ops/_shared/softmax-stable-helpers.wgsl.jinja" %}@compute@workgroup_size(HEAD_DIM_V4,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){
{% if splitQueries %}let queryToken=wg.x;
{% endif %}let h=wg.y;let b=wg.z;let d4=lid.x;
  if (h >= Q_HEADS || d4 >= HEAD_DIM_V4{% if splitQueries %} || queryToken >= Q_SEQ{% endif %}) {
return;}
{% if splitQueries %}let mdBase=((b*Q_SEQ+queryToken)*Q_HEADS+h)*NUM_SPLITS;
{% else %}let mdBase=(b*Q_HEADS+h)*NUM_SPLITS;
{% endif %}var globalMax=-FLT_MAX;for(var s:u32=0u;s<NUM_SPLITS;s=s+1u){
{% if inlinePartialStats is defined and inlinePartialStats %}let statsBase=(mdBase+s)*(HEAD_DIM_V4+1u);globalMax=max(globalMax,partial_out[statsBase+HEAD_DIM_V4].x);
{% else %}globalMax=max(globalMax,partial_stats[mdBase+s].x);
{% endif %}}var globalDenom=0.0;var outV=vec4<f32>(0.0);for(var s:u32=0u;s<NUM_SPLITS;s=s+1u){
{% if inlinePartialStats is defined and inlinePartialStats %}let pBase=(mdBase+s)*(HEAD_DIM_V4+1u);let stats=partial_out[pBase+HEAD_DIM_V4].xy;
{% else %}let stats=partial_stats[mdBase+s];
{% endif %}let w=exp_shift(stats.x,globalMax);globalDenom=globalDenom+stats.y*w;
{% if not (inlinePartialStats is defined and inlinePartialStats) %}{% if splitQueries %}let pBase=(((b*Q_SEQ+queryToken)*Q_HEADS+h)*NUM_SPLITS+s)*HEAD_DIM_V4;
{% else %}let pBase=((b*Q_HEADS+h)*NUM_SPLITS+s)*HEAD_DIM_V4;
{% endif %}{% endif %}outV=outV+partial_out[pBase+d4]*w;}
{% if layout == "bsh" %}{% if splitQueries %}let qBaseV4=(b*Q_SEQ+queryToken)*Q_HIDDEN_V4+h*HEAD_DIM_V4;
{% else %}let qBaseV4=b*Q_HIDDEN_V4+h*HEAD_DIM_V4;
{% endif %}{% elif layout == "layer_cache" %}let qBaseV4=h*HEAD_DIM_V4;
{% else %}{% if splitQueries %}let qBaseV4=((b*Q_HEADS+h)*Q_SEQ+queryToken)*HEAD_DIM_V4;
{% else %}let qBaseV4=(b*Q_HEADS+h)*HEAD_DIM_V4;
{% endif %}{% endif %}{% if zeroEmptyGatedRows | default(false) %}if(globalDenom==0.0){output[qBaseV4+d4]=vec4<{{ scalar }}>(0.0);return;}
{% endif %}var outValue=select(vec4<f32>(0.0),outV/globalDenom,globalDenom>0.0);
{% if hasBias %}let vBiasBase=2u*Q_HIDDEN+h*HEAD_DIM+d4*4u;outValue=outValue+vec4<f32>(bias[vBiasBase],bias[vBiasBase+1u],bias[vBiasBase+2u],bias[vBiasBase+3u]);
{% endif %}{% if safeSigmoidGate is defined and safeSigmoidGate %}let attentionStored=fma(outValue,vec4<f32>(1.0),vec4<f32>(0.0));let gateV=vec4<f32>(gate[qBaseV4+d4]);let sigmoidV=vec4<f32>(sigmoid_safe(gateV.x),sigmoid_safe(gateV.y),sigmoid_safe(gateV.z),sigmoid_safe(gateV.w));let sigmoidStored=fma(sigmoidV,vec4<f32>(1.0),vec4<f32>(0.0));outValue=attentionStored*sigmoidStored;
{% elif hasGate %}let gateV=vec4<f32>(gate[qBaseV4+d4]);outValue=outValue*(vec4<f32>(1.0)/(vec4<f32>(1.0)+exp(-gateV)));
{% endif %}output[qBaseV4+d4]=vec4<{{ scalar }}>(outValue);}`],["ops/_shared/attn-flash-decode-splitk.wgsl.jinja",`{% include "ops/_shared/attn-seqlens-count.wgsl.jinja" %}{% if useSubgroups is not defined %}{% set useSubgroups = true %}{% endif %}{% if splitQueries is not defined %}{% set splitQueries = false %}{% endif %}{% if quantizedCache is not defined %}{% set quantizedCache = false %}{% endif %}{% if cacheSeqlens is not defined %}{% set cacheSeqlens = false %}{% endif %}{% if hasMask is not defined %}{% set hasMask = false %}{% endif %}{% if maskIsBool is not defined %}{% set maskIsBool = false %}{% endif %}{% if hasKeyPadMask is not defined %}{% set hasKeyPadMask = false %}{% endif %}{% if emitQk is not defined %}{% set emitQk = false %}{% endif %}{% if cacheSeqlensIndex is not defined %}{% set cacheSeqlensIndex = "b" %}{% endif %}{% if maskFilterValue is not defined %}{% set maskFilterValue = 0.0 %}{% endif %}{% if qkScalar is not defined %}{% set qkScalar = "f32" %}{% endif %}{% set layer = layer | default(0) %}{% set cacheLen = cacheLen | default(0) %}{% set scale = scale | default("0.0") %}{% if hasRotary is not defined %}{% set hasRotary = false %}{% endif %}{% set qkEps = qkEps | default("0.0") %}{% set qSeq = qSeq | default(0) %}{% set qHiddenV4 = qHiddenV4 | default(0) %}{% set kvHiddenV4 = kvHiddenV4 | default(0) %}{% set hasBias = hasBias is defined and hasBias %}{% set splitKWorkgroupSize = workgroupSizeSpec if workgroupSizeSpec is defined else tunables.WORKGROUP_SIZE %}{% if useSubgroups %}enable subgroups;
{% endif %}{{ env.wgsl.resourceDeclarations }}
{% if layout == "bhsd" %}{% elif layout == "layer_cache" %}{% else %}{% endif %}{% if (fusedQNormRope is defined and fusedQNormRope) or layout != "layer_cache" %}const HEAD_DIM: u32 = {{ headDim }}u;
{% endif %}const HEAD_DIM_V4:u32={{ headDimV4 }}u;const Q_HEADS:u32={{ qNumHeads }}u;const KV_HEADS:u32={{ kvNumHeads }}u;
{% if layout == "bsh" %}const Q_HIDDEN_V4:u32={{ qHiddenV4 }}u;const KV_HIDDEN_V4:u32={{ kvHiddenV4 }}u;
{% elif layout == "layer_cache" %}const LAYER:u32={{ layer }}u;const CACHE_LEN:u32={{ cacheLen }}u;const ATTN_SCALE:f32={{ scale }};
{% endif %}const WG:u32={{ splitKWorkgroupSize }}u;const NUM_SPLITS:u32={{ numSplits }}u;
{% if fusedQNormRope is defined and fusedQNormRope %}{% if hasRotary and headDim % 8 != 0 %}const HALF_DIM: u32 = HEAD_DIM / 2u;
{% endif %}{% if hasRotary and headDim % 8 == 0 %}const HALF_DIM_V4: u32 = HEAD_DIM_V4 / 2u;
{% endif %}const QK_EPS:f32={{ qkEps }};
{% endif %}{% if splitQueries %}const Q_SEQ:u32={{ qSeq }}u;
{% endif %}{% include "ops/_shared/softmax-stable-helpers.wgsl.jinja" %}var<workgroup>q_shared:array<vec4<f32>,HEAD_DIM_V4>;var<workgroup>running_out:array<vec4<f32>,HEAD_DIM_V4>;var<workgroup>probs:array<f32,WG>;
{% if fusedQNormRope is defined and fusedQNormRope %}var<workgroup> fused_q_inv: f32;
{% if useSubgroups %}var<workgroup> fused_q_partials: array<f32, WG>;
{% else %}var<workgroup> fused_q_partials: array<f32, 32>;
{% endif %}{% endif %}{% set coopQk = useSubgroups and headDimV4 >= 8 and not (usesF16 and headDimV4 <= 32) and (allowCooperativeQk if allowCooperativeQk is defined else true) %}{% set jGroups = (splitKWorkgroupSize / headDimV4)|int %}{% set jSplitV = (splitKWorkgroupSize % headDimV4 == 0) and (jGroups >= 2) %}{% if coopQk %}var<workgroup>sval_sh:array<f32,WG>;
{% endif %}{% if jSplitV %}var<workgroup>vacc_sh:array<vec4<f32>,WG>;
{% endif %}{% set combineSubgroups = useSubgroups %}{% include "ops/_shared/softmax-md-combine.wgsl.jinja" %}{% if layout == "layer_cache" %}{% set ATTN_SCALE_OVERRIDE = "ATTN_SCALE" %}{% endif %}{% include "ops/_shared/attn-scale-value.wgsl.jinja" %}{% if quantizedCache %}{% include "ops/_shared/attn-quant-cache-load.wgsl.jinja" %}{{ emit_quant_load4("int8", "key", "key", "k_scale") }} {{ emit_quant_load4("int8", "value", "value", "v_scale") }}
{% else %}fn load_key4(indexV4:u32)->vec4<f32>{return vec4<f32>(key[indexV4]);}fn load_value4(indexV4:u32)->vec4<f32>{return vec4<f32>(value[indexV4]);}
{% endif %}{% if hasBias %}{% include "ops/_shared/attn-load-bias4.wgsl.jinja" %}{% endif %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,
  @builtin(local_invocation_id) lid: vec3<u32>{% if coopQk %},
@builtin(subgroup_size)sgSize:u32,
  @builtin(subgroup_invocation_id) sgLane: u32{% endif %}){
{% if splitQueries %}let queryToken=wg.x/NUM_SPLITS;let split=wg.x%NUM_SPLITS;
{% else %}let split=wg.x;
{% endif %}let h=wg.y;let b=wg.z;
  if (h >= Q_HEADS || split >= NUM_SPLITS{% if splitQueries %} || queryToken >= Q_SEQ{% endif %}{% if layout == "layer_cache" %} || params.past_len >= CACHE_LEN{% endif %}) {
return;}let tid=lid.x;
{% if coopQk %}let physicalWidth=max(sgSize,1u);let population=subgroupBallot(true);let populationCount=countOneBits(population.x)+countOneBits(population.y)+countOneBits(population.z)+countOneBits(population.w);let firstLocalIndex=subgroupBroadcastFirst(tid);let ownsLocalBlock=subgroupAll(sgSize>0u&&populationCount==sgSize&&WG%physicalWidth==0u&&tid/physicalWidth==firstLocalIndex/physicalWidth&&sgLane==tid%physicalWidth);if(subgroupElect()&&!ownsLocalBlock){atomicStore(&leaderMask[0],1u);}workgroupBarrier();if(tid==0u){combinedMD=vec2<f32>(f32(atomicExchange(&leaderMask[0],0u)),0.0);}let coopQkAccepted=workgroupUniformLoad(&combinedMD).x==0.0;
{% endif %}let hKv=h/(Q_HEADS/KV_HEADS);
{% if layout == "layer_cache" %}let kvSeq=params.past_len+1u;
{% else %}let cacheSeq=params.kvSeq;
{% if cacheSeqlens %}let kvSeq={{ attn_seqlens_bound("cacheSeq", cacheSeqlensIndex, "1u" if seqlensQueryFloor | default(false) else "") }};
{% else %}let kvSeq=cacheSeq;
{% endif %}{% endif %}{% if layout == "bsh" %}{% if splitQueries %}let qBaseV4=(b*Q_SEQ+queryToken)*Q_HIDDEN_V4+h*HEAD_DIM_V4;
{% else %}let qBaseV4=b*Q_HIDDEN_V4+h*HEAD_DIM_V4;
{% endif %}let kvBaseV4=b*cacheSeq*KV_HIDDEN_V4+hKv*HEAD_DIM_V4;let kvTokenStrideV4=KV_HIDDEN_V4;
{% elif layout == "layer_cache" %}let qBaseV4=h*HEAD_DIM_V4;let kvBaseV4=(LAYER*CACHE_LEN*KV_HEADS+hKv)*HEAD_DIM_V4;let kvTokenStrideV4=KV_HEADS*HEAD_DIM_V4;
{% else %}{% if splitQueries %}let qBaseV4=((b*Q_HEADS+h)*Q_SEQ+queryToken)*HEAD_DIM_V4;
{% else %}let qBaseV4=(b*Q_HEADS+h)*HEAD_DIM_V4;
{% endif %}let kvBaseV4=(b*KV_HEADS+hKv)*cacheSeq*HEAD_DIM_V4;let kvTokenStrideV4=HEAD_DIM_V4;
{% endif %}{% if hasWindow %}var windowStart:u32=0u;if(kvSeq>params.windowSize){windowStart=kvSeq-params.windowSize;}let activeKeys=kvSeq-windowStart;let keysPerSplit=(activeKeys+NUM_SPLITS-1u)/NUM_SPLITS;let splitStart=windowStart+split*keysPerSplit;
{% else %}{% if minKeysPerChunk is defined %}let evenKeysPerSplit=(kvSeq+NUM_SPLITS-1u)/NUM_SPLITS;let keysPerSplit=max(evenKeysPerSplit,{{ minKeysPerChunk }}u);
{% else %}let keysPerSplit=(kvSeq+NUM_SPLITS-1u)/NUM_SPLITS;
{% endif %}let splitStart=split*keysPerSplit;
{% endif %}var splitEnd=splitStart+keysPerSplit;if(splitEnd>kvSeq){splitEnd=kvSeq;}
{% if fusedQNormRope is defined and fusedQNormRope %}var q_acc=0.0;if(tid<32u){for(var d4=tid;d4<HEAD_DIM_V4;d4=d4+32u){let q4=vec4<f32>(query[qBaseV4+d4]);q_acc=q_acc+dot(q4,q4);} }
{% if useSubgroups %}workgroupBarrier();let q_sg=subgroupAdd(q_acc);fused_q_partials[tid]=select(0.0,q_sg,subgroupElect());workgroupBarrier();if(tid==0u){var q_total=0.0;for(var s=0u;s<WG;s=s+1u){q_total=q_total+fused_q_partials[s];}fused_q_inv=inverseSqrt(q_total/f32(HEAD_DIM)+QK_EPS);}
{% else %}if(tid<32u){fused_q_partials[tid]=q_acc;}workgroupBarrier();var q_stride=16u;loop{if(q_stride==0u){break;}if(tid<q_stride){fused_q_partials[tid]=fused_q_partials[tid]+fused_q_partials[tid+q_stride];}workgroupBarrier();q_stride=q_stride>>1u;}if(tid==0u){fused_q_inv=inverseSqrt(fused_q_partials[0]/f32(HEAD_DIM)+QK_EPS);}
{% endif %}workgroupBarrier();for(var d4=tid;d4<HEAD_DIM_V4;d4=d4+WG){q_shared[d4]=fma(vec4<f32>(query[qBaseV4+d4])*fused_q_inv*q_norm_weight[d4],vec4<f32>(1.0),vec4<f32>(0.0));running_out[d4]=vec4<f32>(0.0);}workgroupBarrier();
{% if hasRotary %}{% if headDim % 8 == 0 %}for(var d4=tid;d4<HALF_DIM_V4;d4=d4+WG){let x0=q_shared[d4];let x1=q_shared[d4+HALF_DIM_V4];let d=d4*4u;let c=vec4<f32>(cos_cache[d],cos_cache[d+1u],cos_cache[d+2u],cos_cache[d+3u]);let s=vec4<f32>(sin_cache[d],sin_cache[d+1u],sin_cache[d+2u],sin_cache[d+3u]);q_shared[d4]=x0*c-x1*s;q_shared[d4+HALF_DIM_V4]=x1*c+x0*s;}
{% else %}if(tid==0u){for(var d=0u;d<HALF_DIM;d=d+1u){let hi_d=d+HALF_DIM;let lo_i=d/4u;let hi_i=hi_d/4u;let lo_lane=d&3u;let hi_lane=hi_d&3u;let x0=q_shared[lo_i][lo_lane];let x1=q_shared[hi_i][hi_lane];let c=cos_cache[d];let s=sin_cache[d];q_shared[lo_i][lo_lane]=x0*c-x1*s;q_shared[hi_i][hi_lane]=x1*c+x0*s;} }
{% endif %}workgroupBarrier();
{% endif %}{% else %}{% include "ops/_shared/attn-q-row-load.wgsl.jinja" %}{% endif %}let scale=scale_value();var runningMax=-FLT_MAX;var runningDenom=0.0;
{% if hasMask and maskIsBool %}{% if splitQueries %}let maskRowBase=b*params.maskBatchStride+h*params.maskHeadStride+queryToken*params.maskSeqStride;
{% else %}let maskRowBase=b*params.maskBatchStride+h*params.maskHeadStride;
{% endif %}{% endif %}var kjBase=splitStart;loop{if(kjBase>=splitEnd){break;}let kj=kjBase+tid;var keyAllowed=kj<splitEnd;
{% if hasMask and maskIsBool %}if(keyAllowed&&attn_mask[maskRowBase+kj]==0u){keyAllowed=false;}
{% endif %}let tileCount=min(WG,splitEnd-kjBase);var score=-FLT_MAX;var m=-FLT_MAX;var dPart=0.0;
{% if coopQk %}if(coopQkAccepted){let sgPerWg=WG/sgSize;let qkRounds=(tileCount+sgPerWg-1u)/sgPerWg;let lane=tid%sgSize;let sgInWg=tid/sgSize;for(var rr:u32=0u;rr<qkRounds;rr=rr+1u){let j=rr*sgPerWg+sgInWg;var accS:f32=0.0;if(j<tileCount){let kRowV4=kvBaseV4+(kjBase+j)*kvTokenStrideV4;for(var d4:u32=lane;d4<HEAD_DIM_V4;d4=d4+sgSize){
          accS = accS + dot(q_shared[d4], load_key4(kRowV4 + d4{% if quantizedCache %}, d4, hKv{% endif %}));
} }let sj=subgroupAdd(accS);if(lane==0u&&j<tileCount){sval_sh[j]=sj;} }workgroupBarrier();if(keyAllowed){score=sval_sh[tid]*scale;m=score;dPart=1.0;} }else{if(keyAllowed){let kRowV4=kvBaseV4+kj*kvTokenStrideV4;var acc:f32=0.0;for(var d4:u32=0u;d4<HEAD_DIM_V4;d4=d4+1u){
        acc = acc + dot(q_shared[d4], load_key4(kRowV4 + d4{% if quantizedCache %}, d4, hKv{% endif %}));
}score=acc*scale;m=score;dPart=1.0;} }
{% else %}if(keyAllowed){let kRowV4=kvBaseV4+kj*kvTokenStrideV4;var acc:f32=0.0;for(var d4:u32=0u;d4<HEAD_DIM_V4;d4=d4+1u){
        acc = acc + dot(q_shared[d4], load_key4(kRowV4 + d4{% if quantizedCache %}, d4, hKv{% endif %}));
}score=acc*scale;m=score;dPart=1.0;}
{% endif %}{% if splitSoftcap is defined and splitSoftcap != 0 %}if(keyAllowed){score={{ splitSoftcap }}*tanh(clamp(score/{{ splitSoftcap }},-30.0,30.0));m=score;}
{% endif %}{% if hasMask and not maskIsBool %}if(keyAllowed){
{% if splitQueries %}let maskQuery=queryToken;
{% else %}let maskQuery=0u;
{% endif %}let maskIndex=b*params.maskBatchStride+h*params.maskHeadStride+maskQuery*params.maskSeqStride+kj;score=score+f32(attn_mask[maskIndex]);m=score;}
{% endif %}{% if hasKeyPadMask %}if(keyAllowed){score=score+select(f32({{ maskFilterValue }}),0.0,key_pad_mask[b*kvSeq+kj]>0);m=score;}
{% endif %}{% if emitQk %}if(keyAllowed){qk[(b*Q_HEADS+h)*kvSeq+kj]={{ qkScalar }}(score);}
{% endif %}let tile=combine_partials(m,dPart,tid);
{% include "ops/_shared/attn-online-merge-tile.wgsl.jinja" %}{% if jSplitV %}const J_GROUPS:u32={{ jGroups }}u;let jg=tid/HEAD_DIM_V4;let d4v=tid%HEAD_DIM_V4;var vacc=vec4<f32>(0.0);var jj=jg;loop{if(jj>=tileCount){break;}
{% if hasMask and maskIsBool %}if(probs[jj]!=0.0){
{% endif %}vacc=vacc+probs[jj]*load_value4(
        kvBaseV4 + (kjBase + jj) * kvTokenStrideV4 + d4v{% if quantizedCache %},
d4v,
        hKv{% endif %});
{% if hasMask and maskIsBool %}}
{% endif %}jj=jj+J_GROUPS;}vacc_sh[tid]=vacc;workgroupBarrier();for(var d4:u32=tid;d4<HEAD_DIM_V4;d4=d4+WG){var a4=running_out[d4]*correction;for(var g:u32=0u;g<J_GROUPS;g=g+1u){a4=a4+vacc_sh[g*HEAD_DIM_V4+d4];}running_out[d4]=a4;}workgroupBarrier();
{% else %}for(var d4:u32=tid;d4<HEAD_DIM_V4;d4=d4+WG){var vSum=vec4<f32>(0.0);for(var i:u32=0u;i<tileCount;i=i+1u){
{% if hasMask and maskIsBool %}if(probs[i]==0.0){continue;}
{% endif %}vSum=vSum+probs[i]*load_value4(
          kvBaseV4 + (kjBase + i) * kvTokenStrideV4 + d4{% if quantizedCache %},
d4,
          hKv{% endif %});}running_out[d4]=running_out[d4]*correction+vSum;}workgroupBarrier();
{% endif %}kjBase=kjBase+WG;}
{% if directOutput is defined and directOutput %}let finalWeight=exp_shift(runningMax,max(-FLT_MAX,runningMax));let finalDenom=runningDenom*finalWeight;for(var d4:u32=tid;d4<HEAD_DIM_V4;d4=d4+WG){let finalOut=running_out[d4]*finalWeight;output[qBaseV4+d4]=vec4<{{ "f16" if usesF16 else "f32" }}>(select(vec4<f32>(0.0),finalOut/finalDenom,finalDenom>0.0));}
{% else %}{% if splitQueries %}let partialBase=(((b*Q_SEQ+queryToken)*Q_HEADS+h)*NUM_SPLITS+split)*HEAD_DIM_V4;
{% else %}{% if inlinePartialStats is defined and inlinePartialStats %}let partialBase=((b*Q_HEADS+h)*NUM_SPLITS+split)*(HEAD_DIM_V4+1u);
{% else %}let partialBase=((b*Q_HEADS+h)*NUM_SPLITS+split)*HEAD_DIM_V4;
{% endif %}{% endif %}for(var d4:u32=tid;d4<HEAD_DIM_V4;d4=d4+WG){partial_out[partialBase+d4]=running_out[d4];}if(tid==0u){
{% if inlinePartialStats is defined and inlinePartialStats %}partial_out[partialBase+HEAD_DIM_V4]=vec4<f32>(runningMax,runningDenom,0.0,0.0);
{% else %}{% if splitQueries %}let mdBase=((b*Q_SEQ+queryToken)*Q_HEADS+h)*NUM_SPLITS+split;
{% else %}let mdBase=(b*Q_HEADS+h)*NUM_SPLITS+split;
{% endif %}partial_stats[mdBase]=vec2<f32>(runningMax,runningDenom);
{% endif %}}
{% endif %}}`],["ops/_shared/attn-flash-prefill-cluster.wgsl.jinja",`{% set clusterPin = clusterPin | default(0) %}{% set validateClusterLayout = validateClusterLayout | default(false) %}{% macro cluster_shuffle_sum(value, width) %}{% for stage in range(log2ceil(width)) %}{{ value }} = {{ value }} + subgroupShuffleXor({{ value }}, {{ pow(2, stage)|int }}u);
{% endfor %}{% endmacro %}{% macro cluster_tree_sum(prefix, offset, count) %}{% if count == 1 %}{{ prefix }}{{ offset }}{% else %}{% set left = ((count + 1) / 2)|int %}({{ cluster_tree_sum(prefix, offset, left) }} + {{ cluster_tree_sum(prefix, offset + left, count - left) }}){% endif %}{% endmacro %}{% include "ops/_shared/attn-seqlens-count.wgsl.jinja" %}{% set sourceProfile = sourceProfile if sourceProfile is defined else 0 %}{% set scaling = scaling | default("0.0") %}{% set qkvStrideV4 = qkvStrideV4 | default(0) %}{% set numSplits = numSplits | default(0) %}{% set QSEQ = "params.seq_len" if sourceProfile == 1 else "params.qSeq" %}{% set KVSEQ = "(params.past_len + params.seq_len)" if sourceProfile == 1 else "params.kvSeq" %}{% set IS_CAUSAL = "1u" if sourceProfile == 1 else "params.isCausal" %}{% set Q_STRIDE = "QKV_STRIDE_V4" if sourceProfile == 1 else "Q_HIDDEN_V4" %}{% set QUERY = "qkv" if sourceProfile == 1 else "query" %}{% set KEY = "cache_keys" if sourceProfile == 1 else "key" %}{% set VALUE = "cache_values" if sourceProfile == 1 else "value" %}{% set OUTPUT = "attn_out" if sourceProfile == 1 else "output" %}{% set HAS_OUTPUT_GATE = outputGate if outputGate is defined else false %}{% set SPLIT_KV = splitKv is defined and splitKv %}{% if sourceProfile == 1 %}{% set ATTN_SCALE_OVERRIDE = scaling %}{% endif %}{% if bakedScale is defined %}{% set ATTN_SCALE_OVERRIDE = bakedScale %}{% endif %}{% if useSubgroups is not defined %}{% set useSubgroups = true %}{% endif %}{% if not useSubgroups %}{% set clusterPin = 0 %}{% endif %}{% if batchNoSgReduction is not defined %}{% set batchNoSgReduction = false %}{% endif %}{% set SHARE_PROB = shareNoSgProb is defined and shareNoSgProb %}{% if hasMask is not defined %}{% set hasMask = false %}{% endif %}{% if maskIsKeyKeep is not defined %}{% set maskIsKeyKeep = false %}{% endif %}{% if maskIsBool is not defined %}{% set maskIsBool = false %}{% endif %}{% if hasSoftcap is not defined %}{% set hasSoftcap = false %}{% endif %}{% if hasHeadSink is not defined %}{% set hasHeadSink = false %}{% endif %}{% set Q_HIDDEN = qHidden | default(0) %}{% if quantCacheFormat is not defined %}{% set quantCacheFormat = "" %}{% endif %}{% if useSeqlens is not defined %}{% set useSeqlens = false %}{% endif %}{% if hasKeyLimit is not defined %}{% set hasKeyLimit = false %}{% endif %}{% if useSeqlens %}{% set KVA = "kvActive" %}{% elif hasKeyLimit %}{% set KVA = "select(" ~ KVSEQ ~ ", min(params.keyLimit, " ~ KVSEQ ~ "), params.keyLimit > 0u)" %}{% else %}{% set KVA = KVSEQ %}{% endif %}{% macro score_expr(part) %}{% if hasSoftcap %}params.softcap * tanh(clamp(({{ part }} * SCALE) / params.softcap, -30.0, 30.0)){% else %}{{ part }} * SCALE{% endif %}{% endmacro %}{% if useSubgroups %}enable subgroups;
{% if clusterPin > 0 %}enable subgroup_size_control;{% endif %}{% endif %}{{ env.wgsl.resourceDeclarations }}
{% set ST = "f16" if usesF16 else "f32" %}{% set FUSED_ROTARY = fusedRotary is defined and fusedRotary %}{% set STAGE_MASK = hasMask and useSubgroups and stageMask is defined and stageMask %}{% set MASK_IS_INT = hasMask and (maskIsKeyKeep or maskIsBool) %}{% set MASK_TILE_TYPE = "u32" if MASK_IS_INT else "f32" %}{% set MASK_TILE_LOAD = "attn_mask[maskIndex]" if MASK_IS_INT else "f32(attn_mask[maskIndex])" %}{% set MASK_TILE_ZERO = "0u" if MASK_IS_INT else "0.0" %}{% set MASK_ELEMENT = "maskValue" if STAGE_MASK else "attn_mask[maskIndex]" %}{% set MASK_ADDITIVE = "maskValue" if STAGE_MASK else "f32(attn_mask[maskIndex])" %}{% set SLICE_COUNT = ((headDimV4 / LPQ) | int) %}{% set QPL = (2 if (not usesF16 and SLICE_COUNT <= 4 and TILE_K <= 16 and TILE_Q % 2 == 0) else 1) if useSubgroups else 1 %}{% macro qn(name, qi) %}{{ name }}{% if qi > 0 %}_q{{ qi }}{% endif %}{% endmacro %}{% macro emit_pair_scores() %}{% for qi in range(QPL) %}{let sc=partv[{{ qi }}];if(kj>={{ qn("minKj", qi) }}&&kj<{{ qn("maxKj", qi) }}){
{% if hasMask %}{% if STAGE_MASK %}let maskValue=mask_tile[(qSub+{{ qi }}u)*TILE_K+kk];
{% else %}let maskIndex=b*params.maskBatchStride+h*params.maskHeadStride+{{ qn("qClamped", qi) }}*params.maskSeqStride+kj;
{% endif %}{% if maskIsKeyKeep %}{{ qn("s", qi) }}[kk]={{ score_expr("sc") }}+(1.0-f32({{ MASK_ELEMENT }}))*MASK_NEG;
{% elif maskIsBool %}if({{ MASK_ELEMENT }}!=0u){ {{ qn("s", qi) }}[kk]={{ score_expr("sc") }};}else{ {{ qn("s", qi) }}[kk]=MASK_NEG;}
{% else %}{{ qn("s", qi) }}[kk]={{ score_expr("sc") }}+{{ MASK_ADDITIVE }};
{% endif %}{% else %}{{ qn("s", qi) }}[kk]={{ score_expr("sc") }};
{% endif %}} }
{% endfor %}{% endmacro %}{% macro emit_tile_softmax() %}{% for qi in range(QPL) %}var {{ qn("tileMax", qi) }}:f32={{ qn("s", qi) }}[0];for(var kk:u32=1u;kk<TILE_K;kk=kk+1u){ {{ qn("tileMax", qi) }}=max({{ qn("tileMax", qi) }},{{ qn("s", qi) }}[kk]);}let {{ qn("newMax", qi) }}=max({{ qn("m", qi) }},{{ qn("tileMax", qi) }});let {{ qn("corr", qi) }}=select(exp({{ qn("m", qi) }}-{{ qn("newMax", qi) }}),0.0,{{ qn("m", qi) }}==NEG_INF);var {{ qn("pSum", qi) }}:f32=0.0;for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let pk=select(0.0,exp({{ qn("s", qi) }}[kk]-{{ qn("newMax", qi) }}),{{ qn("s", qi) }}[kk]!=NEG_INF);{{ qn("s", qi) }}[kk]=pk;{{ qn("pSum", qi) }}={{ qn("pSum", qi) }}+pk;} {{ qn("l", qi) }}={{ qn("l", qi) }}*{{ qn("corr", qi) }}+{{ qn("pSum", qi) }};{{ qn("m", qi) }}={{ qn("newMax", qi) }};
{% endfor %}{% endmacro %}{% set ROPE_LANE_XOR = ((LPQ / 2) | int) %}{% set QL = qLayout if qLayout is defined else layout %}{% set KL = kvLayout if kvLayout is defined else layout %}{% set RIGHT = causalRightAlign is defined and causalRightAlign %}const HEAD_DIM:u32={{ headDim }}u;const HEAD_DIM_V4:u32={{ headDimV4 }}u;
{% if QL != "bhsd" %}const Q_HIDDEN_V4:u32={{ qHiddenV4 }}u;
{% if sourceProfile == 1 %}const QKV_STRIDE_V4: u32 = {{ qkvStrideV4 }}u; 
{% endif %}{% endif %}{% if KL != "bhsd" %}const KV_HIDDEN_V4:u32={{ kvHiddenV4 }}u;
{% endif %}const Q_HEADS:u32={{ qNumHeads }}u;const KV_HEADS:u32={{ kvNumHeads }}u;const TILE_Q:u32={{ TILE_Q }}u;const LPQ:u32={{ LPQ }}u;const SLICE:u32=HEAD_DIM/(4u*LPQ);
{% if FUSED_ROTARY %}const HALF_DIM:u32=HEAD_DIM/2u;const HALF_LPQ:u32=LPQ/2u;
{% endif %}const TILE_K:u32={{ TILE_K }}u;
{% if QPL > 1 %}const QPL:u32={{ QPL }}u;const WG:u32=(TILE_Q/QPL)*LPQ;
{% else %}const WG:u32=TILE_Q*LPQ;
{% endif %}{% if SPLIT_KV %}const NUM_SPLITS:u32={{ numSplits }}u;
{% endif %}{% if MASK_IS_INT %}const NEG_INF:f32=-3.4028234663852886e38;const MASK_NEG:f32={{ "-1e38" if maskIsKeyKeep else "-3.4028234663852886e38" }};
{% else %}const NEG_INF:f32=-3.4028234663852886e38;
{% endif %}var<workgroup>k_tile:array<vec4<{{ ST }}>,TILE_K*(HEAD_DIM/4u)>;var<workgroup>v_tile:array<vec4<{{ ST }}>,TILE_K*(HEAD_DIM/4u)>;
{% if STAGE_MASK %}var<workgroup>mask_tile:array<{{ MASK_TILE_TYPE }},TILE_Q*TILE_K>;
{% endif %}{% if not useSubgroups %}{% if batchNoSgReduction %}var<workgroup>red:array<f32,TILE_K*WG>;
{% else %}var<workgroup>red:array<f32,WG>;
{% endif %}{% endif %}{% include "ops/_shared/attn-scale-value.wgsl.jinja" %}{% if quantCacheFormat %}{% include "ops/_shared/attn-quant-cache-load.wgsl.jinja" %}{{ emit_quant_load4(quantCacheFormat, "key", KEY, "k_scale") }} {{ emit_quant_load4(quantCacheFormat, "value", VALUE, "v_scale") }}
{% endif %}{% if hasBias %}{% include "ops/_shared/attn-load-bias4.wgsl.jinja" %}{% endif %}{% if validateClusterLayout %}fn cluster_native_layout(tid:u32,lane:u32,width:u32)->bool{let mask=subgroupBallot(true);let population=countOneBits(mask.x)+countOneBits(mask.y)+countOneBits(mask.z)+countOneBits(mask.w);let first=subgroupBroadcastFirst(tid);let compatible=width>=LPQ&&width<=WG&&width%LPQ==0u&&WG%width==0u&&population==width&&lane==tid%width&&first==tid-(tid%width);let completeSegment=subgroupAll(compatible);if(tid%LPQ==0u){k_tile[tid/LPQ]=vec4<f32>(select(1.0,0.0,completeSegment),0.0,0.0,0.0);}workgroupBarrier();if(tid==0u){var rejected=0.0;for(var i=0u;i<WG/LPQ;i=i+1u){rejected=max(rejected,k_tile[i].x);}k_tile[0]=vec4<f32>(1.0-rejected,0.0,0.0,0.0);}let accepted=workgroupUniformLoad(&k_tile[0]).x!=0.0;return accepted;}
{% endif %}{% if clusterPin > 0 %}@subgroup_size({{ clusterPin }}){% endif %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,
  @builtin(local_invocation_id) lid: vec3<u32>{% if useSubgroups %},
  @builtin(subgroup_size) sgSize: u32{% if validateClusterLayout %},
  @builtin(subgroup_invocation_id) sgLane: u32{% endif %}{% endif %}){let h=wg.y;
{% if SPLIT_KV %}let b=wg.z/NUM_SPLITS;let split=wg.z%NUM_SPLITS;
{% else %}let b=wg.z;
{% endif %}let tid=lid.x;
{% if useSubgroups %}{% if validateClusterLayout %}let clusterNative=cluster_native_layout(tid,sgLane,sgSize);
{% else %}if(sgSize<LPQ||sgSize%LPQ!=0u){return;}
{% endif %}{% endif %}{% if QPL > 1 %}let qSub=(tid/LPQ)*QPL;
{% else %}let qSub=tid/LPQ;
{% endif %}let lane8=tid%LPQ;
{% for qi in range(QPL) %}  let {{ qn("qIdx", qi) }} = wg.x * TILE_Q + qSub{% if qi > 0 %} + {{ qi }}u{% endif %};
let {{ qn("qValid", qi) }}={{ qn("qIdx", qi) }}<{{ QSEQ }}&&h<Q_HEADS;
{% endfor %}let hKv=h/(Q_HEADS/KV_HEADS);let SCALE=scale_value();
{% for qi in range(QPL) %}let {{ qn("qClamped", qi) }}=min({{ qn("qIdx", qi) }},{{ QSEQ }}-1u);
{% if QL == "bhsd" %}let {{ qn("qBase4", qi) }}=((b*Q_HEADS+h)*{{ QSEQ }}+{{ qn("qClamped", qi) }})*HEAD_DIM_V4+lane8*SLICE;
{% else %}let {{ qn("qBase4", qi) }}=(b*{{ QSEQ }}+{{ qn("qClamped", qi) }})*{{ Q_STRIDE }}+h*HEAD_DIM_V4+lane8*SLICE;
{% endif %}{% for c in range(SLICE_COUNT) %}var {{ qn("qr" ~ c, qi) }}=vec4<f32>({{ QUERY }}[{{ qn("qBase4", qi) }}+{{ c }}u]);
{% if hasBias %}{{ qn("qr" ~ c, qi) }}={{ qn("qr" ~ c, qi) }}+load_bias4(h*HEAD_DIM,lane8*SLICE+{{ c }}u);
{% endif %}var {{ qn("o" ~ c, qi) }}=vec4<f32>(0.0);
{% endfor %}{% endfor %}{% if FUSED_ROTARY %}{% if not useSubgroups %}let qHeadBase4=qBase4-lane8*SLICE;let pairedLane=(lane8+HALF_LPQ)%LPQ;
{% endif %}let ropeChunk=(lane8%HALF_LPQ)*SLICE;let pastLenForRope=select(0u,{{ KVSEQ }}-{{ QSEQ }},{{ KVSEQ }}>={{ QSEQ }});
{% for qi in range(QPL) %}let {{ qn("ropePosition", qi) }}=pastLenForRope+{{ qn("qClamped", qi) }};
{% for c in range(SLICE_COUNT) %}{% if useSubgroups %}let {{ qn("pairedQ" ~ c, qi) }}=subgroupShuffleXor({{ qn("qr" ~ c, qi) }},{{ ROPE_LANE_XOR }}u);
{% else %}let {{ qn("pairedQ" ~ c, qi) }}=vec4<f32>({{ QUERY }}[qHeadBase4+pairedLane*SLICE+{{ c }}u]);
{% endif %}let {{ qn("ropeBase" ~ c, qi) }}={{ qn("ropePosition", qi) }}*HALF_DIM+(ropeChunk+{{ c }}u)*4u;let {{ qn("ropeCos" ~ c, qi) }}=vec4<f32>(f32(cos_cache[{{ qn("ropeBase" ~ c, qi) }}]),f32(cos_cache[{{ qn("ropeBase" ~ c, qi) }}+1u]),f32(cos_cache[{{ qn("ropeBase" ~ c, qi) }}+2u]),f32(cos_cache[{{ qn("ropeBase" ~ c, qi) }}+3u]));let {{ qn("ropeSin" ~ c, qi) }}=vec4<f32>(f32(sin_cache[{{ qn("ropeBase" ~ c, qi) }}]),f32(sin_cache[{{ qn("ropeBase" ~ c, qi) }}+1u]),f32(sin_cache[{{ qn("ropeBase" ~ c, qi) }}+2u]),f32(sin_cache[{{ qn("ropeBase" ~ c, qi) }}+3u]));{{ qn("qr" ~ c, qi) }}=select({{ qn("qr" ~ c, qi) }}*{{ qn("ropeCos" ~ c, qi) }}+{{ qn("pairedQ" ~ c, qi) }}*{{ qn("ropeSin" ~ c, qi) }},{{ qn("qr" ~ c, qi) }}*{{ qn("ropeCos" ~ c, qi) }}-{{ qn("pairedQ" ~ c, qi) }}*{{ qn("ropeSin" ~ c, qi) }},lane8<HALF_LPQ);
{% endfor %}{% endfor %}{% endif %}{% for qi in range(QPL) %}var {{ qn("m", qi) }}:f32=NEG_INF;var {{ qn("l", qi) }}:f32=0.0;
{% endfor %}{% if useSeqlens %}{% if seqlensQueryFloor | default(true) %}{% else %}{% endif %}let kvActive={{ attn_seqlens_bound(KVSEQ, "b", QSEQ if seqlensQueryFloor | default(true) else "") }};
{% endif %}{% if hasCausal or hasWindow %}{% if RIGHT or hasWindow %}let pastLen=select(0u,{{ KVA }}-{{ QSEQ }},{{ KVA }}>={{ QSEQ }});
{% endif %}{% for qi in range(QPL) %}{% if hasCausal %}{% if RIGHT %}var {{ qn("maxKjV", qi) }}=select({{ KVA }},min(pastLen+{{ qn("qIdx", qi) }}+1u,{{ KVA }}),{{ IS_CAUSAL }}!=0u);
{% else %}var {{ qn("maxKjV", qi) }}=select({{ KVA }},min({{ qn("qIdx", qi) }}+1u,{{ KVA }}),{{ IS_CAUSAL }}!=0u);
{% endif %}{% else %}var {{ qn("maxKjV", qi) }}={{ KVA }};
{% endif %}var {{ qn("minKjV", qi) }}:u32=0u;
{% endfor %}let lastQ=min(wg.x*TILE_Q+TILE_Q-1u,{{ QSEQ }}-1u);
{% if hasCausal %}{% if RIGHT %}var wgEndV=select({{ KVA }},min(pastLen+lastQ+1u,{{ KVA }}),{{ IS_CAUSAL }}!=0u);
{% else %}var wgEndV=select({{ KVA }},min(lastQ+1u,{{ KVA }}),{{ IS_CAUSAL }}!=0u);
{% endif %}{% else %}var wgEndV={{ KVA }};
{% endif %}var wgStartV:u32=0u;
{% if hasWindow %}{% for qi in range(QPL) %}let {{ qn("qAbsP1", qi) }}=pastLen+{{ qn("qIdx", qi) }}+1u;{{ qn("maxKjV", qi) }}=min({{ qn("maxKjV", qi) }},{{ qn("qAbsP1", qi) }});if({{ qn("qAbsP1", qi) }}>params.windowSize){ {{ qn("minKjV", qi) }}={{ qn("qAbsP1", qi) }}-params.windowSize;}
{% endfor %}let lastQAbsP1=pastLen+lastQ+1u;wgEndV=min(wgEndV,lastQAbsP1);let firstQAbsP1=pastLen+wg.x*TILE_Q+1u;if(firstQAbsP1>params.windowSize){wgStartV=firstQAbsP1-params.windowSize;}
{% endif %}{% for qi in range(QPL) %}let {{ qn("maxKj", qi) }}={{ qn("maxKjV", qi) }};let {{ qn("minKj", qi) }}={{ qn("minKjV", qi) }};
{% endfor %}let wgEnd=wgEndV;let wgStart=wgStartV;
{% else %}{% if SPLIT_KV %}let splitLen=({{ KVA }}+NUM_SPLITS-1u)/NUM_SPLITS;let splitStart=split*splitLen;let splitEnd=min({{ KVA }},splitStart+splitLen);let maxKj=splitEnd;let minKj=splitStart;let wgEnd=splitEnd;let wgStart=splitStart;
{% else %}let maxKj={{ KVA }};let minKj:u32=0u;let wgEnd={{ KVA }};let wgStart:u32=0u;
{% endif %}{% for qi in range(1, QPL) %}let {{ qn("maxKj", qi) }}=maxKj;let {{ qn("minKj", qi) }}=minKj;
{% endfor %}{% endif %}{% if KL == "bhsd" %}{% if quantCacheFormat == "int4" %}let kvRowStride4=HEAD_DIM_V4*2u;
{% else %}let kvRowStride4=HEAD_DIM_V4;
{% endif %}let kvBatch4=(b*KV_HEADS+hKv)*{{ KVSEQ }}*kvRowStride4;
{% else %}let kvBatch4=b*{{ KVSEQ }}*KV_HIDDEN_V4+hKv*HEAD_DIM_V4;
{% endif %}var kStart:u32=wgStart;loop{if(kStart>=wgEnd){break;}workgroupBarrier();for(var i:u32=tid;i<TILE_K*HEAD_DIM_V4;i=i+WG){let slot=i/HEAD_DIM_V4;let d4=i%HEAD_DIM_V4;let kj=kStart+slot;if(kj<wgEnd){
{% if KL == "bhsd" %}let base4=kvBatch4+kj*kvRowStride4+d4;
{% else %}let base4=kvBatch4+kj*KV_HIDDEN_V4+d4;
{% endif %}{% if quantCacheFormat %}k_tile[i]=vec4<{{ ST }}>(load_key4(base4,d4,hKv));v_tile[i]=vec4<{{ ST }}>(load_value4(base4,d4,hKv));
{% else %}k_tile[i]=vec4<{{ ST }}>({{ KEY }}[base4]);v_tile[i]=vec4<{{ ST }}>({{ VALUE }}[base4]);
{% endif %}}else{k_tile[i]=vec4<{{ ST }}>(0.0);v_tile[i]=vec4<{{ ST }}>(0.0);} }
{% if STAGE_MASK %}for(var i:u32=tid;i<TILE_Q*TILE_K;i=i+WG){let qSlot=i/TILE_K;let kSlot=i%TILE_K;let maskQ=min(wg.x*TILE_Q+qSlot,{{ QSEQ }}-1u);let maskK=kStart+kSlot;if(maskK<wgEnd){let maskIndex=b*params.maskBatchStride+h*params.maskHeadStride+maskQ*params.maskSeqStride+maskK;mask_tile[i]={{ MASK_TILE_LOAD }};}else{mask_tile[i]={{ MASK_TILE_ZERO }};} }
{% endif %}workgroupBarrier();
{% if not useSubgroups and batchNoSgReduction %}{% if not SHARE_PROB %}var s:array<f32,TILE_K>;
{% endif %}for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){
{% if not SHARE_PROB %}s[kk]=NEG_INF;
{% endif %}var part:f32=0.0;let kb=kk*HEAD_DIM_V4+lane8*SLICE;
{% for c in range(SLICE_COUNT) %}part=part+dot(qr{{ c }},vec4<f32>(k_tile[kb+{{ c }}u]));
{% endfor %}red[kk*WG+tid]=part;}workgroupBarrier();let cbase=tid-lane8;for(var kk:u32=lane8;kk<TILE_K;kk=kk+LPQ){
{% for j in range(LPQ) %}let r{{ j }}=red[kk*WG+cbase+{{ j }}u];
{% endfor %}let part={{ cluster_tree_sum("r", 0, LPQ) }};red[kk*WG+cbase]=part;}workgroupBarrier();
{% if SHARE_PROB %}if(lane8==0u){var s:array<f32,TILE_K>;
{% endif %}for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let kj=kStart+kk;let part=red[kk*WG+tid-lane8];
{% if SHARE_PROB %}s[kk]=NEG_INF;
{% endif %}if(kj>=minKj&&kj<maxKj){
{% if hasMask %}let maskIndex=b*params.maskBatchStride+h*params.maskHeadStride+qClamped*params.maskSeqStride+kj;
{% if maskIsBool %}if(attn_mask[maskIndex]!=0u){s[kk]={{ score_expr("part") }};}else{s[kk]=MASK_NEG;}
{% else %}s[kk]={{ score_expr("part") }}+f32(attn_mask[maskIndex]);
{% endif %}{% else %}s[kk]={{ score_expr("part") }};
{% endif %}} }
{% if SHARE_PROB %}{{ emit_tile_softmax() }} for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){red[kk*WG+cbase]=s[kk];}red[cbase+1u]=corr;red[cbase+2u]=m;red[cbase+3u]=l;}workgroupBarrier();let corr=red[cbase+1u];m=red[cbase+2u];l=red[cbase+3u];
{% endif %}{% elif QPL > 1 %}{% for qi in range(QPL) %}var {{ qn("s", qi) }}:array<f32,TILE_K>;
{% endfor %}{% if validateClusterLayout %}if(clusterNative){
{% endif %}for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let kj=kStart+kk;
{% for qi in range(QPL) %}{{ qn("s", qi) }}[kk]=NEG_INF;var {{ qn("part", qi) }}:f32=0.0;
{% endfor %}let kb=kk*HEAD_DIM_V4+lane8*SLICE;
{% for c in range(SLICE_COUNT) %}let kw{{ c }}=vec4<f32>(k_tile[kb+{{ c }}u]);
{% for qi in range(QPL) %}{{ qn("part", qi) }}={{ qn("part", qi) }}+dot({{ qn("qr" ~ c, qi) }},kw{{ c }});
{% endfor %}{% endfor %}      var partv = vec{{ QPL }}<f32>({% for qi in range(QPL) %}{{ qn("part", qi) }}{% if not loop.last %}, {% endif %}{% endfor %});
{{ cluster_shuffle_sum("partv", LPQ) | indent(6, true) }} {{ emit_pair_scores() }} }
{% if validateClusterLayout %}}else{for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){
{% for qi in range(QPL) %}var {{ qn("part", qi) }}:f32=0.0;
{% endfor %}let kb=kk*HEAD_DIM_V4+lane8*SLICE;
{% for c in range(SLICE_COUNT) %}let kw{{ c }}=vec4<f32>(k_tile[kb+{{ c }}u]);
{% for qi in range(QPL) %}{{ qn("part", qi) }}={{ qn("part", qi) }}+dot({{ qn("qr" ~ c, qi) }},kw{{ c }});
{% endfor %}{% endfor %}{% for qi in range(QPL) %}{{ qn("s", qi) }}[kk]={{ qn("part", qi) }};
{% endfor %}}workgroupBarrier();let cbase=tid-lane8;for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let kj=kStart+kk;k_tile[tid]=vec4<f32>(s[kk],s_q1[kk],0.0,0.0);workgroupBarrier();
{% for j in range(LPQ) %}let rv{{ j }}=k_tile[cbase+{{ j }}u].xy;
{% endfor %}let partv={{ cluster_tree_sum("rv", 0, LPQ) }};
{% for qi in range(QPL) %}{{ qn("s", qi) }}[kk]=NEG_INF;
{% endfor %}{{ emit_pair_scores() | indent(2, true) }} workgroupBarrier();} }
{% endif %}{% else %}var s:array<f32,TILE_K>;for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let kj=kStart+kk;s[kk]=NEG_INF;var part:f32=0.0;let kb=kk*HEAD_DIM_V4+lane8*SLICE;
{% for c in range(SLICE_COUNT) %}part=part+dot(qr{{ c }},vec4<f32>(k_tile[kb+{{ c }}u]));
{% endfor %}{% if useSubgroups %}{{ cluster_shuffle_sum("part", LPQ) | indent(6, true) }}
{% else %}red[tid]=part;workgroupBarrier();let cbase=tid-lane8;part=0.0;for(var j:u32=0u;j<LPQ;j=j+1u){part=part+red[cbase+j];}workgroupBarrier();
{% endif %}if(kj>=minKj&&kj<maxKj){
{% if hasMask %}{% if STAGE_MASK %}let maskValue=mask_tile[qSub*TILE_K+kk];
{% else %}let maskIndex=b*params.maskBatchStride+h*params.maskHeadStride+qClamped*params.maskSeqStride+kj;
{% endif %}{% if maskIsKeyKeep %}s[kk]={{ score_expr("part") }}+(1.0-f32({{ MASK_ELEMENT }}))*MASK_NEG;
{% elif maskIsBool %}if({{ MASK_ELEMENT }}!=0u){s[kk]={{ score_expr("part") }};}else{s[kk]=MASK_NEG;}
{% else %}s[kk]={{ score_expr("part") }}+{{ MASK_ADDITIVE }};
{% endif %}{% else %}s[kk]={{ score_expr("part") }};
{% endif %}} }
{% endif %}{% if not SHARE_PROB %}{{ emit_tile_softmax() }}
{% endif %}{% if SHARE_PROB %}{% macro shared_pv(masked) %}for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let weight=red[kk*WG+cbase];let vb=kk*HEAD_DIM_V4+lane8*SLICE;
{% for c in range(SLICE_COUNT) %}          o{{ c }} = o{{ c }} + weight * {% if masked %}select(vec4<f32>(), {% endif %}vec4<f32>(v_tile[vb + {{ c }}u]){% if masked %}, weight != 0.0){% endif %};
{% endfor %}}
{% endmacro %}{% for c in range(SLICE_COUNT) %}o{{ c }}=o{{ c }}*corr;
{% endfor %}let tileInterior=kStart>=minKj&&kStart+TILE_K<=maxKj;if(tileInterior){ {{ shared_pv(false) }} }else{ {{ shared_pv(true) }} }
{% elif QPL > 1 %}    let tileInterior = {% for qi in range(QPL) %}(kStart >= {{ qn("minKj", qi) }} && kStart + TILE_K <= {{ qn("maxKj", qi) }}){% if not loop.last %} && {% endif %}{% endfor %};
{% for c in range(SLICE_COUNT) %}{
{% for qi in range(QPL) %}var {{ qn("acc", qi) }}={{ qn("o" ~ c, qi) }}*{{ qn("corr", qi) }};
{% endfor %}if(tileInterior){for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let vw=vec4<f32>(v_tile[kk*HEAD_DIM_V4+lane8*SLICE+{{ c }}u]);
{% for qi in range(QPL) %}{{ qn("acc", qi) }}={{ qn("acc", qi) }}+{{ qn("s", qi) }}[kk]*vw;
{% endfor %}} }else{for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){let vw=vec4<f32>(v_tile[kk*HEAD_DIM_V4+lane8*SLICE+{{ c }}u]);
{% for qi in range(QPL) %}{{ qn("acc", qi) }}={{ qn("acc", qi) }}+{{ qn("s", qi) }}[kk]*select(vec4<f32>(),vw,{{ qn("s", qi) }}[kk]!=0.0);
{% endfor %}} }
{% for qi in range(QPL) %}{{ qn("o" ~ c, qi) }}={{ qn("acc", qi) }};
{% endfor %}}
{% endfor %}{% else %}let tileInterior=kStart>=minKj&&kStart+TILE_K<=maxKj;
{% for c in range(SLICE_COUNT) %}{var acc=o{{ c }}*corr;if(tileInterior){for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){acc=acc+s[kk]*vec4<f32>(v_tile[kk*HEAD_DIM_V4+lane8*SLICE+{{ c }}u]);} }else{for(var kk:u32=0u;kk<TILE_K;kk=kk+1u){acc=acc+s[kk]*select(vec4<f32>(),vec4<f32>(v_tile[kk*HEAD_DIM_V4+lane8*SLICE+{{ c }}u]),s[kk]!=0.0);} }o{{ c }}=acc;}
{% endfor %}{% endif %}kStart=kStart+TILE_K;}
{% macro attention_value(c, qi) %}{% if hasHeadSink %}select({% endif %}select({{ qn("o" ~ c, qi) }} * inv, vec4<f32>(), zeroSupport){% if hasHeadSink %}, vec4<f32>(bitcast<f32>(sinkBits | 0x00400000u)), !sinkAllowsZero){% endif %}{% if hasBias %} + select(vec4<f32>(), load_bias4(2u * {{ Q_HIDDEN }}u + h * HEAD_DIM, lane8 * SLICE + {{ c }}u), {{ qn("l", qi) }} > 0.0){% endif %}{% endmacro %}{% for qi in range(QPL) %}if({{ qn("qValid", qi) }}){
{% if SPLIT_KV %}let statsBase=((b*{{ QSEQ }}+{{ qn("qIdx", qi) }})*Q_HEADS+h)*NUM_SPLITS+split;let partialBase4=statsBase*HEAD_DIM_V4+lane8*SLICE;
{% for c in range(SLICE_COUNT) %}partial_out[partialBase4+{{ c }}u]=select({{ qn("o" ~ c, qi) }},vec4<f32>(),{{ qn("l", qi) }}==0.0);
{% endfor %}if(lane8==0u){partial_stats[statsBase]=vec2<f32>({{ qn("m", qi) }},{{ qn("l", qi) }});} }
{% else %}{% if QL == "bhsd" %}let outBase4=((b*Q_HEADS+h)*{{ QSEQ }}+{{ qn("qIdx", qi) }})*HEAD_DIM_V4+lane8*SLICE;
{% else %}let outBase4=(b*{{ QSEQ }}+{{ qn("qIdx", qi) }})*Q_HIDDEN_V4+h*HEAD_DIM_V4+lane8*SLICE;
{% endif %}{% if hasHeadSink %}let sink=f32(head_sink[h]);let finalM=max({{ qn("m", qi) }},sink);let accScale=exp({{ qn("m", qi) }}-finalM);let finalDenom=exp(sink-finalM)+{{ qn("l", qi) }}*accScale;let inv=accScale/finalDenom;let denomIsNaN=(bitcast<u32>(finalDenom)&0x7fffffffu)>0x7f800000u;let sinkBits=bitcast<u32>(sink);let sinkAllowsZero=(sinkBits&0x7fffffffu)<0x7f800000u||sinkBits==0xff800000u;let zeroSupport={{ qn("l", qi) }}==0.0&&!denomIsNaN&&sinkAllowsZero;
{% else %}let inv=select(0.0,1.0/{{ qn("l", qi) }},{{ qn("l", qi) }}>0.0);let zeroSupport={{ qn("l", qi) }}==0.0;
{% endif %}{% for c in range(SLICE_COUNT) %}{% if HAS_OUTPUT_GATE %}{let attentionValue={{ attention_value(c, qi) }};let rounded=vec4<f32>(vec4<{{ scalar }}>(attentionValue));let gateValue=vec4<f32>(1.0)/(vec4<f32>(1.0)+exp(-vec4<f32>(gate[outBase4+{{ c }}u])));{{ OUTPUT }}[outBase4+{{ c }}u]=vec4<{{ scalar }}>(rounded*gateValue);}
{% else %}{{ OUTPUT }}[outBase4+{{ c }}u]=vec4<{{ scalar }}>({{ attention_value(c, qi) }});
{% endif %}{% endfor %}}
{% endif %}{% endfor %}}`],["ops/_shared/attn-load-bias4.wgsl.jinja",'{% set widenBias = biasScalarWiden if biasScalarWiden is defined else (inputScalar is defined and inputScalar == "f16") %}{% set BW = "f32(" if widenBias else "" %}{% set BC = ")" if widenBias else "" %}fn load_bias4(base:u32,d4:u32)->vec4<f32>{let offset=base+d4*4u;return vec4<f32>({{ BW }}bias[offset]{{ BC }},{{ BW }}bias[offset+1u]{{ BC }},{{ BW }}bias[offset+2u]{{ BC }},{{ BW }}bias[offset+3u]{{ BC }});}'],["ops/_shared/attn-online-merge-tile.wgsl.jinja","let newMax=max(runningMax,tile.x);let correction=exp_shift(runningMax,newMax);runningDenom=runningDenom*correction+tile.y*exp_shift(tile.x,newMax);runningMax=newMax;var prob=0.0;if(keyAllowed){prob=exp_shift(score,newMax);}probs[tid]=prob;workgroupBarrier();"],["ops/_shared/attn-q-row-load.wgsl.jinja",`{% set hasBias = hasBias is defined and hasBias %}for(var d4=tid;d4<HEAD_DIM_V4;d4=d4+WG){var qv=vec4<f32>(query[qBaseV4+d4]);
{% if hasBias %}qv=qv+load_bias4(h*HEAD_DIM,d4);
{% endif %}q_shared[d4]=qv;running_out[d4]=vec4<f32>(0.0);}workgroupBarrier();`],["ops/_shared/attn-quant-cache-load.wgsl.jinja",`{% macro emit_quant_scale4(kind, scaleBuffer) %}fn {{ kind }}scale4(d4:u32,hk:u32)->vec4<f32>{if(params.perChannel==0u){return vec4<f32>({{ scaleBuffer }}[0]);}let base=hk*HEAD_DIM+d4*4u;return vec4<f32>({{ scaleBuffer }}[base],{{ scaleBuffer }}[base+1u],{{ scaleBuffer }}[base+2u],{{ scaleBuffer }}[base+3u]);
}{% endmacro %}{% macro emit_quant_load4(format, kind, buffer, scaleBuffer) %}{{ emit_quant_scale4(kind, scaleBuffer) }} fn load_{{ kind }}4(indexV4:u32,d4:u32,hk:u32)->vec4<f32>{
{% if format == "int8" %}return vec4<f32>({{ buffer }}[indexV4])*{{ kind }}scale4(d4,hk);
{% else %}let rowBase=indexV4-d4;let lo={{ buffer }}[rowBase+d4*2u];let hi={{ buffer }}[rowBase+d4*2u+1u];let nibbles=vec4<i32>(i32(lo&0xFu),i32((lo>>4u)&0xFu),i32(hi&0xFu),i32((hi>>4u)&0xFu));let signed=nibbles-vec4<i32>(8);return vec4<f32>(signed)*{{ kind }}scale4(d4,hk);
{% endif %}}{% endmacro %}`],["ops/_shared/attn-scale-value.wgsl.jinja",`{% if ATTN_SCALE_DIM is not defined %}{% set ATTN_SCALE_DIM = "HEAD_DIM" %}{% endif %}fn scale_value()->f32{
{% if scaleIsExplicitZero is defined and scaleIsExplicitZero %}return 0.0;
{% elif ATTN_SCALE_OVERRIDE is defined %}return {{ ATTN_SCALE_OVERRIDE }};
{% else %}if(params.scale!=0.0){return params.scale;}return inverseSqrt(f32({{ ATTN_SCALE_DIM }}));
{% endif %}}`],["ops/_shared/attn-seqlens-count.wgsl.jinja",'{% macro attn_seqlens_count(index) %}(u32(max(seqlens_k[{{ index }}], 0)) + select(0u, 1u, seqlens_k[{{ index }}] >= 0)){% endmacro %}{% macro attn_seqlens_bound(capacity, index, queryFloor="") %}min({{ capacity }}, {% if queryFloor %}max({{ queryFloor }}, {{ attn_seqlens_count(index) }}){% else %}{{ attn_seqlens_count(index) }}{% endif %}){% endmacro %}'],["ops/_shared/kv-cache-index.wgsl.jinja","fn cache_index(pos:u32,head:u32,dim:u32)->u32{return(((LAYER*CACHE_LEN+pos)*NUM_KV_HEADS+head)*HEAD_DIM+dim);}"],["ops/_shared/sigmoid-safe.wgsl.jinja","fn sigmoid_safe(x:f32)->f32{if(x>=0.0){let z=exp(-x);return 1.0/(1.0+z);}let z=exp(x);return z/(1.0+z);}"],["ops/_shared/softmax-md-combine-streamed-subgroups.wgsl.jinja",`var<workgroup>partialM:array<f32,WG*{{ mdStreams }}u>;var<workgroup>partialD:array<f32,WG*{{ mdStreams }}u>;fn combine_partials_streams(tid:u32,sgId:u32,foldCount:u32,valid:f32
{% for g in range(mdStreams) %},score_{{ g }}:f32
{% endfor %}){
{% for g in range(mdStreams) %}let subgroupMax_{{ g }}=subgroupMax(score_{{ g }});let subgroupDenom_{{ g }}=subgroupAdd(valid*exp_shift(score_{{ g }},subgroupMax_{{ g }}));
{% endfor %}if(subgroupElect()){
{% for g in range(mdStreams) %}partialM[{{ g }}u*WG+sgId]=subgroupMax_{{ g }};partialD[{{ g }}u*WG+sgId]=subgroupDenom_{{ g }};
{% endfor %}}workgroupBarrier();if(tid==0u){
{% for g in range(mdStreams) %}var m_{{ g }}=-FLT_MAX;var d_{{ g }}=0.0;
{% endfor %}for(var s=0u;s<foldCount;s=s+1u){
{% for g in range(mdStreams) %}let sm_{{ g }}=partialM[{{ g }}u*WG+s];let newM_{{ g }}=max(m_{{ g }},sm_{{ g }});d_{{ g }}=d_{{ g }}*exp_shift(m_{{ g }},newM_{{ g }})+partialD[{{ g }}u*WG+s]*exp_shift(sm_{{ g }},newM_{{ g }});m_{{ g }}=newM_{{ g }};
{% endfor %}}
{% for g in range(mdStreams) %}partialM[{{ g }}u*WG]=m_{{ g }};partialD[{{ g }}u*WG]=d_{{ g }};
{% endfor %}}workgroupBarrier();}`],["ops/_shared/softmax-md-combine.wgsl.jinja",`{% macro subgroup_pair(score, valid, maximum, denominator, pair) %}let {{ maximum }}=subgroupMax({{ score }});let {{ denominator }}=subgroupAdd({{ valid }}*exp_shift({{ score }},{{ maximum }}));let {{ pair }}=vec2<f32>({{ maximum }},{{ denominator }});
{% endmacro %}{% set combineSubgroups = combineSubgroups is defined and combineSubgroups %}{% set mdStreamed = mdStreams is defined %}{% set mdStreams = mdStreams if mdStreams is defined else 1 %}{% if mdWholeSubgroup | default(false) %}{% elif combineSubgroups and mdStreamed %}{% include "ops/_shared/softmax-md-combine-streamed-subgroups.wgsl.jinja" %}{% elif combineSubgroups %}var<workgroup>partialM:array<f32,WG>;var<workgroup>partialD:array<f32,WG>;var<workgroup>leaderMask:array<atomic<u32>,(WG+31u)/32u>;var<workgroup>combinedMD:vec2<f32>;fn combine_partials(m:f32,d:f32,lidx:u32)->vec2<f32>{let sgM=subgroupMax(m);let sgD=subgroupAdd(d*exp_shift(m,sgM));if(subgroupElect()){partialM[lidx]=sgM;partialD[lidx]=sgD;atomicOr(&leaderMask[lidx/32u],1u<<(lidx%32u));}workgroupBarrier();if(lidx==0u){var accM=-FLT_MAX;var accD=0.0;for(var w=0u;w<(WG+31u)/32u;w=w+1u){var bits=atomicExchange(&leaderMask[w],0u);while(bits!=0u){let slot=w*32u+firstTrailingBit(bits);bits=bits&(bits-1u);let mNew=max(accM,partialM[slot]);accD=accD*exp_shift(accM,mNew)+partialD[slot]*exp_shift(partialM[slot],mNew);accM=mNew;} }combinedMD=vec2<f32>(accM,accD);}workgroupBarrier();return combinedMD;}
{% else %}{% set mdExtent = "WG" if mdStreams == 1 else "WG * " ~ mdStreams ~ "u" %}var<workgroup>partialM:array<f32,{{ mdExtent }}>;var<workgroup>partialD:array<f32,{{ mdExtent }}>;
{% if mdStreamed %}fn combine_partials_streams(lidx:u32){workgroupBarrier();var stride=WG/2u;loop{if(stride==0u){break;}if(lidx<stride){
{% for s in range(mdStreams) %}{let slot={{ s }}u*WG+lidx;let m1=partialM[slot];let d1=partialD[slot];let m2=partialM[slot+stride];let d2=partialD[slot+stride];let mNew=max(m1,m2);partialD[slot]=d1*exp_shift(m1,mNew)+d2*exp_shift(m2,mNew);partialM[slot]=mNew;}
{% endfor %}}workgroupBarrier();stride=stride/2u;} }
{% else %}fn combine_partials(m:f32,d:f32,lidx:u32)->vec2<f32>{partialM[lidx]=m;partialD[lidx]=d;workgroupBarrier();var stride=WG/2u;loop{if(stride==0u){break;}if(lidx<stride){let m1=partialM[lidx];let d1=partialD[lidx];let m2=partialM[lidx+stride];let d2=partialD[lidx+stride];let mNew=max(m1,m2);partialD[lidx]=d1*exp_shift(m1,mNew)+d2*exp_shift(m2,mNew);partialM[lidx]=mNew;}workgroupBarrier();stride=stride/2u;}let merged=vec2<f32>(partialM[0],partialD[0]);workgroupBarrier();return merged;}
{% endif %}{% endif %}`],["ops/_shared/softmax-stable-helpers.wgsl.jinja",`{% set stableUsage = stableHelperUsage if stableHelperUsage is defined else "all" %}const FLT_MAX:f32=3.4028234663852886e38;
{% if stableUsage != "constant" %}fn is_finite_f32(value:f32)->bool{return select(false,value<=FLT_MAX,value>=-FLT_MAX);}fn shifted_value(value:f32,maxValue:f32)->f32{let equalFiniteMax=select(false,value==maxValue,is_finite_f32(maxValue));return select(value-maxValue,0.0,equalFiniteMax);}
{% endif %}{% if stableUsage == "all" %}fn exp_shift(value:f32,maxValue:f32)->f32{return exp(shifted_value(value,maxValue));}
{% endif %}`],["ops/_shared/subgroup-local-layout-proof.wgsl.jinja","var<workgroup>subgroupLocalLayoutRejected:atomic<u32>;var<workgroup>subgroupLocalLayoutAcceptedState:u32;fn subgroupLocalLayoutAccepted(localIndex:u32,memberValid:bool)->bool{let valid=subgroupAll(memberValid);if(subgroupElect()&&!valid){atomicStore(&subgroupLocalLayoutRejected,1u);}workgroupBarrier();if(localIndex==0u){subgroupLocalLayoutAcceptedState=select(0u,1u,atomicLoad(&subgroupLocalLayoutRejected)==0u);}return workgroupUniformLoad(&subgroupLocalLayoutAcceptedState)!=0u;}"],["ops/_shared/subgroup-matrix-enable.wgsl.jinja",`enable subgroups;
{% if pinSubgroupSize32 %}enable subgroup_size_control;
{% endif %}enable chromium_experimental_subgroup_matrix;diagnostic(off,chromium.subgroup_matrix_uniformity);`],["ops/_shared/subgroup-matrix-software-8x8.wgsl.jinja",`var<private>matrixSoftwareLane:u32;fn matrixSoftwareMultiplyAccumulate(lhs:array<f32,8>,rhs:array<vec2<f32>,8>,acc:vec2<f32>)->vec2<f32>{var value=acc;for(var k=0u;k<8u;k+=1u){value=fma(vec2<f32>(lhs[k]),rhs[k],value);}return value;}
{% macro matrixType(software, kind, scalar) %}{% if software %}{% if kind == "left" %}array<f32, 8>{% elif kind == "right" %}array<vec2<f32>, 8>{% else %}vec2<{{ scalar }}>{% endif %}{% else %}subgroup_matrix_{{ kind }}<{{ scalar }}, 8, 8>{% endif %}{% endmacro %}{% macro matrixLoad(software, kind, scalar, major, pointer, offset, stride) %}{% if not software %}subgroupMatrixLoad<{{ matrixType(false, kind, scalar) }}, {{ major }}>({{ pointer }}, {{ offset }}, {{ stride }}){% else %}{% if kind == "left" %}array<f32, 8>(
{% for k in range(8) %}f32((*({{ pointer }}))[({{ offset }}) + {% if major == "row_major" %}(matrixSoftwareLane / 4u) * ({{ stride }}) + {{ k }}u{% else %}{{ k }}u * ({{ stride }}) + matrixSoftwareLane / 4u{% endif %}]){{ "," if k < 7 else "" }}
{% endfor %})
{% elif kind == "right" %}array<vec2<f32>, 8>(
{% for k in range(8) %}vec2<f32>(
{% for h in range(2) %}f32((*({{ pointer }}))[({{ offset }}) + {% if major == "row_major" %}{{ k }}u * ({{ stride }}) + (matrixSoftwareLane % 4u) * 2u + {{ h }}u{% else %}((matrixSoftwareLane % 4u) * 2u + {{ h }}u) * ({{ stride }}) + {{ k }}u{% endif %}]){{ "," if h == 0 else "" }}
{% endfor %}){{ "," if k < 7 else "" }}
{% endfor %})
{% else %}vec2<{{ scalar }}>(
{% for h in range(2) %}(*({{ pointer }}))[({{ offset }}) + {{ {"row_major": "(matrixSoftwareLane / 4u) * (" ~ stride ~ ") + (matrixSoftwareLane % 4u) * 2u + " ~ h ~ "u", "col_major": "((matrixSoftwareLane % 4u) * 2u + " ~ h ~ "u) * (" ~ stride ~ ") + matrixSoftwareLane / 4u"}[major] }}]{{ "," if h == 0 else "" }}
{% endfor %})
{% endif %}{% endif %}{% endmacro %}{% macro matrixMma(software, scalar, lhs, rhs, acc) %}{% if not software %}subgroupMatrixMultiplyAccumulate({{ lhs }}, {{ rhs }}, {{ acc }}){% else %}vec2<{{ scalar }}>(matrixSoftwareMultiplyAccumulate({{ lhs }}, {{ rhs }}, vec2<f32>({{ acc }}))){% endif %}{% endmacro %}{% macro matrixStore(software, major, pointer, offset, value, stride) %}{% if not software %}subgroupMatrixStore<{{ major }}>({{ pointer }}, {{ offset }}, {{ value }}, {{ stride }});{% else %}{
let matrixPointer={{ pointer }};let matrixValue={{ value }};
{% for h in range(2) %}  (*matrixPointer)[({{ offset }}) + {{ {"row_major": "(matrixSoftwareLane / 4u) * (" ~ stride ~ ") + (matrixSoftwareLane % 4u) * 2u + " ~ h ~ "u", "col_major": "((matrixSoftwareLane % 4u) * 2u + " ~ h ~ "u) * (" ~ stride ~ ") + matrixSoftwareLane / 4u"}[major] }}] = matrixValue.{{ "x" if h == 0 else "y" }};
{% endfor %}}{% endif %}{% endmacro %}`],["ops/_shared/subgroup-matrix-topology.wgsl.jinja","var<workgroup>matrixRoleCount:atomic<u32>;var<workgroup>matrixRejected:atomic<u32>;var<workgroup>matrixAccepted:u32;fn claimMatrixSubgroupRole(width:u32,requiredWidth:u32)->u32{let population=subgroupBallot(true);let populationCount=countOneBits(population.x)+countOneBits(population.y)+countOneBits(population.z)+countOneBits(population.w);var claimed=0u;if(subgroupElect()){claimed=atomicAdd(&matrixRoleCount,1u);if(width!=requiredWidth||populationCount!=requiredWidth){atomicStore(&matrixRejected,1u);} }workgroupBarrier();return subgroupBroadcastFirst(claimed);}fn matrixTopologyAccepted(localIndex:u32,bankCount:u32)->bool{if(localIndex==0u){matrixAccepted=select(0u,1u,atomicLoad(&matrixRejected)==0u&&atomicLoad(&matrixRoleCount)==bankCount);}return workgroupUniformLoad(&matrixAccepted)!=0u;}"],["ops/_shared/arg-reduction-packed-key-compare.wgsl.jinja","fn better_key(candidate:vec2<u32>,incumbent:vec2<u32>)->bool{return candidate.y>incumbent.y||(candidate.y==incumbent.y&&candidate.x>incumbent.x);}"],["ops/_shared/arg-reduction-packed-key.wgsl.jinja",`{% set work = scalar if scalar == "u32" or scalar == "i32" else "f32" %}fn candidate_key(value:{{ work }},index:u32)->vec2<u32>{
{% if scalar == "u32" %}var score=value;
{% elif scalar == "i32" %}var score=bitcast<u32>(value)^0x80000000u;
{% else %}let original_bits=bitcast<u32>(value);if((original_bits&0x7fffffffu)>0x7f800000u){return vec2<u32>(0u);}let bits=select(original_bits,0u,value==0.0);var score=select(~bits,bits^0x80000000u,(bits&0x80000000u)==0u);
{% endif %}{% if modeSpec == "min" %}score=~score;
{% endif %}let tie={{ "index + 1u" if selectLastIndex else "0xffffffffu - index" }};return vec2<u32>(tie,score);}`],["ops/_shared/float-is-nan.wgsl.jinja","fn is_nan_f32(value:f32)->bool{let bits=bitcast<u32>(value);return(bits&0x7f800000u)==0x7f800000u&&(bits&0x007fffffu)!=0u;}"],["ops/_shared/float-neg-inf.wgsl.jinja","fn negative_infinity()->f32{var bits=0xff800000u;return bitcast<f32>(bits);}"],["ops/_shared/matmul-argmax-reduce.wgsl.jinja",`{% set trackValidity = trackValidity is defined and trackValidity %}{{ env.wgsl.resourceDeclarations }} const NUM_BLOCKS:u32={{ numBlocks }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const OUTPUT_OFFSET:u32={{ outputOffset }}u;
{% if not trackValidity %}const F32_MIN:f32={{ f32Min | default("-3.4028234663852886e38") }};
{% endif %}{% if trackValidity %}{% include "ops/_shared/float-is-nan.wgsl.jinja" %}const INVALID_INDEX:u32=0xffffffffu;
{% if useSubgroups %}{% include "ops/_shared/float-neg-inf.wgsl.jinja" %}{% endif %}{% endif %}{% if useSubgroups %}var<workgroup>partial_values:array<f32,{{ workgroupSize }}>;var<workgroup>partial_ids:array<u32,{{ workgroupSize }}>;
{% if trackValidity %}var<workgroup>partial_valid:array<u32,{{ workgroupSize }}>;
{% endif %}{% else %}var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;
{% if trackValidity %}var<workgroup>best_valid:array<u32,{{ workgroupSize }}>;
{% endif %}{% endif %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(local_invocation_id)lid3:vec3<u32>){let lid=lid3.x;
{% if trackValidity %}var value=0.0;var index=INVALID_INDEX;var has_best=false;
{% else %}var value=F32_MIN;var index=0u;
{% endif %}for(var i=lid;i<NUM_BLOCKS;i=i+WORKGROUP_SIZE){let v=best_values_in[i];let id=best_ids_in[i];
{% if trackValidity %}if(!is_nan_f32(v)&&(!has_best||v>value||(v==value&&id<index))){
{% else %}if(v>value||(v==value&&id<index)){
{% endif %}value=v;index=id;
{% if trackValidity %}has_best=true;
{% endif %}} }
{% if useSubgroups %}{% if trackValidity %}let sg_best=subgroupMax(select(negative_infinity(),value,has_best));let sg_token=subgroupMin(select(INVALID_INDEX,index,has_best&&value==sg_best));let sg_valid=subgroupMax(select(0u,1u,has_best));
{% else %}let sg_best=subgroupMax(value);let sg_token=subgroupMin(select(0xffffffffu,index,value==sg_best));
{% endif %}let elected=subgroupElect();
{% if trackValidity %}partial_values[lid]=select(negative_infinity(),sg_best,elected);partial_ids[lid]=select(INVALID_INDEX,sg_token,elected);partial_valid[lid]=select(0u,sg_valid,elected);
{% else %}partial_values[lid]=select(F32_MIN,sg_best,elected);partial_ids[lid]=select(0u,sg_token,elected);
{% endif %}workgroupBarrier();if(lid==0u){
{% if trackValidity %}var best_value=0.0;var best_index=INVALID_INDEX;var has_valid=false;
{% else %}var best_value=F32_MIN;var best_index=0u;
{% endif %}for(var i=0u;i<WORKGROUP_SIZE;i=i+1u){let v=partial_values[i];let id=partial_ids[i];
{% if trackValidity %}if(partial_valid[i]!=0u&&(!has_valid||v>best_value||(v==best_value&&id<best_index))){
{% else %}if(v>best_value||(v==best_value&&id<best_index)){
{% endif %}best_value=v;best_index=id;
{% if trackValidity %}has_valid=true;
{% endif %}} }token_out[OUTPUT_OFFSET]=best_index;}
{% else %}best_values[lid]=value;best_indices[lid]=index;
{% if trackValidity %}best_valid[lid]=select(0u,1u,has_best);
{% endif %}workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){
{% if trackValidity %}let other_valid=best_valid[lid+stride];
{% endif %}let other_value=best_values[lid+stride];let other_index=best_indices[lid+stride];
{% if trackValidity %}if(other_valid!=0u&&(best_valid[lid]==0u||other_value>best_values[lid]||(other_value==best_values[lid]&&other_index<best_indices[lid]))){
{% else %}if(other_value>best_values[lid]||(other_value==best_values[lid]&&other_index<best_indices[lid])){
{% endif %}best_values[lid]=other_value;best_indices[lid]=other_index;
{% if trackValidity %}best_valid[lid]=1u;
{% endif %}} }workgroupBarrier();stride=stride/2u;}if(lid==0u){token_out[OUTPUT_OFFSET]=best_indices[0];}
{% endif %}}`],["ops/_shared/decode-entry-point.wgsl.jinja",'fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ",\\n  @builtin(subgroup_id) sg_id: u32,\\n  @builtin(num_subgroups) sg_count: u32" if (entrySubgroupIds | default(false)) else "" }}){'],["ops/_shared/iq-grid-iq1s.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}const IQ_GRID_0{{ iqSfx }}:array<vec2<u32>,2047>=array<vec2<u32>,2047>(vec2<u32>(4294967295u,4294967295u),vec2<u32>(4294967041u,4294967295u),vec2<u32>(4294901760u,4294967295u),vec2<u32>(4294902271u,4294967295u),vec2<u32>(4294902017u,4294967295u),vec2<u32>(4278255360u,4294967295u),vec2<u32>(4278190080u,4294967295u),vec2<u32>(4278321151u,4294967295u),vec2<u32>(4278320897u,4294967295u),vec2<u32>(4278256127u,4294967295u),vec2<u32>(4278255873u,4294967295u),vec2<u32>(16711680u,4294967295u),vec2<u32>(65280u,4294967295u),vec2<u32>(255u,4294967295u),vec2<u32>(1u,4294967295u),vec2<u32>(65536u,4294967295u),vec2<u32>(33554431u,4294967295u),vec2<u32>(33554177u,4294967295u),vec2<u32>(33489407u,4294967295u),vec2<u32>(33489153u,4294967295u),vec2<u32>(16777216u,4294967295u),vec2<u32>(16908287u,4294967295u),vec2<u32>(16908033u,4294967295u),vec2<u32>(16843263u,4294967295u),vec2<u32>(16843009u,4294967295u),vec2<u32>(4294902015u,4294967040u),vec2<u32>(4294901760u,4294967040u),vec2<u32>(4278255360u,4294967040u),vec2<u32>(4278190335u,4294967040u),vec2<u32>(4278190081u,4294967040u),vec2<u32>(4278190336u,4294967040u),vec2<u32>(4278190337u,4294967040u),vec2<u32>(4278255616u,4294967040u),vec2<u32>(16776960u,4294967040u),vec2<u32>(16711681u,4294967040u),vec2<u32>(16711936u,4294967040u),vec2<u32>(65281u,4294967040u),vec2<u32>(0u,4294967040u),vec2<u32>(257u,4294967040u),vec2<u32>(130816u,4294967040u),vec2<u32>(65791u,4294967040u),vec2<u32>(65537u,4294967040u),vec2<u32>(66047u,4294967040u),vec2<u32>(33488896u,4294967040u),vec2<u32>(16842496u,4294967040u),vec2<u32>(16777471u,4294967040u),vec2<u32>(16777217u,4294967040u),vec2<u32>(16842752u,4294967040u),vec2<u32>(4294967295u,4294967041u),vec2<u32>(4294967041u,4294967041u),vec2<u32>(4294902271u,4294967041u),vec2<u32>(4294902017u,4294967041u),vec2<u32>(4278190080u,4294967041u),vec2<u32>(4278321151u,4294967041u),vec2<u32>(4278320897u,4294967041u),vec2<u32>(4278256127u,4294967041u),vec2<u32>(4278255873u,4294967041u),vec2<u32>(16711680u,4294967041u),vec2<u32>(65280u,4294967041u),vec2<u32>(256u,4294967041u),vec2<u32>(65791u,4294967041u),vec2<u32>(65792u,4294967041u),vec2<u32>(33554431u,4294967041u),vec2<u32>(33554177u,4294967041u),vec2<u32>(33489407u,4294967041u),vec2<u32>(33489153u,4294967041u),vec2<u32>(16842496u,4294967041u),vec2<u32>(16777216u,4294967041u),vec2<u32>(16777472u,4294967041u),vec2<u32>(16908287u,4294967041u),vec2<u32>(16908033u,4294967041u),vec2<u32>(16843263u,4294967041u),vec2<u32>(16843009u,4294967041u),vec2<u32>(4278255360u,4294902015u),vec2<u32>(4278190335u,4294902015u),vec2<u32>(4278190081u,4294902015u),vec2<u32>(4278255616u,4294902015u),vec2<u32>(16776960u,4294902015u),vec2<u32>(16711936u,4294902015u),vec2<u32>(0u,4294902015u),vec2<u32>(257u,4294902015u),vec2<u32>(65791u,4294902015u),vec2<u32>(65536u,4294902015u),vec2<u32>(16842496u,4294902015u),vec2<u32>(16777472u,4294902015u),vec2<u32>(16842752u,4294902015u),vec2<u32>(4294967040u,4294901760u),vec2<u32>(4294902015u,4294901760u),vec2<u32>(4294901760u,4294901760u),vec2<u32>(4294901761u,4294901760u),vec2<u32>(4278190080u,4294901760u),vec2<u32>(4278190591u,4294901760u),vec2<u32>(4278190337u,4294901760u),vec2<u32>(4278255872u,4294901760u),vec2<u32>(16777215u,4294901760u),vec2<u32>(16711680u,4294901760u),vec2<u32>(16711937u,4294901760u),vec2<u32>(65535u,4294901760u),vec2<u32>(65280u,4294901760u),vec2<u32>(255u,4294901760u),vec2<u32>(0u,4294901760u),vec2<u32>(1u,4294901760u),vec2<u32>(256u,4294901760u),vec2<u32>(131071u,4294901760u),vec2<u32>(130817u,4294901760u),vec2<u32>(65536u,4294901760u),vec2<u32>(66047u,4294901760u),vec2<u32>(65793u,4294901760u),vec2<u32>(33554176u,4294901760u),vec2<u32>(16842496u,4294901760u),vec2<u32>(16777216u,4294901760u),vec2<u32>(16777727u,4294901760u),vec2<u32>(16777473u,4294901760u),vec2<u32>(16908032u,4294901760u),vec2<u32>(16843007u,4294901760u),vec2<u32>(16842752u,4294901760u),vec2<u32>(16842753u,4294901760u),vec2<u32>(16843008u,4294901760u),vec2<u32>(4278190335u,4294901761u),vec2<u32>(4278190336u,4294901761u),vec2<u32>(16776960u,4294901761u),vec2<u32>(16711935u,4294901761u),vec2<u32>(65535u,4294901761u),vec2<u32>(65281u,4294901761u),vec2<u32>(0u,4294901761u),vec2<u32>(511u,4294901761u),vec2<u32>(131071u,4294901761u),vec2<u32>(130816u,4294901761u),vec2<u32>(65537u,4294901761u),vec2<u32>(65792u,4294901761u),vec2<u32>(33488896u,4294901761u),vec2<u32>(16842496u,4294901761u),vec2<u32>(16777471u,4294901761u),vec2<u32>(16777472u,4294901761u),vec2<u32>(4294967295u,4294902271u),vec2<u32>(4294967041u,4294902271u),vec2<u32>(4294902271u,4294902271u),vec2<u32>(4294902017u,4294902271u),vec2<u32>(4278190080u,4294902271u),vec2<u32>(4278321151u,4294902271u),vec2<u32>(4278320897u,4294902271u),vec2<u32>(4278256127u,4294902271u),vec2<u32>(4278255873u,4294902271u),vec2<u32>(16711680u,4294902271u),vec2<u32>(65280u,4294902271u),vec2<u32>(1u,4294902271u),vec2<u32>(65536u,4294902271u),vec2<u32>(33554431u,4294902271u),vec2<u32>(33554177u,4294902271u),vec2<u32>(33489407u,4294902271u),vec2<u32>(33489153u,4294902271u),vec2<u32>(16777216u,4294902271u),vec2<u32>(16908287u,4294902271u),vec2<u32>(16908033u,4294902271u),vec2<u32>(16843263u,4294902271u),vec2<u32>(16843009u,4294902271u),vec2<u32>(4294901760u,4294902016u),vec2<u32>(4278255360u,4294902016u),vec2<u32>(4278190335u,4294902016u),vec2<u32>(4278190336u,4294902016u),vec2<u32>(4278255871u,4294902016u),vec2<u32>(4278255616u,4294902016u),vec2<u32>(16776960u,4294902016u),vec2<u32>(65535u,4294902016u),vec2<u32>(65280u,4294902016u),vec2<u32>(0u,4294902016u),vec2<u32>(130816u,4294902016u),vec2<u32>(65791u,4294902016u),vec2<u32>(65792u,4294902016u),vec2<u32>(16842496u,4294902016u),vec2<u32>(16777471u,4294902016u),vec2<u32>(16777217u,4294902016u),vec2<u32>(16777472u,4294902016u),vec2<u32>(16842752u,4294902016u),vec2<u32>(4294967295u,4294902017u),vec2<u32>(4294967041u,4294902017u),vec2<u32>(4294902271u,4294902017u),vec2<u32>(4294902017u,4294902017u),vec2<u32>(4278190080u,4294902017u),vec2<u32>(4278321151u,4294902017u),vec2<u32>(4278320897u,4294902017u),vec2<u32>(4278256127u,4294902017u),vec2<u32>(4278255873u,4294902017u),vec2<u32>(16711680u,4294902017u),vec2<u32>(65280u,4294902017u),vec2<u32>(256u,4294902017u),vec2<u32>(130816u,4294902017u),vec2<u32>(65536u,4294902017u),vec2<u32>(33554431u,4294902017u),vec2<u32>(33554177u,4294902017u),vec2<u32>(33488896u,4294902017u),vec2<u32>(33489407u,4294902017u),vec2<u32>(33489153u,4294902017u),vec2<u32>(16777216u,4294902017u),vec2<u32>(16908287u,4294902017u),vec2<u32>(16908033u,4294902017u),vec2<u32>(16843263u,4294902017u),vec2<u32>(16843009u,4294902017u),vec2<u32>(4278255615u,4278255615u),vec2<u32>(4278255360u,4278255615u),vec2<u32>(4278190335u,4278255615u),vec2<u32>(4278190336u,4278255615u),vec2<u32>(4278255871u,4278255615u),vec2<u32>(4278255616u,4278255615u),vec2<u32>(16776960u,4278255615u),vec2<u32>(16711935u,4278255615u),vec2<u32>(65535u,4278255615u),vec2<u32>(0u,4278255615u),vec2<u32>(511u,4278255615u),vec2<u32>(130816u,4278255615u),vec2<u32>(65791u,4278255615u),vec2<u32>(65536u,4278255615u),vec2<u32>(65792u,4278255615u),vec2<u32>(16842496u,4278255615u),vec2<u32>(16777471u,4278255615u),vec2<u32>(16777217u,4278255615u),vec2<u32>(16908032u,4278255615u),vec2<u32>(16842752u,4278255615u),vec2<u32>(4294967040u,4278255360u),vec2<u32>(4294902015u,4278255360u),vec2<u32>(4294901761u,4278255360u),vec2<u32>(4294902016u,4278255360u),vec2<u32>(4278255615u,4278255360u),vec2<u32>(4278255361u,4278255360u),vec2<u32>(4278190080u,4278255360u),vec2<u32>(4278190591u,4278255360u),vec2<u32>(4278320896u,4278255360u),vec2<u32>(4278255871u,4278255360u),vec2<u32>(4278255872u,4278255360u),vec2<u32>(16711680u,4278255360u),vec2<u32>(16711937u,4278255360u),vec2<u32>(65535u,4278255360u),vec2<u32>(65280u,4278255360u),vec2<u32>(65281u,4278255360u),vec2<u32>(255u,4278255360u),vec2<u32>(0u,4278255360u),vec2<u32>(1u,4278255360u),vec2<u32>(256u,4278255360u),vec2<u32>(131071u,4278255360u),vec2<u32>(65536u,4278255360u),vec2<u32>(33489151u,4278255360u),vec2<u32>(16842497u,4278255360u),vec2<u32>(16777216u,4278255360u),vec2<u32>(16908032u,4278255360u),vec2<u32>(16843007u,4278255360u),vec2<u32>(4278255360u,4278255361u),vec2<u32>(4278190335u,4278255361u),vec2<u32>(4278190081u,4278255361u),vec2<u32>(4278255616u,4278255361u),vec2<u32>(16777215u,4278255361u),vec2<u32>(16711681u,4278255361u),vec2<u32>(16711936u,4278255361u),vec2<u32>(65281u,4278255361u),vec2<u32>(0u,4278255361u),vec2<u32>(511u,4278255361u),vec2<u32>(257u,4278255361u),vec2<u32>(65791u,4278255361u),vec2<u32>(65537u,4278255361u),vec2<u32>(33488896u,4278255361u),vec2<u32>(16842496u,4278255361u),vec2<u32>(16777471u,4278255361u),vec2<u32>(16777217u,4278255361u),vec2<u32>(16842752u,4278255361u),vec2<u32>(4294967040u,4278190335u),vec2<u32>(4294901761u,4278190335u),vec2<u32>(4294902016u,4278190335u),vec2<u32>(4278190335u,4278190335u),vec2<u32>(4278190080u,4278190335u),vec2<u32>(4278190591u,4278190335u),vec2<u32>(4278190336u,4278190335u),vec2<u32>(4278320896u,4278190335u),vec2<u32>(4278255617u,4278190335u),vec2<u32>(16776960u,4278190335u),vec2<u32>(16711680u,4278190335u),vec2<u32>(16711681u,4278190335u),vec2<u32>(16712191u,4278190335u),vec2<u32>(16711937u,4278190335u),vec2<u32>(65280u,4278190335u),vec2<u32>(255u,4278190335u),vec2<u32>(0u,4278190335u),vec2<u32>(1u,4278190335u),vec2<u32>(256u,4278190335u),vec2<u32>(130817u,4278190335u),vec2<u32>(65536u,4278190335u),vec2<u32>(66047u,4278190335u),vec2<u32>(33489151u,4278190335u),vec2<u32>(33489152u,4278190335u),vec2<u32>(16842751u,4278190335u),vec2<u32>(16777471u,4278190335u),vec2<u32>(16777216u,4278190335u),vec2<u32>(16777727u,4278190335u),vec2<u32>(16777472u,4278190335u),vec2<u32>(16777473u,4278190335u),vec2<u32>(16908032u,4278190335u),vec2<u32>(16843007u,4278190335u),vec2<u32>(16842752u,4278190335u),vec2<u32>(16843008u,4278190335u),vec2<u32>(4294967041u,4278190080u),vec2<u32>(4294901760u,4278190080u),vec2<u32>(4294902017u,4278190080u),vec2<u32>(4278255360u,4278190080u),vec2<u32>(4278190335u,4278190080u),vec2<u32>(4278190080u,4278190080u),vec2<u32>(4278190081u,4278190080u),vec2<u32>(4278190336u,4278190080u),vec2<u32>(4278321151u,4278190080u),vec2<u32>(4278320897u,4278190080u),vec2<u32>(4278255616u,4278190080u),vec2<u32>(4278256127u,4278190080u),vec2<u32>(4278255873u,4278190080u),vec2<u32>(16776960u,4278190080u),vec2<u32>(16711935u,4278190080u),vec2<u32>(16711680u,4278190080u),vec2<u32>(16711681u,4278190080u),vec2<u32>(65280u,4278190080u),vec2<u32>(65281u,4278190080u),vec2<u32>(255u,4278190080u),vec2<u32>(0u,4278190080u),vec2<u32>(1u,4278190080u),vec2<u32>(256u,4278190080u),vec2<u32>(257u,4278190080u),vec2<u32>(130816u,4278190080u),vec2<u32>(65791u,4278190080u),vec2<u32>(65536u,4278190080u),vec2<u32>(65537u,4278190080u),vec2<u32>(65792u,4278190080u),vec2<u32>(33554431u,4278190080u),vec2<u32>(33554177u,4278190080u),vec2<u32>(33489151u,4278190080u),vec2<u32>(33488896u,4278190080u),vec2<u32>(33489407u,4278190080u),vec2<u32>(33489153u,4278190080u),vec2<u32>(16842751u,4278190080u),vec2<u32>(16842496u,4278190080u),vec2<u32>(16777471u,4278190080u),vec2<u32>(16777216u,4278190080u),vec2<u32>(16777217u,4278190080u),vec2<u32>(16777472u,4278190080u),vec2<u32>(16777473u,4278190080u),vec2<u32>(16908287u,4278190080u),vec2<u32>(16908033u,4278190080u),vec2<u32>(16842752u,4278190080u),vec2<u32>(4294967040u,4278190081u),vec2<u32>(4294902015u,4278190081u),vec2<u32>(4294901760u,4278190081u),vec2<u32>(4294901761u,4278190081u),vec2<u32>(4278190080u,4278190081u),vec2<u32>(4278190081u,4278190081u),vec2<u32>(4278190591u,4278190081u),vec2<u32>(4278190337u,4278190081u),vec2<u32>(4278320896u,4278190081u),vec2<u32>(4278255617u,4278190081u),vec2<u32>(16777215u,4278190081u),vec2<u32>(16776961u,4278190081u),vec2<u32>(16711935u,4278190081u),vec2<u32>(16711680u,4278190081u),vec2<u32>(16712191u,4278190081u),vec2<u32>(16711937u,4278190081u),vec2<u32>(65280u,4278190081u),vec2<u32>(0u,4278190081u),vec2<u32>(1u,4278190081u),vec2<u32>(511u,4278190081u),vec2<u32>(256u,4278190081u),vec2<u32>(130816u,4278190081u),vec2<u32>(65791u,4278190081u),vec2<u32>(65536u,4278190081u),vec2<u32>(66047u,4278190081u),vec2<u32>(65792u,4278190081u),vec2<u32>(65793u,4278190081u),vec2<u32>(33488897u,4278190081u),vec2<u32>(33489153u,4278190081u),vec2<u32>(16842497u,4278190081u),vec2<u32>(16777216u,4278190081u),vec2<u32>(16843007u,4278190081u),vec2<u32>(16843008u,4278190081u),vec2<u32>(4278255360u,4278190591u),vec2<u32>(4278190081u,4278190591u),vec2<u32>(4278255616u,4278190591u),vec2<u32>(16776960u,4278190591u),vec2<u32>(16711935u,4278190591u),vec2<u32>(16711681u,4278190591u),vec2<u32>(16711936u,4278190591u),vec2<u32>(65535u,4278190591u),vec2<u32>(0u,4278190591u),vec2<u32>(511u,4278190591u),vec2<u32>(257u,4278190591u),vec2<u32>(131071u,4278190591u),vec2<u32>(130816u,4278190591u),vec2<u32>(65791u,4278190591u),vec2<u32>(65537u,4278190591u),vec2<u32>(65792u,4278190591u),vec2<u32>(33488896u,4278190591u),vec2<u32>(16842496u,4278190591u),vec2<u32>(16777471u,4278190591u),vec2<u32>(16842752u,4278190591u),vec2<u32>(4278255615u,4278190336u),vec2<u32>(4278255361u,4278190336u),vec2<u32>(4278190080u,4278190336u),vec2<u32>(4278190337u,4278190336u),vec2<u32>(4278320896u,4278190336u),vec2<u32>(4278255616u,4278190336u),vec2<u32>(16776961u,4278190336u),vec2<u32>(16711935u,4278190336u),vec2<u32>(16711680u,4278190336u),vec2<u32>(16712191u,4278190336u),vec2<u32>(65280u,4278190336u),vec2<u32>(255u,4278190336u),vec2<u32>(0u,4278190336u),vec2<u32>(1u,4278190336u),vec2<u32>(256u,4278190336u),vec2<u32>(257u,4278190336u),vec2<u32>(131071u,4278190336u),vec2<u32>(65536u,4278190336u),vec2<u32>(65793u,4278190336u),vec2<u32>(33489152u,4278190336u),vec2<u32>(16842496u,4278190336u),vec2<u32>(16842497u,4278190336u),vec2<u32>(16777216u,4278190336u),vec2<u32>(16777727u,4278190336u),vec2<u32>(16908032u,4278190336u),vec2<u32>(16842753u,4278190336u),vec2<u32>(16843008u,4278190336u),vec2<u32>(4294902016u,4278190337u),vec2<u32>(4278190081u,4278190337u),vec2<u32>(4278255871u,4278190337u),vec2<u32>(4278255617u,4278190337u),vec2<u32>(16711935u,4278190337u),vec2<u32>(16711681u,4278190337u),vec2<u32>(16711936u,4278190337u),vec2<u32>(65535u,4278190337u),vec2<u32>(65281u,4278190337u),vec2<u32>(0u,4278190337u),vec2<u32>(511u,4278190337u),vec2<u32>(130816u,4278190337u),vec2<u32>(65537u,4278190337u),vec2<u32>(65792u,4278190337u),vec2<u32>(33488896u,4278190337u),vec2<u32>(16842496u,4278190337u),vec2<u32>(16777217u,4278190337u),vec2<u32>(16777473u,4278190337u),vec2<u32>(4294967295u,4278321151u),vec2<u32>(4294967041u,4278321151u),vec2<u32>(4294902271u,4278321151u),vec2<u32>(4294902017u,4278321151u),vec2<u32>(4278190080u,4278321151u),vec2<u32>(4278321151u,4278321151u),vec2<u32>(4278320897u,4278321151u),vec2<u32>(4278255616u,4278321151u),vec2<u32>(4278256127u,4278321151u),vec2<u32>(4278255873u,4278321151u),vec2<u32>(16711680u,4278321151u),vec2<u32>(65280u,4278321151u),vec2<u32>(256u,4278321151u),vec2<u32>(130816u,4278321151u),vec2<u32>(65536u,4278321151u),vec2<u32>(33554431u,4278321151u),vec2<u32>(33554177u,4278321151u),vec2<u32>(33489407u,4278321151u),vec2<u32>(33489153u,4278321151u),vec2<u32>(16777216u,4278321151u),vec2<u32>(16908287u,4278321151u),vec2<u32>(16908033u,4278321151u),vec2<u32>(16842752u,4278321151u),vec2<u32>(16843263u,4278321151u),vec2<u32>(16843009u,4278321151u),vec2<u32>(4294901760u,4278320896u),vec2<u32>(4278255360u,4278320896u),vec2<u32>(4278190335u,4278320896u),vec2<u32>(4278190336u,4278320896u),vec2<u32>(4278255616u,4278320896u),vec2<u32>(16776961u,4278320896u),vec2<u32>(16711935u,4278320896u),vec2<u32>(16711936u,4278320896u),vec2<u32>(0u,4278320896u),vec2<u32>(511u,4278320896u),vec2<u32>(257u,4278320896u),vec2<u32>(130816u,4278320896u),vec2<u32>(65791u,4278320896u),vec2<u32>(65536u,4278320896u),vec2<u32>(65537u,4278320896u),vec2<u32>(33488896u,4278320896u),vec2<u32>(16842751u,4278320896u),vec2<u32>(16777217u,4278320896u),vec2<u32>(16777472u,4278320896u),vec2<u32>(16842752u,4278320896u),vec2<u32>(4294967040u,4278320897u),vec2<u32>(4294902271u,4278320897u),vec2<u32>(4294902017u,4278320897u),vec2<u32>(4278255360u,4278320897u),vec2<u32>(4278190080u,4278320897u),vec2<u32>(4278321151u,4278320897u),vec2<u32>(4278320897u,4278320897u),vec2<u32>(4278256127u,4278320897u),vec2<u32>(4278255873u,4278320897u),vec2<u32>(16711680u,4278320897u),vec2<u32>(65280u,4278320897u),vec2<u32>(1u,4278320897u),vec2<u32>(256u,4278320897u),vec2<u32>(65536u,4278320897u),vec2<u32>(33554176u,4278320897u),vec2<u32>(33489407u,4278320897u),vec2<u32>(33489153u,4278320897u),vec2<u32>(16842496u,4278320897u),vec2<u32>(16777216u,4278320897u),vec2<u32>(16908287u,4278320897u),vec2<u32>(16908033u,4278320897u),vec2<u32>(16843263u,4278320897u),vec2<u32>(16843009u,4278320897u),vec2<u32>(4294901760u,4278255871u),vec2<u32>(4278190335u,4278255871u),vec2<u32>(4278190081u,4278255871u),vec2<u32>(4278190336u,4278255871u),vec2<u32>(4278255616u,4278255871u),vec2<u32>(16711935u,4278255871u),vec2<u32>(16711680u,4278255871u),vec2<u32>(16711681u,4278255871u),vec2<u32>(16711936u,4278255871u),vec2<u32>(65281u,4278255871u),vec2<u32>(0u,4278255871u),vec2<u32>(511u,4278255871u),vec2<u32>(257u,4278255871u),vec2<u32>(65537u,4278255871u),vec2<u32>(33488896u,4278255871u),vec2<u32>(16842496u,4278255871u),vec2<u32>(16777471u,4278255871u),vec2<u32>(16777472u,4278255871u),vec2<u32>(16908032u,4278255871u),vec2<u32>(16842752u,4278255871u),vec2<u32>(4294902016u,4278255616u),vec2<u32>(4278190080u,4278255616u),vec2<u32>(4278320896u,4278255616u),vec2<u32>(4278255872u,4278255616u),vec2<u32>(16777215u,4278255616u),vec2<u32>(16711680u,4278255616u),vec2<u32>(16712191u,4278255616u),vec2<u32>(65280u,4278255616u),vec2<u32>(255u,4278255616u),vec2<u32>(0u,4278255616u),vec2<u32>(256u,4278255616u),vec2<u32>(130817u,4278255616u),vec2<u32>(65536u,4278255616u),vec2<u32>(66047u,4278255616u),vec2<u32>(33489152u,4278255616u),vec2<u32>(16842751u,4278255616u),vec2<u32>(16777471u,4278255616u),vec2<u32>(16777216u,4278255616u),vec2<u32>(16777727u,4278255616u),vec2<u32>(16777473u,4278255616u),vec2<u32>(16908032u,4278255616u),vec2<u32>(16843007u,4278255616u),vec2<u32>(16842753u,4278255616u),vec2<u32>(16843008u,4278255616u),vec2<u32>(4294901760u,4278255617u),vec2<u32>(4278255615u,4278255617u),vec2<u32>(4278255361u,4278255617u),vec2<u32>(4278190336u,4278255617u),vec2<u32>(4278255616u,4278255617u),vec2<u32>(16776960u,4278255617u),vec2<u32>(16711936u,4278255617u),vec2<u32>(0u,4278255617u),vec2<u32>(131071u,4278255617u),vec2<u32>(130816u,4278255617u),vec2<u32>(65792u,4278255617u),vec2<u32>(33489151u,4278255617u),vec2<u32>(33488897u,4278255617u),vec2<u32>(16842751u,4278255617u),vec2<u32>(16777473u,4278255617u),vec2<u32>(4294967295u,4278256127u),vec2<u32>(4294967041u,4278256127u),vec2<u32>(4294902271u,4278256127u),vec2<u32>(4294902017u,4278256127u),vec2<u32>(4278190080u,4278256127u),vec2<u32>(4278321151u,4278256127u),vec2<u32>(4278320897u,4278256127u),vec2<u32>(4278256127u,4278256127u),vec2<u32>(4278255873u,4278256127u),vec2<u32>(16711680u,4278256127u),vec2<u32>(65280u,4278256127u),vec2<u32>(255u,4278256127u),vec2<u32>(65536u,4278256127u),vec2<u32>(33554431u,4278256127u),vec2<u32>(33554177u,4278256127u),vec2<u32>(33489407u,4278256127u),vec2<u32>(33489153u,4278256127u),vec2<u32>(16908287u,4278256127u),vec2<u32>(16908033u,4278256127u),vec2<u32>(16843263u,4278256127u),vec2<u32>(16843009u,4278256127u),vec2<u32>(4294902016u,4278255872u),vec2<u32>(4278255360u,4278255872u),vec2<u32>(4278190335u,4278255872u),vec2<u32>(4278190336u,4278255872u),vec2<u32>(4278255616u,4278255872u),vec2<u32>(16711681u,4278255872u),vec2<u32>(16711936u,4278255872u),vec2<u32>(65281u,4278255872u),vec2<u32>(0u,4278255872u),vec2<u32>(130816u,4278255872u),vec2<u32>(65791u,4278255872u),vec2<u32>(65537u,4278255872u),vec2<u32>(65792u,4278255872u),vec2<u32>(33488896u,4278255872u),vec2<u32>(16842751u,4278255872u),vec2<u32>(16777217u,4278255872u),vec2<u32>(16777472u,4278255872u),vec2<u32>(16843007u,4278255872u),vec2<u32>(16842752u,4278255872u),vec2<u32>(4294967295u,4278255873u),vec2<u32>(4294967041u,4278255873u),vec2<u32>(4294902271u,4278255873u),vec2<u32>(4294902017u,4278255873u),vec2<u32>(4278321151u,4278255873u),vec2<u32>(4278320897u,4278255873u),vec2<u32>(4278256127u,4278255873u),vec2<u32>(4278255873u,4278255873u),vec2<u32>(16711680u,4278255873u),vec2<u32>(65280u,4278255873u),vec2<u32>(1u,4278255873u),vec2<u32>(256u,4278255873u),vec2<u32>(65536u,4278255873u),vec2<u32>(33554431u,4278255873u),vec2<u32>(33554177u,4278255873u),vec2<u32>(33489407u,4278255873u),vec2<u32>(33489153u,4278255873u),vec2<u32>(16777216u,4278255873u),vec2<u32>(16908287u,4278255873u),vec2<u32>(16908033u,4278255873u),vec2<u32>(16843263u,4278255873u),vec2<u32>(16843009u,4278255873u),vec2<u32>(4294901760u,16777215u),vec2<u32>(4278255360u,16777215u),vec2<u32>(4278190081u,16777215u),vec2<u32>(4278255616u,16777215u),vec2<u32>(16711936u,16777215u),vec2<u32>(65281u,16777215u),vec2<u32>(0u,16777215u),vec2<u32>(511u,16777215u),vec2<u32>(257u,16777215u),vec2<u32>(130816u,16777215u),vec2<u32>(65791u,16777215u),vec2<u32>(65537u,16777215u),vec2<u32>(16777471u,16777215u),vec2<u32>(16777472u,16777215u),vec2<u32>(16908032u,16777215u),vec2<u32>(16842753u,16777215u),vec2<u32>(4294967295u,16776960u),vec2<u32>(4294967040u,16776960u),vec2<u32>(4294902015u,16776960u),vec2<u32>(4294901761u,16776960u),vec2<u32>(4294902016u,16776960u),vec2<u32>(4278255361u,16776960u),vec2<u32>(4278190080u,16776960u),vec2<u32>(4278190081u,16776960u),vec2<u32>(4278190591u,16776960u),vec2<u32>(4278190337u,16776960u),vec2<u32>(4278320896u,16776960u),vec2<u32>(4278255617u,16776960u),vec2<u32>(4278255872u,16776960u),vec2<u32>(16711680u,16776960u),vec2<u32>(16712191u,16776960u),vec2<u32>(16711937u,16776960u),vec2<u32>(65280u,16776960u),vec2<u32>(255u,16776960u),vec2<u32>(0u,16776960u),vec2<u32>(1u,16776960u),vec2<u32>(256u,16776960u),vec2<u32>(257u,16776960u),vec2<u32>(65536u,16776960u),vec2<u32>(66047u,16776960u),vec2<u32>(65793u,16776960u),vec2<u32>(33554176u,16776960u),vec2<u32>(33489151u,16776960u),vec2<u32>(33488897u,16776960u),vec2<u32>(16842751u,16776960u),vec2<u32>(16842497u,16776960u),vec2<u32>(16777216u,16776960u),vec2<u32>(16908287u,16776960u),vec2<u32>(16908032u,16776960u),vec2<u32>(16908033u,16776960u),vec2<u32>(4294901760u,16776961u),vec2<u32>(4278255360u,16776961u),vec2<u32>(4278190335u,16776961u),vec2<u32>(4278190081u,16776961u),vec2<u32>(4278255616u,16776961u),vec2<u32>(16776960u,16776961u),vec2<u32>(65281u,16776961u),vec2<u32>(0u,16776961u),vec2<u32>(257u,16776961u),vec2<u32>(65791u,16776961u),vec2<u32>(65792u,16776961u),vec2<u32>(33489152u,16776961u),vec2<u32>(16777471u,16776961u),vec2<u32>(16842752u,16776961u),vec2<u32>(4294967040u,16711935u),vec2<u32>(4278190080u,16711935u),vec2<u32>(4278190336u,16711935u),vec2<u32>(4278255872u,16711935u),vec2<u32>(16711680u,16711935u),vec2<u32>(16712191u,16711935u),vec2<u32>(16711937u,16711935u),vec2<u32>(65280u,16711935u),vec2<u32>(255u,16711935u),vec2<u32>(0u,16711935u),vec2<u32>(1u,16711935u),vec2<u32>(130816u,16711935u),vec2<u32>(130817u,16711935u),vec2<u32>(65536u,16711935u),vec2<u32>(66047u,16711935u),vec2<u32>(65793u,16711935u),vec2<u32>(33554176u,16711935u),vec2<u32>(33488897u,16711935u),vec2<u32>(33489152u,16711935u),vec2<u32>(16842751u,16711935u),vec2<u32>(16842497u,16711935u),vec2<u32>(16777216u,16711935u),vec2<u32>(16908287u,16711935u),vec2<u32>(16908032u,16711935u),vec2<u32>(16843008u,16711935u),vec2<u32>(4294967040u,16711680u),vec2<u32>(4294967041u,16711680u),vec2<u32>(4294901760u,16711680u),vec2<u32>(4294902017u,16711680u),vec2<u32>(4278255360u,16711680u),vec2<u32>(4278190335u,16711680u),vec2<u32>(4278190080u,16711680u),vec2<u32>(4278190081u,16711680u),vec2<u32>(4278190336u,16711680u),vec2<u32>(4278321151u,16711680u),vec2<u32>(4278255616u,16711680u),vec2<u32>(4278255873u,16711680u),vec2<u32>(16776960u,16711680u),vec2<u32>(16711935u,16711680u),vec2<u32>(16711680u,16711680u),vec2<u32>(16711681u,16711680u),vec2<u32>(16711936u,16711680u),vec2<u32>(65535u,16711680u),vec2<u32>(65280u,16711680u),vec2<u32>(255u,16711680u),vec2<u32>(0u,16711680u),vec2<u32>(1u,16711680u),vec2<u32>(511u,16711680u),vec2<u32>(256u,16711680u),vec2<u32>(130816u,16711680u),vec2<u32>(65791u,16711680u),vec2<u32>(65536u,16711680u),vec2<u32>(65537u,16711680u),vec2<u32>(65792u,16711680u),vec2<u32>(33554177u,16711680u),vec2<u32>(33489151u,16711680u),vec2<u32>(33488896u,16711680u),vec2<u32>(33489407u,16711680u),vec2<u32>(16842496u,16711680u),vec2<u32>(16777471u,16711680u),vec2<u32>(16777216u,16711680u),vec2<u32>(16777217u,16711680u),vec2<u32>(16777472u,16711680u),vec2<u32>(16777473u,16711680u),vec2<u32>(16842752u,16711680u),vec2<u32>(16843263u,16711680u),vec2<u32>(16843009u,16711680u),vec2<u32>(4294967040u,16711681u),vec2<u32>(4294901760u,16711681u),vec2<u32>(4294902016u,16711681u),vec2<u32>(4278190335u,16711681u),vec2<u32>(4278190080u,16711681u),vec2<u32>(4278190591u,16711681u),vec2<u32>(4278190337u,16711681u),vec2<u32>(4278320896u,16711681u),vec2<u32>(4278255871u,16711681u),vec2<u32>(4278255872u,16711681u),vec2<u32>(16777215u,16711681u),vec2<u32>(16776961u,16711681u),vec2<u32>(16711680u,16711681u),vec2<u32>(16712191u,16711681u),vec2<u32>(65535u,16711681u),vec2<u32>(65280u,16711681u),vec2<u32>(65281u,16711681u),vec2<u32>(0u,16711681u),vec2<u32>(1u,16711681u),vec2<u32>(256u,16711681u),vec2<u32>(130817u,16711681u),vec2<u32>(65536u,16711681u),vec2<u32>(66047u,16711681u),vec2<u32>(33554176u,16711681u),vec2<u32>(33488896u,16711681u),vec2<u32>(33489153u,16711681u),vec2<u32>(16777471u,16711681u),vec2<u32>(16777216u,16711681u),vec2<u32>(16908032u,16711681u),vec2<u32>(16843007u,16711681u),vec2<u32>(16842753u,16711681u),vec2<u32>(4294901760u,16712191u),vec2<u32>(4278255360u,16712191u),vec2<u32>(4278190080u,16712191u),vec2<u32>(4278190337u,16712191u),vec2<u32>(4278255616u,16712191u),vec2<u32>(16776961u,16712191u),vec2<u32>(16711936u,16712191u),vec2<u32>(65535u,16712191u),vec2<u32>(0u,16712191u),vec2<u32>(511u,16712191u),vec2<u32>(130816u,16712191u),vec2<u32>(65791u,16712191u),vec2<u32>(65537u,16712191u),vec2<u32>(65792u,16712191u),vec2<u32>(33488896u,16712191u),vec2<u32>(16842496u,16712191u),vec2<u32>(16777471u,16712191u),vec2<u32>(16777217u,16712191u),vec2<u32>(16777472u,16712191u),vec2<u32>(16842752u,16712191u),vec2<u32>(4294967040u,16711936u),vec2<u32>(4294901760u,16711936u),vec2<u32>(4294901761u,16711936u),vec2<u32>(4294902017u,16711936u),vec2<u32>(4278255615u,16711936u),vec2<u32>(4278190335u,16711936u),vec2<u32>(4278190080u,16711936u),vec2<u32>(4278190591u,16711936u),vec2<u32>(4278320896u,16711936u),vec2<u32>(4278255871u,16711936u),vec2<u32>(4278255617u,16711936u),vec2<u32>(16777215u,16711936u),vec2<u32>(16711680u,16711936u),vec2<u32>(16711937u,16711936u),vec2<u32>(65280u,16711936u),vec2<u32>(65281u,16711936u),vec2<u32>(255u,16711936u),vec2<u32>(0u,16711936u),vec2<u32>(1u,16711936u),vec2<u32>(256u,16711936u),vec2<u32>(131071u,16711936u),vec2<u32>(130817u,16711936u),vec2<u32>(65536u,16711936u),vec2<u32>(65537u,16711936u),vec2<u32>(65793u,16711936u),vec2<u32>(33488897u,16711936u),vec2<u32>(33489152u,16711936u),vec2<u32>(16842497u,16711936u),vec2<u32>(16777216u,16711936u),vec2<u32>(16777217u,16711936u),vec2<u32>(16777727u,16711936u),vec2<u32>(16908032u,16711936u),vec2<u32>(16843007u,16711936u),vec2<u32>(16842753u,16711936u),vec2<u32>(16843008u,16711936u),vec2<u32>(4278190081u,16711937u),vec2<u32>(16711935u,16711937u),vec2<u32>(16711681u,16711937u),vec2<u32>(16711936u,16711937u),vec2<u32>(0u,16711937u),vec2<u32>(511u,16711937u),vec2<u32>(257u,16711937u),vec2<u32>(65791u,16711937u),vec2<u32>(65792u,16711937u),vec2<u32>(16777471u,16711937u),vec2<u32>(16842752u,16711937u),vec2<u32>(4294967040u,65535u),vec2<u32>(4294902015u,65535u),vec2<u32>(4294901760u,65535u),vec2<u32>(4294901761u,65535u),vec2<u32>(4294902016u,65535u),vec2<u32>(4278255361u,65535u),vec2<u32>(4278190080u,65535u),vec2<u32>(4278190337u,65535u),vec2<u32>(4278320896u,65535u),vec2<u32>(4278255871u,65535u),vec2<u32>(4278255872u,65535u),vec2<u32>(16777215u,65535u),vec2<u32>(16711680u,65535u),vec2<u32>(16712191u,65535u),vec2<u32>(65280u,65535u),vec2<u32>(255u,65535u),vec2<u32>(0u,65535u),vec2<u32>(1u,65535u),vec2<u32>(256u,65535u),vec2<u32>(65536u,65535u),vec2<u32>(66047u,65535u),vec2<u32>(33488897u,65535u),vec2<u32>(33489152u,65535u),vec2<u32>(16777216u,65535u),vec2<u32>(16777727u,65535u),vec2<u32>(16908287u,65535u),vec2<u32>(16908032u,65535u),vec2<u32>(16842753u,65535u),vec2<u32>(16843008u,65535u),vec2<u32>(4294901760u,65280u),vec2<u32>(4294902271u,65280u),vec2<u32>(4294902016u,65280u),vec2<u32>(4294902017u,65280u),vec2<u32>(4278255360u,65280u),vec2<u32>(4278190335u,65280u),vec2<u32>(4278190080u,65280u),vec2<u32>(4278190081u,65280u),vec2<u32>(4278190591u,65280u),vec2<u32>(4278190336u,65280u),vec2<u32>(4278321151u,65280u),vec2<u32>(4278255616u,65280u),vec2<u32>(4278255617u,65280u),vec2<u32>(4278256127u,65280u),vec2<u32>(4278255873u,65280u),vec2<u32>(16776960u,65280u),vec2<u32>(16711935u,65280u),vec2<u32>(16711680u,65280u),vec2<u32>(16711681u,65280u),vec2<u32>(16711936u,65280u),vec2<u32>(65535u,65280u),vec2<u32>(65280u,65280u),vec2<u32>(65281u,65280u),vec2<u32>(255u,65280u),vec2<u32>(0u,65280u),vec2<u32>(1u,65280u),vec2<u32>(511u,65280u),vec2<u32>(256u,65280u),vec2<u32>(257u,65280u),vec2<u32>(130816u,65280u),vec2<u32>(65791u,65280u),vec2<u32>(65536u,65280u),vec2<u32>(65537u,65280u),vec2<u32>(65792u,65280u),vec2<u32>(33554177u,65280u),vec2<u32>(33488896u,65280u),vec2<u32>(16842496u,65280u),vec2<u32>(16777471u,65280u),vec2<u32>(16777216u,65280u),vec2<u32>(16777217u,65280u),vec2<u32>(16777472u,65280u),vec2<u32>(16908287u,65280u),vec2<u32>(16842752u,65280u),vec2<u32>(16843009u,65280u),vec2<u32>(4294967040u,65281u),vec2<u32>(4294901761u,65281u),vec2<u32>(4278255361u,65281u),vec2<u32>(4278190080u,65281u),vec2<u32>(4278190337u,65281u),vec2<u32>(4278320896u,65281u),vec2<u32>(4278255871u,65281u),vec2<u32>(16776961u,65281u),vec2<u32>(16711680u,65281u),vec2<u32>(16711937u,65281u),vec2<u32>(65280u,65281u),vec2<u32>(255u,65281u),vec2<u32>(0u,65281u),vec2<u32>(1u,65281u),vec2<u32>(256u,65281u),vec2<u32>(130817u,65281u),vec2<u32>(65536u,65281u),vec2<u32>(33488896u,65281u),vec2<u32>(16842751u,65281u),vec2<u32>(16842497u,65281u),vec2<u32>(16777216u,65281u),vec2<u32>(16777472u,65281u),vec2<u32>(16777473u,65281u),vec2<u32>(16843007u,65281u),vec2<u32>(4294902015u,255u),vec2<u32>(4294901760u,255u),vec2<u32>(4278255360u,255u),vec2<u32>(4278190335u,255u),vec2<u32>(4278190080u,255u),vec2<u32>(4278190081u,255u),vec2<u32>(4278190591u,255u),vec2<u32>(4278190336u,255u),vec2<u32>(4278320896u,255u),vec2<u32>(4278255616u,255u),vec2<u32>(4278256127u,255u),vec2<u32>(4278255873u,255u),vec2<u32>(16776960u,255u),vec2<u32>(16711935u,255u),vec2<u32>(16711680u,255u),vec2<u32>(16711681u,255u),vec2<u32>(16711936u,255u),vec2<u32>(16711937u,255u),vec2<u32>(65535u,255u),vec2<u32>(65280u,255u),vec2<u32>(255u,255u),vec2<u32>(0u,255u),vec2<u32>(1u,255u),vec2<u32>(511u,255u),vec2<u32>(256u,255u),vec2<u32>(257u,255u),vec2<u32>(130816u,255u),vec2<u32>(130817u,255u),vec2<u32>(65791u,255u),vec2<u32>(65536u,255u),vec2<u32>(65537u,255u),vec2<u32>(65792u,255u),vec2<u32>(33554431u,255u),vec2<u32>(33489407u,255u),vec2<u32>(33489153u,255u),vec2<u32>(16842496u,255u),vec2<u32>(16777471u,255u),vec2<u32>(16777216u,255u),vec2<u32>(16777217u,255u),vec2<u32>(16777472u,255u),vec2<u32>(16908032u,255u),vec2<u32>(16843007u,255u),vec2<u32>(16842752u,255u),vec2<u32>(16843009u,255u),vec2<u32>(4294967040u,0u),vec2<u32>(4294967041u,0u),vec2<u32>(4294902015u,0u),vec2<u32>(4294901760u,0u),vec2<u32>(4294901761u,0u),vec2<u32>(4294902016u,0u),vec2<u32>(4278255615u,0u),vec2<u32>(4278255360u,0u),vec2<u32>(4278255361u,0u),vec2<u32>(4278190335u,0u),vec2<u32>(4278190080u,0u),vec2<u32>(4278190081u,0u),vec2<u32>(4278190336u,0u),vec2<u32>(4278190337u,0u),vec2<u32>(4278320896u,0u),vec2<u32>(4278255871u,0u),vec2<u32>(4278255616u,0u),vec2<u32>(4278255617u,0u),vec2<u32>(4278255872u,0u),vec2<u32>(16777215u,0u),vec2<u32>(16776960u,0u),vec2<u32>(16776961u,0u),vec2<u32>(16711935u,0u),vec2<u32>(16711680u,0u),vec2<u32>(16711681u,0u),vec2<u32>(16712191u,0u),vec2<u32>(16711936u,0u),vec2<u32>(65535u,0u),vec2<u32>(65280u,0u),vec2<u32>(65281u,0u),vec2<u32>(255u,0u),vec2<u32>(0u,0u),vec2<u32>(1u,0u),vec2<u32>(511u,0u),vec2<u32>(256u,0u),vec2<u32>(257u,0u),vec2<u32>(131071u,0u),vec2<u32>(130816u,0u),vec2<u32>(65791u,0u),vec2<u32>(65536u,0u),vec2<u32>(65537u,0u),vec2<u32>(66047u,0u),vec2<u32>(65792u,0u),vec2<u32>(65793u,0u),vec2<u32>(33554176u,0u),vec2<u32>(33489151u,0u),vec2<u32>(33488896u,0u),vec2<u32>(33489152u,0u),vec2<u32>(33489153u,0u),vec2<u32>(16842751u,0u),vec2<u32>(16842496u,0u),vec2<u32>(16777471u,0u),vec2<u32>(16777216u,0u),vec2<u32>(16777217u,0u),vec2<u32>(16777727u,0u),vec2<u32>(16777472u,0u),vec2<u32>(16908032u,0u),vec2<u32>(16843007u,0u),vec2<u32>(16842752u,0u),vec2<u32>(16842753u,0u),vec2<u32>(16843008u,0u),vec2<u32>(4294967295u,1u),vec2<u32>(4294967040u,1u),vec2<u32>(4294967041u,1u),vec2<u32>(4294902015u,1u),vec2<u32>(4294901761u,1u),vec2<u32>(4294902271u,1u),vec2<u32>(4294902016u,1u),vec2<u32>(4278255360u,1u),vec2<u32>(4278190335u,1u),vec2<u32>(4278190080u,1u),vec2<u32>(4278190591u,1u),vec2<u32>(4278190336u,1u),vec2<u32>(4278321151u,1u),vec2<u32>(4278320896u,1u),vec2<u32>(4278320897u,1u),vec2<u32>(4278255871u,1u),vec2<u32>(4278255616u,1u),vec2<u32>(4278255617u,1u),vec2<u32>(4278256127u,1u),vec2<u32>(4278255872u,1u),vec2<u32>(16776960u,1u),vec2<u32>(16711680u,1u),vec2<u32>(16711681u,1u),vec2<u32>(16712191u,1u),vec2<u32>(16711936u,1u),vec2<u32>(16711937u,1u),vec2<u32>(65535u,1u),vec2<u32>(65280u,1u),vec2<u32>(65281u,1u),vec2<u32>(255u,1u),vec2<u32>(0u,1u),vec2<u32>(1u,1u),vec2<u32>(511u,1u),vec2<u32>(256u,1u),vec2<u32>(257u,1u),vec2<u32>(130816u,1u),vec2<u32>(65791u,1u),vec2<u32>(65536u,1u),vec2<u32>(65792u,1u),vec2<u32>(33554177u,1u),vec2<u32>(33488896u,1u),vec2<u32>(33488897u,1u),vec2<u32>(33489407u,1u),vec2<u32>(33489152u,1u),vec2<u32>(33489153u,1u),vec2<u32>(16842496u,1u),vec2<u32>(16777216u,1u),vec2<u32>(16777473u,1u),vec2<u32>(16908033u,1u),vec2<u32>(16842752u,1u),vec2<u32>(16842753u,1u),vec2<u32>(16843263u,1u),vec2<u32>(16843008u,1u),vec2<u32>(4294902015u,511u),vec2<u32>(4294901760u,511u),vec2<u32>(4294901761u,511u),vec2<u32>(4294902016u,511u),vec2<u32>(4278255615u,511u),vec2<u32>(4278190080u,511u),vec2<u32>(4278190591u,511u),vec2<u32>(4278320896u,511u),vec2<u32>(4278255873u,511u),vec2<u32>(16711680u,511u),vec2<u32>(16712191u,511u),vec2<u32>(16711937u,511u),vec2<u32>(65280u,511u),vec2<u32>(255u,511u),vec2<u32>(0u,511u),vec2<u32>(1u,511u),vec2<u32>(511u,511u),vec2<u32>(256u,511u),vec2<u32>(131071u,511u),vec2<u32>(130817u,511u),vec2<u32>(65791u,511u),vec2<u32>(65536u,511u),vec2<u32>(33554177u,511u),vec2<u32>(33489152u,511u),vec2<u32>(16842751u,511u),vec2<u32>(16842497u,511u),vec2<u32>(16777216u,511u),vec2<u32>(16777727u,511u),vec2<u32>(16908032u,511u),vec2<u32>(16843008u,511u),vec2<u32>(4294967040u,256u),vec2<u32>(4294967041u,256u),vec2<u32>(4294901760u,256u),vec2<u32>(4294902017u,256u),vec2<u32>(4278255360u,256u),vec2<u32>(4278190335u,256u),vec2<u32>(4278190080u,256u),vec2<u32>(4278190081u,256u),vec2<u32>(4278190336u,256u),vec2<u32>(4278255616u,256u),vec2<u32>(16776960u,256u),vec2<u32>(16711935u,256u),vec2<u32>(16711680u,256u),vec2<u32>(16711681u,256u),vec2<u32>(16711936u,256u),vec2<u32>(65535u,256u),vec2<u32>(65280u,256u),vec2<u32>(65281u,256u),vec2<u32>(255u,256u),vec2<u32>(0u,256u),vec2<u32>(1u,256u),vec2<u32>(511u,256u),vec2<u32>(256u,256u),vec2<u32>(257u,256u),vec2<u32>(130816u,256u),vec2<u32>(65791u,256u),vec2<u32>(65536u,256u),vec2<u32>(65537u,256u),vec2<u32>(65792u,256u),vec2<u32>(33554176u,256u),vec2<u32>(33488896u,256u),vec2<u32>(33489152u,256u),vec2<u32>(16842496u,256u),vec2<u32>(16777471u,256u),vec2<u32>(16777216u,256u),vec2<u32>(16777217u,256u),vec2<u32>(16777727u,256u),vec2<u32>(16777472u,256u),vec2<u32>(16842752u,256u),vec2<u32>(4294902015u,257u),vec2<u32>(4294902271u,257u),vec2<u32>(4278190080u,257u),vec2<u32>(4278190337u,257u),vec2<u32>(4278321151u,257u),vec2<u32>(4278255616u,257u),vec2<u32>(4278255617u,257u),vec2<u32>(4278255872u,257u),vec2<u32>(16711680u,257u),vec2<u32>(16712191u,257u),vec2<u32>(16711936u,257u),vec2<u32>(65280u,257u),vec2<u32>(0u,257u),vec2<u32>(1u,257u),vec2<u32>(511u,257u),vec2<u32>(256u,257u),vec2<u32>(130817u,257u),vec2<u32>(65536u,257u),vec2<u32>(66047u,257u),vec2<u32>(65793u,257u),vec2<u32>(33554176u,257u),vec2<u32>(33489153u,257u),vec2<u32>(16842497u,257u),vec2<u32>(16777216u,257u),vec2<u32>(16777217u,257u),vec2<u32>(16777727u,257u),vec2<u32>(16777473u,257u),vec2<u32>(16908032u,257u),vec2<u32>(4294901760u,131071u),vec2<u32>(4278190335u,131071u),vec2<u32>(4278190081u,131071u),vec2<u32>(4278190336u,131071u),vec2<u32>(4278255616u,131071u),vec2<u32>(16711935u,131071u),vec2<u32>(65535u,131071u),vec2<u32>(0u,131071u),vec2<u32>(1u,131071u),vec2<u32>(511u,131071u),vec2<u32>(257u,131071u),vec2<u32>(130816u,131071u),vec2<u32>(65791u,131071u),vec2<u32>(65537u,131071u),vec2<u32>(65792u,131071u),vec2<u32>(33554176u,131071u),vec2<u32>(16777217u,131071u),vec2<u32>(16842752u,131071u),vec2<u32>(4294967040u,130816u),vec2<u32>(4294902015u,130816u),vec2<u32>(4294901761u,130816u),vec2<u32>(4294902016u,130816u),vec2<u32>(4278255361u,130816u),vec2<u32>(4278190080u,130816u),vec2<u32>(4278320896u,130816u),vec2<u32>(4278320897u,130816u),vec2<u32>(4278255617u,130816u),vec2<u32>(4278255872u,130816u),vec2<u32>(16711680u,130816u),vec2<u32>(16711936u,130816u),vec2<u32>(65280u,130816u),vec2<u32>(0u,130816u),vec2<u32>(1u,130816u),vec2<u32>(256u,130816u),vec2<u32>(65536u,130816u),vec2<u32>(65537u,130816u),vec2<u32>(65793u,130816u),vec2<u32>(33489151u,130816u),vec2<u32>(33489153u,130816u),vec2<u32>(16842497u,130816u),vec2<u32>(16777216u,130816u),vec2<u32>(16908032u,130816u),vec2<u32>(16842753u,130816u),vec2<u32>(16843008u,130816u),vec2<u32>(4278255360u,130817u),vec2<u32>(4278190081u,130817u),vec2<u32>(4278190336u,130817u),vec2<u32>(16777215u,130817u),vec2<u32>(16776960u,130817u),vec2<u32>(16711681u,130817u),vec2<u32>(0u,130817u),vec2<u32>(1u,130817u),vec2<u32>(511u,130817u),vec2<u32>(131071u,130817u),vec2<u32>(33488896u,130817u),vec2<u32>(16842496u,130817u),vec2<u32>(16777217u,130817u),vec2<u32>(16842752u,130817u),vec2<u32>(4278255360u,65791u),vec2<u32>(4278255361u,65791u),vec2<u32>(4278190080u,65791u),vec2<u32>(4278190081u,65791u),vec2<u32>(4278190337u,65791u),vec2<u32>(4278320896u,65791u),vec2<u32>(4278255617u,65791u),vec2<u32>(4278255872u,65791u),vec2<u32>(16777215u,65791u),vec2<u32>(16776961u,65791u),vec2<u32>(16711680u,65791u),vec2<u32>(16712191u,65791u),vec2<u32>(16711937u,65791u),vec2<u32>(65280u,65791u),vec2<u32>(255u,65791u),vec2<u32>(0u,65791u),vec2<u32>(1u,65791u),vec2<u32>(256u,65791u),vec2<u32>(257u,65791u),vec2<u32>(131071u,65791u),vec2<u32>(130817u,65791u),vec2<u32>(65536u,65791u),vec2<u32>(33489151u,65791u),vec2<u32>(33488896u,65791u),vec2<u32>(33489152u,65791u),vec2<u32>(16842751u,65791u),vec2<u32>(16842497u,65791u),vec2<u32>(16777471u,65791u),vec2<u32>(16777216u,65791u),vec2<u32>(16777217u,65791u),vec2<u32>(16777727u,65791u),vec2<u32>(16777473u,65791u),vec2<u32>(16908032u,65791u),vec2<u32>(16843007u,65791u),vec2<u32>(16843008u,65791u),vec2<u32>(4294901760u,65536u),vec2<u32>(4294902271u,65536u),vec2<u32>(4294902017u,65536u),vec2<u32>(4278255360u,65536u),vec2<u32>(4278190080u,65536u),vec2<u32>(4278190081u,65536u),vec2<u32>(4278190336u,65536u),vec2<u32>(16711935u,65536u),vec2<u32>(16711680u,65536u),vec2<u32>(16711681u,65536u),vec2<u32>(16711936u,65536u),vec2<u32>(65535u,65536u),vec2<u32>(65280u,65536u),vec2<u32>(255u,65536u),vec2<u32>(0u,65536u),vec2<u32>(1u,65536u),vec2<u32>(256u,65536u),vec2<u32>(130816u,65536u),vec2<u32>(65791u,65536u),vec2<u32>(65536u,65536u),vec2<u32>(65537u,65536u),vec2<u32>(65792u,65536u),vec2<u32>(33488897u,65536u),vec2<u32>(33489152u,65536u),vec2<u32>(33489153u,65536u),vec2<u32>(16842496u,65536u),vec2<u32>(16777216u,65536u),vec2<u32>(16777217u,65536u),vec2<u32>(16777472u,65536u),vec2<u32>(16777473u,65536u),vec2<u32>(16908033u,65536u),vec2<u32>(16842752u,65536u),vec2<u32>(16842753u,65536u),vec2<u32>(16843263u,65536u),vec2<u32>(4294967041u,65537u),vec2<u32>(4294902016u,65537u),vec2<u32>(4278190080u,65537u),vec2<u32>(4278321151u,65537u),vec2<u32>(4278255617u,65537u),vec2<u32>(4278256127u,65537u),vec2<u32>(4278255872u,65537u),vec2<u32>(16777215u,65537u),vec2<u32>(16711680u,65537u),vec2<u32>(16712191u,65537u),vec2<u32>(16711937u,65537u),vec2<u32>(65280u,65537u),vec2<u32>(255u,65537u),vec2<u32>(0u,65537u),vec2<u32>(1u,65537u),vec2<u32>(511u,65537u),vec2<u32>(257u,65537u),vec2<u32>(131071u,65537u),vec2<u32>(65536u,65537u),vec2<u32>(66047u,65537u),vec2<u32>(33554431u,65537u),vec2<u32>(33554177u,65537u),vec2<u32>(33488896u,65537u),vec2<u32>(33489153u,65537u),vec2<u32>(16777471u,65537u),vec2<u32>(16777217u,65537u),vec2<u32>(16777727u,65537u),vec2<u32>(16777472u,65537u),vec2<u32>(16908287u,65537u),vec2<u32>(16843007u,65537u),vec2<u32>(16842753u,65537u),vec2<u32>(16843009u,65537u),vec2<u32>(4278190081u,66047u),vec2<u32>(4278190336u,66047u),vec2<u32>(4278255616u,66047u),vec2<u32>(16776960u,66047u),vec2<u32>(65281u,66047u),vec2<u32>(0u,66047u),vec2<u32>(257u,66047u),vec2<u32>(130816u,66047u),vec2<u32>(65792u,66047u),vec2<u32>(33488896u,66047u),vec2<u32>(16842496u,66047u),vec2<u32>(16777727u,66047u),vec2<u32>(16842753u,66047u),vec2<u32>(4294967040u,65792u),vec2<u32>(4294902015u,65792u),vec2<u32>(4278255615u,65792u),vec2<u32>(4278190080u,65792u),vec2<u32>(4278320896u,65792u),vec2<u32>(4278255871u,65792u),vec2<u32>(4278255617u,65792u),vec2<u32>(4278255872u,65792u),vec2<u32>(16777215u,65792u),vec2<u32>(16776960u,65792u),vec2<u32>(16711680u,65792u),vec2<u32>(16711681u,65792u),vec2<u32>(16712191u,65792u),vec2<u32>(65280u,65792u),vec2<u32>(255u,65792u),vec2<u32>(0u,65792u),vec2<u32>(1u,65792u),vec2<u32>(256u,65792u),vec2<u32>(131071u,65792u),vec2<u32>(65536u,65792u),vec2<u32>(65793u,65792u),vec2<u32>(33554177u,65792u),vec2<u32>(33489151u,65792u),vec2<u32>(33489153u,65792u),vec2<u32>(16777216u,65792u),vec2<u32>(16908032u,65792u),vec2<u32>(16843007u,65792u),vec2<u32>(16842752u,65792u),vec2<u32>(16843008u,65792u),vec2<u32>(4278255360u,65793u),vec2<u32>(4278190081u,65793u),vec2<u32>(4278190591u,65793u),vec2<u32>(16776960u,65793u),vec2<u32>(16711935u,65793u),vec2<u32>(16711936u,65793u),vec2<u32>(65535u,65793u),vec2<u32>(0u,65793u),vec2<u32>(511u,65793u),vec2<u32>(257u,65793u),vec2<u32>(65791u,65793u),vec2<u32>(65536u,65793u),vec2<u32>(65792u,65793u),vec2<u32>(33488897u,65793u),vec2<u32>(16777471u,65793u),vec2<u32>(16777727u,65793u),vec2<u32>(16777473u,65793u),vec2<u32>(16842753u,65793u),vec2<u32>(4294967295u,33554431u),vec2<u32>(4294967041u,33554431u),vec2<u32>(4294902271u,33554431u),vec2<u32>(4294902017u,33554431u),vec2<u32>(4278321151u,33554431u),vec2<u32>(4278320897u,33554431u),vec2<u32>(4278256127u,33554431u),vec2<u32>(4278255873u,33554431u),vec2<u32>(16711680u,33554431u),vec2<u32>(65535u,33554431u),vec2<u32>(65280u,33554431u),vec2<u32>(255u,33554431u),vec2<u32>(1u,33554431u),vec2<u32>(256u,33554431u),vec2<u32>(65536u,33554431u),vec2<u32>(33554431u,33554431u),vec2<u32>(33554177u,33554431u),vec2<u32>(33489407u,33554431u),vec2<u32>(33489153u,33554431u),vec2<u32>(16777216u,33554431u),vec2<u32>(16908287u,33554431u),vec2<u32>(16908033u,33554431u),vec2<u32>(16843263u,33554431u),vec2<u32>(16843009u,33554431u),vec2<u32>(4294901760u,33554176u),vec2<u32>(4278255360u,33554176u),vec2<u32>(4278190335u,33554176u),vec2<u32>(4278190081u,33554176u),vec2<u32>(4278190336u,33554176u),vec2<u32>(4278255616u,33554176u),vec2<u32>(16776960u,33554176u),vec2<u32>(16711935u,33554176u),vec2<u32>(16711936u,33554176u),vec2<u32>(65535u,33554176u),vec2<u32>(65281u,33554176u),vec2<u32>(0u,33554176u),vec2<u32>(1u,33554176u),vec2<u32>(511u,33554176u),vec2<u32>(256u,33554176u),vec2<u32>(65791u,33554176u),vec2<u32>(65537u,33554176u),vec2<u32>(65792u,33554176u),vec2<u32>(33488896u,33554176u),vec2<u32>(33489152u,33554176u),vec2<u32>(16777471u,33554176u),vec2<u32>(16777217u,33554176u),vec2<u32>(16777472u,33554176u),vec2<u32>(16842752u,33554176u),vec2<u32>(4294967295u,33554177u),vec2<u32>(4294967041u,33554177u),vec2<u32>(4294902271u,33554177u),vec2<u32>(4294902017u,33554177u),vec2<u32>(4278190080u,33554177u),vec2<u32>(4278321151u,33554177u),vec2<u32>(4278320897u,33554177u),vec2<u32>(4278256127u,33554177u),vec2<u32>(4278255873u,33554177u),vec2<u32>(65280u,33554177u),vec2<u32>(255u,33554177u),vec2<u32>(256u,33554177u),vec2<u32>(65536u,33554177u),vec2<u32>(33554431u,33554177u),vec2<u32>(33554177u,33554177u),vec2<u32>(33489407u,33554177u),vec2<u32>(33489153u,33554177u),vec2<u32>(16777216u,33554177u),vec2<u32>(16908287u,33554177u),vec2<u32>(16908033u,33554177u),vec2<u32>(16843263u,33554177u),vec2<u32>(16843009u,33554177u),vec2<u32>(4278190335u,33489151u),vec2<u32>(4278190336u,33489151u),vec2<u32>(16776960u,33489151u),vec2<u32>(16711935u,33489151u),vec2<u32>(65280u,33489151u),vec2<u32>(0u,33489151u),vec2<u32>(257u,33489151u),vec2<u32>(130816u,33489151u),vec2<u32>(65791u,33489151u),vec2<u32>(65792u,33489151u),vec2<u32>(16777471u,33489151u),vec2<u32>(16777472u,33489151u),vec2<u32>(4294967040u,33488896u),vec2<u32>(4294902016u,33488896u),vec2<u32>(4278255361u,33488896u),vec2<u32>(4278190080u,33488896u),vec2<u32>(4278190337u,33488896u),vec2<u32>(4278255617u,33488896u),vec2<u32>(4278255872u,33488896u),vec2<u32>(16777215u,33488896u),vec2<u32>(16776960u,33488896u),vec2<u32>(16711680u,33488896u),vec2<u32>(16712191u,33488896u),vec2<u32>(65280u,33488896u),vec2<u32>(255u,33488896u),vec2<u32>(0u,33488896u),vec2<u32>(1u,33488896u),vec2<u32>(256u,33488896u),vec2<u32>(257u,33488896u),vec2<u32>(65536u,33488896u),vec2<u32>(65537u,33488896u),vec2<u32>(66047u,33488896u),vec2<u32>(65793u,33488896u),vec2<u32>(33554176u,33488896u),vec2<u32>(33489151u,33488896u),vec2<u32>(33488897u,33488896u),vec2<u32>(33489152u,33488896u),vec2<u32>(16842751u,33488896u),vec2<u32>(16842497u,33488896u),vec2<u32>(16777216u,33488896u),vec2<u32>(16777727u,33488896u),vec2<u32>(16842753u,33488896u),vec2<u32>(4278255360u,33488897u),vec2<u32>(4278190081u,33488897u),vec2<u32>(4278190336u,33488897u),vec2<u32>(4278255616u,33488897u),vec2<u32>(16776960u,33488897u),vec2<u32>(16711935u,33488897u),vec2<u32>(16711936u,33488897u),vec2<u32>(16711937u,33488897u),vec2<u32>(65535u,33488897u),vec2<u32>(0u,33488897u),vec2<u32>(256u,33488897u),vec2<u32>(257u,33488897u),vec2<u32>(130816u,33488897u),vec2<u32>(65537u,33488897u),vec2<u32>(65793u,33488897u),vec2<u32>(33488896u,33488897u),vec2<u32>(16842496u,33488897u),vec2<u32>(16777473u,33488897u),vec2<u32>(16843007u,33488897u),vec2<u32>(4294967295u,33489407u),vec2<u32>(4294967041u,33489407u),vec2<u32>(4294902271u,33489407u),vec2<u32>(4294902017u,33489407u),vec2<u32>(4278190080u,33489407u),vec2<u32>(4278321151u,33489407u),vec2<u32>(4278320897u,33489407u),vec2<u32>(4278256127u,33489407u),vec2<u32>(4278255873u,33489407u),vec2<u32>(16776960u,33489407u),vec2<u32>(16711680u,33489407u),vec2<u32>(65280u,33489407u),vec2<u32>(255u,33489407u),vec2<u32>(256u,33489407u),vec2<u32>(65536u,33489407u),vec2<u32>(65792u,33489407u),vec2<u32>(33554431u,33489407u),vec2<u32>(33554177u,33489407u),vec2<u32>(33489407u,33489407u),vec2<u32>(33489153u,33489407u),vec2<u32>(16777216u,33489407u),vec2<u32>(16908287u,33489407u),vec2<u32>(16908033u,33489407u),vec2<u32>(16843263u,33489407u),vec2<u32>(16843009u,33489407u),vec2<u32>(4294901760u,33489152u),vec2<u32>(4294901761u,33489152u),vec2<u32>(4278255360u,33489152u),vec2<u32>(4278190335u,33489152u),vec2<u32>(4278190081u,33489152u),vec2<u32>(4278255616u,33489152u),vec2<u32>(16776960u,33489152u),vec2<u32>(16711935u,33489152u),vec2<u32>(16711681u,33489152u),vec2<u32>(16711936u,33489152u),vec2<u32>(65535u,33489152u),vec2<u32>(65281u,33489152u),vec2<u32>(0u,33489152u),vec2<u32>(257u,33489152u),vec2<u32>(130816u,33489152u),vec2<u32>(65791u,33489152u),vec2<u32>(33488896u,33489152u),vec2<u32>(16777217u,33489152u),vec2<u32>(16777472u,33489152u),vec2<u32>(16842752u,33489152u),vec2<u32>(4294967295u,33489153u),vec2<u32>(4294967041u,33489153u),vec2<u32>(4294902271u,33489153u),vec2<u32>(4294902017u,33489153u),vec2<u32>(4278190080u,33489153u),vec2<u32>(4278321151u,33489153u),vec2<u32>(4278320897u,33489153u),vec2<u32>(4278256127u,33489153u),vec2<u32>(4278255873u,33489153u),vec2<u32>(16711680u,33489153u),vec2<u32>(65280u,33489153u),vec2<u32>(255u,33489153u),vec2<u32>(1u,33489153u),vec2<u32>(33554431u,33489153u),vec2<u32>(33554177u,33489153u),vec2<u32>(33489407u,33489153u),vec2<u32>(33489153u,33489153u),vec2<u32>(16777216u,33489153u),vec2<u32>(16908287u,33489153u),vec2<u32>(16908033u,33489153u),vec2<u32>(16843263u,33489153u),vec2<u32>(16843009u,33489153u),vec2<u32>(4294901760u,16842751u),vec2<u32>(4278255360u,16842751u),vec2<u32>(4278190081u,16842751u),vec2<u32>(4278190591u,16842751u),vec2<u32>(4278190336u,16842751u),vec2<u32>(4278255616u,16842751u),vec2<u32>(16776960u,16842751u),vec2<u32>(16711681u,16842751u),vec2<u32>(16711936u,16842751u),vec2<u32>(0u,16842751u),vec2<u32>(511u,16842751u),vec2<u32>(257u,16842751u),vec2<u32>(65792u,16842751u),vec2<u32>(65793u,16842751u),vec2<u32>(33488896u,16842751u),vec2<u32>(16842496u,16842751u),vec2<u32>(16777471u,16842751u),vec2<u32>(16777217u,16842751u),vec2<u32>(16777472u,16842751u),vec2<u32>(16842752u,16842751u),vec2<u32>(4294967040u,16842496u),vec2<u32>(4294902015u,16842496u),vec2<u32>(4294901761u,16842496u),vec2<u32>(4294902016u,16842496u),vec2<u32>(4278255615u,16842496u),vec2<u32>(4278190080u,16842496u),vec2<u32>(4278190591u,16842496u),vec2<u32>(4278190337u,16842496u),vec2<u32>(4278320896u,16842496u),vec2<u32>(4278255871u,16842496u),vec2<u32>(4278255617u,16842496u),vec2<u32>(4278255872u,16842496u),vec2<u32>(16777215u,16842496u),vec2<u32>(16711680u,16842496u),vec2<u32>(65535u,16842496u),vec2<u32>(65280u,16842496u),vec2<u32>(255u,16842496u),vec2<u32>(0u,16842496u),vec2<u32>(1u,16842496u),vec2<u32>(256u,16842496u),vec2<u32>(130817u,16842496u),vec2<u32>(65536u,16842496u),vec2<u32>(33489151u,16842496u),vec2<u32>(33488897u,16842496u),vec2<u32>(16842497u,16842496u),vec2<u32>(16777216u,16842496u),vec2<u32>(16777727u,16842496u),vec2<u32>(16908032u,16842496u),vec2<u32>(16843007u,16842496u),vec2<u32>(16842753u,16842496u),vec2<u32>(16843008u,16842496u),vec2<u32>(4294901760u,16842497u),vec2<u32>(4278255360u,16842497u),vec2<u32>(4278190335u,16842497u),vec2<u32>(4278190336u,16842497u),vec2<u32>(4278255616u,16842497u),vec2<u32>(16711935u,16842497u),vec2<u32>(16711681u,16842497u),vec2<u32>(16711936u,16842497u),vec2<u32>(65535u,16842497u),vec2<u32>(65281u,16842497u),vec2<u32>(0u,16842497u),vec2<u32>(511u,16842497u),vec2<u32>(65537u,16842497u),vec2<u32>(65792u,16842497u),vec2<u32>(33488896u,16842497u),vec2<u32>(16777471u,16842497u),vec2<u32>(16777217u,16842497u),vec2<u32>(16843008u,16842497u),vec2<u32>(4294967040u,16777471u),vec2<u32>(4294902015u,16777471u),vec2<u32>(4294901761u,16777471u),vec2<u32>(4278255615u,16777471u),vec2<u32>(4278190080u,16777471u),vec2<u32>(4278190591u,16777471u),vec2<u32>(4278255617u,16777471u),vec2<u32>(16777215u,16777471u),vec2<u32>(16711937u,16777471u),vec2<u32>(65280u,16777471u),vec2<u32>(255u,16777471u),vec2<u32>(0u,16777471u),vec2<u32>(1u,16777471u),vec2<u32>(511u,16777471u),vec2<u32>(256u,16777471u),vec2<u32>(131071u,16777471u),vec2<u32>(130816u,16777471u),vec2<u32>(130817u,16777471u),vec2<u32>(65536u,16777471u),vec2<u32>(33489151u,16777471u),vec2<u32>(33488897u,16777471u),vec2<u32>(16842497u,16777471u),vec2<u32>(16777471u,16777471u),vec2<u32>(16777216u,16777471u),vec2<u32>(16777727u,16777471u),vec2<u32>(16908032u,16777471u),vec2<u32>(16843008u,16777471u),vec2<u32>(4294967295u,16777216u),vec2<u32>(4294901760u,16777216u),vec2<u32>(4294902271u,16777216u),vec2<u32>(4294902017u,16777216u),vec2<u32>(4278255615u,16777216u),vec2<u32>(4278255360u,16777216u),vec2<u32>(4278190335u,16777216u),vec2<u32>(4278190080u,16777216u),vec2<u32>(4278190081u,16777216u),vec2<u32>(4278190336u,16777216u),vec2<u32>(4278320896u,16777216u),vec2<u32>(4278255616u,16777216u),vec2<u32>(4278255872u,16777216u),vec2<u32>(4278255873u,16777216u),vec2<u32>(16776960u,16777216u),vec2<u32>(16711935u,16777216u),vec2<u32>(16711680u,16777216u),vec2<u32>(16711681u,16777216u),vec2<u32>(16711936u,16777216u),vec2<u32>(65535u,16777216u),vec2<u32>(65280u,16777216u),vec2<u32>(65281u,16777216u),vec2<u32>(255u,16777216u),vec2<u32>(0u,16777216u),vec2<u32>(1u,16777216u),vec2<u32>(511u,16777216u),vec2<u32>(256u,16777216u),vec2<u32>(257u,16777216u),vec2<u32>(130816u,16777216u),vec2<u32>(65791u,16777216u),vec2<u32>(65536u,16777216u),vec2<u32>(65537u,16777216u),vec2<u32>(65792u,16777216u),vec2<u32>(33554176u,16777216u),vec2<u32>(33488896u,16777216u),vec2<u32>(33489407u,16777216u),vec2<u32>(16842496u,16777216u),vec2<u32>(16842497u,16777216u),vec2<u32>(16777471u,16777216u),vec2<u32>(16777216u,16777216u),vec2<u32>(16777217u,16777216u),vec2<u32>(16777472u,16777216u),vec2<u32>(16777473u,16777216u),vec2<u32>(16908287u,16777216u),vec2<u32>(16908033u,16777216u),vec2<u32>(16842752u,16777216u),vec2<u32>(16843263u,16777216u),vec2<u32>(16843009u,16777216u),vec2<u32>(4294967040u,16777217u),vec2<u32>(4294902015u,16777217u),vec2<u32>(4278255615u,16777217u),vec2<u32>(4278190080u,16777217u),vec2<u32>(4278190336u,16777217u),vec2<u32>(4278321151u,16777217u),vec2<u32>(4278255617u,16777217u),vec2<u32>(4278255872u,16777217u),vec2<u32>(16711680u,16777217u),vec2<u32>(16712191u,16777217u),vec2<u32>(16711936u,16777217u),vec2<u32>(65280u,16777217u),vec2<u32>(65281u,16777217u),vec2<u32>(0u,16777217u),vec2<u32>(1u,16777217u),vec2<u32>(256u,16777217u),vec2<u32>(65536u,16777217u),vec2<u32>(66047u,16777217u),vec2<u32>(33554177u,16777217u),vec2<u32>(33489151u,16777217u),vec2<u32>(33489152u,16777217u),vec2<u32>(33489153u,16777217u),vec2<u32>(16842497u,16777217u),vec2<u32>(16777471u,16777217u),vec2<u32>(16777216u,16777217u),vec2<u32>(16843007u,16777217u),vec2<u32>(16842753u,16777217u),vec2<u32>(16843008u,16777217u),vec2<u32>(4294901760u,16777727u),vec2<u32>(4278190081u,16777727u),vec2<u32>(4278190336u,16777727u),vec2<u32>(4278255616u,16777727u),vec2<u32>(16776960u,16777727u),vec2<u32>(16711681u,16777727u),vec2<u32>(65535u,16777727u),vec2<u32>(65281u,16777727u),vec2<u32>(0u,16777727u),vec2<u32>(1u,16777727u),vec2<u32>(257u,16777727u),vec2<u32>(65791u,16777727u),vec2<u32>(65536u,16777727u),vec2<u32>(33488896u,16777727u),vec2<u32>(16842496u,16777727u),vec2<u32>(16777217u,16777727u),vec2<u32>(16777472u,16777727u),vec2<u32>(16842752u,16777727u),vec2<u32>(4294902015u,16777472u),vec2<u32>(4294901761u,16777472u),vec2<u32>(4294902016u,16777472u),vec2<u32>(4278255615u,16777472u),vec2<u32>(4278255361u,16777472u),vec2<u32>(4278190080u,16777472u),vec2<u32>(4278190591u,16777472u),vec2<u32>(4278190337u,16777472u),vec2<u32>(4278321151u,16777472u),vec2<u32>(4278320896u,16777472u),vec2<u32>(4278255871u,16777472u),vec2<u32>(4278255617u,16777472u),vec2<u32>(16777215u,16777472u),vec2<u32>(16776961u,16777472u),vec2<u32>(16711680u,16777472u),vec2<u32>(16712191u,16777472u),vec2<u32>(16711937u,16777472u),vec2<u32>(65280u,16777472u),vec2<u32>(255u,16777472u),vec2<u32>(0u,16777472u),vec2<u32>(1u,16777472u),vec2<u32>(256u,16777472u),vec2<u32>(130817u,16777472u),vec2<u32>(65536u,16777472u),vec2<u32>(65537u,16777472u),vec2<u32>(65793u,16777472u),vec2<u32>(33554176u,16777472u),vec2<u32>(33489151u,16777472u),vec2<u32>(16842751u,16777472u),vec2<u32>(16842497u,16777472u),vec2<u32>(16777216u,16777472u),vec2<u32>(16777473u,16777472u),vec2<u32>(16908032u,16777472u),vec2<u32>(16842753u,16777472u),vec2<u32>(4294901760u,16777473u),vec2<u32>(4278190080u,16777473u),vec2<u32>(4278255616u,16777473u),vec2<u32>(16711935u,16777473u),vec2<u32>(16711681u,16777473u),vec2<u32>(16711936u,16777473u),vec2<u32>(65535u,16777473u),vec2<u32>(0u,16777473u),vec2<u32>(511u,16777473u),vec2<u32>(130816u,16777473u),vec2<u32>(33488896u,16777473u),vec2<u32>(16842496u,16777473u),vec2<u32>(16777471u,16777473u),vec2<u32>(16777216u,16777473u),vec2<u32>(16777217u,16777473u),vec2<u32>(4294967295u,16908287u),vec2<u32>(4294967041u,16908287u),vec2<u32>(4294902271u,16908287u),vec2<u32>(4294902017u,16908287u),vec2<u32>(4278190080u,16908287u),vec2<u32>(4278321151u,16908287u),vec2<u32>(4278320897u,16908287u),vec2<u32>(4278256127u,16908287u),vec2<u32>(4278255873u,16908287u),vec2<u32>(16711680u,16908287u),vec2<u32>(65280u,16908287u),vec2<u32>(255u,16908287u),vec2<u32>(1u,16908287u),vec2<u32>(256u,16908287u),vec2<u32>(33554431u,16908287u),vec2<u32>(33554177u,16908287u),vec2<u32>(33489407u,16908287u),vec2<u32>(33489153u,16908287u),vec2<u32>(16777216u,16908287u),vec2<u32>(16908287u,16908287u),vec2<u32>(16908033u,16908287u),vec2<u32>(16843263u,16908287u),vec2<u32>(16843009u,16908287u),vec2<u32>(4294901760u,16908032u),vec2<u32>(4294902016u,16908032u),vec2<u32>(4278255360u,16908032u),vec2<u32>(4278190335u,16908032u),vec2<u32>(4278190081u,16908032u),vec2<u32>(4278190336u,16908032u),vec2<u32>(4278190337u,16908032u),vec2<u32>(16711681u,16908032u),vec2<u32>(16711936u,16908032u),vec2<u32>(65280u,16908032u),vec2<u32>(0u,16908032u),vec2<u32>(511u,16908032u),vec2<u32>(257u,16908032u),vec2<u32>(130816u,16908032u),vec2<u32>(65791u,16908032u),vec2<u32>(33488896u,16908032u),vec2<u32>(16842751u,16908032u),vec2<u32>(16842497u,16908032u),vec2<u32>(16777217u,16908032u),vec2<u32>(16777472u,16908032u),vec2<u32>(4294967041u,16908033u),vec2<u32>(4294902271u,16908033u),vec2<u32>(4294902017u,16908033u),vec2<u32>(4278255615u,16908033u),vec2<u32>(4278190336u,16908033u),vec2<u32>(4278320897u,16908033u),vec2<u32>(4278256127u,16908033u),vec2<u32>(4278255873u,16908033u),vec2<u32>(16711680u,16908033u),vec2<u32>(65280u,16908033u),vec2<u32>(1u,16908033u),vec2<u32>(256u,16908033u),vec2<u32>(65536u,16908033u),vec2<u32>(33554431u,16908033u),vec2<u32>(33554177u,16908033u),vec2<u32>(33489407u,16908033u),vec2<u32>(33489153u,16908033u),vec2<u32>(16777216u,16908033u),vec2<u32>(16908287u,16908033u),vec2<u32>(16908033u,16908033u),vec2<u32>(16843263u,16908033u),vec2<u32>(16843009u,16908033u),vec2<u32>(4278190336u,16843007u),vec2<u32>(4278255616u,16843007u),vec2<u32>(16776960u,16843007u),vec2<u32>(16711935u,16843007u),vec2<u32>(65535u,16843007u),vec2<u32>(255u,16843007u),vec2<u32>(0u,16843007u),vec2<u32>(511u,16843007u),vec2<u32>(257u,16843007u),vec2<u32>(130816u,16843007u),vec2<u32>(65536u,16843007u),vec2<u32>(65537u,16843007u),vec2<u32>(66047u,16843007u),vec2<u32>(65792u,16843007u),vec2<u32>(33488896u,16843007u),vec2<u32>(4294901761u,16842752u),vec2<u32>(4294902016u,16842752u),vec2<u32>(4278255615u,16842752u),vec2<u32>(4278255361u,16842752u),vec2<u32>(4278190080u,16842752u),vec2<u32>(4278190591u,16842752u),vec2<u32>(4278255617u,16842752u),vec2<u32>(4278255872u,16842752u),vec2<u32>(16776961u,16842752u),vec2<u32>(16711680u,16842752u),vec2<u32>(65280u,16842752u),vec2<u32>(255u,16842752u),vec2<u32>(0u,16842752u),vec2<u32>(1u,16842752u),vec2<u32>(256u,16842752u),vec2<u32>(65536u,16842752u),vec2<u32>(65793u,16842752u),vec2<u32>(33554176u,16842752u),vec2<u32>(33489151u,16842752u),vec2<u32>(33488896u,16842752u),vec2<u32>(33488897u,16842752u),vec2<u32>(33489152u,16842752u),vec2<u32>(16842497u,16842752u),vec2<u32>(16777216u,16842752u),vec2<u32>(16777727u,16842752u),vec2<u32>(4294901760u,16842753u),vec2<u32>(4278255360u,16842753u),vec2<u32>(4278190081u,16842753u),vec2<u32>(4278190337u,16842753u),vec2<u32>(4278320896u,16842753u),vec2<u32>(4278255616u,16842753u),vec2<u32>(16711935u,16842753u),vec2<u32>(16711681u,16842753u),vec2<u32>(16711937u,16842753u),vec2<u32>(65281u,16842753u),vec2<u32>(0u,16842753u),vec2<u32>(1u,16842753u),vec2<u32>(511u,16842753u),vec2<u32>(131071u,16842753u),vec2<u32>(130817u,16842753u),vec2<u32>(33488897u,16842753u),vec2<u32>(16842751u,16842753u),vec2<u32>(16777216u,16842753u),vec2<u32>(16777217u,16842753u),vec2<u32>(16777472u,16842753u),vec2<u32>(16908032u,16842753u),vec2<u32>(16843007u,16842753u),vec2<u32>(16842753u,16842753u),vec2<u32>(4294967295u,16843263u),vec2<u32>(4294967041u,16843263u),vec2<u32>(4294902271u,16843263u),vec2<u32>(4294902017u,16843263u),vec2<u32>(4278321151u,16843263u),vec2<u32>(4278320897u,16843263u),vec2<u32>(4278256127u,16843263u),vec2<u32>(4278255873u,16843263u),vec2<u32>(65280u,16843263u),vec2<u32>(255u,16843263u),vec2<u32>(1u,16843263u),vec2<u32>(256u,16843263u),vec2<u32>(33554431u,16843263u),vec2<u32>(33554177u,16843263u),vec2<u32>(33489407u,16843263u),vec2<u32>(33489153u,16843263u),vec2<u32>(16777216u,16843263u),vec2<u32>(16908287u,16843263u),vec2<u32>(16908033u,16843263u),vec2<u32>(16843263u,16843263u),vec2<u32>(16843009u,16843263u),vec2<u32>(4294901760u,16843008u),vec2<u32>(4278190335u,16843008u),vec2<u32>(4278190336u,16843008u),vec2<u32>(4278320896u,16843008u),vec2<u32>(4278255616u,16843008u),vec2<u32>(16776960u,16843008u),vec2<u32>(65535u,16843008u),vec2<u32>(0u,16843008u),vec2<u32>(257u,16843008u),vec2<u32>(130816u,16843008u),vec2<u32>(65537u,16843008u),vec2<u32>(65792u,16843008u),vec2<u32>(16842751u,16843008u),vec2<u32>(16777217u,16843008u),vec2<u32>(4294967295u,16843009u),vec2<u32>(4294967041u,16843009u),vec2<u32>(4294902271u,16843009u),vec2<u32>(4294902017u,16843009u),vec2<u32>(4278321151u,16843009u),vec2<u32>(4278320897u,16843009u),vec2<u32>(4278256127u,16843009u),vec2<u32>(4278255873u,16843009u),vec2<u32>(65280u,16843009u),vec2<u32>(255u,16843009u),vec2<u32>(1u,16843009u),vec2<u32>(33554431u,16843009u),vec2<u32>(33554177u,16843009u),vec2<u32>(33489407u,16843009u),vec2<u32>(33489153u,16843009u),vec2<u32>(16777216u,16843009u),vec2<u32>(16908287u,16843009u),vec2<u32>(16908033u,16843009u),vec2<u32>(16843263u,16843009u));const IQ_GRID_1{{ iqSfx }}:array<vec2<u32>,1>=array<vec2<u32>,1>(vec2<u32>(16843009u,16843009u));
{% if not (iqInline | default(false)) %}fn iq_grid{{ iqSfx }}(i:u32)->vec2<u32>{if(i<2047u){return IQ_GRID_0{{ iqSfx }}[i];}return IQ_GRID_1{{ iqSfx }}[i-2047u];}
{% endif %}{% macro iq1grid_at(i, sfx="") %}select(IQ_GRID_1{{ sfx }}[0u], IQ_GRID_0{{ sfx }}[{{ i }}], {{ i }} < 2047u){% endmacro %}`],["ops/_shared/iq-grid-iq2s.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}const IQ_GRID_0{{ iqSfx }}:array<vec2<u32>,1024>=array<vec2<u32>,1024>(vec2<u32>(134744072u,134744072u),vec2<u32>(134744107u,134744072u),vec2<u32>(134748441u,134744072u),vec2<u32>(134753032u,134744072u),vec2<u32>(134753067u,134744072u),vec2<u32>(135858201u,134744072u),vec2<u32>(135862536u,134744072u),vec2<u32>(135862571u,134744072u),vec2<u32>(135867161u,134744072u),vec2<u32>(137037832u,134744072u),vec2<u32>(137037867u,134744072u),vec2<u32>(137042201u,134744072u),vec2<u32>(137046792u,134744072u),vec2<u32>(419956761u,134744072u),vec2<u32>(419961096u,134744072u),vec2<u32>(419961131u,134744072u),vec2<u32>(419965721u,134744072u),vec2<u32>(421070856u,134744072u),vec2<u32>(421070891u,134744072u),vec2<u32>(421075225u,134744072u),vec2<u32>(421079816u,134744072u),vec2<u32>(422250521u,134744072u),vec2<u32>(422254856u,134744072u),vec2<u32>(422254891u,134744072u),vec2<u32>(422259481u,134744072u),vec2<u32>(721946632u,134744072u),vec2<u32>(721946667u,134744072u),vec2<u32>(721951001u,134744072u),vec2<u32>(721955592u,134744072u),vec2<u32>(723060761u,134744072u),vec2<u32>(723065096u,134744072u),vec2<u32>(724240392u,134744072u),vec2<u32>(724244761u,134744072u),vec2<u32>(724249387u,134744072u),vec2<u32>(134744089u,134744089u),vec2<u32>(134748424u,134744089u),vec2<u32>(134748459u,134744089u),vec2<u32>(134753049u,134744089u),vec2<u32>(135858184u,134744089u),vec2<u32>(135858219u,134744089u),vec2<u32>(135862553u,134744089u),vec2<u32>(135867144u,134744089u),vec2<u32>(137037849u,134744089u),vec2<u32>(137042184u,134744089u),vec2<u32>(419956744u,134744089u),vec2<u32>(419956779u,134744089u),vec2<u32>(419961113u,134744089u),vec2<u32>(419965704u,134744089u),vec2<u32>(421070873u,134744089u),vec2<u32>(421075208u,134744089u),vec2<u32>(421075243u,134744089u),vec2<u32>(421079833u,134744089u),vec2<u32>(422250504u,134744089u),vec2<u32>(422254873u,134744089u),vec2<u32>(422259464u,134744089u),vec2<u32>(721946649u,134744089u),vec2<u32>(721950984u,134744089u),vec2<u32>(723060744u,134744089u),vec2<u32>(723060779u,134744089u),vec2<u32>(723065113u,134744089u),vec2<u32>(724240409u,134744089u),vec2<u32>(724244744u,134744089u),vec2<u32>(134744072u,134744107u),vec2<u32>(134744107u,134744107u),vec2<u32>(134748441u,134744107u),vec2<u32>(134753032u,134744107u),vec2<u32>(135858201u,134744107u),vec2<u32>(135862536u,134744107u),vec2<u32>(137037832u,134744107u),vec2<u32>(137046827u,134744107u),vec2<u32>(419956761u,134744107u),vec2<u32>(419961096u,134744107u),vec2<u32>(419961131u,134744107u),vec2<u32>(419965721u,134744107u),vec2<u32>(421070856u,134744107u),vec2<u32>(421075225u,134744107u),vec2<u32>(721946632u,134744107u),vec2<u32>(721951001u,134744107u),vec2<u32>(721955627u,134744107u),vec2<u32>(723065096u,134744107u),vec2<u32>(724240427u,134744107u),vec2<u32>(134744089u,134748424u),vec2<u32>(134748424u,134748424u),vec2<u32>(134748459u,134748424u),vec2<u32>(134753049u,134748424u),vec2<u32>(135858184u,134748424u),vec2<u32>(135858219u,134748424u),vec2<u32>(135862553u,134748424u),vec2<u32>(135867144u,134748424u),vec2<u32>(137037849u,134748424u),vec2<u32>(137042184u,134748424u),vec2<u32>(137042219u,134748424u),vec2<u32>(137046809u,134748424u),vec2<u32>(419956744u,134748424u),vec2<u32>(419956779u,134748424u),vec2<u32>(419961113u,134748424u),vec2<u32>(419965704u,134748424u),vec2<u32>(419965739u,134748424u),vec2<u32>(421070873u,134748424u),vec2<u32>(421075208u,134748424u),vec2<u32>(421075243u,134748424u),vec2<u32>(421079833u,134748424u),vec2<u32>(422250504u,134748424u),vec2<u32>(422250539u,134748424u),vec2<u32>(422254873u,134748424u),vec2<u32>(721946649u,134748424u),vec2<u32>(721950984u,134748424u),vec2<u32>(721951019u,134748424u),vec2<u32>(721955609u,134748424u),vec2<u32>(723060744u,134748424u),vec2<u32>(723065113u,134748424u),vec2<u32>(723069704u,134748424u),vec2<u32>(724240409u,134748424u),vec2<u32>(724244744u,134748424u),vec2<u32>(134744072u,134748441u),vec2<u32>(134744107u,134748441u),vec2<u32>(134748441u,134748441u),vec2<u32>(134753032u,134748441u),vec2<u32>(134753067u,134748441u),vec2<u32>(135858201u,134748441u),vec2<u32>(135862536u,134748441u),vec2<u32>(135862571u,134748441u),vec2<u32>(135867161u,134748441u),vec2<u32>(137037832u,134748441u),vec2<u32>(137042201u,134748441u),vec2<u32>(137046792u,134748441u),vec2<u32>(419956761u,134748441u),vec2<u32>(419961096u,134748441u),vec2<u32>(419961131u,134748441u),vec2<u32>(419965721u,134748441u),vec2<u32>(421070856u,134748441u),vec2<u32>(421070891u,134748441u),vec2<u32>(421075225u,134748441u),vec2<u32>(421079816u,134748441u),vec2<u32>(422250521u,134748441u),vec2<u32>(422254856u,134748441u),vec2<u32>(721946632u,134748441u),vec2<u32>(721946667u,134748441u),vec2<u32>(721951001u,134748441u),vec2<u32>(721955592u,134748441u),vec2<u32>(723060761u,134748441u),vec2<u32>(723065096u,134748441u),vec2<u32>(724240392u,134748441u),vec2<u32>(134744089u,134748459u),vec2<u32>(134748424u,134748459u),vec2<u32>(134748459u,134748459u),vec2<u32>(134753049u,134748459u),vec2<u32>(135858184u,134748459u),vec2<u32>(135862553u,134748459u),vec2<u32>(419956744u,134748459u),vec2<u32>(419961113u,134748459u),vec2<u32>(419965704u,134748459u),vec2<u32>(421070873u,134748459u),vec2<u32>(421075208u,134748459u),vec2<u32>(422250504u,134748459u),vec2<u32>(721946649u,134748459u),vec2<u32>(721950984u,134748459u),vec2<u32>(723060744u,134748459u),vec2<u32>(134744072u,134753032u),vec2<u32>(134744107u,134753032u),vec2<u32>(134748441u,134753032u),vec2<u32>(134753032u,134753032u),vec2<u32>(135858201u,134753032u),vec2<u32>(135862536u,134753032u),vec2<u32>(135862571u,134753032u),vec2<u32>(135867161u,134753032u),vec2<u32>(137037832u,134753032u),vec2<u32>(137042201u,134753032u),vec2<u32>(137046827u,134753032u),vec2<u32>(419956761u,134753032u),vec2<u32>(419961096u,134753032u),vec2<u32>(419961131u,134753032u),vec2<u32>(419965721u,134753032u),vec2<u32>(421070856u,134753032u),vec2<u32>(421070891u,134753032u),vec2<u32>(421075225u,134753032u),vec2<u32>(421079816u,134753032u),vec2<u32>(422250521u,134753032u),vec2<u32>(422254856u,134753032u),vec2<u32>(721946632u,134753032u),vec2<u32>(721951001u,134753032u),vec2<u32>(723065096u,134753032u),vec2<u32>(724249387u,134753032u),vec2<u32>(134744089u,134753049u),vec2<u32>(134748424u,134753049u),vec2<u32>(135858184u,134753049u),vec2<u32>(135858219u,134753049u),vec2<u32>(135862553u,134753049u),vec2<u32>(135867144u,134753049u),vec2<u32>(137037849u,134753049u),vec2<u32>(419956744u,134753049u),vec2<u32>(419961113u,134753049u),vec2<u32>(419965704u,134753049u),vec2<u32>(421070873u,134753049u),vec2<u32>(421075208u,134753049u),vec2<u32>(422250504u,134753049u),vec2<u32>(721946649u,134753049u),vec2<u32>(723060744u,134753049u),vec2<u32>(134744072u,134753067u),vec2<u32>(135858201u,134753067u),vec2<u32>(135862536u,134753067u),vec2<u32>(137037867u,134753067u),vec2<u32>(137046792u,134753067u),vec2<u32>(137046827u,134753067u),vec2<u32>(421070856u,134753067u),vec2<u32>(723069721u,134753067u),vec2<u32>(134744089u,135858184u),vec2<u32>(134748424u,135858184u),vec2<u32>(134748459u,135858184u),vec2<u32>(134753049u,135858184u),vec2<u32>(135858184u,135858184u),vec2<u32>(135858219u,135858184u),vec2<u32>(135862553u,135858184u),vec2<u32>(135867144u,135858184u),vec2<u32>(137037849u,135858184u),vec2<u32>(137042184u,135858184u),vec2<u32>(137042219u,135858184u),vec2<u32>(419956744u,135858184u),vec2<u32>(419956779u,135858184u),vec2<u32>(419961113u,135858184u),vec2<u32>(419965704u,135858184u),vec2<u32>(421070873u,135858184u),vec2<u32>(421075208u,135858184u),vec2<u32>(421075243u,135858184u),vec2<u32>(421079833u,135858184u),vec2<u32>(422250504u,135858184u),vec2<u32>(422250539u,135858184u),vec2<u32>(422254873u,135858184u),vec2<u32>(422259464u,135858184u),vec2<u32>(721946649u,135858184u),vec2<u32>(721950984u,135858184u),vec2<u32>(721951019u,135858184u),vec2<u32>(723060744u,135858184u),vec2<u32>(723065113u,135858184u),vec2<u32>(723069704u,135858184u),vec2<u32>(724240409u,135858184u),vec2<u32>(724244744u,135858184u),vec2<u32>(134744072u,135858201u),vec2<u32>(134744107u,135858201u),vec2<u32>(134748441u,135858201u),vec2<u32>(134753032u,135858201u),vec2<u32>(134753067u,135858201u),vec2<u32>(135858201u,135858201u),vec2<u32>(135862536u,135858201u),vec2<u32>(135862571u,135858201u),vec2<u32>(135867161u,135858201u),vec2<u32>(137037832u,135858201u),vec2<u32>(137037867u,135858201u),vec2<u32>(137042201u,135858201u),vec2<u32>(137046792u,135858201u),vec2<u32>(419956761u,135858201u),vec2<u32>(419961096u,135858201u),vec2<u32>(419961131u,135858201u),vec2<u32>(419965721u,135858201u),vec2<u32>(421070856u,135858201u),vec2<u32>(421070891u,135858201u),vec2<u32>(421075225u,135858201u),vec2<u32>(421079816u,135858201u),vec2<u32>(422250521u,135858201u),vec2<u32>(422254856u,135858201u),vec2<u32>(721946632u,135858201u),vec2<u32>(721946667u,135858201u),vec2<u32>(721951001u,135858201u),vec2<u32>(721955592u,135858201u),vec2<u32>(723060761u,135858201u),vec2<u32>(723065096u,135858201u),vec2<u32>(134744089u,135858219u),vec2<u32>(134748424u,135858219u),vec2<u32>(134753049u,135858219u),vec2<u32>(135858184u,135858219u),vec2<u32>(135862553u,135858219u),vec2<u32>(137037849u,135858219u),vec2<u32>(137042184u,135858219u),vec2<u32>(419956744u,135858219u),vec2<u32>(419961113u,135858219u),vec2<u32>(421070873u,135858219u),vec2<u32>(421075208u,135858219u),vec2<u32>(721946649u,135858219u),vec2<u32>(721950984u,135858219u),vec2<u32>(723060744u,135858219u),vec2<u32>(134744072u,135862536u),vec2<u32>(134744107u,135862536u),vec2<u32>(134748441u,135862536u),vec2<u32>(134753032u,135862536u),vec2<u32>(135858201u,135862536u),vec2<u32>(135862536u,135862536u),vec2<u32>(135862571u,135862536u),vec2<u32>(135867161u,135862536u),vec2<u32>(137037832u,135862536u),vec2<u32>(137042201u,135862536u),vec2<u32>(137046792u,135862536u),vec2<u32>(419956761u,135862536u),vec2<u32>(419961096u,135862536u),vec2<u32>(419961131u,135862536u),vec2<u32>(419965721u,135862536u),vec2<u32>(421070856u,135862536u),vec2<u32>(421070891u,135862536u),vec2<u32>(421075225u,135862536u),vec2<u32>(421079816u,135862536u),vec2<u32>(422250521u,135862536u),vec2<u32>(422254856u,135862536u),vec2<u32>(721946632u,135862536u),vec2<u32>(721946667u,135862536u),vec2<u32>(721951001u,135862536u),vec2<u32>(721955592u,135862536u),vec2<u32>(723060761u,135862536u),vec2<u32>(723065096u,135862536u),vec2<u32>(724240392u,135862536u),vec2<u32>(134744089u,135862553u),vec2<u32>(134748424u,135862553u),vec2<u32>(134748459u,135862553u),vec2<u32>(134753049u,135862553u),vec2<u32>(135858184u,135862553u),vec2<u32>(135858219u,135862553u),vec2<u32>(135862553u,135862553u),vec2<u32>(135867144u,135862553u),vec2<u32>(137037849u,135862553u),vec2<u32>(137042184u,135862553u),vec2<u32>(419956744u,135862553u),vec2<u32>(419956779u,135862553u),vec2<u32>(419961113u,135862553u),vec2<u32>(419965704u,135862553u),vec2<u32>(421070873u,135862553u),vec2<u32>(421075208u,135862553u),vec2<u32>(422250504u,135862553u),vec2<u32>(721946649u,135862553u),vec2<u32>(721950984u,135862553u),vec2<u32>(723060744u,135862553u),vec2<u32>(134744072u,135862571u),vec2<u32>(134748441u,135862571u),vec2<u32>(134753032u,135862571u),vec2<u32>(135858201u,135862571u),vec2<u32>(135862536u,135862571u),vec2<u32>(137037832u,135862571u),vec2<u32>(419956761u,135862571u),vec2<u32>(419961096u,135862571u),vec2<u32>(421070856u,135862571u),vec2<u32>(721946632u,135862571u),vec2<u32>(724249387u,135862571u),vec2<u32>(134744089u,135867144u),vec2<u32>(134748424u,135867144u),vec2<u32>(134748459u,135867144u),vec2<u32>(134753049u,135867144u),vec2<u32>(135858184u,135867144u),vec2<u32>(135862553u,135867144u),vec2<u32>(135867144u,135867144u),vec2<u32>(137037849u,135867144u),vec2<u32>(419956744u,135867144u),vec2<u32>(419956779u,135867144u),vec2<u32>(419961113u,135867144u),vec2<u32>(419965704u,135867144u),vec2<u32>(421070873u,135867144u),vec2<u32>(421075208u,135867144u),vec2<u32>(422250504u,135867144u),vec2<u32>(721946649u,135867144u),vec2<u32>(721950984u,135867144u),vec2<u32>(134744072u,135867161u),vec2<u32>(134744107u,135867161u),vec2<u32>(134748441u,135867161u),vec2<u32>(134753032u,135867161u),vec2<u32>(135858201u,135867161u),vec2<u32>(135862536u,135867161u),vec2<u32>(137037832u,135867161u),vec2<u32>(419956761u,135867161u),vec2<u32>(419961096u,135867161u),vec2<u32>(421070856u,135867161u),vec2<u32>(422259481u,135867161u),vec2<u32>(724240427u,135867161u),vec2<u32>(134748424u,135867179u),vec2<u32>(135858184u,135867179u),vec2<u32>(419956744u,135867179u),vec2<u32>(421075243u,135867179u),vec2<u32>(134744072u,137037832u),vec2<u32>(134744107u,137037832u),vec2<u32>(134748441u,137037832u),vec2<u32>(134753032u,137037832u),vec2<u32>(135858201u,137037832u),vec2<u32>(135862536u,137037832u),vec2<u32>(135862571u,137037832u),vec2<u32>(135867161u,137037832u),vec2<u32>(137037832u,137037832u),vec2<u32>(137042201u,137037832u),vec2<u32>(137046827u,137037832u),vec2<u32>(419956761u,137037832u),vec2<u32>(419961096u,137037832u),vec2<u32>(421070856u,137037832u),vec2<u32>(421070891u,137037832u),vec2<u32>(421075225u,137037832u),vec2<u32>(422254856u,137037832u),vec2<u32>(721946632u,137037832u),vec2<u32>(721955627u,137037832u),vec2<u32>(723065096u,137037832u),vec2<u32>(724249387u,137037832u),vec2<u32>(134744089u,137037849u),vec2<u32>(134748424u,137037849u),vec2<u32>(135858184u,137037849u),vec2<u32>(135858219u,137037849u),vec2<u32>(135862553u,137037849u),vec2<u32>(137037849u,137037849u),vec2<u32>(419956744u,137037849u),vec2<u32>(419956779u,137037849u),vec2<u32>(419961113u,137037849u),vec2<u32>(421070873u,137037849u),vec2<u32>(421075208u,137037849u),vec2<u32>(422250504u,137037849u),vec2<u32>(721946649u,137037849u),vec2<u32>(721950984u,137037849u),vec2<u32>(723060744u,137037849u),vec2<u32>(134744072u,137037867u),vec2<u32>(134753067u,137037867u),vec2<u32>(137037867u,137037867u),vec2<u32>(137046792u,137037867u),vec2<u32>(137046827u,137037867u),vec2<u32>(419961096u,137037867u),vec2<u32>(421070856u,137037867u),vec2<u32>(721955592u,137037867u),vec2<u32>(721955627u,137037867u),vec2<u32>(724249352u,137037867u),vec2<u32>(134744089u,137042184u),vec2<u32>(134748424u,137042184u),vec2<u32>(134748459u,137042184u),vec2<u32>(134753049u,137042184u),vec2<u32>(135858184u,137042184u),vec2<u32>(135862553u,137042184u),vec2<u32>(135867144u,137042184u),vec2<u32>(137037849u,137042184u),vec2<u32>(137042184u,137042184u),vec2<u32>(419956744u,137042184u),vec2<u32>(419956779u,137042184u),vec2<u32>(419961113u,137042184u),vec2<u32>(419965704u,137042184u),vec2<u32>(421070873u,137042184u),vec2<u32>(421075208u,137042184u),vec2<u32>(422250504u,137042184u),vec2<u32>(721946649u,137042184u),vec2<u32>(721950984u,137042184u),vec2<u32>(723060744u,137042184u),vec2<u32>(134744072u,137042201u),vec2<u32>(134748441u,137042201u),vec2<u32>(134753032u,137042201u),vec2<u32>(135858201u,137042201u),vec2<u32>(135862536u,137042201u),vec2<u32>(137037832u,137042201u),vec2<u32>(419956761u,137042201u),vec2<u32>(419961096u,137042201u),vec2<u32>(421070856u,137042201u),vec2<u32>(422254891u,137042201u),vec2<u32>(721946632u,137042201u),vec2<u32>(134744089u,137042219u),vec2<u32>(134748424u,137042219u),vec2<u32>(135858184u,137042219u),vec2<u32>(419956744u,137042219u),vec2<u32>(421079833u,137042219u),vec2<u32>(134744072u,137046792u),vec2<u32>(134748441u,137046792u),vec2<u32>(135858201u,137046792u),vec2<u32>(135862536u,137046792u),vec2<u32>(419956761u,137046792u),vec2<u32>(419961096u,137046792u),vec2<u32>(421070856u,137046792u),vec2<u32>(721955627u,137046792u),vec2<u32>(724249387u,137046792u),vec2<u32>(134744089u,137046809u),vec2<u32>(134748424u,137046809u),vec2<u32>(135858184u,137046809u),vec2<u32>(723065113u,137046809u),vec2<u32>(134753067u,137046827u),vec2<u32>(137037867u,137046827u),vec2<u32>(422254856u,137046827u),vec2<u32>(721955592u,137046827u),vec2<u32>(721955627u,137046827u),vec2<u32>(134744089u,419956744u),vec2<u32>(134748424u,419956744u),vec2<u32>(134748459u,419956744u),vec2<u32>(134753049u,419956744u),vec2<u32>(135858184u,419956744u),vec2<u32>(135858219u,419956744u),vec2<u32>(135862553u,419956744u),vec2<u32>(135867144u,419956744u),vec2<u32>(135867179u,419956744u),vec2<u32>(137037849u,419956744u),vec2<u32>(137042184u,419956744u),vec2<u32>(137042219u,419956744u),vec2<u32>(419956744u,419956744u),vec2<u32>(419956779u,419956744u),vec2<u32>(419961113u,419956744u),vec2<u32>(419965704u,419956744u),vec2<u32>(419965739u,419956744u),vec2<u32>(421070873u,419956744u),vec2<u32>(421075208u,419956744u),vec2<u32>(421075243u,419956744u),vec2<u32>(421079833u,419956744u),vec2<u32>(422250504u,419956744u),vec2<u32>(422250539u,419956744u),vec2<u32>(422254873u,419956744u),vec2<u32>(721946649u,419956744u),vec2<u32>(721950984u,419956744u),vec2<u32>(723060744u,419956744u),vec2<u32>(723065113u,419956744u),vec2<u32>(723069704u,419956744u),vec2<u32>(724240409u,419956744u),vec2<u32>(724244744u,419956744u),vec2<u32>(134744072u,419956761u),vec2<u32>(134744107u,419956761u),vec2<u32>(134748441u,419956761u),vec2<u32>(134753032u,419956761u),vec2<u32>(135858201u,419956761u),vec2<u32>(135862536u,419956761u),vec2<u32>(135862571u,419956761u),vec2<u32>(135867161u,419956761u),vec2<u32>(137037832u,419956761u),vec2<u32>(137037867u,419956761u),vec2<u32>(137042201u,419956761u),vec2<u32>(419956761u,419956761u),vec2<u32>(419961096u,419956761u),vec2<u32>(419961131u,419956761u),vec2<u32>(419965721u,419956761u),vec2<u32>(421070856u,419956761u),vec2<u32>(421070891u,419956761u),vec2<u32>(421075225u,419956761u),vec2<u32>(421079816u,419956761u),vec2<u32>(422250521u,419956761u),vec2<u32>(422254856u,419956761u),vec2<u32>(721946632u,419956761u),vec2<u32>(721946667u,419956761u),vec2<u32>(721951001u,419956761u),vec2<u32>(721955592u,419956761u),vec2<u32>(723060761u,419956761u),vec2<u32>(723065096u,419956761u),vec2<u32>(724240392u,419956761u),vec2<u32>(134744089u,419956779u),vec2<u32>(134748424u,419956779u),vec2<u32>(135858184u,419956779u),vec2<u32>(135858219u,419956779u),vec2<u32>(135862553u,419956779u),vec2<u32>(135867144u,419956779u),vec2<u32>(137042184u,419956779u),vec2<u32>(419956744u,419956779u),vec2<u32>(419961113u,419956779u),vec2<u32>(419965704u,419956779u),vec2<u32>(421070873u,419956779u),vec2<u32>(421075208u,419956779u),vec2<u32>(422250504u,419956779u),vec2<u32>(721946649u,419956779u),vec2<u32>(721950984u,419956779u),vec2<u32>(134744072u,419961096u),vec2<u32>(134744107u,419961096u),vec2<u32>(134748441u,419961096u),vec2<u32>(134753032u,419961096u),vec2<u32>(134753067u,419961096u),vec2<u32>(135858201u,419961096u),vec2<u32>(135862536u,419961096u),vec2<u32>(135862571u,419961096u),vec2<u32>(135867161u,419961096u),vec2<u32>(137037832u,419961096u),vec2<u32>(137037867u,419961096u),vec2<u32>(137042201u,419961096u),vec2<u32>(137046792u,419961096u),vec2<u32>(419956761u,419961096u),vec2<u32>(419961096u,419961096u),vec2<u32>(419961131u,419961096u),vec2<u32>(419965721u,419961096u),vec2<u32>(421070856u,419961096u),vec2<u32>(421070891u,419961096u),vec2<u32>(421075225u,419961096u),vec2<u32>(421079816u,419961096u),vec2<u32>(422250521u,419961096u),vec2<u32>(422254856u,419961096u),vec2<u32>(721946632u,419961096u),vec2<u32>(721946667u,419961096u),vec2<u32>(721951001u,419961096u),vec2<u32>(721955592u,419961096u),vec2<u32>(723060761u,419961096u),vec2<u32>(723065096u,419961096u),vec2<u32>(724240392u,419961096u),vec2<u32>(134744089u,419961113u),vec2<u32>(134748424u,419961113u),vec2<u32>(134748459u,419961113u),vec2<u32>(134753049u,419961113u),vec2<u32>(135858184u,419961113u),vec2<u32>(135858219u,419961113u),vec2<u32>(135862553u,419961113u),vec2<u32>(135867144u,419961113u),vec2<u32>(137037849u,419961113u),vec2<u32>(137042184u,419961113u),vec2<u32>(419956744u,419961113u),vec2<u32>(419956779u,419961113u),vec2<u32>(419961113u,419961113u),vec2<u32>(419965704u,419961113u),vec2<u32>(421070873u,419961113u),vec2<u32>(421075208u,419961113u),vec2<u32>(422250504u,419961113u),vec2<u32>(422259499u,419961113u),vec2<u32>(721946649u,419961113u),vec2<u32>(721950984u,419961113u),vec2<u32>(723060744u,419961113u),vec2<u32>(134744072u,419961131u),vec2<u32>(134744107u,419961131u),vec2<u32>(134748441u,419961131u),vec2<u32>(134753032u,419961131u),vec2<u32>(135858201u,419961131u),vec2<u32>(135862536u,419961131u),vec2<u32>(137037832u,419961131u),vec2<u32>(419956761u,419961131u),vec2<u32>(419961096u,419961131u),vec2<u32>(421070856u,419961131u),vec2<u32>(721946632u,419961131u),vec2<u32>(724244761u,419961131u),vec2<u32>(134744089u,419965704u),vec2<u32>(134748424u,419965704u),vec2<u32>(134753049u,419965704u),vec2<u32>(135858184u,419965704u),vec2<u32>(135858219u,419965704u),vec2<u32>(135862553u,419965704u),vec2<u32>(135867144u,419965704u),vec2<u32>(137037849u,419965704u),vec2<u32>(137042184u,419965704u),vec2<u32>(419956744u,419965704u),vec2<u32>(419956779u,419965704u),vec2<u32>(419961113u,419965704u),vec2<u32>(419965704u,419965704u),vec2<u32>(421070873u,419965704u),vec2<u32>(421075208u,419965704u),vec2<u32>(422250504u,419965704u),vec2<u32>(721950984u,419965704u),vec2<u32>(723060744u,419965704u),vec2<u32>(134744072u,419965721u),vec2<u32>(134744107u,419965721u),vec2<u32>(134748441u,419965721u),vec2<u32>(134753032u,419965721u),vec2<u32>(135858201u,419965721u),vec2<u32>(135862536u,419965721u),vec2<u32>(137037832u,419965721u),vec2<u32>(419956761u,419965721u),vec2<u32>(419961096u,419965721u),vec2<u32>(421070856u,419965721u),vec2<u32>(721946632u,419965721u),vec2<u32>(723065131u,419965721u),vec2<u32>(134744089u,419965739u),vec2<u32>(134748424u,419965739u),vec2<u32>(135858184u,419965739u),vec2<u32>(419956744u,419965739u),vec2<u32>(134744072u,421070856u),vec2<u32>(134744107u,421070856u),vec2<u32>(134748441u,421070856u),vec2<u32>(134753032u,421070856u),vec2<u32>(135858201u,421070856u),vec2<u32>(135862536u,421070856u),vec2<u32>(135862571u,421070856u),vec2<u32>(135867161u,421070856u),vec2<u32>(137037832u,421070856u),vec2<u32>(137037867u,421070856u),vec2<u32>(137042201u,421070856u),vec2<u32>(137046792u,421070856u),vec2<u32>(419956761u,421070856u),vec2<u32>(419961096u,421070856u),vec2<u32>(419961131u,421070856u),vec2<u32>(419965721u,421070856u),vec2<u32>(421070856u,421070856u),vec2<u32>(421070891u,421070856u),vec2<u32>(421075225u,421070856u),vec2<u32>(421079816u,421070856u),vec2<u32>(422250521u,421070856u),vec2<u32>(422254856u,421070856u),vec2<u32>(721946632u,421070856u),vec2<u32>(721946667u,421070856u),vec2<u32>(721951001u,421070856u),vec2<u32>(721955592u,421070856u),vec2<u32>(723060761u,421070856u),vec2<u32>(723065096u,421070856u),vec2<u32>(134744089u,421070873u),vec2<u32>(134748424u,421070873u),vec2<u32>(134748459u,421070873u),vec2<u32>(134753049u,421070873u),vec2<u32>(135858184u,421070873u),vec2<u32>(135858219u,421070873u),vec2<u32>(135862553u,421070873u),vec2<u32>(135867144u,421070873u),vec2<u32>(137037849u,421070873u),vec2<u32>(137042184u,421070873u),vec2<u32>(419956744u,421070873u),vec2<u32>(419956779u,421070873u),vec2<u32>(419961113u,421070873u),vec2<u32>(419965704u,421070873u),vec2<u32>(421070873u,421070873u),vec2<u32>(421075208u,421070873u),vec2<u32>(422250504u,421070873u),vec2<u32>(721946649u,421070873u),vec2<u32>(721950984u,421070873u),vec2<u32>(723060744u,421070873u),vec2<u32>(134744072u,421070891u),vec2<u32>(134748441u,421070891u),vec2<u32>(134753032u,421070891u),vec2<u32>(135858201u,421070891u),vec2<u32>(135862536u,421070891u),vec2<u32>(137037832u,421070891u),vec2<u32>(419956761u,421070891u),vec2<u32>(419961096u,421070891u),vec2<u32>(421070856u,421070891u),vec2<u32>(422259481u,421070891u),vec2<u32>(721946632u,421070891u),vec2<u32>(134744089u,421075208u),vec2<u32>(134748424u,421075208u),vec2<u32>(134748459u,421075208u),vec2<u32>(134753049u,421075208u),vec2<u32>(135858184u,421075208u),vec2<u32>(135858219u,421075208u),vec2<u32>(135862553u,421075208u),vec2<u32>(135867144u,421075208u),vec2<u32>(137037849u,421075208u),vec2<u32>(137042184u,421075208u),vec2<u32>(419956744u,421075208u),vec2<u32>(419956779u,421075208u),vec2<u32>(419961113u,421075208u),vec2<u32>(419965704u,421075208u),vec2<u32>(421070873u,421075208u),vec2<u32>(421075208u,421075208u),vec2<u32>(422250504u,421075208u),vec2<u32>(721946649u,421075208u),vec2<u32>(721950984u,421075208u),vec2<u32>(723060744u,421075208u),vec2<u32>(134744072u,421075225u),vec2<u32>(134744107u,421075225u),vec2<u32>(134748441u,421075225u),vec2<u32>(134753032u,421075225u),vec2<u32>(135858201u,421075225u),vec2<u32>(135862536u,421075225u),vec2<u32>(137037832u,421075225u),vec2<u32>(419956761u,421075225u),vec2<u32>(419961096u,421075225u),vec2<u32>(421070856u,421075225u),vec2<u32>(721946632u,421075225u),vec2<u32>(134744089u,421075243u),vec2<u32>(134748424u,421075243u),vec2<u32>(135858184u,421075243u),vec2<u32>(137042219u,421075243u),vec2<u32>(419956744u,421075243u),vec2<u32>(134744072u,421079816u),vec2<u32>(134744107u,421079816u),vec2<u32>(134748441u,421079816u),vec2<u32>(134753032u,421079816u),vec2<u32>(135858201u,421079816u),vec2<u32>(135862536u,421079816u),vec2<u32>(137037832u,421079816u),vec2<u32>(419956761u,421079816u),vec2<u32>(419961096u,421079816u),vec2<u32>(421070856u,421079816u),vec2<u32>(421079851u,421079816u),vec2<u32>(721946632u,421079816u),vec2<u32>(134744089u,421079833u),vec2<u32>(134748424u,421079833u),vec2<u32>(135858184u,421079833u),vec2<u32>(419956744u,421079833u),vec2<u32>(134744072u,421079851u),vec2<u32>(135867161u,421079851u),vec2<u32>(721951001u,421079851u),vec2<u32>(724249352u,421079851u),vec2<u32>(134744089u,422250504u),vec2<u32>(134748424u,422250504u),vec2<u32>(134748459u,422250504u),vec2<u32>(135858184u,422250504u),vec2<u32>(135858219u,422250504u),vec2<u32>(135862553u,422250504u),vec2<u32>(135867144u,422250504u),vec2<u32>(137037849u,422250504u),vec2<u32>(137042184u,422250504u),vec2<u32>(419956744u,422250504u),vec2<u32>(419961113u,422250504u),vec2<u32>(419965704u,422250504u),vec2<u32>(421070873u,422250504u),vec2<u32>(421075208u,422250504u),vec2<u32>(422250504u,422250504u),vec2<u32>(721950984u,422250504u),vec2<u32>(723060744u,422250504u),vec2<u32>(134744072u,422250521u),vec2<u32>(134744107u,422250521u),vec2<u32>(134748441u,422250521u),vec2<u32>(134753032u,422250521u),vec2<u32>(135858201u,422250521u),vec2<u32>(135862536u,422250521u),vec2<u32>(137037832u,422250521u),vec2<u32>(419956761u,422250521u),vec2<u32>(419961096u,422250521u),vec2<u32>(421070856u,422250521u),vec2<u32>(721946632u,422250521u),vec2<u32>(723069721u,422250521u),vec2<u32>(134748424u,422250539u),vec2<u32>(135858184u,422250539u),vec2<u32>(419956744u,422250539u),vec2<u32>(421075243u,422250539u),vec2<u32>(724240409u,422250539u),vec2<u32>(134744072u,422254856u),vec2<u32>(134748441u,422254856u),vec2<u32>(134753032u,422254856u),vec2<u32>(135858201u,422254856u),vec2<u32>(135862536u,422254856u),vec2<u32>(137037832u,422254856u),vec2<u32>(419956761u,422254856u),vec2<u32>(419961096u,422254856u),vec2<u32>(421070856u,422254856u),vec2<u32>(721946632u,422254856u),vec2<u32>(134744089u,422254873u),vec2<u32>(134748424u,422254873u),vec2<u32>(135858184u,422254873u),vec2<u32>(419956744u,422254873u),vec2<u32>(419965739u,422254873u),vec2<u32>(422259464u,422254873u),vec2<u32>(723060779u,422254873u),vec2<u32>(134744072u,422254891u),vec2<u32>(723065096u,422254891u),vec2<u32>(134744089u,422259464u),vec2<u32>(134748424u,422259464u),vec2<u32>(135858184u,422259464u),vec2<u32>(422254873u,422259464u),vec2<u32>(723069704u,422259464u),vec2<u32>(134744072u,422259481u),vec2<u32>(137046827u,422259481u),vec2<u32>(419956779u,422259499u),vec2<u32>(724240409u,422259499u),vec2<u32>(134744072u,721946632u),vec2<u32>(134744107u,721946632u),vec2<u32>(134748441u,721946632u),vec2<u32>(134753032u,721946632u),vec2<u32>(135858201u,721946632u),vec2<u32>(135862536u,721946632u),vec2<u32>(135867161u,721946632u),vec2<u32>(137037832u,721946632u),vec2<u32>(137042201u,721946632u),vec2<u32>(419956761u,721946632u),vec2<u32>(419961096u,721946632u),vec2<u32>(421070856u,721946632u),vec2<u32>(421070891u,721946632u),vec2<u32>(421075225u,721946632u),vec2<u32>(421079816u,721946632u),vec2<u32>(422250521u,721946632u),vec2<u32>(721946632u,721946632u),vec2<u32>(721951001u,721946632u),vec2<u32>(723060761u,721946632u),vec2<u32>(723065096u,721946632u),vec2<u32>(134744089u,721946649u),vec2<u32>(134748424u,721946649u),vec2<u32>(134753049u,721946649u),vec2<u32>(135858184u,721946649u),vec2<u32>(135858219u,721946649u),vec2<u32>(135862553u,721946649u),vec2<u32>(135867144u,721946649u),vec2<u32>(137037849u,721946649u),vec2<u32>(137042184u,721946649u),vec2<u32>(419956744u,721946649u),vec2<u32>(419956779u,721946649u),vec2<u32>(419961113u,721946649u),vec2<u32>(419965704u,721946649u),vec2<u32>(421070873u,721946649u),vec2<u32>(421075208u,721946649u),vec2<u32>(721946649u,721946649u),vec2<u32>(721950984u,721946649u),vec2<u32>(723060744u,721946649u),vec2<u32>(724249369u,721946649u),vec2<u32>(134744072u,721946667u),vec2<u32>(134748441u,721946667u),vec2<u32>(134753067u,721946667u),vec2<u32>(135858201u,721946667u),vec2<u32>(135862536u,721946667u),vec2<u32>(419956761u,721946667u),vec2<u32>(419961096u,721946667u),vec2<u32>(421070856u,721946667u),vec2<u32>(134744089u,721950984u),vec2<u32>(134748424u,721950984u),vec2<u32>(134748459u,721950984u),vec2<u32>(134753049u,721950984u),vec2<u32>(135858184u,721950984u),vec2<u32>(135858219u,721950984u),vec2<u32>(135862553u,721950984u),vec2<u32>(135867144u,721950984u),vec2<u32>(137037849u,721950984u),vec2<u32>(419956744u,721950984u),vec2<u32>(419956779u,721950984u),vec2<u32>(419961113u,721950984u),vec2<u32>(419965704u,721950984u),vec2<u32>(421070873u,721950984u),vec2<u32>(421075208u,721950984u),vec2<u32>(422250504u,721950984u),vec2<u32>(721946649u,721950984u),vec2<u32>(721950984u,721950984u),vec2<u32>(723060744u,721950984u),vec2<u32>(134744072u,721951001u),vec2<u32>(134744107u,721951001u),vec2<u32>(134748441u,721951001u),vec2<u32>(134753032u,721951001u),vec2<u32>(135858201u,721951001u),vec2<u32>(135862536u,721951001u),vec2<u32>(137037832u,721951001u),vec2<u32>(419956761u,721951001u),vec2<u32>(419961096u,721951001u),vec2<u32>(421070856u,721951001u),vec2<u32>(721946632u,721951001u),vec2<u32>(721955627u,721951001u),vec2<u32>(134744089u,721951019u),vec2<u32>(134748424u,721951019u),vec2<u32>(135858184u,721951019u),vec2<u32>(137046809u,721951019u),vec2<u32>(419956744u,721951019u),vec2<u32>(134744072u,721955592u),vec2<u32>(134748441u,721955592u),vec2<u32>(135858201u,721955592u),vec2<u32>(135862536u,721955592u),vec2<u32>(419956761u,721955592u),vec2<u32>(419961096u,721955592u),vec2<u32>(421070856u,721955592u),vec2<u32>(724240427u,721955592u),vec2<u32>(134744089u,721955609u),vec2<u32>(134748424u,721955609u),vec2<u32>(419956744u,721955609u),vec2<u32>(422254873u,721955609u),vec2<u32>(137037867u,721955627u),vec2<u32>(421079816u,721955627u),vec2<u32>(421079851u,721955627u),vec2<u32>(721946667u,721955627u),vec2<u32>(724240427u,721955627u),vec2<u32>(134744089u,723060744u),vec2<u32>(134748424u,723060744u),vec2<u32>(134753049u,723060744u),vec2<u32>(135858184u,723060744u),vec2<u32>(135858219u,723060744u),vec2<u32>(135862553u,723060744u),vec2<u32>(135867144u,723060744u),vec2<u32>(137042184u,723060744u),vec2<u32>(419956744u,723060744u),vec2<u32>(419956779u,723060744u),vec2<u32>(419961113u,723060744u),vec2<u32>(419965704u,723060744u),vec2<u32>(421070873u,723060744u),vec2<u32>(421075208u,723060744u),vec2<u32>(422250504u,723060744u),vec2<u32>(721946649u,723060744u),vec2<u32>(721950984u,723060744u),vec2<u32>(723060744u,723060744u),vec2<u32>(134744072u,723060761u),vec2<u32>(134748441u,723060761u),vec2<u32>(135858201u,723060761u),vec2<u32>(135862536u,723060761u),vec2<u32>(419956761u,723060761u),vec2<u32>(419961096u,723060761u),vec2<u32>(421070856u,723060761u),vec2<u32>(421079851u,723060761u),vec2<u32>(134744089u,723060779u),vec2<u32>(134748424u,723060779u),vec2<u32>(135858184u,723060779u),vec2<u32>(419956744u,723060779u),vec2<u32>(724244779u,723060779u),vec2<u32>(134744072u,723065096u),vec2<u32>(134744107u,723065096u),vec2<u32>(134748441u,723065096u),vec2<u32>(134753032u,723065096u),vec2<u32>(135858201u,723065096u),vec2<u32>(135862536u,723065096u),vec2<u32>(137037832u,723065096u),vec2<u32>(419956761u,723065096u),vec2<u32>(419961096u,723065096u),vec2<u32>(421070856u,723065096u),vec2<u32>(721946632u,723065096u),vec2<u32>(723065131u,723065096u),vec2<u32>(134744089u,723065113u),vec2<u32>(134748424u,723065113u),vec2<u32>(135858184u,723065113u),vec2<u32>(419956744u,723065113u),vec2<u32>(723069704u,723065113u),vec2<u32>(724240409u,723065113u),vec2<u32>(134744072u,723065131u),vec2<u32>(419961131u,723065131u),vec2<u32>(422254856u,723065131u),vec2<u32>(134744089u,723069704u),vec2<u32>(134748424u,723069704u),vec2<u32>(135858184u,723069704u),vec2<u32>(137042219u,723069704u),vec2<u32>(419956744u,723069704u),vec2<u32>(724249369u,723069704u),vec2<u32>(134744072u,723069721u),vec2<u32>(419965721u,723069721u),vec2<u32>(421070891u,723069721u),vec2<u32>(723060744u,723069739u),vec2<u32>(134744072u,724240392u),vec2<u32>(134748441u,724240392u),vec2<u32>(134753067u,724240392u),vec2<u32>(135862536u,724240392u),vec2<u32>(137037867u,724240392u),vec2<u32>(137046827u,724240392u),vec2<u32>(419956761u,724240392u),vec2<u32>(419961096u,724240392u),vec2<u32>(421070856u,724240392u),vec2<u32>(724240427u,724240392u),vec2<u32>(724249387u,724240392u),vec2<u32>(419956744u,724240409u),vec2<u32>(422254873u,724240409u),vec2<u32>(134744107u,724240427u),vec2<u32>(134753067u,724240427u),vec2<u32>(137037867u,724240427u),vec2<u32>(137046792u,724240427u),vec2<u32>(137046827u,724240427u),vec2<u32>(721946667u,724240427u),vec2<u32>(721955592u,724240427u),vec2<u32>(721955627u,724240427u),vec2<u32>(724249352u,724240427u),vec2<u32>(134744089u,724244744u),vec2<u32>(134748424u,724244744u),vec2<u32>(135858184u,724244744u),vec2<u32>(419956744u,724244744u),vec2<u32>(721955609u,724244744u),vec2<u32>(724244744u,724244744u),vec2<u32>(134744072u,724244761u),vec2<u32>(135867161u,724244761u),vec2<u32>(421070873u,724244779u),vec2<u32>(134753067u,724249352u),vec2<u32>(137046792u,724249352u),vec2<u32>(724240427u,724249352u),vec2<u32>(421075208u,724249369u),vec2<u32>(721951019u,724249369u),vec2<u32>(134753032u,724249387u),vec2<u32>(134753067u,724249387u),vec2<u32>(137037832u,724249387u),vec2<u32>(137037867u,724249387u),vec2<u32>(137046792u,724249387u),vec2<u32>(721955592u,724249387u),vec2<u32>(724249387u,724249387u));
{% if not (iqInline | default(false)) %}fn iq_grid{{ iqSfx }}(i:u32)->vec2<u32>{return IQ_GRID_0{{ iqSfx }}[i];}
{% endif %}{% macro iq2sLo_at(i, sfx="") %}vec4<f32>(unpack4xU8(IQ_GRID_0{{ sfx }}[{{ i }}].x)){% endmacro %}{% macro iq2sHi_at(i, sfx="") %}vec4<f32>(unpack4xU8(IQ_GRID_0{{ sfx }}[{{ i }}].y)){% endmacro %}`],["ops/_shared/iq-grid-iq2xs-mag.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}const IQ_MAG8_0{{ iqSfx }}:array<vec2<u32>,1024>=array<vec2<u32>,1024>(vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837984u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365265984u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837984u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1312837184u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1312837184u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1312837984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1312837984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1365264384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1365265984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365265984u,1365265984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365266784u,1207979584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365264384u,1312837984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1312837184u,1312837184u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1365266784u,1365264384u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312837984u,1312837184u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207979584u,1312837984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1312837984u,1207980384u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365266784u,1365266784u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1365265984u,1365264384u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1312837984u,1365265984u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1365266784u,1312837984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1312837984u,1365265984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1312835584u,1312837984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837984u,1207979584u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1312837984u,1207979584u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1312837984u,1207979584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1312837984u,1365264384u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365265984u,1207979584u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1365265984u,1207979584u),vec2<u32>(1312837984u,1312837984u),vec2<u32>(1365265984u,1207979584u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1365266784u,1207979584u),vec2<u32>(1365265984u,1365266784u),vec2<u32>(1365266784u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1312837184u,1312837984u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1365265984u,1207980384u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1207980384u,1312837184u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1365265984u,1365265984u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312837984u,1207980384u),vec2<u32>(1365266784u,1207979584u),vec2<u32>(1312837984u,1207980384u),vec2<u32>(1312837984u,1365266784u),vec2<u32>(1312837984u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1312837184u,1312837184u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1207980384u,1365266784u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1365264384u,1312837984u),vec2<u32>(1365265984u,1207980384u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1365265984u,1207980384u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365266784u,1207980384u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1365266784u,1207980384u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1365266784u,1207980384u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1365266784u,1207980384u),vec2<u32>(1365266784u,1365264384u),vec2<u32>(1365266784u,1207980384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365266784u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312837184u,1312837984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1312837184u,1365264384u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207980384u,1365266784u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1365265984u,1207980384u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1365265984u,1207979584u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1365266784u,1365265984u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837984u,1312835584u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1312837984u,1312835584u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1312837984u,1312835584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1312837984u,1312835584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207980384u,1312837984u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1365265984u,1312837984u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365266784u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1312837184u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207980384u,1312837184u),vec2<u32>(1365266784u,1365266784u),vec2<u32>(1207980384u,1312837184u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207979584u,1365266784u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1312837184u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312837184u,1312837184u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1312837184u,1312837184u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1312837184u,1312837184u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1312837984u,1312837184u),vec2<u32>(1365264384u,1312837984u),vec2<u32>(1312837984u,1312837184u),vec2<u32>(1207979584u,1365266784u),vec2<u32>(1312837984u,1312837184u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1365265984u,1365265984u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1365266784u,1207979584u),vec2<u32>(1365265984u,1312837184u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365265984u,1312837184u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1365265984u,1312837184u),vec2<u32>(1312837184u,1365264384u),vec2<u32>(1365266784u,1312837184u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207980384u,1312837984u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1312837984u,1365264384u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1365265984u,1365266784u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1312837984u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207980384u,1312837984u),vec2<u32>(1365266784u,1312835584u),vec2<u32>(1207980384u,1312837984u),vec2<u32>(1207980384u,1365265984u),vec2<u32>(1207980384u,1312837984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1312837984u),vec2<u32>(1312837984u,1207979584u),vec2<u32>(1312835584u,1312837984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312837184u,1312837984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312837184u,1312837984u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1312837184u,1312837984u),vec2<u32>(1312835584u,1365266784u),vec2<u32>(1312837184u,1312837984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365264384u,1312837984u),vec2<u32>(1365266784u,1312837984u),vec2<u32>(1365264384u,1312837984u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1365265984u,1312837984u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1365266784u,1312837984u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1365266784u,1312837984u),vec2<u32>(1207980384u,1312837984u),vec2<u32>(1365266784u,1312837984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1365266784u,1207980384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1365266784u,1365266784u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1365265984u,1312837184u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1365265984u,1365264384u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1365264384u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1312837184u,1365264384u),vec2<u32>(1312837184u,1365266784u),vec2<u32>(1312837184u,1365264384u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1312837984u,1365264384u),vec2<u32>(1365266784u,1312837984u),vec2<u32>(1312837984u,1365264384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1365266784u,1312837184u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1365265984u,1365264384u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1365266784u,1365264384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1365266784u,1365264384u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1365266784u,1365264384u),vec2<u32>(1312837984u,1365265984u),vec2<u32>(1365266784u,1365264384u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1365266784u,1365264384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1312837984u,1312837184u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1365265984u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1207979584u,1365265984u),vec2<u32>(1312835584u,1312837984u),vec2<u32>(1207979584u,1365265984u),vec2<u32>(1312837984u,1312837184u),vec2<u32>(1207980384u,1365265984u),vec2<u32>(1365265984u,1365264384u),vec2<u32>(1207980384u,1365265984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1365265984u,1207980384u),vec2<u32>(1312837184u,1365265984u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1312837184u,1365265984u),vec2<u32>(1207980384u,1365265984u),vec2<u32>(1312837184u,1365265984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1312837984u,1365265984u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1365264384u,1365265984u),vec2<u32>(1312837984u,1365266784u),vec2<u32>(1365264384u,1365265984u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1365265984u,1365265984u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1365266784u,1365265984u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1365266784u,1365265984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1365266784u,1207980384u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1207979584u,1365266784u),vec2<u32>(1365265984u,1312837184u),vec2<u32>(1207979584u,1365266784u),vec2<u32>(1312837984u,1365266784u),vec2<u32>(1207979584u,1365266784u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1365266784u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207980384u,1365266784u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207980384u,1365266784u),vec2<u32>(1365266784u,1207980384u),vec2<u32>(1207980384u,1365266784u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207980384u,1365266784u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1207980384u,1365266784u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1365266784u),vec2<u32>(1312837184u,1365265984u),vec2<u32>(1312835584u,1365266784u),vec2<u32>(1312837184u,1312837984u),vec2<u32>(1312837984u,1365266784u),vec2<u32>(1365264384u,1365265984u),vec2<u32>(1312837984u,1365266784u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365265984u,1365266784u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1365265984u,1365266784u),vec2<u32>(1312837984u,1365264384u),vec2<u32>(1365265984u,1365266784u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1365266784u,1365266784u),vec2<u32>(1365266784u,1207980384u),vec2<u32>(1365266784u,1365266784u),vec2<u32>(1207979584u,1365265984u),vec2<u32>(1365266784u,1365266784u),vec2<u32>(1365266784u,1365266784u),vec2<u32>(1365266784u,1365266784u));fn iq_f16x4{{ iqSfx }}(e:vec2<u32>)->vec4<f32>{return vec4<f32>(unpack2x16float(e.x),unpack2x16float(e.y));}
{% macro iq2magLo_at(i, sfx="") %}iq_f16x4{{ sfx }}(IQ_MAG8_0{{ sfx }}[{{ i }} * 2u]){% endmacro %}{% macro iq2magHi_at(i, sfx="") %}iq_f16x4{{ sfx }}(IQ_MAG8_0{{ sfx }}[{{ i }} * 2u + 1u]){% endmacro %}`],["ops/_shared/iq-grid-iq2xs.wgsl.jinja",'{% set iqSfx = iqSfx | default("") %}const IQ_GRID_0{{ iqSfx }}:array<vec2<u32>,512>=array<vec2<u32>,512>(vec2<u32>(134744072u,134744072u),vec2<u32>(134744107u,134744072u),vec2<u32>(134748441u,134744072u),vec2<u32>(134753032u,134744072u),vec2<u32>(134753067u,134744072u),vec2<u32>(135858201u,134744072u),vec2<u32>(135862536u,134744072u),vec2<u32>(135862571u,134744072u),vec2<u32>(135867161u,134744072u),vec2<u32>(137037832u,134744072u),vec2<u32>(137037867u,134744072u),vec2<u32>(137042201u,134744072u),vec2<u32>(137046792u,134744072u),vec2<u32>(419956761u,134744072u),vec2<u32>(419961096u,134744072u),vec2<u32>(419961131u,134744072u),vec2<u32>(419965721u,134744072u),vec2<u32>(421070856u,134744072u),vec2<u32>(421070891u,134744072u),vec2<u32>(421075225u,134744072u),vec2<u32>(421079816u,134744072u),vec2<u32>(422250521u,134744072u),vec2<u32>(422254856u,134744072u),vec2<u32>(721946632u,134744072u),vec2<u32>(721946667u,134744072u),vec2<u32>(721951001u,134744072u),vec2<u32>(721955592u,134744072u),vec2<u32>(723060761u,134744072u),vec2<u32>(723065096u,134744072u),vec2<u32>(723069721u,134744072u),vec2<u32>(724240392u,134744072u),vec2<u32>(134744089u,134744089u),vec2<u32>(134748424u,134744089u),vec2<u32>(134748459u,134744089u),vec2<u32>(134753049u,134744089u),vec2<u32>(135858184u,134744089u),vec2<u32>(135858219u,134744089u),vec2<u32>(135862553u,134744089u),vec2<u32>(135867144u,134744089u),vec2<u32>(135867179u,134744089u),vec2<u32>(137037849u,134744089u),vec2<u32>(137042184u,134744089u),vec2<u32>(419956744u,134744089u),vec2<u32>(419956779u,134744089u),vec2<u32>(419961113u,134744089u),vec2<u32>(419965704u,134744089u),vec2<u32>(421070873u,134744089u),vec2<u32>(421075208u,134744089u),vec2<u32>(422250504u,134744089u),vec2<u32>(422259464u,134744089u),vec2<u32>(721946649u,134744089u),vec2<u32>(721950984u,134744089u),vec2<u32>(723060744u,134744089u),vec2<u32>(134744072u,134744107u),vec2<u32>(134744107u,134744107u),vec2<u32>(134748441u,134744107u),vec2<u32>(134753032u,134744107u),vec2<u32>(135858201u,134744107u),vec2<u32>(135862536u,134744107u),vec2<u32>(137037832u,134744107u),vec2<u32>(419956761u,134744107u),vec2<u32>(419961096u,134744107u),vec2<u32>(421070856u,134744107u),vec2<u32>(421075225u,134744107u),vec2<u32>(721946632u,134744107u),vec2<u32>(721955627u,134744107u),vec2<u32>(134744089u,134748424u),vec2<u32>(134748424u,134748424u),vec2<u32>(134748459u,134748424u),vec2<u32>(134753049u,134748424u),vec2<u32>(135858184u,134748424u),vec2<u32>(135858219u,134748424u),vec2<u32>(135862553u,134748424u),vec2<u32>(135867144u,134748424u),vec2<u32>(137037849u,134748424u),vec2<u32>(137042184u,134748424u),vec2<u32>(419956744u,134748424u),vec2<u32>(419956779u,134748424u),vec2<u32>(419961113u,134748424u),vec2<u32>(419965704u,134748424u),vec2<u32>(421070873u,134748424u),vec2<u32>(421075208u,134748424u),vec2<u32>(421075243u,134748424u),vec2<u32>(422250504u,134748424u),vec2<u32>(721946649u,134748424u),vec2<u32>(721950984u,134748424u),vec2<u32>(723060744u,134748424u),vec2<u32>(134744072u,134748441u),vec2<u32>(134744107u,134748441u),vec2<u32>(134748441u,134748441u),vec2<u32>(134753032u,134748441u),vec2<u32>(135858201u,134748441u),vec2<u32>(135862536u,134748441u),vec2<u32>(137037832u,134748441u),vec2<u32>(419956761u,134748441u),vec2<u32>(419961096u,134748441u),vec2<u32>(421070856u,134748441u),vec2<u32>(422250521u,134748441u),vec2<u32>(721946632u,134748441u),vec2<u32>(134744089u,134748459u),vec2<u32>(134748424u,134748459u),vec2<u32>(135858184u,134748459u),vec2<u32>(137042219u,134748459u),vec2<u32>(419956744u,134748459u),vec2<u32>(419956779u,134748459u),vec2<u32>(721950984u,134748459u),vec2<u32>(134744072u,134753032u),vec2<u32>(134744107u,134753032u),vec2<u32>(134748441u,134753032u),vec2<u32>(134753032u,134753032u),vec2<u32>(134753067u,134753032u),vec2<u32>(135858201u,134753032u),vec2<u32>(135862536u,134753032u),vec2<u32>(137037832u,134753032u),vec2<u32>(137042201u,134753032u),vec2<u32>(419956761u,134753032u),vec2<u32>(419961096u,134753032u),vec2<u32>(421070856u,134753032u),vec2<u32>(421079816u,134753032u),vec2<u32>(721946632u,134753032u),vec2<u32>(724240392u,134753032u),vec2<u32>(724249387u,134753032u),vec2<u32>(134744089u,134753049u),vec2<u32>(134748424u,134753049u),vec2<u32>(135858184u,134753049u),vec2<u32>(419956744u,134753049u),vec2<u32>(721946649u,134753049u),vec2<u32>(721955609u,134753049u),vec2<u32>(134744072u,134753067u),vec2<u32>(137037832u,134753067u),vec2<u32>(137046792u,134753067u),vec2<u32>(723065131u,134753067u),vec2<u32>(724240392u,134753067u),vec2<u32>(134744089u,135858184u),vec2<u32>(134748424u,135858184u),vec2<u32>(134748459u,135858184u),vec2<u32>(134753049u,135858184u),vec2<u32>(135858184u,135858184u),vec2<u32>(135858219u,135858184u),vec2<u32>(135862553u,135858184u),vec2<u32>(135867144u,135858184u),vec2<u32>(137037849u,135858184u),vec2<u32>(137042184u,135858184u),vec2<u32>(419956744u,135858184u),vec2<u32>(419956779u,135858184u),vec2<u32>(419961113u,135858184u),vec2<u32>(419965704u,135858184u),vec2<u32>(421070873u,135858184u),vec2<u32>(421075208u,135858184u),vec2<u32>(422250504u,135858184u),vec2<u32>(422259499u,135858184u),vec2<u32>(721946649u,135858184u),vec2<u32>(721950984u,135858184u),vec2<u32>(723060744u,135858184u),vec2<u32>(134744072u,135858201u),vec2<u32>(134744107u,135858201u),vec2<u32>(134748441u,135858201u),vec2<u32>(134753032u,135858201u),vec2<u32>(135858201u,135858201u),vec2<u32>(135862536u,135858201u),vec2<u32>(137037832u,135858201u),vec2<u32>(419956761u,135858201u),vec2<u32>(419961096u,135858201u),vec2<u32>(421070856u,135858201u),vec2<u32>(721946632u,135858201u),vec2<u32>(723065096u,135858201u),vec2<u32>(723065131u,135858201u),vec2<u32>(134744089u,135858219u),vec2<u32>(134748424u,135858219u),vec2<u32>(134748459u,135858219u),vec2<u32>(135858184u,135858219u),vec2<u32>(419956744u,135858219u),vec2<u32>(422250504u,135858219u),vec2<u32>(134744072u,135862536u),vec2<u32>(134744107u,135862536u),vec2<u32>(134748441u,135862536u),vec2<u32>(134753032u,135862536u),vec2<u32>(135858201u,135862536u),vec2<u32>(135862536u,135862536u),vec2<u32>(137037832u,135862536u),vec2<u32>(419956761u,135862536u),vec2<u32>(419961096u,135862536u),vec2<u32>(419965721u,135862536u),vec2<u32>(421070856u,135862536u),vec2<u32>(422254856u,135862536u),vec2<u32>(721946632u,135862536u),vec2<u32>(134744089u,135862553u),vec2<u32>(134748424u,135862553u),vec2<u32>(135858184u,135862553u),vec2<u32>(419956744u,135862553u),vec2<u32>(134744072u,135862571u),vec2<u32>(135862536u,135862571u),vec2<u32>(419965721u,135862571u),vec2<u32>(134744089u,135867144u),vec2<u32>(134748424u,135867144u),vec2<u32>(135858184u,135867144u),vec2<u32>(135858219u,135867144u),vec2<u32>(419956744u,135867144u),vec2<u32>(421075208u,135867144u),vec2<u32>(721951019u,135867144u),vec2<u32>(134744072u,135867161u),vec2<u32>(134748441u,135867161u),vec2<u32>(422254891u,135867161u),vec2<u32>(421070873u,135867179u),vec2<u32>(724249369u,135867179u),vec2<u32>(134744072u,137037832u),vec2<u32>(134744107u,137037832u),vec2<u32>(134748441u,137037832u),vec2<u32>(134753032u,137037832u),vec2<u32>(134753067u,137037832u),vec2<u32>(135858201u,137037832u),vec2<u32>(135862536u,137037832u),vec2<u32>(137037832u,137037832u),vec2<u32>(419956761u,137037832u),vec2<u32>(419961096u,137037832u),vec2<u32>(421070856u,137037832u),vec2<u32>(721946632u,137037832u),vec2<u32>(724240392u,137037832u),vec2<u32>(134744089u,137037849u),vec2<u32>(134748424u,137037849u),vec2<u32>(135858184u,137037849u),vec2<u32>(419956744u,137037849u),vec2<u32>(419965704u,137037849u),vec2<u32>(422254873u,137037849u),vec2<u32>(134744072u,137037867u),vec2<u32>(137037867u,137037867u),vec2<u32>(721946632u,137037867u),vec2<u32>(724249352u,137037867u),vec2<u32>(134744089u,137042184u),vec2<u32>(134748424u,137042184u),vec2<u32>(135858184u,137042184u),vec2<u32>(137046809u,137042184u),vec2<u32>(419956744u,137042184u),vec2<u32>(134744072u,137042201u),vec2<u32>(419956761u,137042201u),vec2<u32>(421070891u,137042201u),vec2<u32>(723069721u,137042201u),vec2<u32>(134744089u,137042219u),vec2<u32>(135867179u,137042219u),vec2<u32>(724244779u,137042219u),vec2<u32>(134744072u,137046792u),vec2<u32>(134753032u,137046792u),vec2<u32>(134753067u,137046792u),vec2<u32>(137037832u,137046792u),vec2<u32>(421075225u,137046792u),vec2<u32>(721955592u,137046792u),vec2<u32>(724240427u,137046792u),vec2<u32>(422259464u,137046809u),vec2<u32>(723060744u,137046809u),vec2<u32>(134753032u,137046827u),vec2<u32>(137037832u,137046827u),vec2<u32>(721946667u,137046827u),vec2<u32>(721955592u,137046827u),vec2<u32>(721955627u,137046827u),vec2<u32>(134744089u,419956744u),vec2<u32>(134748424u,419956744u),vec2<u32>(134748459u,419956744u),vec2<u32>(134753049u,419956744u),vec2<u32>(135858184u,419956744u),vec2<u32>(135858219u,419956744u),vec2<u32>(135862553u,419956744u),vec2<u32>(135867144u,419956744u),vec2<u32>(137037849u,419956744u),vec2<u32>(137042184u,419956744u),vec2<u32>(419956744u,419956744u),vec2<u32>(419956779u,419956744u),vec2<u32>(419961113u,419956744u),vec2<u32>(419965704u,419956744u),vec2<u32>(419965739u,419956744u),vec2<u32>(421070873u,419956744u),vec2<u32>(421075208u,419956744u),vec2<u32>(422250504u,419956744u),vec2<u32>(422254873u,419956744u),vec2<u32>(721946649u,419956744u),vec2<u32>(721950984u,419956744u),vec2<u32>(723060744u,419956744u),vec2<u32>(134744072u,419956761u),vec2<u32>(134744107u,419956761u),vec2<u32>(134748441u,419956761u),vec2<u32>(134753032u,419956761u),vec2<u32>(135858201u,419956761u),vec2<u32>(135862536u,419956761u),vec2<u32>(137037832u,419956761u),vec2<u32>(419956761u,419956761u),vec2<u32>(419961096u,419956761u),vec2<u32>(421070856u,419956761u),vec2<u32>(721946632u,419956761u),vec2<u32>(721951001u,419956761u),vec2<u32>(724240427u,419956761u),vec2<u32>(134744089u,419956779u),vec2<u32>(134748424u,419956779u),vec2<u32>(135858184u,419956779u),vec2<u32>(135858219u,419956779u),vec2<u32>(137046809u,419956779u),vec2<u32>(419956744u,419956779u),vec2<u32>(134744072u,419961096u),vec2<u32>(134744107u,419961096u),vec2<u32>(134748441u,419961096u),vec2<u32>(134753032u,419961096u),vec2<u32>(135858201u,419961096u),vec2<u32>(135862536u,419961096u),vec2<u32>(135867161u,419961096u),vec2<u32>(137037832u,419961096u),vec2<u32>(419956761u,419961096u),vec2<u32>(419961096u,419961096u),vec2<u32>(421070856u,419961096u),vec2<u32>(721946632u,419961096u),vec2<u32>(723065096u,419961096u),vec2<u32>(134744089u,419961113u),vec2<u32>(134748424u,419961113u),vec2<u32>(135858184u,419961113u),vec2<u32>(137042184u,419961113u),vec2<u32>(419956744u,419961113u),vec2<u32>(723069739u,419961113u),vec2<u32>(134744072u,419961131u),vec2<u32>(134753067u,419961131u),vec2<u32>(419961096u,419961131u),vec2<u32>(421070856u,419961131u),vec2<u32>(134744089u,419965704u),vec2<u32>(134748424u,419965704u),vec2<u32>(135858184u,419965704u),vec2<u32>(419956744u,419965704u),vec2<u32>(419961113u,419965704u),vec2<u32>(421075208u,419965704u),vec2<u32>(422250539u,419965704u),vec2<u32>(134744072u,419965721u),vec2<u32>(135858201u,419965721u),vec2<u32>(419961096u,419965721u),vec2<u32>(421070856u,419965721u),vec2<u32>(422259481u,419965721u),vec2<u32>(134748424u,419965739u),vec2<u32>(134744072u,421070856u),vec2<u32>(134744107u,421070856u),vec2<u32>(134748441u,421070856u),vec2<u32>(134753032u,421070856u),vec2<u32>(135858201u,421070856u),vec2<u32>(135862536u,421070856u),vec2<u32>(137037832u,421070856u),vec2<u32>(137046792u,421070856u),vec2<u32>(419956761u,421070856u),vec2<u32>(419961096u,421070856u),vec2<u32>(421070856u,421070856u),vec2<u32>(721946632u,421070856u),vec2<u32>(134744089u,421070873u),vec2<u32>(134748424u,421070873u),vec2<u32>(135858184u,421070873u),vec2<u32>(135862553u,421070873u),vec2<u32>(419956744u,421070873u),vec2<u32>(419956779u,421070873u),vec2<u32>(134744072u,421070891u),vec2<u32>(419961096u,421070891u),vec2<u32>(724249387u,421070891u),vec2<u32>(134744089u,421075208u),vec2<u32>(134748424u,421075208u),vec2<u32>(135858184u,421075208u),vec2<u32>(137037849u,421075208u),vec2<u32>(419956744u,421075208u),vec2<u32>(422250504u,421075208u),vec2<u32>(721946649u,421075208u),vec2<u32>(724240409u,421075208u),vec2<u32>(134744072u,421075225u),vec2<u32>(134753032u,421075225u),vec2<u32>(721946632u,421075225u),vec2<u32>(721955592u,421075225u),vec2<u32>(137037849u,421075243u),vec2<u32>(422259464u,421075243u),vec2<u32>(724240409u,421075243u),vec2<u32>(134744072u,421079816u),vec2<u32>(135862536u,421079816u),vec2<u32>(419956761u,421079816u),vec2<u32>(421070856u,421079816u),vec2<u32>(723069721u,421079816u),vec2<u32>(135867179u,421079833u),vec2<u32>(419956744u,421079833u),vec2<u32>(419956779u,421079833u),vec2<u32>(721951001u,421079851u),vec2<u32>(134744089u,422250504u),vec2<u32>(134748424u,422250504u),vec2<u32>(135858184u,422250504u),vec2<u32>(419956744u,422250504u),vec2<u32>(421075208u,422250504u),vec2<u32>(422250539u,422250504u),vec2<u32>(721951019u,422250504u),vec2<u32>(724249369u,422250504u),vec2<u32>(134744072u,422250521u),vec2<u32>(137042184u,422250539u),vec2<u32>(419965739u,422250539u),vec2<u32>(723060779u,422250539u),vec2<u32>(134744072u,422254856u),vec2<u32>(135862571u,422254856u),vec2<u32>(135858184u,422254873u),vec2<u32>(419956744u,422254873u),vec2<u32>(419961113u,422254873u),vec2<u32>(724244744u,422254873u),vec2<u32>(134744089u,422259464u),vec2<u32>(422259499u,422259464u),vec2<u32>(137042201u,422259481u),vec2<u32>(134748459u,422259499u),vec2<u32>(421075208u,422259499u),vec2<u32>(422250539u,422259499u),vec2<u32>(134744072u,721946632u),vec2<u32>(134744107u,721946632u),vec2<u32>(134748441u,721946632u),vec2<u32>(134753032u,721946632u),vec2<u32>(135858201u,721946632u),vec2<u32>(135862536u,721946632u),vec2<u32>(137037832u,721946632u),vec2<u32>(137046827u,721946632u),vec2<u32>(419956761u,721946632u),vec2<u32>(419961096u,721946632u),vec2<u32>(421070856u,721946632u),vec2<u32>(721946632u,721946632u),vec2<u32>(721946667u,721946632u),vec2<u32>(724249352u,721946632u),vec2<u32>(724249387u,721946632u),vec2<u32>(134744089u,721946649u),vec2<u32>(134748424u,721946649u),vec2<u32>(134748459u,721946649u),vec2<u32>(135858184u,721946649u),vec2<u32>(419956744u,721946649u),vec2<u32>(421070873u,721946649u),vec2<u32>(421079833u,721946649u),vec2<u32>(134744072u,721946667u),vec2<u32>(137037832u,721946667u),vec2<u32>(721946632u,721946667u),vec2<u32>(721946667u,721946667u),vec2<u32>(724240392u,721946667u),vec2<u32>(724249352u,721946667u),vec2<u32>(134744089u,721950984u),vec2<u32>(134748424u,721950984u),vec2<u32>(135858184u,721950984u),vec2<u32>(135858219u,721950984u),vec2<u32>(135862553u,721950984u),vec2<u32>(419956744u,721950984u),vec2<u32>(422250504u,721950984u),vec2<u32>(721955609u,721950984u),vec2<u32>(134744072u,721951001u),vec2<u32>(419961096u,721951001u),vec2<u32>(724244761u,721951001u),vec2<u32>(135867144u,721951019u),vec2<u32>(422259499u,721951019u),vec2<u32>(134744072u,721955592u),vec2<u32>(134753032u,721955592u),vec2<u32>(137042201u,721955592u),vec2<u32>(421079851u,721955592u),vec2<u32>(721946632u,721955592u),vec2<u32>(721946667u,721955592u),vec2<u32>(724249352u,721955592u),vec2<u32>(134748459u,721955609u),vec2<u32>(137037867u,721955627u),vec2<u32>(721946632u,721955627u),vec2<u32>(721955592u,721955627u),vec2<u32>(723065131u,721955627u),vec2<u32>(724249352u,721955627u),vec2<u32>(134744089u,723060744u),vec2<u32>(134748424u,723060744u),vec2<u32>(135858184u,723060744u),vec2<u32>(419956744u,723060744u),vec2<u32>(421075243u,723060744u),vec2<u32>(721950984u,723060744u),vec2<u32>(134744072u,723060761u),vec2<u32>(137037867u,723060761u),vec2<u32>(422254856u,723060761u),vec2<u32>(421075243u,723060779u),vec2<u32>(721955609u,723060779u),vec2<u32>(134744072u,723065096u),vec2<u32>(134748441u,723065096u),vec2<u32>(419961096u,723065096u),vec2<u32>(421070856u,723065096u),vec2<u32>(421079816u,723065096u),vec2<u32>(137046809u,723065113u),vec2<u32>(723060744u,723065113u),vec2<u32>(723060779u,723065113u),vec2<u32>(419956761u,723065131u),vec2<u32>(421070873u,723069704u),vec2<u32>(724244779u,723069704u),vec2<u32>(419965721u,723069721u),vec2<u32>(135862553u,723069739u),vec2<u32>(422250504u,723069739u),vec2<u32>(134744072u,724240392u),vec2<u32>(134744107u,724240392u),vec2<u32>(134753032u,724240392u),vec2<u32>(134753067u,724240392u),vec2<u32>(137037832u,724240392u),vec2<u32>(137046827u,724240392u),vec2<u32>(724240392u,724240392u),vec2<u32>(421070873u,724240409u),vec2<u32>(421079833u,724240409u),vec2<u32>(724244779u,724240409u),vec2<u32>(134744072u,724240427u),vec2<u32>(134744107u,724240427u),vec2<u32>(134753032u,724240427u),vec2<u32>(137046827u,724240427u),vec2<u32>(721946632u,724240427u),vec2<u32>(724240392u,724240427u),vec2<u32>(419956744u,724244744u),vec2<u32>(723065113u,724244744u),vec2<u32>(422254873u,724244779u),vec2<u32>(723069704u,724244779u),vec2<u32>(134753067u,724249352u),vec2<u32>(137037832u,724249352u),vec2<u32>(137037867u,724249352u),vec2<u32>(137046792u,724249352u),vec2<u32>(724240392u,724249352u),vec2<u32>(724249352u,724249352u),vec2<u32>(134748424u,724249369u),vec2<u32>(721950984u,724249369u),vec2<u32>(721951019u,724249369u),vec2<u32>(137046792u,724249387u),vec2<u32>(137046827u,724249387u),vec2<u32>(723060761u,724249387u),vec2<u32>(724249387u,724249387u));fn iq_grid{{ iqSfx }}(i:u32)->vec2<u32>{return IQ_GRID_0{{ iqSfx }}[i];}'],["ops/_shared/iq-grid-iq2xxs-mag.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}const IQ_MAG8_0{{ iqSfx }}:array<vec2<u32>,512>=array<vec2<u32>,512>(vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365266784u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1312837984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1312837984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365266784u,1365264384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1365266784u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365264384u,1365265984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1365266784u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1312837984u,1312835584u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1365265984u,1312837984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1207979584u,1365265984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1312835584u,1365266784u),vec2<u32>(1312837984u,1207977984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207980384u,1312837184u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312837184u,1365265984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1365266784u,1312837184u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1312835584u,1207979584u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1207980384u,1312837984u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837984u,1207979584u),vec2<u32>(1312837984u,1207979584u),vec2<u32>(1312837984u,1207979584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365265984u,1207979584u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1365265984u,1207979584u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1365265984u,1207979584u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1365266784u,1207979584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207979584u,1312837984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1365265984u,1207980384u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1312837984u,1312837184u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1312835584u,1312837984u),vec2<u32>(1312837184u,1207980384u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1312837984u,1207980384u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1365264384u,1207980384u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1365266784u,1207980384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207979584u,1207980384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312837984u,1312837184u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207979584u,1312837984u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1312837184u,1365264384u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1312837984u,1312837184u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1365264384u,1312837984u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1365265984u,1365265984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837984u,1312835584u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1365264384u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1207979584u,1312837984u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1365265984u,1312835584u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1365266784u,1312835584u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1365266784u,1312835584u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1365265984u,1207979584u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1312835584u,1365266784u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1207979584u,1365265984u),vec2<u32>(1207980384u,1312837184u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1207980384u,1365265984u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1312837184u,1312837184u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312837984u,1312837184u),vec2<u32>(1312835584u,1312837184u),vec2<u32>(1312837984u,1312837184u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1365265984u,1207979584u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1312835584u,1312837984u),vec2<u32>(1365264384u,1312837184u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1365265984u,1312837184u),vec2<u32>(1365264384u,1207977984u),vec2<u32>(1365266784u,1312837184u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1365264384u,1312837984u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207979584u,1312837984u),vec2<u32>(1312837184u,1312837184u),vec2<u32>(1207979584u,1312837984u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207980384u,1312837984u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1207980384u,1312837984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1312837984u),vec2<u32>(1312837184u,1207977984u),vec2<u32>(1312835584u,1312837984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312837184u,1312837984u),vec2<u32>(1207980384u,1207979584u),vec2<u32>(1312837184u,1312837984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1312837184u,1312837984u),vec2<u32>(1207980384u,1312835584u),vec2<u32>(1365264384u,1312837984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1365266784u,1207977984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207979584u,1312835584u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1207980384u,1365264384u),vec2<u32>(1207979584u,1207977984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1312837184u,1207979584u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1312835584u,1365264384u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312837184u,1365264384u),vec2<u32>(1312837984u,1312835584u),vec2<u32>(1312837184u,1365264384u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1312837184u,1365264384u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1312837984u,1365264384u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1312837984u,1365264384u),vec2<u32>(1207977984u,1312837984u),vec2<u32>(1312837984u,1365264384u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1365264384u,1365264384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365265984u,1365264384u),vec2<u32>(1207979584u,1207979584u),vec2<u32>(1365266784u,1365264384u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1207977984u,1207979584u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1312835584u,1207980384u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1207977984u,1312835584u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1207979584u,1365266784u),vec2<u32>(1207977984u,1365265984u),vec2<u32>(1312837984u,1207979584u),vec2<u32>(1207979584u,1365265984u),vec2<u32>(1207977984u,1365264384u),vec2<u32>(1207979584u,1365265984u),vec2<u32>(1312837184u,1312835584u),vec2<u32>(1207980384u,1365265984u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1207980384u,1207980384u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1312835584u,1312835584u),vec2<u32>(1312835584u,1365265984u),vec2<u32>(1207979584u,1312837184u),vec2<u32>(1312837184u,1365265984u),vec2<u32>(1207979584u,1365264384u),vec2<u32>(1365264384u,1365265984u),vec2<u32>(1207977984u,1207980384u),vec2<u32>(1365265984u,1365265984u),vec2<u32>(1207980384u,1207977984u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1312837184u,1365264384u),vec2<u32>(1207977984u,1365266784u),vec2<u32>(1365265984u,1207977984u),vec2<u32>(1207979584u,1365266784u),vec2<u32>(1207977984u,1207977984u),vec2<u32>(1207980384u,1365266784u),vec2<u32>(1365264384u,1207979584u),vec2<u32>(1312835584u,1365266784u),vec2<u32>(1207977984u,1312837184u),vec2<u32>(1365264384u,1365266784u),vec2<u32>(1312835584u,1207977984u),vec2<u32>(1365265984u,1365266784u));
{% if iqInline | default(false) %}fn iq_f16x4{{ iqSfx }}(e:vec2<u32>)->vec4<f32>{return vec4<f32>(unpack2x16float(e.x),unpack2x16float(e.y));}
{% else %}fn iq2magLo{{ iqSfx }}(i:u32)->vec4<f32>{let c=i*2u;return vec4<f32>(unpack2x16float(IQ_MAG8_0{{ iqSfx }}[c].x),unpack2x16float(IQ_MAG8_0{{ iqSfx }}[c].y));}fn iq2magHi{{ iqSfx }}(i:u32)->vec4<f32>{let c=i*2u+1u;return vec4<f32>(unpack2x16float(IQ_MAG8_0{{ iqSfx }}[c].x),unpack2x16float(IQ_MAG8_0{{ iqSfx }}[c].y));}
{% endif %}{% macro iq2magLo_at(i, sfx="") %}iq_f16x4{{ sfx }}(IQ_MAG8_0{{ sfx }}[{{ i }} * 2u]){% endmacro %}{% macro iq2magHi_at(i, sfx="") %}iq_f16x4{{ sfx }}(IQ_MAG8_0{{ sfx }}[{{ i }} * 2u + 1u]){% endmacro %}`],["ops/_shared/iq-grid-iq2xxs.wgsl.jinja",'{% set iqSfx = iqSfx | default("") %}const IQ_GRID_0{{ iqSfx }}:array<vec2<u32>,256>=array<vec2<u32>,256>(vec2<u32>(134744072u,134744072u),vec2<u32>(134744107u,134744072u),vec2<u32>(134748441u,134744072u),vec2<u32>(134753032u,134744072u),vec2<u32>(134753067u,134744072u),vec2<u32>(135858201u,134744072u),vec2<u32>(135862536u,134744072u),vec2<u32>(137037832u,134744072u),vec2<u32>(137037867u,134744072u),vec2<u32>(137046792u,134744072u),vec2<u32>(137046827u,134744072u),vec2<u32>(419956761u,134744072u),vec2<u32>(419961096u,134744072u),vec2<u32>(421070856u,134744072u),vec2<u32>(421079816u,134744072u),vec2<u32>(422250521u,134744072u),vec2<u32>(422254856u,134744072u),vec2<u32>(721946632u,134744072u),vec2<u32>(721946667u,134744072u),vec2<u32>(721955627u,134744072u),vec2<u32>(724240427u,134744072u),vec2<u32>(134744089u,134744089u),vec2<u32>(134748424u,134744089u),vec2<u32>(135858184u,134744089u),vec2<u32>(135862553u,134744089u),vec2<u32>(419956744u,134744089u),vec2<u32>(721950984u,134744089u),vec2<u32>(723069704u,134744089u),vec2<u32>(134744072u,134744107u),vec2<u32>(134744107u,134744107u),vec2<u32>(137037867u,134744107u),vec2<u32>(721946667u,134744107u),vec2<u32>(134744089u,134748424u),vec2<u32>(134748424u,134748424u),vec2<u32>(135858184u,134748424u),vec2<u32>(137037849u,134748424u),vec2<u32>(137042184u,134748424u),vec2<u32>(419956744u,134748424u),vec2<u32>(419956779u,134748424u),vec2<u32>(419965704u,134748424u),vec2<u32>(422250504u,134748424u),vec2<u32>(721946649u,134748424u),vec2<u32>(721950984u,134748424u),vec2<u32>(723060744u,134748424u),vec2<u32>(724244744u,134748424u),vec2<u32>(134744072u,134748441u),vec2<u32>(134744107u,134748441u),vec2<u32>(134753032u,134748441u),vec2<u32>(137037832u,134748441u),vec2<u32>(419961131u,134748441u),vec2<u32>(422259481u,134748441u),vec2<u32>(721946632u,134748441u),vec2<u32>(723060761u,134748441u),vec2<u32>(134753049u,134748459u),vec2<u32>(135858184u,134748459u),vec2<u32>(419956744u,134748459u),vec2<u32>(721950984u,134748459u),vec2<u32>(724244744u,134748459u),vec2<u32>(134744072u,134753032u),vec2<u32>(134748441u,134753032u),vec2<u32>(134753032u,134753032u),vec2<u32>(135862536u,134753032u),vec2<u32>(137046792u,134753032u),vec2<u32>(419956761u,134753032u),vec2<u32>(419961096u,134753032u),vec2<u32>(421070856u,134753032u),vec2<u32>(421070891u,134753032u),vec2<u32>(721955592u,134753032u),vec2<u32>(134748424u,134753049u),vec2<u32>(419956744u,134753049u),vec2<u32>(134744107u,134753067u),vec2<u32>(135862536u,134753067u),vec2<u32>(134744089u,135858184u),vec2<u32>(134748424u,135858184u),vec2<u32>(135858184u,135858184u),vec2<u32>(137037849u,135858184u),vec2<u32>(419956744u,135858184u),vec2<u32>(422250504u,135858184u),vec2<u32>(721950984u,135858184u),vec2<u32>(723060744u,135858184u),vec2<u32>(723065113u,135858184u),vec2<u32>(134744072u,135858201u),vec2<u32>(134753032u,135858201u),vec2<u32>(137037832u,135858201u),vec2<u32>(421070856u,135858201u),vec2<u32>(421079851u,135858201u),vec2<u32>(721946632u,135858201u),vec2<u32>(137042184u,135858219u),vec2<u32>(419961113u,135858219u),vec2<u32>(134744072u,135862536u),vec2<u32>(134753032u,135862536u),vec2<u32>(137037832u,135862536u),vec2<u32>(137042201u,135862536u),vec2<u32>(419965721u,135862536u),vec2<u32>(721946632u,135862536u),vec2<u32>(135867144u,135862553u),vec2<u32>(422250539u,135862553u),vec2<u32>(134744072u,135862571u),vec2<u32>(135862571u,135862571u),vec2<u32>(134744089u,135867144u),vec2<u32>(134748424u,135867144u),vec2<u32>(135858184u,135867144u),vec2<u32>(419956744u,135867144u),vec2<u32>(721946649u,135867144u),vec2<u32>(134744072u,135867161u),vec2<u32>(134748441u,135867161u),vec2<u32>(724240392u,135867161u),vec2<u32>(421070873u,135867179u),vec2<u32>(134744072u,137037832u),vec2<u32>(134744107u,137037832u),vec2<u32>(134753067u,137037832u),vec2<u32>(419961096u,137037832u),vec2<u32>(422250521u,137037832u),vec2<u32>(721946632u,137037832u),vec2<u32>(721946667u,137037832u),vec2<u32>(137046809u,137037849u),vec2<u32>(419965704u,137037849u),vec2<u32>(134744072u,137037867u),vec2<u32>(134744107u,137037867u),vec2<u32>(134744089u,137042184u),vec2<u32>(134748424u,137042184u),vec2<u32>(135858184u,137042184u),vec2<u32>(419956744u,137042184u),vec2<u32>(421075243u,137042184u),vec2<u32>(134744072u,137042201u),vec2<u32>(419956761u,137042201u),vec2<u32>(422254856u,137042201u),vec2<u32>(723060744u,137042219u),vec2<u32>(134753032u,137046792u),vec2<u32>(137037832u,137046792u),vec2<u32>(723065096u,137046792u),vec2<u32>(419961096u,137046827u),vec2<u32>(134744089u,419956744u),vec2<u32>(134748424u,419956744u),vec2<u32>(135858184u,419956744u),vec2<u32>(135867144u,419956744u),vec2<u32>(137037849u,419956744u),vec2<u32>(137042184u,419956744u),vec2<u32>(419956744u,419956744u),vec2<u32>(419965704u,419956744u),vec2<u32>(421075243u,419956744u),vec2<u32>(422250504u,419956744u),vec2<u32>(721946649u,419956744u),vec2<u32>(721950984u,419956744u),vec2<u32>(723060744u,419956744u),vec2<u32>(134744072u,419956761u),vec2<u32>(137037832u,419956761u),vec2<u32>(422250521u,419956761u),vec2<u32>(721946632u,419956761u),vec2<u32>(721951001u,419956761u),vec2<u32>(134744089u,419956779u),vec2<u32>(135858184u,419956779u),vec2<u32>(419965704u,419956779u),vec2<u32>(421075243u,419956779u),vec2<u32>(422259464u,419956779u),vec2<u32>(134744072u,419961096u),vec2<u32>(134753032u,419961096u),vec2<u32>(137037832u,419961096u),vec2<u32>(721946632u,419961096u),vec2<u32>(723069721u,419961096u),vec2<u32>(135858219u,419961113u),vec2<u32>(137042184u,419961113u),vec2<u32>(134744072u,419961131u),vec2<u32>(134744089u,419965704u),vec2<u32>(134748424u,419965704u),vec2<u32>(135858184u,419965704u),vec2<u32>(419956744u,419965704u),vec2<u32>(419961113u,419965704u),vec2<u32>(134744072u,419965721u),vec2<u32>(421079816u,419965721u),vec2<u32>(422250521u,419965721u),vec2<u32>(721946667u,419965721u),vec2<u32>(419961113u,419965739u),vec2<u32>(723060744u,419965739u),vec2<u32>(134744072u,421070856u),vec2<u32>(134753032u,421070856u),vec2<u32>(135858201u,421070856u),vec2<u32>(135867161u,421070856u),vec2<u32>(137037832u,421070856u),vec2<u32>(721946632u,421070856u),vec2<u32>(721955592u,421070856u),vec2<u32>(134748424u,421070873u),vec2<u32>(419956779u,421070873u),vec2<u32>(724244744u,421070873u),vec2<u32>(723060761u,421070891u),vec2<u32>(723060744u,421075208u),vec2<u32>(723060779u,421075208u),vec2<u32>(134753067u,421075225u),vec2<u32>(134744089u,421075243u),vec2<u32>(421075208u,421075243u),vec2<u32>(134744072u,421079816u),vec2<u32>(135858201u,421079816u),vec2<u32>(135867161u,421079816u),vec2<u32>(422254856u,421079816u),vec2<u32>(419956744u,421079833u),vec2<u32>(134753032u,421079851u),vec2<u32>(134748424u,422250504u),vec2<u32>(135858184u,422250504u),vec2<u32>(419956744u,422250504u),vec2<u32>(422259464u,422250504u),vec2<u32>(134744072u,422250521u),vec2<u32>(421075225u,422250521u),vec2<u32>(135867144u,422250539u),vec2<u32>(422250504u,422250539u),vec2<u32>(134744072u,422254856u),vec2<u32>(134748441u,422254856u),vec2<u32>(135858184u,422254873u),vec2<u32>(135858219u,422254873u),vec2<u32>(721950984u,422254873u),vec2<u32>(419956779u,422259464u),vec2<u32>(134744072u,721946632u),vec2<u32>(134744107u,721946632u),vec2<u32>(134753067u,721946632u),vec2<u32>(419956761u,721946632u),vec2<u32>(721946667u,721946632u),vec2<u32>(134748424u,721946649u),vec2<u32>(135867144u,721946649u),vec2<u32>(419956744u,721946649u),vec2<u32>(135858201u,721946667u),vec2<u32>(134744089u,721950984u),vec2<u32>(134748424u,721950984u),vec2<u32>(135858184u,721950984u),vec2<u32>(135862553u,721950984u),vec2<u32>(419956744u,721950984u),vec2<u32>(422250504u,721950984u),vec2<u32>(134744072u,721951001u),vec2<u32>(419961131u,721951001u),vec2<u32>(723065096u,721951001u),vec2<u32>(134753049u,721951019u),vec2<u32>(419956744u,721951019u),vec2<u32>(422250504u,721951019u),vec2<u32>(134744107u,721955592u),vec2<u32>(134748424u,721955609u),vec2<u32>(135858201u,721955627u),vec2<u32>(134748424u,723060744u),vec2<u32>(135858184u,723060744u),vec2<u32>(137042184u,723060744u),vec2<u32>(419956744u,723060744u),vec2<u32>(724240409u,723060744u),vec2<u32>(135862571u,723060761u),vec2<u32>(721946632u,723060761u),vec2<u32>(419961113u,723060779u),vec2<u32>(134744072u,723065096u),vec2<u32>(137037867u,723065096u),vec2<u32>(419961096u,723065096u),vec2<u32>(421070873u,723065113u),vec2<u32>(721946649u,723069704u),vec2<u32>(137037832u,723069721u),vec2<u32>(134744107u,724240392u),vec2<u32>(421070856u,724240392u),vec2<u32>(721951001u,724240392u),vec2<u32>(134753049u,724240409u),vec2<u32>(134744072u,724240427u),vec2<u32>(135867144u,724244744u),vec2<u32>(421070856u,724249352u),vec2<u32>(134748424u,724249369u));fn iq_grid{{ iqSfx }}(i:u32)->vec2<u32>{return IQ_GRID_0{{ iqSfx }}[i];}'],["ops/_shared/iq-grid-iq3s.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}const IQ_MAG{{ iqSfx }}:array<vec2<u32>,512>=array<vec2<u32>,512>(vec2<u32>(1006648320u,1006648320u),vec2<u32>(1006649856u,1006648320u),vec2<u32>(1006650624u,1006648320u),vec2<u32>(1006651776u,1006648320u),vec2<u32>(1006652288u,1006648320u),vec2<u32>(1107311616u,1006648320u),vec2<u32>(1107313152u,1006648320u),vec2<u32>(1107313920u,1006648320u),vec2<u32>(1107314816u,1006648320u),vec2<u32>(1107315328u,1006648320u),vec2<u32>(1157643264u,1006648320u),vec2<u32>(1157644800u,1006648320u),vec2<u32>(1157646720u,1006648320u),vec2<u32>(1191200512u,1006648320u),vec2<u32>(1216363520u,1006648320u),vec2<u32>(1216365824u,1006648320u),vec2<u32>(1216366976u,1006648320u),vec2<u32>(1216367488u,1006648320u),vec2<u32>(1233142272u,1006648320u),vec2<u32>(1233143552u,1006648320u),vec2<u32>(1249917952u,1006648320u),vec2<u32>(1249920256u,1006648320u),vec2<u32>(1266696704u,1006648320u),vec2<u32>(1266698368u,1006648320u),vec2<u32>(1266699136u,1006648320u),vec2<u32>(1006648320u,1006649856u),vec2<u32>(1006649856u,1006649856u),vec2<u32>(1006650624u,1006649856u),vec2<u32>(1006651520u,1006649856u),vec2<u32>(1107311616u,1006649856u),vec2<u32>(1107313152u,1006649856u),vec2<u32>(1107315072u,1006649856u),vec2<u32>(1157643264u,1006649856u),vec2<u32>(1157646080u,1006649856u),vec2<u32>(1157647232u,1006649856u),vec2<u32>(1191199232u,1006649856u),vec2<u32>(1191201152u,1006649856u),vec2<u32>(1216366720u,1006649856u),vec2<u32>(1249919488u,1006649856u),vec2<u32>(1249921408u,1006649856u),vec2<u32>(1266697472u,1006649856u),vec2<u32>(1006648320u,1006650624u),vec2<u32>(1006649856u,1006650624u),vec2<u32>(1006651776u,1006650624u),vec2<u32>(1006652288u,1006650624u),vec2<u32>(1107311616u,1006650624u),vec2<u32>(1107314432u,1006650624u),vec2<u32>(1107315328u,1006650624u),vec2<u32>(1157644800u,1006650624u),vec2<u32>(1157646720u,1006650624u),vec2<u32>(1191197696u,1006650624u),vec2<u32>(1191200896u,1006650624u),vec2<u32>(1216365824u,1006650624u),vec2<u32>(1216366976u,1006650624u),vec2<u32>(1216367488u,1006650624u),vec2<u32>(1233142272u,1006650624u),vec2<u32>(1233143552u,1006650624u),vec2<u32>(1266695168u,1006650624u),vec2<u32>(1266697984u,1006650624u),vec2<u32>(1006651136u,1006651136u),vec2<u32>(1107313152u,1006651136u),vec2<u32>(1107315072u,1006651136u),vec2<u32>(1157643264u,1006651136u),vec2<u32>(1157645568u,1006651136u),vec2<u32>(1191199232u,1006651136u),vec2<u32>(1191200512u,1006651136u),vec2<u32>(1191201408u,1006651136u),vec2<u32>(1216366720u,1006651136u),vec2<u32>(1233140736u,1006651136u),vec2<u32>(1233143040u,1006651136u),vec2<u32>(1249921920u,1006651136u),vec2<u32>(1266696704u,1006651136u),vec2<u32>(1266698624u,1006651136u),vec2<u32>(1006648320u,1006651520u),vec2<u32>(1107314432u,1006651520u),vec2<u32>(1107315584u,1006651520u),vec2<u32>(1157644800u,1006651520u),vec2<u32>(1157646464u,1006651520u),vec2<u32>(1191200000u,1006651520u),vec2<u32>(1216363520u,1006651520u),vec2<u32>(1216366336u,1006651520u),vec2<u32>(1233142272u,1006651520u),vec2<u32>(1266695168u,1006651520u),vec2<u32>(1006650624u,1006651776u),vec2<u32>(1006651520u,1006651776u),vec2<u32>(1157643264u,1006651776u),vec2<u32>(1157645568u,1006651776u),vec2<u32>(1157646976u,1006651776u),vec2<u32>(1191200512u,1006651776u),vec2<u32>(1216365056u,1006651776u),vec2<u32>(1216366976u,1006651776u),vec2<u32>(1216367488u,1006651776u),vec2<u32>(1249921664u,1006651776u),vec2<u32>(1266697984u,1006651776u),vec2<u32>(1006652032u,1006652032u),vec2<u32>(1107313152u,1006652032u),vec2<u32>(1107314432u,1006652032u),vec2<u32>(1191199232u,1006652032u),vec2<u32>(1233143040u,1006652032u),vec2<u32>(1266696704u,1006652032u),vec2<u32>(1006648320u,1006652288u),vec2<u32>(1006650624u,1006652288u),vec2<u32>(1006651520u,1006652288u),vec2<u32>(1157643264u,1006652288u),vec2<u32>(1157645568u,1006652288u),vec2<u32>(1157646976u,1006652288u),vec2<u32>(1191200512u,1006652288u),vec2<u32>(1233140736u,1006652288u),vec2<u32>(1233143936u,1006652288u),vec2<u32>(1006648320u,1107311616u),vec2<u32>(1006649856u,1107311616u),vec2<u32>(1006650624u,1107311616u),vec2<u32>(1006651520u,1107311616u),vec2<u32>(1107311616u,1107311616u),vec2<u32>(1107313152u,1107311616u),vec2<u32>(1107314432u,1107311616u),vec2<u32>(1107315072u,1107311616u),vec2<u32>(1107315584u,1107311616u),vec2<u32>(1157643264u,1107311616u),vec2<u32>(1157645568u,1107311616u),vec2<u32>(1191199232u,1107311616u),vec2<u32>(1191200896u,1107311616u),vec2<u32>(1191201408u,1107311616u),vec2<u32>(1233143936u,1107311616u),vec2<u32>(1233144448u,1107311616u),vec2<u32>(1249919488u,1107311616u),vec2<u32>(1266697472u,1107311616u),vec2<u32>(1006648320u,1107313152u),vec2<u32>(1006649856u,1107313152u),vec2<u32>(1006651136u,1107313152u),vec2<u32>(1006652032u,1107313152u),vec2<u32>(1107311616u,1107313152u),vec2<u32>(1107314816u,1107313152u),vec2<u32>(1157644800u,1107313152u),vec2<u32>(1191197696u,1107313152u),vec2<u32>(1191200512u,1107313152u),vec2<u32>(1216365056u,1107313152u),vec2<u32>(1233140736u,1107313152u),vec2<u32>(1233143040u,1107313152u),vec2<u32>(1266695168u,1107313152u),vec2<u32>(1266698880u,1107313152u),vec2<u32>(1006648320u,1107313920u),vec2<u32>(1107313920u,1107313920u),vec2<u32>(1107315072u,1107313920u),vec2<u32>(1107315584u,1107313920u),vec2<u32>(1157643264u,1107313920u),vec2<u32>(1157646464u,1107313920u),vec2<u32>(1191200000u,1107313920u),vec2<u32>(1216363520u,1107313920u),vec2<u32>(1216366336u,1107313920u),vec2<u32>(1233144192u,1107313920u),vec2<u32>(1249917952u,1107313920u),vec2<u32>(1266697472u,1107313920u),vec2<u32>(1006649856u,1107314432u),vec2<u32>(1006651520u,1107314432u),vec2<u32>(1006652288u,1107314432u),vec2<u32>(1107311616u,1107314432u),vec2<u32>(1107314432u,1107314432u),vec2<u32>(1157644800u,1107314432u),vec2<u32>(1157647232u,1107314432u),vec2<u32>(1191197696u,1107314432u),vec2<u32>(1191200896u,1107314432u),vec2<u32>(1216365056u,1107314432u),vec2<u32>(1249920256u,1107314432u),vec2<u32>(1266695168u,1107314432u),vec2<u32>(1006651136u,1107314816u),vec2<u32>(1006651776u,1107314816u),vec2<u32>(1107313920u,1107314816u),vec2<u32>(1107314816u,1107314816u),vec2<u32>(1191199232u,1107314816u),vec2<u32>(1191200512u,1107314816u),vec2<u32>(1216365824u,1107314816u),vec2<u32>(1216367232u,1107314816u),vec2<u32>(1233140736u,1107314816u),vec2<u32>(1233143936u,1107314816u),vec2<u32>(1006649856u,1107315072u),vec2<u32>(1107311616u,1107315072u),vec2<u32>(1107314432u,1107315072u),vec2<u32>(1157644800u,1107315072u),vec2<u32>(1191197696u,1107315072u),vec2<u32>(1191200000u,1107315072u),vec2<u32>(1233142272u,1107315072u),vec2<u32>(1157643264u,1107315328u),vec2<u32>(1157646464u,1107315328u),vec2<u32>(1157647232u,1107315328u),vec2<u32>(1216366720u,1107315328u),vec2<u32>(1216367232u,1107315328u),vec2<u32>(1006649856u,1107315584u),vec2<u32>(1006651136u,1107315584u),vec2<u32>(1107311616u,1107315584u),vec2<u32>(1107313920u,1107315584u),vec2<u32>(1157644800u,1107315584u),vec2<u32>(1191201152u,1107315584u),vec2<u32>(1216365056u,1107315584u),vec2<u32>(1249920256u,1107315584u),vec2<u32>(1266695168u,1107315584u),vec2<u32>(1006648320u,1157643264u),vec2<u32>(1006649856u,1157643264u),vec2<u32>(1006651136u,1157643264u),vec2<u32>(1006651776u,1157643264u),vec2<u32>(1006652288u,1157643264u),vec2<u32>(1107311616u,1157643264u),vec2<u32>(1107313920u,1157643264u),vec2<u32>(1107314816u,1157643264u),vec2<u32>(1107315328u,1157643264u),vec2<u32>(1157644800u,1157643264u),vec2<u32>(1157646080u,1157643264u),vec2<u32>(1157647232u,1157643264u),vec2<u32>(1191197696u,1157643264u),vec2<u32>(1191200000u,1157643264u),vec2<u32>(1216365056u,1157643264u),vec2<u32>(1216366336u,1157643264u),vec2<u32>(1216366976u,1157643264u),vec2<u32>(1233140736u,1157643264u),vec2<u32>(1233143040u,1157643264u),vec2<u32>(1249921920u,1157643264u),vec2<u32>(1266695168u,1157643264u),vec2<u32>(1266697984u,1157643264u),vec2<u32>(1266698624u,1157643264u),vec2<u32>(1006648320u,1157644800u),vec2<u32>(1006650624u,1157644800u),vec2<u32>(1107311616u,1157644800u),vec2<u32>(1107314432u,1157644800u),vec2<u32>(1107315584u,1157644800u),vec2<u32>(1157645568u,1157644800u),vec2<u32>(1157646720u,1157644800u),vec2<u32>(1191199232u,1157644800u),vec2<u32>(1191200896u,1157644800u),vec2<u32>(1216365824u,1157644800u),vec2<u32>(1233142272u,1157644800u),vec2<u32>(1006649856u,1157645568u),vec2<u32>(1006651520u,1157645568u),vec2<u32>(1006652288u,1157645568u),vec2<u32>(1157644800u,1157645568u),vec2<u32>(1157646080u,1157645568u),vec2<u32>(1191197696u,1157645568u),vec2<u32>(1191201664u,1157645568u),vec2<u32>(1216365056u,1157645568u),vec2<u32>(1233143552u,1157645568u),vec2<u32>(1233144704u,1157645568u),vec2<u32>(1266696704u,1157645568u),vec2<u32>(1266698368u,1157645568u),vec2<u32>(1006648320u,1157646080u),vec2<u32>(1006650624u,1157646080u),vec2<u32>(1006651776u,1157646080u),vec2<u32>(1107313152u,1157646080u),vec2<u32>(1157645568u,1157646080u),vec2<u32>(1157646464u,1157646080u),vec2<u32>(1191199232u,1157646080u),vec2<u32>(1191200512u,1157646080u),vec2<u32>(1216365824u,1157646080u),vec2<u32>(1233140736u,1157646080u),vec2<u32>(1249921664u,1157646080u),vec2<u32>(1006649856u,1157646464u),vec2<u32>(1006652288u,1157646464u),vec2<u32>(1157643264u,1157646464u),vec2<u32>(1157646080u,1157646464u),vec2<u32>(1191200000u,1157646464u),vec2<u32>(1191201152u,1157646464u),vec2<u32>(1216365056u,1157646464u),vec2<u32>(1266697472u,1157646464u),vec2<u32>(1266698624u,1157646464u),vec2<u32>(1006651520u,1157646720u),vec2<u32>(1107313152u,1157646720u),vec2<u32>(1157645568u,1157646720u),vec2<u32>(1191201664u,1157646720u),vec2<u32>(1216363520u,1157646720u),vec2<u32>(1233143552u,1157646720u),vec2<u32>(1266695168u,1157646720u),vec2<u32>(1006648320u,1157646976u),vec2<u32>(1006650624u,1157646976u),vec2<u32>(1006652288u,1157646976u),vec2<u32>(1157644800u,1157646976u),vec2<u32>(1233144192u,1157646976u),vec2<u32>(1249919488u,1157646976u),vec2<u32>(1006651776u,1157647232u),vec2<u32>(1107313152u,1157647232u),vec2<u32>(1157646976u,1157647232u),vec2<u32>(1191197696u,1157647232u),vec2<u32>(1216366336u,1157647232u),vec2<u32>(1233140736u,1157647232u),vec2<u32>(1006650624u,1191197696u),vec2<u32>(1107313152u,1191197696u),vec2<u32>(1107314432u,1191197696u),vec2<u32>(1107315072u,1191197696u),vec2<u32>(1107315584u,1191197696u),vec2<u32>(1157645568u,1191197696u),vec2<u32>(1191199232u,1191197696u),vec2<u32>(1191200512u,1191197696u),vec2<u32>(1191201152u,1191197696u),vec2<u32>(1216365824u,1191197696u),vec2<u32>(1216366720u,1191197696u),vec2<u32>(1216367488u,1191197696u),vec2<u32>(1233142272u,1191197696u),vec2<u32>(1249920768u,1191197696u),vec2<u32>(1266696704u,1191197696u),vec2<u32>(1006649856u,1191199232u),vec2<u32>(1006651136u,1191199232u),vec2<u32>(1006651776u,1191199232u),vec2<u32>(1107314816u,1191199232u),vec2<u32>(1157644800u,1191199232u),vec2<u32>(1157646080u,1191199232u),vec2<u32>(1216363520u,1191199232u),vec2<u32>(1249917952u,1191199232u),vec2<u32>(1266697472u,1191199232u),vec2<u32>(1266698880u,1191199232u),vec2<u32>(1006648320u,1191200000u),vec2<u32>(1107313920u,1191200000u),vec2<u32>(1157643264u,1191200000u),vec2<u32>(1191200000u,1191200000u),vec2<u32>(1191200896u,1191200000u),vec2<u32>(1233140736u,1191200000u),vec2<u32>(1006649856u,1191200512u),vec2<u32>(1107311616u,1191200512u),vec2<u32>(1107314816u,1191200512u),vec2<u32>(1157644800u,1191200512u),vec2<u32>(1157646080u,1191200512u),vec2<u32>(1157647232u,1191200512u),vec2<u32>(1191197696u,1191200512u),vec2<u32>(1216365056u,1191200512u),vec2<u32>(1216366336u,1191200512u),vec2<u32>(1216367488u,1191200512u),vec2<u32>(1233144192u,1191200512u),vec2<u32>(1266697984u,1191200512u),vec2<u32>(1006651136u,1191200896u),vec2<u32>(1107313152u,1191200896u),vec2<u32>(1107315328u,1191200896u),vec2<u32>(1157645568u,1191200896u),vec2<u32>(1191199232u,1191200896u),vec2<u32>(1233143040u,1191200896u),vec2<u32>(1249917952u,1191200896u),vec2<u32>(1249921152u,1191200896u),vec2<u32>(1006649856u,1191201152u),vec2<u32>(1107311616u,1191201152u),vec2<u32>(1107313920u,1191201152u),vec2<u32>(1157646720u,1191201152u),vec2<u32>(1191200000u,1191201152u),vec2<u32>(1216366720u,1191201152u),vec2<u32>(1233144448u,1191201152u),vec2<u32>(1266697984u,1191201152u),vec2<u32>(1107315328u,1191201408u),vec2<u32>(1216365056u,1191201408u),vec2<u32>(1006649856u,1191201664u),vec2<u32>(1006651136u,1191201664u),vec2<u32>(1157643264u,1191201664u),vec2<u32>(1157645568u,1191201664u),vec2<u32>(1191201152u,1191201664u),vec2<u32>(1006648320u,1216363520u),vec2<u32>(1006651520u,1216363520u),vec2<u32>(1107313920u,1216363520u),vec2<u32>(1157643264u,1216363520u),vec2<u32>(1157646464u,1216363520u),vec2<u32>(1157647232u,1216363520u),vec2<u32>(1191200000u,1216363520u),vec2<u32>(1216365056u,1216363520u),vec2<u32>(1233140736u,1216363520u),vec2<u32>(1266695168u,1216363520u),vec2<u32>(1006650624u,1216365056u),vec2<u32>(1006652288u,1216365056u),vec2<u32>(1107313152u,1216365056u),vec2<u32>(1107314432u,1216365056u),vec2<u32>(1157645568u,1216365056u),vec2<u32>(1191197696u,1216365056u),vec2<u32>(1191201152u,1216365056u),vec2<u32>(1216366336u,1216365056u),vec2<u32>(1233142272u,1216365056u),vec2<u32>(1233144192u,1216365056u),vec2<u32>(1006649856u,1216365824u),vec2<u32>(1006651136u,1216365824u),vec2<u32>(1107311616u,1216365824u),vec2<u32>(1107315072u,1216365824u),vec2<u32>(1157644800u,1216365824u),vec2<u32>(1191200512u,1216365824u),vec2<u32>(1216363520u,1216365824u),vec2<u32>(1233144704u,1216365824u),vec2<u32>(1249920256u,1216365824u),vec2<u32>(1266695168u,1216365824u),vec2<u32>(1006651520u,1216366336u),vec2<u32>(1107313152u,1216366336u),vec2<u32>(1107314432u,1216366336u),vec2<u32>(1157643264u,1216366336u),vec2<u32>(1157645568u,1216366336u),vec2<u32>(1191199232u,1216366336u),vec2<u32>(1191201152u,1216366336u),vec2<u32>(1006648320u,1216366720u),vec2<u32>(1006650624u,1216366720u),vec2<u32>(1157646464u,1216366720u),vec2<u32>(1191201664u,1216366720u),vec2<u32>(1216363520u,1216366720u),vec2<u32>(1266696704u,1216366720u),vec2<u32>(1006651776u,1216366976u),vec2<u32>(1006652288u,1216366976u),vec2<u32>(1157644800u,1216366976u),vec2<u32>(1249920256u,1216366976u),vec2<u32>(1107314432u,1216367232u),vec2<u32>(1191200896u,1216367232u),vec2<u32>(1249917952u,1216367232u),vec2<u32>(1107311616u,1216367488u),vec2<u32>(1107315072u,1216367488u),vec2<u32>(1191197696u,1216367488u),vec2<u32>(1216366336u,1216367488u),vec2<u32>(1233142272u,1216367488u),vec2<u32>(1006650624u,1233140736u),vec2<u32>(1107311616u,1233140736u),vec2<u32>(1107314816u,1233140736u),vec2<u32>(1157645568u,1233140736u),vec2<u32>(1216363520u,1233140736u),vec2<u32>(1216366720u,1233140736u),vec2<u32>(1216367488u,1233140736u),vec2<u32>(1233143040u,1233140736u),vec2<u32>(1249921664u,1233140736u),vec2<u32>(1266698368u,1233140736u),vec2<u32>(1006649856u,1233142272u),vec2<u32>(1006651136u,1233142272u),vec2<u32>(1006651776u,1233142272u),vec2<u32>(1107313920u,1233142272u),vec2<u32>(1157644800u,1233142272u),vec2<u32>(1191200000u,1233142272u),vec2<u32>(1266697472u,1233142272u),vec2<u32>(1006648320u,1233143040u),vec2<u32>(1107313152u,1233143040u),vec2<u32>(1157646080u,1233143040u),vec2<u32>(1191197696u,1233143040u),vec2<u32>(1191201408u,1233143040u),vec2<u32>(1233143552u,1233143040u),vec2<u32>(1006650624u,1233143552u),vec2<u32>(1006652288u,1233143552u),vec2<u32>(1107311616u,1233143552u),vec2<u32>(1157647232u,1233143552u),vec2<u32>(1216366720u,1233143552u),vec2<u32>(1233142272u,1233143552u),vec2<u32>(1249921408u,1233143552u),vec2<u32>(1266697984u,1233143552u),vec2<u32>(1006649856u,1233143936u),vec2<u32>(1006651520u,1233143936u),vec2<u32>(1157643264u,1233143936u),vec2<u32>(1191200000u,1233143936u),vec2<u32>(1216367232u,1233143936u),vec2<u32>(1107313920u,1233144192u),vec2<u32>(1157646976u,1233144192u),vec2<u32>(1233142272u,1233144192u),vec2<u32>(1233143552u,1233144192u),vec2<u32>(1216365824u,1233144448u),vec2<u32>(1006650624u,1233144704u),vec2<u32>(1006651520u,1233144704u),vec2<u32>(1157645568u,1233144704u),vec2<u32>(1107313152u,1249917952u),vec2<u32>(1107314432u,1249917952u),vec2<u32>(1107315072u,1249917952u),vec2<u32>(1191199232u,1249917952u),vec2<u32>(1191200512u,1249917952u),vec2<u32>(1249917952u,1249917952u),vec2<u32>(1006648320u,1249919488u),vec2<u32>(1157643264u,1249919488u),vec2<u32>(1157647232u,1249919488u),vec2<u32>(1249921152u,1249919488u),vec2<u32>(1107313920u,1249920256u),vec2<u32>(1191200896u,1249920256u),vec2<u32>(1216365824u,1249920256u),vec2<u32>(1233144192u,1249920256u),vec2<u32>(1249920256u,1249920256u),vec2<u32>(1266695168u,1249920256u),vec2<u32>(1006648320u,1249920768u),vec2<u32>(1107314816u,1249920768u),vec2<u32>(1157644800u,1249920768u),vec2<u32>(1216363520u,1249920768u),vec2<u32>(1157646720u,1249921152u),vec2<u32>(1216366336u,1249921152u),vec2<u32>(1249920256u,1249921152u),vec2<u32>(1006648320u,1249921408u),vec2<u32>(1006651136u,1249921408u),vec2<u32>(1191200896u,1249921408u),vec2<u32>(1249917952u,1249921408u),vec2<u32>(1006651776u,1249921664u),vec2<u32>(1216363520u,1249921664u),vec2<u32>(1107313152u,1249921920u),vec2<u32>(1107314432u,1249921920u),vec2<u32>(1006648320u,1266695168u),vec2<u32>(1006651520u,1266695168u),vec2<u32>(1006652288u,1266695168u),vec2<u32>(1157643264u,1266695168u),vec2<u32>(1157645568u,1266695168u),vec2<u32>(1191201408u,1266695168u),vec2<u32>(1216363520u,1266695168u),vec2<u32>(1233143936u,1266695168u),vec2<u32>(1249920256u,1266695168u),vec2<u32>(1006650624u,1266696704u),vec2<u32>(1107313152u,1266696704u),vec2<u32>(1157646464u,1266696704u),vec2<u32>(1216366336u,1266696704u),vec2<u32>(1216366976u,1266696704u),vec2<u32>(1006649856u,1266697472u),vec2<u32>(1006651520u,1266697472u),vec2<u32>(1107311616u,1266697472u),vec2<u32>(1107315328u,1266697472u),vec2<u32>(1157644800u,1266697472u),vec2<u32>(1191197696u,1266697472u),vec2<u32>(1233142272u,1266697472u),vec2<u32>(1006650624u,1266697984u),vec2<u32>(1191200000u,1266697984u),vec2<u32>(1191201152u,1266697984u),vec2<u32>(1233143552u,1266697984u),vec2<u32>(1006649856u,1266698368u),vec2<u32>(1006651776u,1266698368u),vec2<u32>(1107314432u,1266698368u),vec2<u32>(1157643264u,1266698368u),vec2<u32>(1233140736u,1266698368u),vec2<u32>(1157645568u,1266698624u),vec2<u32>(1216365824u,1266698624u),vec2<u32>(1006650624u,1266698880u),vec2<u32>(1191199232u,1266698880u),vec2<u32>(1006648320u,1266699136u));
{% if iqInline | default(false) %}fn iq_f16x4{{ iqSfx }}(e:vec2<u32>)->vec4<f32>{return vec4<f32>(unpack2x16float(e.x),unpack2x16float(e.y));}
{% else %}fn iq3mag{{ iqSfx }}(i:u32)->vec4<f32>{let e=IQ_MAG{{ iqSfx }}[i];return vec4<f32>(unpack2x16float(e.x),unpack2x16float(e.y));}
{% endif %}{% macro iq3mag_at(i, sfx="") %}iq_f16x4{{ sfx }}(IQ_MAG{{ sfx }}[{{ i }}]){% endmacro %}`],["ops/_shared/iq-grid-iq3xxs.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}const IQ_MAG{{ iqSfx }}:array<vec2<u32>,256>=array<vec2<u32>,256>(vec2<u32>(1140868096u,1140868096u),vec2<u32>(1140870400u,1140868096u),vec2<u32>(1140871296u,1140868096u),vec2<u32>(1241532928u,1140868096u),vec2<u32>(1241534208u,1140868096u),vec2<u32>(1241535424u,1140868096u),vec2<u32>(1291863040u,1140868096u),vec2<u32>(1291865344u,1140868096u),vec2<u32>(1325419008u,1140868096u),vec2<u32>(1350585600u,1140868096u),vec2<u32>(1405112064u,1140868096u),vec2<u32>(1405112704u,1140868096u),vec2<u32>(1140869632u,1140869632u),vec2<u32>(1140870912u,1140869632u),vec2<u32>(1241531392u,1140869632u),vec2<u32>(1241533696u,1140869632u),vec2<u32>(1291864576u,1140869632u),vec2<u32>(1291866496u,1140869632u),vec2<u32>(1325417472u,1140869632u),vec2<u32>(1325419776u,1140869632u),vec2<u32>(1350584832u,1140869632u),vec2<u32>(1367363712u,1140869632u),vec2<u32>(1405109248u,1140869632u),vec2<u32>(1140868096u,1140870400u),vec2<u32>(1140870400u,1140870400u),vec2<u32>(1140871296u,1140870400u),vec2<u32>(1241532928u,1140870400u),vec2<u32>(1291863040u,1140870400u),vec2<u32>(1291865344u,1140870400u),vec2<u32>(1325419008u,1140870400u),vec2<u32>(1325420288u,1140870400u),vec2<u32>(1325421504u,1140870400u),vec2<u32>(1367362048u,1140870400u),vec2<u32>(1367364544u,1140870400u),vec2<u32>(1405112704u,1140870400u),vec2<u32>(1140869632u,1140870912u),vec2<u32>(1140872128u,1140870912u),vec2<u32>(1241531392u,1140870912u),vec2<u32>(1241533696u,1140870912u),vec2<u32>(1291866496u,1140870912u),vec2<u32>(1405109248u,1140870912u),vec2<u32>(1241534208u,1140871296u),vec2<u32>(1325421504u,1140871296u),vec2<u32>(1350586496u,1140871296u),vec2<u32>(1367364544u,1140871296u),vec2<u32>(1405112064u,1140871296u),vec2<u32>(1405112704u,1140871296u),vec2<u32>(1140869632u,1140871552u),vec2<u32>(1140872128u,1140871552u),vec2<u32>(1325419776u,1140871552u),vec2<u32>(1367362816u,1140871552u),vec2<u32>(1325420928u,1140871808u),vec2<u32>(1384140928u,1140871808u),vec2<u32>(1241531392u,1140872128u),vec2<u32>(1241534592u,1140872128u),vec2<u32>(1241535104u,1140872128u),vec2<u32>(1350586112u,1140872128u),vec2<u32>(1384139264u,1140872128u),vec2<u32>(1140869632u,1241531392u),vec2<u32>(1140870912u,1241531392u),vec2<u32>(1241531392u,1241531392u),vec2<u32>(1241533696u,1241531392u),vec2<u32>(1291864576u,1241531392u),vec2<u32>(1291865856u,1241531392u),vec2<u32>(1325417472u,1241531392u),vec2<u32>(1325419776u,1241531392u),vec2<u32>(1325420672u,1241531392u),vec2<u32>(1350587328u,1241531392u),vec2<u32>(1367360512u,1241531392u),vec2<u32>(1140868096u,1241532928u),vec2<u32>(1140870400u,1241532928u),vec2<u32>(1241532928u,1241532928u),vec2<u32>(1291863040u,1241532928u),vec2<u32>(1291865344u,1241532928u),vec2<u32>(1140869632u,1241533696u),vec2<u32>(1140870912u,1241533696u),vec2<u32>(1241531392u,1241533696u),vec2<u32>(1241533696u,1241533696u),vec2<u32>(1291864576u,1241533696u),vec2<u32>(1325417472u,1241533696u),vec2<u32>(1405111552u,1241533696u),vec2<u32>(1140868096u,1241534208u),vec2<u32>(1140870400u,1241534208u),vec2<u32>(1291863040u,1241534208u),vec2<u32>(1325419008u,1241534208u),vec2<u32>(1350587008u,1241534208u),vec2<u32>(1384141440u,1241534208u),vec2<u32>(1140869632u,1241534592u),vec2<u32>(1140871552u,1241534592u),vec2<u32>(1367360512u,1241534592u),vec2<u32>(1291863040u,1241534848u),vec2<u32>(1291866240u,1241534848u),vec2<u32>(1350587008u,1241534848u),vec2<u32>(1405110784u,1241534848u),vec2<u32>(1140871552u,1241535104u),vec2<u32>(1291865344u,1241535424u),vec2<u32>(1350583296u,1241535424u),vec2<u32>(1140868096u,1291863040u),vec2<u32>(1140870400u,1291863040u),vec2<u32>(1241532928u,1291863040u),vec2<u32>(1241534208u,1291863040u),vec2<u32>(1291863040u,1291863040u),vec2<u32>(1291865344u,1291863040u),vec2<u32>(1291866752u,1291863040u),vec2<u32>(1325419008u,1291863040u),vec2<u32>(1350585600u,1291863040u),vec2<u32>(1140869632u,1291864576u),vec2<u32>(1140870912u,1291864576u),vec2<u32>(1140871552u,1291864576u),vec2<u32>(1241531392u,1291864576u),vec2<u32>(1241533696u,1291864576u),vec2<u32>(1291864576u,1291864576u),vec2<u32>(1325417472u,1291864576u),vec2<u32>(1384140544u,1291864576u),vec2<u32>(1384141760u,1291864576u),vec2<u32>(1405109248u,1291864576u),vec2<u32>(1140868096u,1291865344u),vec2<u32>(1140870400u,1291865344u),vec2<u32>(1241532928u,1291865344u),vec2<u32>(1241535424u,1291865344u),vec2<u32>(1291863040u,1291865344u),vec2<u32>(1291865344u,1291865344u),vec2<u32>(1325421504u,1291865344u),vec2<u32>(1350583296u,1291865344u),vec2<u32>(1367363968u,1291865344u),vec2<u32>(1140869632u,1291865856u),vec2<u32>(1241531392u,1291865856u),vec2<u32>(1241534592u,1291865856u),vec2<u32>(1405109248u,1291865856u),vec2<u32>(1405112448u,1291865856u),vec2<u32>(1325420928u,1291866240u),vec2<u32>(1367363328u,1291866240u),vec2<u32>(1140870912u,1291866496u),vec2<u32>(1291867072u,1291866496u),vec2<u32>(1350584832u,1291866496u),vec2<u32>(1405112448u,1291866496u),vec2<u32>(1140869632u,1291867072u),vec2<u32>(1140870912u,1291867072u),vec2<u32>(1241535104u,1291867072u),vec2<u32>(1350586752u,1291867072u),vec2<u32>(1140869632u,1325417472u),vec2<u32>(1241531392u,1325417472u),vec2<u32>(1241533696u,1325417472u),vec2<u32>(1291864576u,1325417472u),vec2<u32>(1291865856u,1325417472u),vec2<u32>(1367360512u,1325417472u),vec2<u32>(1384141184u,1325417472u),vec2<u32>(1405111552u,1325417472u),vec2<u32>(1140868096u,1325419008u),vec2<u32>(1140870400u,1325419008u),vec2<u32>(1291863040u,1325419008u),vec2<u32>(1325419008u,1325419008u),vec2<u32>(1350586496u,1325419008u),vec2<u32>(1350587008u,1325419008u),vec2<u32>(1140869632u,1325419776u),vec2<u32>(1140870912u,1325419776u),vec2<u32>(1241531392u,1325419776u),vec2<u32>(1291866496u,1325419776u),vec2<u32>(1367362816u,1325419776u),vec2<u32>(1405111552u,1325419776u),vec2<u32>(1241532928u,1325420288u),vec2<u32>(1325420288u,1325420288u),vec2<u32>(1325417472u,1325420672u),vec2<u32>(1350587328u,1325420672u),vec2<u32>(1405111552u,1325420672u),vec2<u32>(1140868096u,1325420928u),vec2<u32>(1140871808u,1325420928u),vec2<u32>(1291865344u,1325420928u),vec2<u32>(1367363968u,1325420928u),vec2<u32>(1241534592u,1325421184u),vec2<u32>(1325421184u,1325421184u),vec2<u32>(1384140544u,1325421184u),vec2<u32>(1325420288u,1325421504u),vec2<u32>(1384137728u,1325421504u),vec2<u32>(1140871296u,1350583296u),vec2<u32>(1241535424u,1350583296u),vec2<u32>(1325420928u,1350583296u),vec2<u32>(1325421504u,1350583296u),vec2<u32>(1367363328u,1350583296u),vec2<u32>(1367364544u,1350583296u),vec2<u32>(1405112448u,1350584832u),vec2<u32>(1291863040u,1350585600u),vec2<u32>(1325421504u,1350585600u),vec2<u32>(1350583296u,1350585600u),vec2<u32>(1384137728u,1350585600u),vec2<u32>(1384141440u,1350585600u),vec2<u32>(1140872128u,1350586112u),vec2<u32>(1350586752u,1350586112u),vec2<u32>(1140871296u,1350586496u),vec2<u32>(1367362048u,1350586496u),vec2<u32>(1384140928u,1350586496u),vec2<u32>(1291866496u,1350586752u),vec2<u32>(1350586112u,1350586752u),vec2<u32>(1405109248u,1350586752u),vec2<u32>(1140871552u,1350587328u),vec2<u32>(1241531392u,1350587328u),vec2<u32>(1241533696u,1350587328u),vec2<u32>(1325417472u,1350587328u),vec2<u32>(1241533696u,1367360512u),vec2<u32>(1350584832u,1367360512u),vec2<u32>(1405109248u,1367360512u),vec2<u32>(1140868096u,1367362048u),vec2<u32>(1140871808u,1367362048u),vec2<u32>(1291866752u,1367362048u),vec2<u32>(1367363968u,1367362048u),vec2<u32>(1241534592u,1367362816u),vec2<u32>(1325419776u,1367362816u),vec2<u32>(1405111552u,1367362816u),vec2<u32>(1140870400u,1367363328u),vec2<u32>(1367363328u,1367363328u),vec2<u32>(1241531392u,1367363712u),vec2<u32>(1291865856u,1367363712u),vec2<u32>(1291867072u,1367363712u),vec2<u32>(1405111552u,1367363712u),vec2<u32>(1140870400u,1367363968u),vec2<u32>(1325419008u,1367363968u),vec2<u32>(1367360512u,1367364224u),vec2<u32>(1291866240u,1367364544u),vec2<u32>(1350585600u,1367364544u),vec2<u32>(1291866240u,1384137728u),vec2<u32>(1350586496u,1384137728u),vec2<u32>(1350587008u,1384137728u),vec2<u32>(1384140928u,1384137728u),vec2<u32>(1291864576u,1384139264u),vec2<u32>(1384139264u,1384139264u),vec2<u32>(1241535424u,1384140032u),vec2<u32>(1384140928u,1384140032u),vec2<u32>(1325417472u,1384140544u),vec2<u32>(1325421184u,1384140544u),vec2<u32>(1350586496u,1384140928u),vec2<u32>(1140871552u,1384141184u),vec2<u32>(1367362816u,1384141184u),vec2<u32>(1325420288u,1384141440u),vec2<u32>(1140870912u,1384141760u),vec2<u32>(1291864576u,1384141760u),vec2<u32>(1140870912u,1405109248u),vec2<u32>(1140871552u,1405109248u),vec2<u32>(1140872128u,1405109248u),vec2<u32>(1241531392u,1405109248u),vec2<u32>(1325419776u,1405109248u),vec2<u32>(1367362816u,1405109248u),vec2<u32>(1291866752u,1405110784u),vec2<u32>(1350583296u,1405110784u),vec2<u32>(1241533696u,1405111552u),vec2<u32>(1350586752u,1405111552u),vec2<u32>(1367362816u,1405111552u),vec2<u32>(1140868096u,1405112064u),vec2<u32>(1241534848u,1405112064u),vec2<u32>(1325420288u,1405112064u),vec2<u32>(1384137728u,1405112064u),vec2<u32>(1291864576u,1405112448u),vec2<u32>(1350584832u,1405112448u),vec2<u32>(1140868096u,1405112704u),vec2<u32>(1140870400u,1405112704u),vec2<u32>(1291866240u,1405112704u),vec2<u32>(1325417472u,1405112960u));
{% if iqInline | default(false) %}fn iq_f16x4{{ iqSfx }}(e:vec2<u32>)->vec4<f32>{return vec4<f32>(unpack2x16float(e.x),unpack2x16float(e.y));}
{% else %}fn iq3mag{{ iqSfx }}(i:u32)->vec4<f32>{let e=IQ_MAG{{ iqSfx }}[i];return vec4<f32>(unpack2x16float(e.x),unpack2x16float(e.y));}
{% endif %}{% macro iq3mag_at(i, sfx="") %}iq_f16x4{{ sfx }}(IQ_MAG{{ sfx }}[{{ i }}]){% endmacro %}`],["ops/_shared/iq-grid.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}{% set iq2xxs = iq2xxs | default(false) %}{% set iq2xs = iq2xs | default(false) %}{% set iq2s = iq2s | default(false) %}{% set iq3s = iq3s | default(false) %}{% set iq3xxs = iq3xxs | default(false) %}{% set iqGridBound = iqGridBound | default(false) %}{% set iq1m = iq1m is defined and iq1m %}{% set iqGridMag = iqGridMag | default(iqDot | default(false)) %}{% set iqGridBase = iqGridBase | default(0) %}{% set iqGridMag8 = iqGridMag and (iq2xxs or iq2xs) and not iqGridBound %}{% set iqInline = (iqInline | default(false)) and not iqGridBound and not (iqGridInBits | default(false)) %}{% set iq2magUnpacked = false %}{% if iqGridInBits | default(false) %}fn iq_grid{{ iqSfx }}(i:u32)->vec2<u32>{return vec2<u32>(q4_bits[{{ iqGridBase }}u+2u*i],q4_bits[{{ iqGridBase }}u+2u*i+1u]);}
{% elif iqGridBound %}fn iq_grid{{ iqSfx }}(i:u32)->vec2<u32>{return {{ iqGridBinding | default("iq_grid_data") }}[i];}
{% if iq3xxs or iq3s %}fn iq3mag{{ iqSfx }}(i:u32)->vec4<f32>{let e=iq_grid{{ iqSfx }}(i>>1u);return vec4<f32>(unpack4xU8(select(e.x,e.y,(i&1u)==1u)));}
{% endif %}{% set iq2magUnpacked = iqGridMag and (iq2xxs or iq2xs or iq2s) %}{% elif iq1m %}{% include "ops/_shared/iq-grid-iq1s.wgsl.jinja" %}{% elif iq2xxs %}{% if iqGridMag8 %}{% include "ops/_shared/iq-grid-iq2xxs-mag.wgsl.jinja" %}{% else %}{% include "ops/_shared/iq-grid-iq2xxs.wgsl.jinja" %}{% endif %}{% elif iq2xs %}{% if iqGridMag8 %}{% include "ops/_shared/iq-grid-iq2xs-mag.wgsl.jinja" %}{% else %}{% include "ops/_shared/iq-grid-iq2xs.wgsl.jinja" %}{% endif %}{% elif iq2s %}{% include "ops/_shared/iq-grid-iq2s.wgsl.jinja" %}{% set iq2magUnpacked = iqGridMag and not iqInline %}{% elif iq3xxs %}{% include "ops/_shared/iq-grid-iq3xxs.wgsl.jinja" %}{% elif iq3s %}{% include "ops/_shared/iq-grid-iq3s.wgsl.jinja" %}{% endif %}{% if iq2magUnpacked %}fn iq2magLo{{ iqSfx }}(i:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(iq_grid{{ iqSfx }}(i).x));}fn iq2magHi{{ iqSfx }}(i:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(iq_grid{{ iqSfx }}(i).y));}
{% endif %}`],["ops/_shared/iq-inline-dot.wgsl.jinja",`{% set iqAtGrid1 = iq1grid_at if iq1grid_at is defined else none %}{% set iqAtMagLo = iq2magLo_at if iq2magLo_at is defined else none %}{% set iqAtMagHi = iq2magHi_at if iq2magHi_at is defined else none %}{% set iqAtIq2sLo = iq2sLo_at if iq2sLo_at is defined else none %}{% set iqAtIq2sHi = iq2sHi_at if iq2sHi_at is defined else none %}{% set iqAtMag4 = iq3mag_at if iq3mag_at is defined else none %}{% macro iq_dot_acc(P, acc, kind) %}{% if kind == "two" %}{{ acc }}={{ acc }}+(dlA{{ P }}*((dot(g{{ P }}_0,x0)+dot(g{{ P }}_1,x1))+(dot(g{{ P }}_2,x2)+dot(g{{ P }}_3,x3)))+dlB{{ P }}*((dot(g{{ P }}_4,x4)+dot(g{{ P }}_5,x5))+(dot(g{{ P }}_6,x6)+dot(g{{ P }}_7,x7))));
{% else %}{{ acc }}={{ acc }}+db{{ P }}*((dot(g{{ P }}_0,x0)+dot(g{{ P }}_1,x1))+(dot(g{{ P }}_2,x2)+dot(g{{ P }}_3,x3))+(dot(g{{ P }}_4,x4)+dot(g{{ P }}_5,x5))+(dot(g{{ P }}_6,x6)+dot(g{{ P }}_7,x7)));
{% endif %}{% endmacro %}{% macro iq1_regs(P, q, m, d, sfx="") %}let iq1q{{ P }}={{ q }};let iq1m{{ P }}={{ m }};let iq1d{{ P }}={{ d }};let dlA{{ P }}=iq1d{{ P }}*f32(2u*((iq1m{{ P }}>>16u)&7u)+1u);let dlB{{ P }}=iq1d{{ P }}*f32(2u*((iq1m{{ P }}>>19u)&7u)+1u);let qh0_{{ P }}=iq1m{{ P }}&0xFFu;let qh1_{{ P }}=(iq1m{{ P }}>>8u)&0xFFu;
{% for c in range(4) %}{% set lo = ["iq1q" ~ P ~ " & 0xFFu", "(iq1q" ~ P ~ " >> 8u) & 0xFFu", "(iq1q" ~ P ~ " >> 16u) & 0xFFu", "iq1q" ~ P ~ " >> 24u"][c] %}{% set qh = ("qh0_" ~ P) if c < 2 else ("qh1_" ~ P) %}{% set hi = "(" ~ qh ~ (" << 8u)" if c % 2 == 0 else " << 4u)") ~ " & 0x700u" %}{% set neg = "(" ~ qh ~ (" & 0x08u)" if c % 2 == 0 else " & 0x80u)") ~ " != 0u" %}let gi{{ P }}_{{ c }}=({{ lo }})|({{ hi }});let gg{{ P }}_{{ c }}={{ iqAtGrid1("gi" ~ P ~ "_" ~ c, sfx) }};let dd{{ P }}_{{ c }}=select(0.125,-0.125,{{ neg }});let g{{ P }}_{{ 2 * c }}=vec4<f32>(unpack4xI8(gg{{ P }}_{{ c }}.x))+dd{{ P }}_{{ c }};let g{{ P }}_{{ 2 * c + 1 }}=vec4<f32>(unpack4xI8(gg{{ P }}_{{ c }}.y))+dd{{ P }}_{{ c }};
{% endfor %}{% endmacro %}{% macro iq1s_regs(P, q, mt, d, sfx="") %}let iq1q{{ P }}={{ q }};let iq1m{{ P }}={{ mt }};let dlA{{ P }}={{ d }}*f32(2u*(iq1m{{ P }}>>13u)+1u);let dlB{{ P }}=dlA{{ P }};let iq1n{{ P }}=select(0.125,-0.125,(iq1m{{ P }}&0x1000u)!=0u);
{% for c in range(4) %}{% set lo = ["iq1q" ~ P ~ " & 0xFFu", "(iq1q" ~ P ~ " >> 8u) & 0xFFu", "(iq1q" ~ P ~ " >> 16u) & 0xFFu", "iq1q" ~ P ~ " >> 24u"][c] %}{% set hi = ["(iq1m" ~ P ~ " << 8u) & 0x700u", "(iq1m" ~ P ~ " << 5u) & 0x700u", "(iq1m" ~ P ~ " << 2u) & 0x700u", "(iq1m" ~ P ~ " >> 1u) & 0x700u"][c] %}let gi{{ P }}_{{ c }}=({{ lo }})|({{ hi }});let gg{{ P }}_{{ c }}={{ iqAtGrid1("gi" ~ P ~ "_" ~ c, sfx) }};let g{{ P }}_{{ 2 * c }}=vec4<f32>(unpack4xI8(gg{{ P }}_{{ c }}.x))+iq1n{{ P }};let g{{ P }}_{{ 2 * c + 1 }}=vec4<f32>(unpack4xI8(gg{{ P }}_{{ c }}.y))+iq1n{{ P }};
{% endfor %}{% endmacro %}{% macro iq2xxs_regs(P, q, m, d, sfx="") %}let iq2q{{ P }}={{ q }};let iq2m{{ P }}={{ m }};let db{{ P }}={{ d }}*(0.5+f32(iq2m{{ P }}>>28u))*0.25;
{% for c in range(4) %}{% set gi = ["iq2q" ~ P ~ " & 0xFFu", "(iq2q" ~ P ~ " >> 8u) & 0xFFu", "(iq2q" ~ P ~ " >> 16u) & 0xFFu", "iq2q" ~ P ~ " >> 24u"][c] %}{% set sv = ["iq2m" ~ P ~ " & 127u", "(iq2m" ~ P ~ " >> 7u) & 127u", "(iq2m" ~ P ~ " >> 14u) & 127u", "(iq2m" ~ P ~ " >> 21u) & 127u"][c] %}let gi{{ P }}_{{ c }}={{ gi }};let s{{ P }}_{{ c }}=ksg({{ sv }});let g{{ P }}_{{ 2 * c }}=sgx({{ iqAtMagLo("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_LO);let g{{ P }}_{{ 2 * c + 1 }}=sgx({{ iqAtMagHi("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_HI);
{% endfor %}{% endmacro %}{% macro iq2xs_regs(P, w0, w1, sc, d, sfx="") %}let iq2w0_{{ P }}={{ w0 }};let iq2w1_{{ P }}={{ w1 }};let iq2sc{{ P }}={{ sc }};let iq2d{{ P }}={{ d }};let dlA{{ P }}=iq2d{{ P }}*(0.5+f32(iq2sc{{ P }}&0xFu))*0.25;let dlB{{ P }}=iq2d{{ P }}*(0.5+f32(iq2sc{{ P }}>>4u))*0.25;
{% for c in range(4) %}{% set word = ("iq2w0_" ~ P) if c < 2 else ("iq2w1_" ~ P) %}{% set gi = (word ~ " & 0x1FFu") if c % 2 == 0 else ("(" ~ word ~ " >> 16u) & 0x1FFu") %}{% set sv = ("(" ~ word ~ " >> 9u) & 127u") if c % 2 == 0 else ("(" ~ word ~ " >> 25u) & 127u") %}let gi{{ P }}_{{ c }}={{ gi }};let s{{ P }}_{{ c }}=ksg({{ sv }});let g{{ P }}_{{ 2 * c }}=sgx({{ iqAtMagLo("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_LO);let g{{ P }}_{{ 2 * c + 1 }}=sgx({{ iqAtMagHi("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_HI);
{% endfor %}{% endmacro %}{% macro iq2s_regs(P, qw, sw, qh, sc, d, sfx="") %}let iq2qw{{ P }}={{ qw }};let iq2sw{{ P }}={{ sw }};let iq2qh{{ P }}={{ qh }};let iq2sc{{ P }}={{ sc }};let iq2d{{ P }}={{ d }};let dlA{{ P }}=iq2d{{ P }}*(0.5+f32(iq2sc{{ P }}&0xFu))*0.25;let dlB{{ P }}=iq2d{{ P }}*(0.5+f32(iq2sc{{ P }}>>4u))*0.25;
{% for c in range(4) %}{% set qwn = "iq2qw" ~ P %}{% set qhn = "iq2qh" ~ P %}{% set gi = [
"("~qwn~"&0xFFu)|(("~qhn~"<<8u)&0x300u)","(("~qwn~">>8u)&0xFFu)|(("~qhn~"<<6u)&0x300u)","(("~qwn~">>16u)&0xFFu)|(("~qhn~"<<4u)&0x300u)",
  "(" ~ qwn ~ " >> 24u) | ((" ~ qhn ~ " << 2u) & 0x300u)"][c] %}{% set sv = ("iq2sw" ~ P) if c == 0 else ("iq2sw" ~ P ~ " >> " ~ (8 * c) ~ "u") %}let gi{{ P }}_{{ c }}={{ gi }};let s{{ P }}_{{ c }}={{ sv }};let g{{ P }}_{{ 2 * c }}=sgx({{ iqAtIq2sLo("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_LO);let g{{ P }}_{{ 2 * c + 1 }}=sgx({{ iqAtIq2sHi("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_HI);
{% endfor %}{% endmacro %}{% macro iq3xxs_regs(P, w0, w1, sw, d, sfx="") %}let iq3w0_{{ P }}={{ w0 }};let iq3w1_{{ P }}={{ w1 }};let iq3sw{{ P }}={{ sw }};let db{{ P }}={{ d }}*(0.5+f32(iq3sw{{ P }}>>28u))*0.5;
{% for c in range(4) %}{% set word = "iq3w0_" ~ P if c < 2 else "iq3w1_" ~ P %}{% set lo = 16 * (c % 2) %}let s{{ P }}_{{ c }}=ksg((iq3sw{{ P }}>>{{ 7 * c }}u)&127u);let g{{ P }}_{{ 2 * c }}=sgx({{ iqAtMag4("(" ~ word ~ " >> " ~ lo ~ "u) & 0xFFu", sfx) }},s{{ P }}_{{ c }},SGX_LO);let g{{ P }}_{{ 2 * c + 1 }}=sgx({{ iqAtMag4(("(" ~ word ~ " >> " ~ (lo + 8) ~ "u) & 0xFFu") if lo == 0 else (word ~ " >> " ~ (lo + 8) ~ "u"), sfx) }},s{{ P }}_{{ c }},SGX_HI);
{% endfor %}{% endmacro %}{% macro iq3xxs_dot_at(P, acc, w0, w1, sw, d, sfx="") %}let iq3w0_{{ P }}={{ w0 }};let iq3w1_{{ P }}={{ w1 }};let iq3sw{{ P }}={{ sw }};let db{{ P }}={{ d }}*(0.5+f32(iq3sw{{ P }}>>28u))*0.5;
{% for c in range(4) %}{% set word = "iq3w0_" ~ P if c < 2 else "iq3w1_" ~ P %}{% set ia = (word ~ " & 0xFFu") if c % 2 == 0 else ("(" ~ word ~ " >> 16u) & 0xFFu") %}{% set ib = ("(" ~ word ~ " >> 8u) & 0xFFu") if c % 2 == 0 else (word ~ " >> 24u") %}let s{{ P }}_{{ c }}=ksg({{ ("iq3sw" ~ P ~ " & 127u") if c == 0 else ("(iq3sw" ~ P ~ " >> " ~ (7 * c) ~ "u) & 127u") }});let c{{ P }}_{{ c }}=dot(sgx({{ iqAtMag4(ia, sfx) }},s{{ P }}_{{ c }},SGX_LO),x{{ 2 * c }})+dot(sgx({{ iqAtMag4(ib, sfx) }},s{{ P }}_{{ c }},SGX_HI),x{{ 2 * c + 1 }});
{% endfor %}{{ acc }}={{ acc }}+db{{ P }}*(c{{ P }}_0+c{{ P }}_1+c{{ P }}_2+c{{ P }}_3);
{% endmacro %}{% macro iq3s_dot_at(P, acc, w0, w1, sw, qh, sc, d, sfx="") %}let iq3w0_{{ P }}={{ w0 }};let iq3w1_{{ P }}={{ w1 }};let iq3sw{{ P }}={{ sw }};let iq3qh{{ P }}={{ qh }};let db{{ P }}={{ d }}*(1.0+2.0*f32({{ sc }}));
{% for j in range(8) %}{% set word = ("iq3w0_" ~ P) if j < 4 else ("iq3w1_" ~ P) %}{% set byte = ["(" ~ word ~ " & 0xFFu)", "((" ~ word ~ " >> 8u) & 0xFFu)", "((" ~ word ~ " >> 16u) & 0xFFu)", "(" ~ word ~ " >> 24u)"][j % 4] %}let c{{ P }}_{{ j }}=dot(sgx({{ iqAtMag4(byte ~ " | ((iq3qh" ~ P ~ " << " ~ (8 - j) ~ "u) & 0x100u)", sfx) }},{{ ("iq3sw" ~ P) if j == 0 else ("iq3sw" ~ P ~ " >> " ~ (4 * j) ~ "u") }},SGX_LO),x{{ j }});
{% endfor %}{{ acc }}={{ acc }}+db{{ P }}*(c{{ P }}_0+c{{ P }}_1+c{{ P }}_2+c{{ P }}_3+c{{ P }}_4+c{{ P }}_5+c{{ P }}_6+c{{ P }}_7);
{% endmacro %}{% macro iq2xxs_dot_at(P, acc, q, m, d, sfx="") %}let iq2q{{ P }}={{ q }};let iq2m{{ P }}={{ m }};let db{{ P }}={{ d }}*(0.5+f32(iq2m{{ P }}>>28u))*0.25;
{% for c in range(4) %}{% set gi = ["iq2q" ~ P ~ " & 0xFFu", "(iq2q" ~ P ~ " >> 8u) & 0xFFu", "(iq2q" ~ P ~ " >> 16u) & 0xFFu", "iq2q" ~ P ~ " >> 24u"][c] %}{% set sv = ["iq2m" ~ P ~ " & 127u", "(iq2m" ~ P ~ " >> 7u) & 127u", "(iq2m" ~ P ~ " >> 14u) & 127u", "(iq2m" ~ P ~ " >> 21u) & 127u"][c] %}let gi{{ P }}_{{ c }}={{ gi }};let s{{ P }}_{{ c }}=ksg({{ sv }});let c{{ P }}_{{ c }}=dot(sgx({{ iqAtMagLo("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_LO),x{{ 2 * c }})+dot(sgx({{ iqAtMagHi("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_HI),x{{ 2 * c + 1 }});
{% endfor %}{{ acc }}={{ acc }}+db{{ P }}*(c{{ P }}_0+c{{ P }}_1+c{{ P }}_2+c{{ P }}_3);
{% endmacro %}{% macro iq2xs_dot_at(P, acc, w0, w1, sc, d, sfx="") %}let iq2w0_{{ P }}={{ w0 }};let iq2w1_{{ P }}={{ w1 }};let iq2sc{{ P }}={{ sc }};let iq2d{{ P }}={{ d }};let dlA{{ P }}=iq2d{{ P }}*(0.5+f32(iq2sc{{ P }}&0xFu))*0.25;let dlB{{ P }}=iq2d{{ P }}*(0.5+f32(iq2sc{{ P }}>>4u))*0.25;
{% for c in range(4) %}{% set word = ("iq2w0_" ~ P) if c < 2 else ("iq2w1_" ~ P) %}{% set gi = (word ~ " & 0x1FFu") if c % 2 == 0 else ("(" ~ word ~ " >> 16u) & 0x1FFu") %}{% set sv = ("(" ~ word ~ " >> 9u) & 127u") if c % 2 == 0 else ("(" ~ word ~ " >> 25u) & 127u") %}let gi{{ P }}_{{ c }}={{ gi }};let s{{ P }}_{{ c }}=ksg({{ sv }});let c{{ P }}_{{ c }}=dot(sgx({{ iqAtMagLo("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_LO),x{{ 2 * c }})+dot(sgx({{ iqAtMagHi("gi" ~ P ~ "_" ~ c, sfx) }},s{{ P }}_{{ c }},SGX_HI),x{{ 2 * c + 1 }});
{% endfor %}{{ acc }}={{ acc }}+(dlA{{ P }}*(c{{ P }}_0+c{{ P }}_1)+dlB{{ P }}*(c{{ P }}_2+c{{ P }}_3));
{% endmacro %}{% macro iq2s_dot_at(P, acc, qw, sw, qh, sc, d, sfx="") %}let iq2qw{{ P }}={{ qw }};let iq2sw{{ P }}={{ sw }};let iq2qh{{ P }}={{ qh }};let iq2sc{{ P }}={{ sc }};let iq2d{{ P }}={{ d }};let dlA{{ P }}=iq2d{{ P }}*(0.5+f32(iq2sc{{ P }}&0xFu))*0.25;let dlB{{ P }}=iq2d{{ P }}*(0.5+f32(iq2sc{{ P }}>>4u))*0.25;
{% for c in range(4) %}{% set qwn = "iq2qw" ~ P %}{% set qhn = "iq2qh" ~ P %}{% set gi = [
"("~qwn~"&0xFFu)|(("~qhn~"<<8u)&0x300u)","(("~qwn~">>8u)&0xFFu)|(("~qhn~"<<6u)&0x300u)","(("~qwn~">>16u)&0xFFu)|(("~qhn~"<<4u)&0x300u)",
  "(" ~ qwn ~ " >> 24u) | ((" ~ qhn ~ " << 2u) & 0x300u)"][c] %}let gi{{ P }}_{{ c }}={{ gi }};let c{{ P }}_{{ c }}=dot(sgx({{ iqAtIq2sLo("gi" ~ P ~ "_" ~ c, sfx) }},{{ ("iq2sw" ~ P) if c == 0 else ("iq2sw" ~ P ~ " >> " ~ (8 * c) ~ "u") }},SGX_LO),x{{ 2 * c }})+dot(sgx({{ iqAtIq2sHi("gi" ~ P ~ "_" ~ c, sfx) }},{{ ("iq2sw" ~ P) if c == 0 else ("iq2sw" ~ P ~ " >> " ~ (8 * c) ~ "u") }},SGX_HI),x{{ 2 * c + 1 }});
{% endfor %}{{ acc }}={{ acc }}+(dlA{{ P }}*(c{{ P }}_0+c{{ P }}_1)+dlB{{ P }}*(c{{ P }}_2+c{{ P }}_3));
{% endmacro %}{% macro iq1_dot_at(P, acc, q, m, d, sfx="") %}let iq1q{{ P }}={{ q }};let iq1m{{ P }}={{ m }};let iq1d{{ P }}={{ d }};let dlA{{ P }}=iq1d{{ P }}*f32(2u*((iq1m{{ P }}>>16u)&7u)+1u);let dlB{{ P }}=iq1d{{ P }}*f32(2u*((iq1m{{ P }}>>19u)&7u)+1u);let qh0_{{ P }}=iq1m{{ P }}&0xFFu;let qh1_{{ P }}=(iq1m{{ P }}>>8u)&0xFFu;
{% for c in range(4) %}{% set lo = ["iq1q" ~ P ~ " & 0xFFu", "(iq1q" ~ P ~ " >> 8u) & 0xFFu", "(iq1q" ~ P ~ " >> 16u) & 0xFFu", "iq1q" ~ P ~ " >> 24u"][c] %}{% set qh = ("qh0_" ~ P) if c < 2 else ("qh1_" ~ P) %}{% set hi = "(" ~ qh ~ (" << 8u)" if c % 2 == 0 else " << 4u)") ~ " & 0x700u" %}{% set neg = "(" ~ qh ~ (" & 0x08u)" if c % 2 == 0 else " & 0x80u)") ~ " != 0u" %}let gi{{ P }}_{{ c }}=({{ lo }})|({{ hi }});let gg{{ P }}_{{ c }}={{ iqAtGrid1("gi" ~ P ~ "_" ~ c, sfx) }};let dd{{ P }}_{{ c }}=select(0.125,-0.125,{{ neg }});let c{{ P }}_{{ c }}=dot(vec4<f32>(unpack4xI8(gg{{ P }}_{{ c }}.x))+dd{{ P }}_{{ c }},x{{ 2 * c }})+dot(vec4<f32>(unpack4xI8(gg{{ P }}_{{ c }}.y))+dd{{ P }}_{{ c }},x{{ 2 * c + 1 }});
{% endfor %}{{ acc }}={{ acc }}+(dlA{{ P }}*(c{{ P }}_0+c{{ P }}_1)+dlB{{ P }}*(c{{ P }}_2+c{{ P }}_3));
{% endmacro %}{% macro iq1s_dot_at(P, acc, q, mt, d, sfx="") %}let iq1q{{ P }}={{ q }};let iq1m{{ P }}={{ mt }};let dl{{ P }}={{ d }}*f32(2u*(iq1m{{ P }}>>13u)+1u);let neg{{ P }}=(iq1m{{ P }}&0x1000u)!=0u;
{% for c in range(4) %}{% set lo = ["iq1q" ~ P ~ " & 0xFFu", "(iq1q" ~ P ~ " >> 8u) & 0xFFu", "(iq1q" ~ P ~ " >> 16u) & 0xFFu", "iq1q" ~ P ~ " >> 24u"][c] %}{% set hi = ["(iq1m" ~ P ~ " << 8u) & 0x700u", "(iq1m" ~ P ~ " << 5u) & 0x700u", "(iq1m" ~ P ~ " << 2u) & 0x700u", "(iq1m" ~ P ~ " >> 1u) & 0x700u"][c] %}let gi{{ P }}_{{ c }}=({{ lo }})|({{ hi }});let gg{{ P }}_{{ c }}={{ iqAtGrid1("gi" ~ P ~ "_" ~ c, sfx) }};let dd{{ P }}_{{ c }}=select(0.125,-0.125,neg{{ P }});let c{{ P }}_{{ c }}=dot(vec4<f32>(unpack4xI8(gg{{ P }}_{{ c }}.x))+dd{{ P }}_{{ c }},x{{ 2 * c }})+dot(vec4<f32>(unpack4xI8(gg{{ P }}_{{ c }}.y))+dd{{ P }}_{{ c }},x{{ 2 * c + 1 }});
{% endfor %}{{ acc }}={{ acc }}+(dl{{ P }}*(c{{ P }}_0+c{{ P }}_1)+dl{{ P }}*(c{{ P }}_2+c{{ P }}_3));
{% endmacro %}{% macro iq1_dense_dot(P, acc, md, q, meta, d) %}{% if iqInline | default(false) %}{{ (iq1_dot_at(P, acc, q, meta, d) if md else iq1s_dot_at(P, acc, q, meta, d)) | trim }}
{% elif md %}{{ acc }}={{ acc }}+iq1dot(vec2<u32>({{ q }},{{ meta }}),{{ d }},x0,x1,x2,x3,x4,x5,x6,x7);
{% else %}{{ acc }}={{ acc }}+iq1sdot({{ q }},{{ meta }},{{ d }},x0,x1,x2,x3,x4,x5,x6,x7);
{% endif %}{% endmacro %}`],["ops/_shared/iq1-dequant-helpers.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}fn iq1cell{{ iqSfx }}(lo:u32,hi:u32,neg:bool,xa:vec4<f32>,xb:vec4<f32>)->f32{let g=iq_grid{{ iqSfx }}(lo|hi);let delta=select(0.125,-0.125,neg);return dot(vec4<f32>(unpack4xI8(g.x))+delta,xa)+dot(vec4<f32>(unpack4xI8(g.y))+delta,xb);}
{% set iq1mLive = iq1mLive | default((iq1m | default(false)) and not (iq1s | default(false))) %}{% if iq1mLive %}fn iq1dot{{ iqSfx }}(m:vec2<u32>,d:f32,x0:vec4<f32>,x1:vec4<f32>,x2:vec4<f32>,x3:vec4<f32>,x4:vec4<f32>,x5:vec4<f32>,x6:vec4<f32>,x7:vec4<f32>)->f32{let dl1=d*f32(2u*((m.y>>16u)&7u)+1u);let dl2=d*f32(2u*((m.y>>19u)&7u)+1u);let qh0=m.y&0xFFu;let qh1=(m.y>>8u)&0xFFu;return dl1*(iq1cell{{ iqSfx }}(m.x&0xFFu,(qh0<<8u)&0x700u,(qh0&0x08u)!=0u,x0,x1)+iq1cell{{ iqSfx }}((m.x>>8u)&0xFFu,(qh0<<4u)&0x700u,(qh0&0x80u)!=0u,x2,x3))+dl2*(iq1cell{{ iqSfx }}((m.x>>16u)&0xFFu,(qh1<<8u)&0x700u,(qh1&0x08u)!=0u,x4,x5)+iq1cell{{ iqSfx }}(m.x>>24u,(qh1<<4u)&0x700u,(qh1&0x80u)!=0u,x6,x7));}
{% endif %}{% if iq1s | default(false) %}fn iq1sdot{{ iqSfx }}(qs:u32,mt:u32,d:f32,x0:vec4<f32>,x1:vec4<f32>,x2:vec4<f32>,x3:vec4<f32>,x4:vec4<f32>,x5:vec4<f32>,x6:vec4<f32>,x7:vec4<f32>)->f32{let dl=d*f32(2u*(mt>>13u)+1u);let neg=(mt&0x1000u)!=0u;return dl*(iq1cell{{ iqSfx }}(qs&0xFFu,(mt<<8u)&0x700u,neg,x0,x1)+iq1cell{{ iqSfx }}((qs>>8u)&0xFFu,(mt<<5u)&0x700u,neg,x2,x3))+dl*(iq1cell{{ iqSfx }}((qs>>16u)&0xFFu,(mt<<2u)&0x700u,neg,x4,x5)+iq1cell{{ iqSfx }}(qs>>24u,(mt>>1u)&0x700u,neg,x6,x7));}
{% endif %}`],["ops/_shared/iqx-raw-dequant-helpers.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}{% set iqDot = iqDot | default(false) %}{% set iqPrimitives = iqPrimitives if iqPrimitives is defined else true %}{% set iqFusedDot = iqDot and not (iqInline | default(false)) %}{% set iqInferredPrimitiveMode =
"explicit8"if iq2s|default(false)else"explicit4"if iq3s|default(false)else"none"if iq1m|default(false)else"parity8"
%}{% set iqPrimitiveMode = iqPrimitiveMode | default(iqInferredPrimitiveMode) %}{% if iqPrimitives and iqPrimitiveMode != "none" %}fn sgx(v:vec4<f32>,s:u32,sh:vec4<u32>)->vec4<f32>{return bitcast<vec4<f32>>(bitcast<vec4<u32>>(v)^((vec4<u32>(s)<<sh)&vec4<u32>(0x80000000u)));}const SGX_LO:vec4<u32>=vec4<u32>(31u,30u,29u,28u);
{% if iqPrimitiveMode != "explicit4" %}const SGX_HI:vec4<u32>=vec4<u32>(27u,26u,25u,24u);
{% endif %}{% if iqPrimitiveMode == "parity8" %}fn ksg(sv:u32)->u32{return sv|((countOneBits(sv)&1u)<<7u);}
{% endif %}{% endif %}{% if iqFusedDot and iq1m and not ((hoistFormat | default("")) == "iq1") %}{% include "ops/_shared/iq1-dequant-helpers.wgsl.jinja" %}{% elif iqFusedDot and iq2xxs and not ((hoistFormat | default("")) == "iq2xxs") %}fn iq2cell{{ iqSfx }}(gi:u32,sv:u32,xa:vec4<f32>,xb:vec4<f32>)->f32{let s=ksg(sv);return dot(sgx(iq2magLo{{ iqSfx }}(gi),s,SGX_LO),xa)+dot(sgx(iq2magHi{{ iqSfx }}(gi),s,SGX_HI),xb);}fn iq2dot{{ iqSfx }}(m:vec2<u32>,d:f32,x0:vec4<f32>,x1:vec4<f32>,x2:vec4<f32>,x3:vec4<f32>,x4:vec4<f32>,x5:vec4<f32>,x6:vec4<f32>,x7:vec4<f32>)->f32{let db=d*(0.5+f32(m.y>>28u))*0.25;return db*(iq2cell{{ iqSfx }}(m.x&0xFFu,m.y&127u,x0,x1)+iq2cell{{ iqSfx }}((m.x>>8u)&0xFFu,(m.y>>7u)&127u,x2,x3)+iq2cell{{ iqSfx }}((m.x>>16u)&0xFFu,(m.y>>14u)&127u,x4,x5)+iq2cell{{ iqSfx }}(m.x>>24u,(m.y>>21u)&127u,x6,x7));}
{% elif iqFusedDot and iq2xs and not ((hoistFormat | default("")) == "iq2xs") %}fn iq2xscell{{ iqSfx }}(gi:u32,sv:u32,xa:vec4<f32>,xb:vec4<f32>)->f32{let s=ksg(sv);return dot(sgx(iq2magLo{{ iqSfx }}(gi),s,SGX_LO),xa)+dot(sgx(iq2magHi{{ iqSfx }}(gi),s,SGX_HI),xb);}fn iq2xsdot{{ iqSfx }}(w0:u32,w1:u32,sc:u32,d:f32,x0:vec4<f32>,x1:vec4<f32>,x2:vec4<f32>,x3:vec4<f32>,x4:vec4<f32>,x5:vec4<f32>,x6:vec4<f32>,x7:vec4<f32>)->f32{let dlo=d*(0.5+f32(sc&0xFu))*0.25;let dhi=d*(0.5+f32(sc>>4u))*0.25;return dlo*(iq2xscell{{ iqSfx }}(w0&0x1FFu,(w0>>9u)&127u,x0,x1)+iq2xscell{{ iqSfx }}((w0>>16u)&0x1FFu,(w0>>25u)&127u,x2,x3))+dhi*(iq2xscell{{ iqSfx }}(w1&0x1FFu,(w1>>9u)&127u,x4,x5)+iq2xscell{{ iqSfx }}((w1>>16u)&0x1FFu,(w1>>25u)&127u,x6,x7));}
{% elif iqFusedDot and iq2s and not ((hoistFormat | default("")) == "iq2s") %}fn iq2scell{{ iqSfx }}(gi:u32,s:u32,xa:vec4<f32>,xb:vec4<f32>)->f32{return dot(sgx(iq2magLo{{ iqSfx }}(gi),s,SGX_LO),xa)+dot(sgx(iq2magHi{{ iqSfx }}(gi),s,SGX_HI),xb);}fn iq2sdot{{ iqSfx }}(qw:u32,sw:u32,qh:u32,sc:u32,d:f32,x0:vec4<f32>,x1:vec4<f32>,x2:vec4<f32>,x3:vec4<f32>,x4:vec4<f32>,x5:vec4<f32>,x6:vec4<f32>,x7:vec4<f32>)->f32{let dlo=d*(0.5+f32(sc&0xFu))*0.25;let dhi=d*(0.5+f32(sc>>4u))*0.25;return dlo*(iq2scell{{ iqSfx }}((qw&0xFFu)|((qh<<8u)&0x300u),sw,x0,x1)+iq2scell{{ iqSfx }}(((qw>>8u)&0xFFu)|((qh<<6u)&0x300u),sw>>8u,x2,x3))+dhi*(iq2scell{{ iqSfx }}(((qw>>16u)&0xFFu)|((qh<<4u)&0x300u),sw>>16u,x4,x5)+iq2scell{{ iqSfx }}((qw>>24u)|((qh<<2u)&0x300u),sw>>24u,x6,x7));}
{% elif iqFusedDot and iq3xxs and not ((hoistFormat | default("")) == "iq3xxs") %}fn iq3cell{{ iqSfx }}(ia:u32,ib:u32,sv:u32,xa:vec4<f32>,xb:vec4<f32>)->f32{let s=ksg(sv);return dot(sgx(iq3mag{{ iqSfx }}(ia),s,SGX_LO),xa)+dot(sgx(iq3mag{{ iqSfx }}(ib),s,SGX_HI),xb);}fn iq3dot{{ iqSfx }}(w0:u32,w1:u32,sw:u32,d:f32,x0:vec4<f32>,x1:vec4<f32>,x2:vec4<f32>,x3:vec4<f32>,x4:vec4<f32>,x5:vec4<f32>,x6:vec4<f32>,x7:vec4<f32>)->f32{let db=d*(0.5+f32(sw>>28u))*0.5;return db*(iq3cell{{ iqSfx }}(w0&0xFFu,(w0>>8u)&0xFFu,sw&127u,x0,x1)+iq3cell{{ iqSfx }}((w0>>16u)&0xFFu,w0>>24u,(sw>>7u)&127u,x2,x3)+iq3cell{{ iqSfx }}(w1&0xFFu,(w1>>8u)&0xFFu,(sw>>14u)&127u,x4,x5)+iq3cell{{ iqSfx }}((w1>>16u)&0xFFu,w1>>24u,(sw>>21u)&127u,x6,x7));}
{% elif iqFusedDot and iq3s %}fn iq3scell{{ iqSfx }}(gi:u32,s:u32,xa:vec4<f32>)->f32{return dot(sgx(iq3mag{{ iqSfx }}(gi),s,SGX_LO),xa);}fn iq3sdot{{ iqSfx }}(w0:u32,w1:u32,sw:u32,qh:u32,sc:u32,d:f32,x0:vec4<f32>,x1:vec4<f32>,x2:vec4<f32>,x3:vec4<f32>,x4:vec4<f32>,x5:vec4<f32>,x6:vec4<f32>,x7:vec4<f32>)->f32{let db=d*(1.0+2.0*f32(sc));return db*(iq3scell{{ iqSfx }}((w0&0xFFu)|((qh<<8u)&0x100u),sw,x0)+iq3scell{{ iqSfx }}(((w0>>8u)&0xFFu)|((qh<<7u)&0x100u),sw>>4u,x1)+iq3scell{{ iqSfx }}(((w0>>16u)&0xFFu)|((qh<<6u)&0x100u),sw>>8u,x2)+iq3scell{{ iqSfx }}((w0>>24u)|((qh<<5u)&0x100u),sw>>12u,x3)+iq3scell{{ iqSfx }}((w1&0xFFu)|((qh<<4u)&0x100u),sw>>16u,x4)+iq3scell{{ iqSfx }}(((w1>>8u)&0xFFu)|((qh<<3u)&0x100u),sw>>20u,x5)+iq3scell{{ iqSfx }}(((w1>>16u)&0xFFu)|((qh<<2u)&0x100u),sw>>24u,x6)+iq3scell{{ iqSfx }}((w1>>24u)|((qh<<1u)&0x100u),sw>>28u,x7));}
{% endif %}{% include "ops/_shared/iq-inline-dot.wgsl.jinja" %}`],["ops/_shared/lut2-block-scale.wgsl.jinja",`{% macro lut2_native_scale(block, metadata, codebook) %}{% if codebook == 9 %}unpack2x16float({{ metadata }}[({{ block }}) >> 3u])[(({{ block }}) >> 2u) & 1u]{% else %}unpack2x16float({{ metadata }}[(({{ block }}) >> 3u) * 3u + ((({{ block }}) & 7u) >> 2u)])[(({{ block }}) >> 1u) & 1u]{% endif %}{% endmacro %}{% macro lut2_block_scale(block, prefix, name, lo, hi, delta, per16, metadata, suffix, codebook) %}{% set scale = "lutScale" ~ suffix %}{% if codebook in [8, 9] %}let {{ name }}={{ lut2_native_scale(block, metadata, codebook) }};let {{ delta }}=vec4<u32>(0u);
{% elif per16 %}let {{ prefix }}mb=(({{ block }})>>3u)*5u;let {{ prefix }}dl=bitcast<f32>({{ metadata }}[{{ prefix }}mb+4u]);let {{ prefix }}cw={{ metadata }}[{{ prefix }}mb+((({{ block }})&7u)>>1u)];let {{ prefix }}cs=(({{ block }})&1u)*16u;let {{ prefix }}bLo=({{ prefix }}cw>>{{ prefix }}cs)&255u;let {{ prefix }}bHi=({{ prefix }}cw>>({{ prefix }}cs+8u))&255u;let {{ lo }}={{ prefix }}dl*{{ scale }}({{ prefix }}bLo&15u);let {{ hi }}={{ prefix }}dl*{{ scale }}({{ prefix }}bHi&15u);let {{ delta }}=((vec4<u32>({{ prefix }}bLo,{{ prefix }}bLo,{{ prefix }}bHi,{{ prefix }}bHi)>>vec4<u32>(4u,5u,4u,5u))&vec4<u32>(1u))*8u;
{% else %}let {{ prefix }}mb=(({{ block }})>>3u)*3u;let {{ prefix }}b=({{ metadata }}[{{ prefix }}mb+((({{ block }})&7u)>>2u)]>>((({{ block }})&3u)*8u))&255u;let {{ name }}=bitcast<f32>({{ metadata }}[{{ prefix }}mb+2u])*{{ scale }}({{ prefix }}b&15u);let {{ delta }}=vec4<u32>((({{ prefix }}b>>4u)&1u)*8u);
{% endif %}{% endmacro %}`],["ops/_shared/packed-4x8-compat.wgsl.jinja",`{% if not env.device.wgslLanguageFeatures.has("packed_4x8_integer_dot_product") %}{% if (packed4x8CompatUnpackI8 | default(false)) or (packed4x8CompatUnpackU8 | default(false)) or (packed4x8CompatPackI8 | default(false)) or (packed4x8CompatDotI8 | default(false)) %}{% endif %}{% if packed4x8CompatUnpackI8 | default(false) %}@must_use fn unpack4xI8(e:u32)->vec4<i32>{let bytes=(vec4<u32>(e)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(0xffu);return vec4<i32>(bytes^vec4<u32>(0x80u))-vec4<i32>(128i);}
{% endif %}{% if packed4x8CompatUnpackU8 | default(false) %}@must_use fn unpack4xU8(e:u32)->vec4<u32>{return(vec4<u32>(e)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(0xffu);}
{% endif %}{% if packed4x8CompatPackI8 | default(false) %}@must_use fn pack4xI8(e:vec4<i32>)->u32{let bytes=bitcast<vec4<u32>>(e)&vec4<u32>(0xffu);return bytes.x|(bytes.y<<8u)|(bytes.z<<16u)|(bytes.w<<24u);}
{% endif %}{% if packed4x8CompatDotI8 | default(false) %}@must_use fn dot4I8Packed(e1:u32,e2:u32)->i32{let shifts=vec4<u32>(0u,8u,16u,24u);let a_bytes=(vec4<u32>(e1)>>shifts)&vec4<u32>(0xffu);let b_bytes=(vec4<u32>(e2)>>shifts)&vec4<u32>(0xffu);let a=vec4<i32>(a_bytes^vec4<u32>(0x80u))-vec4<i32>(128i);let b=vec4<i32>(b_bytes^vec4<u32>(0x80u))-vec4<i32>(128i);return dot(a,b);}
{% endif %}{% endif %}`],["ops/_shared/q1-dot-helpers.wgsl.jinja",`{% include "ops/_shared/q1-signs.wgsl.jinja" %}fn q1_dot_word_t4(block_offset:u32,tile:u32,blocks_per_row:u32,col_block:u32,word_id:u32,a0:vec4<f32>,a1:vec4<f32>,a2:vec4<f32>,a3:vec4<f32>,a4:vec4<f32>,a5:vec4<f32>,a6:vec4<f32>,a7:vec4<f32>)->vec4<f32>{let group_idx=tile*blocks_per_row+col_block;let bit_vec=q1_bits[block_offset+group_idx*4u+word_id];
{% if scaleScalar == "f32" %}let s0=q1_scales[block_offset+group_idx*4u+0u];let s1=q1_scales[block_offset+group_idx*4u+1u];let s2=q1_scales[block_offset+group_idx*4u+2u];let s3=q1_scales[block_offset+group_idx*4u+3u];
{% else %}let s0=f32(q1_scales[block_offset+group_idx*4u+0u]);let s1=f32(q1_scales[block_offset+group_idx*4u+1u]);let s2=f32(q1_scales[block_offset+group_idx*4u+2u]);let s3=f32(q1_scales[block_offset+group_idx*4u+3u]);
{% endif %}let unscaled0=dot(a0,q1_signs4_unit(bit_vec.x,0u))+dot(a1,q1_signs4_unit(bit_vec.x,4u))+dot(a2,q1_signs4_unit(bit_vec.x,8u))+dot(a3,q1_signs4_unit(bit_vec.x,12u))+dot(a4,q1_signs4_unit(bit_vec.x,16u))+dot(a5,q1_signs4_unit(bit_vec.x,20u))+dot(a6,q1_signs4_unit(bit_vec.x,24u))+dot(a7,q1_signs4_unit(bit_vec.x,28u));let unscaled1=dot(a0,q1_signs4_unit(bit_vec.y,0u))+dot(a1,q1_signs4_unit(bit_vec.y,4u))+dot(a2,q1_signs4_unit(bit_vec.y,8u))+dot(a3,q1_signs4_unit(bit_vec.y,12u))+dot(a4,q1_signs4_unit(bit_vec.y,16u))+dot(a5,q1_signs4_unit(bit_vec.y,20u))+dot(a6,q1_signs4_unit(bit_vec.y,24u))+dot(a7,q1_signs4_unit(bit_vec.y,28u));let unscaled2=dot(a0,q1_signs4_unit(bit_vec.z,0u))+dot(a1,q1_signs4_unit(bit_vec.z,4u))+dot(a2,q1_signs4_unit(bit_vec.z,8u))+dot(a3,q1_signs4_unit(bit_vec.z,12u))+dot(a4,q1_signs4_unit(bit_vec.z,16u))+dot(a5,q1_signs4_unit(bit_vec.z,20u))+dot(a6,q1_signs4_unit(bit_vec.z,24u))+dot(a7,q1_signs4_unit(bit_vec.z,28u));let unscaled3=dot(a0,q1_signs4_unit(bit_vec.w,0u))+dot(a1,q1_signs4_unit(bit_vec.w,4u))+dot(a2,q1_signs4_unit(bit_vec.w,8u))+dot(a3,q1_signs4_unit(bit_vec.w,12u))+dot(a4,q1_signs4_unit(bit_vec.w,16u))+dot(a5,q1_signs4_unit(bit_vec.w,20u))+dot(a6,q1_signs4_unit(bit_vec.w,24u))+dot(a7,q1_signs4_unit(bit_vec.w,28u));return vec4<f32>(s0*unscaled0,s1*unscaled1,s2*unscaled2,s3*unscaled3);}`],["ops/_shared/q1-hidden-rmsnorm-load.wgsl.jinja",`let col_block=word_index/4u;let word_id=word_index%4u;let base4=word_index*8u;
{% for i in range(8) %}let h{{ i }}={{ "vec4<f32>(" if hiddenVec4 != "vec4<f32>" else "" }}hidden[base4{{ " + " ~ i ~ "u" if i else "" }}]{{ ")" if hiddenVec4 != "vec4<f32>" else "" }};
{% endfor %}ss+=dot(h0,h0)+dot(h1,h1)+dot(h2,h2)+dot(h3,h3)+dot(h4,h4)+dot(h5,h5)+dot(h6,h6)+dot(h7,h7);let g_base=W_NORM_VEC4+base4;
{% for i in range(8) %}let n{{ i }}=h{{ i }}*{{ "vec4<f32>(" if normWeightVec4 != "vec4<f32>" else "" }}norm_weight[g_base{{ " + " ~ i ~ "u" if i else "" }}]{{ ")" if normWeightVec4 != "vec4<f32>" else "" }};
{% endfor %}`],["ops/_shared/q1-signs.wgsl.jinja","fn q1_signs4_unit(word:u32,shift:u32)->vec4<f32>{let bits=(vec4<u32>(word)>>vec4<u32>(shift,shift+1u,shift+2u,shift+3u))&vec4<u32>(1u);return select(vec4<f32>(-1.0),vec4<f32>(1.0),bits==vec4<u32>(1u));}"],["ops/_shared/q4-block-dot-flags.wgsl.jinja",'{% set sBits = "q4_bits" %}{% set sScales = "q4_scales" %}{% set sMeta = packMeta %}{% set sQuantBits = quantBits %}{% set sHasMin = hasMin %}{% set sIq = iq | default(false) %}{% set sQ5 = (q5 | default(false)) and (q5Route | default(false)) %}{% set sQ6k = q6k | default(false) %}{% set sQ5k = q5k | default(false) %}{% set sQ3k = q3k | default(false) %}{% set sQ2k = q2k | default(false) %}{% set sQ4knc = q4knCompact | default(false) %}{% set sLut16 = lut16 | default(false) %}{% set sLutc = lutc | default(false) %}{% set sLutc16 = lutc16 | default(false) %}{% set sLut2 = lut2 | default(false) %}{% set sScalesPk2 = scalesPk2 | default(false) %}{% set sQ8Pair = false %}{% set sIq1m = iq1m | default(false) %}{% set sIq1s = iq1s | default(false) %}{% set sIq1md = iq1md | default(false) %}{% set sIq2xxs = iq2xxs | default(false) %}{% set sIq2xs = iq2xs | default(false) %}{% set sIq2s = iq2s | default(false) %}{% set sIq3xxs = iq3xxs | default(false) %}{% set sIq3s = iq3s | default(false) %}{% set sLutId = lutId | default(0) %}{% set sLutSfx = "" %}{% set sIqSfx = "" %}{% set sPairView = false %}{% set sIqInline = iqInline | default(false) %}'],["ops/_shared/q4-block-dot.wgsl.jinja",`{% include "ops/_shared/lut2-block-scale.wgsl.jinja" %}{% include "ops/_shared/iq-inline-dot.wgsl.jinja" %}{% set rawPair = sBits ~ "[" ~ sPre ~ "blk]" if sPairView else "vec2<u32>(" ~ sBits ~ "[" ~ sPre ~ "w2], " ~ sBits ~ "[" ~ sPre ~ "w2 + 1u])" %}let {{ sPre }}blk={{ sBlk }};
{% if sIq1m %}{% if sIq1s %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}sbw={{ sPre }}sb*12u;let {{ sPre }}mt=({{ sBits }}[{{ sPre }}sbw+dqMo]>>dqSh)&0xFFFFu;let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}sb]);
{% if sIqInline %}{{ iq1s_dot_at(sPre, sAcc, sBits ~ "[" ~ sPre ~ "sbw + dqK]", sPre ~ "mt", sPre ~ "d", sIqSfx) | trim }}
{% else %}{{ sAcc }}={{ sAcc }}+iq1sdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+dqK],{{ sPre }}mt,{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% endif %}{% elif sIq1md %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}sbw={{ sPre }}sb*14u;let {{ sPre }}mt=(({{ sBits }}[{{ sPre }}sbw+dqMo]>>dqSh)&0xFFFFu)|((({{ sBits }}[{{ sPre }}sbw+dqCo]>>dqCs)&0x3Fu)<<16u);let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}sb]);
{% if sIqInline %}{{ iq1_dot_at(sPre, sAcc, sBits ~ "[" ~ sPre ~ "sbw + dqK]", sPre ~ "mt", sPre ~ "d", sIqSfx) | trim }}
{% else %}{{ sAcc }}={{ sAcc }}+iq1dot{{ sIqSfx }}(vec2<u32>({{ sBits }}[{{ sPre }}sbw+dqK],{{ sPre }}mt),{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% endif %}{% else %}{% if not sPairView %}let {{ sPre }}w2={{ sPre }}blk*2u;
{% endif %}let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}blk>>3u]);
{% if sIqInline %}let {{ sPre }}pr={{ rawPair }};{{ iq1_dot_at(sPre, sAcc, sPre ~ "pr.x", sPre ~ "pr.y", sPre ~ "d", sIqSfx) | trim }}
{% else %}{{ sAcc }}={{ sAcc }}+iq1dot{{ sIqSfx }}({{ rawPair }},{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% endif %}{% endif %}{% elif sIq2xxs %}{% if not sPairView %}let {{ sPre }}w2={{ sPre }}blk*2u;
{% endif %}let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}blk>>3u]);
{% if sIqInline %}let {{ sPre }}pr={{ rawPair }};{{ iq2xxs_dot_at(sPre, sAcc, sPre ~ "pr.x", sPre ~ "pr.y", sPre ~ "d", sIqSfx) | trim }}
{% else %}{{ sAcc }}={{ sAcc }}+iq2dot{{ sIqSfx }}({{ rawPair }},{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% endif %}{% elif sIq2xs %}let {{ sPre }}sbw=({{ sPre }}blk>>3u)*18u;let {{ sPre }}k={{ sPre }}blk&7u;
{% if sIqInline %}{{ iq2xs_dot_at(sPre, sAcc, sBits ~ "[" ~ sPre ~ "sbw + 2u * " ~ sPre ~ "k]", sBits ~ "[" ~ sPre ~ "sbw + 2u * " ~ sPre ~ "k + 1u]",
      "(" ~ sBits ~ "[" ~ sPre ~ "sbw + 16u + (" ~ sPre ~ "k >> 2u)] >> ((" ~ sPre ~ "k & 3u) * 8u)) & 0xFFu",
      "f32(" ~ sScales ~ "[" ~ sPre ~ "blk >> 3u])", sIqSfx) | trim }}
{% else %}{{ sAcc }}={{ sAcc }}+iq2xsdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k],{{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k+1u],({{ sBits }}[{{ sPre }}sbw+16u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,f32({{ sScales }}[{{ sPre }}blk>>3u]),x0,x1,x2,x3,x4,x5,x6,x7);
{% endif %}{% elif sIq2s %}let {{ sPre }}sbw=({{ sPre }}blk>>3u)*20u;let {{ sPre }}k={{ sPre }}blk&7u;
{% if sIqInline %}{{ iq2s_dot_at(sPre, sAcc, sBits ~ "[" ~ sPre ~ "sbw + 2u * " ~ sPre ~ "k]", sBits ~ "[" ~ sPre ~ "sbw + 2u * " ~ sPre ~ "k + 1u]",
      "(" ~ sBits ~ "[" ~ sPre ~ "sbw + 16u + (" ~ sPre ~ "k >> 2u)] >> ((" ~ sPre ~ "k & 3u) * 8u)) & 0xFFu",
      "(" ~ sBits ~ "[" ~ sPre ~ "sbw + 18u + (" ~ sPre ~ "k >> 2u)] >> ((" ~ sPre ~ "k & 3u) * 8u)) & 0xFFu",
      "f32(" ~ sScales ~ "[" ~ sPre ~ "blk >> 3u])", sIqSfx) | trim }}
{% else %}{{ sAcc }}={{ sAcc }}+iq2sdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k],{{ sBits }}[{{ sPre }}sbw+2u*{{ sPre }}k+1u],({{ sBits }}[{{ sPre }}sbw+16u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,({{ sBits }}[{{ sPre }}sbw+18u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,f32({{ sScales }}[{{ sPre }}blk>>3u]),x0,x1,x2,x3,x4,x5,x6,x7);
{% endif %}{% elif sIq3xxs %}let {{ sPre }}b3={{ sPre }}blk*3u;let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}blk>>3u]);
{% if sIqInline %}{{ iq3xxs_dot_at(sPre, sAcc, sBits ~ "[" ~ sPre ~ "b3]", sBits ~ "[" ~ sPre ~ "b3 + 1u]", sBits ~ "[" ~ sPre ~ "b3 + 2u]", sPre ~ "d", sIqSfx) | trim }}
{% else %}{{ sAcc }}={{ sAcc }}+iq3dot{{ sIqSfx }}({{ sBits }}[{{ sPre }}b3],{{ sBits }}[{{ sPre }}b3+1u],{{ sBits }}[{{ sPre }}b3+2u],{{ sPre }}d,x0,x1,x2,x3,x4,x5,x6,x7);
{% endif %}{% elif sIq3s %}let {{ sPre }}sbw=({{ sPre }}blk>>3u)*27u;let {{ sPre }}k={{ sPre }}blk&7u;
{% if sIqInline %}{{ iq3s_dot_at(sPre, sAcc, sBits ~ "[" ~ sPre ~ "sbw + 3u * " ~ sPre ~ "k]", sBits ~ "[" ~ sPre ~ "sbw + 3u * " ~ sPre ~ "k + 1u]", sBits ~ "[" ~ sPre ~ "sbw + 3u * " ~ sPre ~ "k + 2u]",
      "(" ~ sBits ~ "[" ~ sPre ~ "sbw + 24u + (" ~ sPre ~ "k >> 2u)] >> ((" ~ sPre ~ "k & 3u) * 8u)) & 0xFFu", "(" ~ sBits ~ "[" ~ sPre ~ "sbw + 26u] >> (" ~ sPre ~ "k * 4u)) & 0xFu",
      "f32(" ~ sScales ~ "[" ~ sPre ~ "blk >> 3u])", sIqSfx) | trim }}
{% else %}{{ sAcc }}={{ sAcc }}+iq3sdot{{ sIqSfx }}({{ sBits }}[{{ sPre }}sbw+3u*{{ sPre }}k],{{ sBits }}[{{ sPre }}sbw+3u*{{ sPre }}k+1u],{{ sBits }}[{{ sPre }}sbw+3u*{{ sPre }}k+2u],({{ sBits }}[{{ sPre }}sbw+24u+({{ sPre }}k>>2u)]>>(({{ sPre }}k&3u)*8u))&0xFFu,({{ sBits }}[{{ sPre }}sbw+26u]>>({{ sPre }}k*4u))&0xFu,f32({{ sScales }}[{{ sPre }}blk>>3u]),x0,x1,x2,x3,x4,x5,x6,x7);
{% endif %}{% elif sQ2k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*5u;let {{ sPre }}cv={{ sBits }}[{{ sPre }}g+({{ sPre }}j>>1u)];let {{ sPre }}cLo={{ sPre }}cv[({{ sPre }}j&1u)*2u];let {{ sPre }}cHi={{ sPre }}cv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}sw={{ sBits }}[{{ sPre }}g+4u][{{ sPre }}j>>1u];let {{ sPre }}bLo=({{ sPre }}sw>>(({{ sPre }}j&1u)*16u))&0xFFu;let {{ sPre }}bHi=({{ sPre }}sw>>(({{ sPre }}j&1u)*16u+8u))&0xFFu;let {{ sPre }}d=f32({{ sScales }}[{{ sPre }}sb*2u]);let {{ sPre }}dm=f32({{ sScales }}[{{ sPre }}sb*2u+1u]);{{ sAcc }}={{ sAcc }}+({{ sPre }}d*f32({{ sPre }}bLo&15u))*(dot(x0,q2w({{ sPre }}cLo,0u))+dot(x1,q2w({{ sPre }}cLo,1u))+dot(x2,q2w({{ sPre }}cLo,2u))+dot(x3,q2w({{ sPre }}cLo,3u)))-({{ sPre }}dm*f32({{ sPre }}bLo>>4u))*xsLo+({{ sPre }}d*f32({{ sPre }}bHi&15u))*(dot(x4,q2w({{ sPre }}cHi,0u))+dot(x5,q2w({{ sPre }}cHi,1u))+dot(x6,q2w({{ sPre }}cHi,2u))+dot(x7,q2w({{ sPre }}cHi,3u)))-({{ sPre }}dm*f32({{ sPre }}bHi>>4u))*xsHi;
{% elif sQ4knc %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}smBase={{ sPre }}sb*5u;let {{ sPre }}dm=unpack2x16float({{ sMeta }}[{{ sPre }}smBase]);let {{ sPre }}smCarrier={{ sMeta }}[{{ sPre }}smBase+1u+({{ sPre }}j>>1u)];let {{ sPre }}smPair={{ sPre }}smCarrier>>(({{ sPre }}j&1u)*12u);let {{ sPre }}sc={{ sPre }}smPair&63u;let {{ sPre }}mn=({{ sPre }}smPair>>6u)&63u;let {{ sPre }}dl={{ sPre }}dm.x*f32({{ sPre }}sc);let {{ sPre }}ml={{ sPre }}dm.y*f32({{ sPre }}mn);let {{ sPre }}bias=8.0*{{ sPre }}dl-{{ sPre }}ml;let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}dl*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7))+{{ sPre }}bias*sa;
{% elif sHasMin %}{% if sScalesPk2 %}let {{ sPre }}sp=unpack2x16float({{ sScales }}[{{ sPre }}blk]);let {{ sPre }}s={{ sPre }}sp.x;let {{ sPre }}bias={{ sPre }}sp.y;
{% else %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk*2u]);let {{ sPre }}bias=f32({{ sScales }}[{{ sPre }}blk*2u+1u]);
{% endif %}let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7))+{{ sPre }}bias*sa;
{% elif sQ5 %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk]);let {{ sPre }}g=({{ sPre }}blk>>2u)*5u;let {{ sPre }}ln={{ sPre }}blk&3u;let {{ sPre }}w={{ sBits }}[{{ sPre }}g+{{ sPre }}ln];let {{ sPre }}qh={{ sBits }}[{{ sPre }}g+4u][{{ sPre }}ln];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q5wp({{ sPre }}w.x,{{ sPre }}qh,0u,x0,x4)+q5wp({{ sPre }}w.y,{{ sPre }}qh,1u,x1,x5)+q5wp({{ sPre }}w.z,{{ sPre }}qh,2u,x2,x6)+q5wp({{ sPre }}w.w,{{ sPre }}qh,3u,x3,x7));
{% elif sQ6k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*13u;let {{ sPre }}words={{ sBits }}[{{ sPre }}g+{{ sPre }}j];let {{ sPre }}hv={{ sBits }}[{{ sPre }}g+8u+({{ sPre }}j>>1u)];let {{ sPre }}hLo={{ sPre }}hv[({{ sPre }}j&1u)*2u];let {{ sPre }}hHi={{ sPre }}hv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}s8=unpack4xI8({{ sBits }}[{{ sPre }}g+12u][{{ sPre }}j>>1u]);let {{ sPre }}d6=f32({{ sScales }}[{{ sPre }}sb]);let {{ sPre }}scLo={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u]);let {{ sPre }}scHi={{ sPre }}d6*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u+1u]);{{ sAcc }}={{ sAcc }}+{{ sPre }}scLo*(dot(x0,q6_lo({{ sPre }}words.x,{{ sPre }}hLo,0u))+dot(x1,q6_lo({{ sPre }}words.y,{{ sPre }}hLo,1u))+dot(x2,q6_lo({{ sPre }}words.z,{{ sPre }}hLo,2u))+dot(x3,q6_lo({{ sPre }}words.w,{{ sPre }}hLo,3u)))+{{ sPre }}scHi*(dot(x4,q6_hi({{ sPre }}words.x,{{ sPre }}hHi,0u))+dot(x5,q6_hi({{ sPre }}words.y,{{ sPre }}hHi,1u))+dot(x6,q6_hi({{ sPre }}words.z,{{ sPre }}hHi,2u))+dot(x7,q6_hi({{ sPre }}words.w,{{ sPre }}hHi,3u)));
{% elif sQ3k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*7u;let {{ sPre }}cv={{ sBits }}[{{ sPre }}g+({{ sPre }}j>>1u)];let {{ sPre }}cLo={{ sPre }}cv[({{ sPre }}j&1u)*2u];let {{ sPre }}cHi={{ sPre }}cv[({{ sPre }}j&1u)*2u+1u];let {{ sPre }}hw={{ sBits }}[{{ sPre }}g+4u+({{ sPre }}j>>2u)][{{ sPre }}j&3u];let {{ sPre }}s8=unpack4xI8({{ sBits }}[{{ sPre }}g+6u][{{ sPre }}j>>1u]);let {{ sPre }}d3=f32({{ sScales }}[{{ sPre }}sb]);let {{ sPre }}scLo={{ sPre }}d3*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u]);let {{ sPre }}scHi={{ sPre }}d3*f32({{ sPre }}s8[({{ sPre }}j&1u)*2u+1u]);{{ sAcc }}={{ sAcc }}+{{ sPre }}scLo*(dot(x0,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,0u))+dot(x1,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,1u))+dot(x2,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,2u))+dot(x3,q3w_lo({{ sPre }}cLo,{{ sPre }}hw,3u)))+{{ sPre }}scHi*(dot(x4,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,0u))+dot(x5,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,1u))+dot(x6,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,2u))+dot(x7,q3w_hi({{ sPre }}cHi,{{ sPre }}hw,3u)));
{% elif sQ5k %}let {{ sPre }}sb={{ sPre }}blk>>3u;let {{ sPre }}j={{ sPre }}blk&7u;let {{ sPre }}g={{ sPre }}sb*11u;let {{ sPre }}words={{ sBits }}[{{ sPre }}g+{{ sPre }}j];let {{ sPre }}qh={{ sBits }}[{{ sPre }}g+8u+({{ sPre }}j>>2u)][{{ sPre }}j&3u];let {{ sPre }}smv={{ sBits }}[{{ sPre }}g+10u];let {{ sPre }}sc=f32(({{ sPre }}smv[{{ sPre }}j>>2u]>>(({{ sPre }}j&3u)*8u))&0xFFu);let {{ sPre }}mn=f32(({{ sPre }}smv[2u+({{ sPre }}j>>2u)]>>(({{ sPre }}j&3u)*8u))&0xFFu);let {{ sPre }}scEff=f32({{ sScales }}[{{ sPre }}sb*2u])*{{ sPre }}sc;let {{ sPre }}mEff=f32({{ sScales }}[{{ sPre }}sb*2u+1u])*{{ sPre }}mn;{{ sAcc }}={{ sAcc }}-{{ sPre }}mEff*sa+{{ sPre }}scEff*(dot(x0,q5k_lo({{ sPre }}words.x,{{ sPre }}qh&15u))+dot(x1,q5k_lo({{ sPre }}words.y,({{ sPre }}qh>>4u)&15u))+dot(x2,q5k_lo({{ sPre }}words.z,({{ sPre }}qh>>8u)&15u))+dot(x3,q5k_lo({{ sPre }}words.w,({{ sPre }}qh>>12u)&15u))+dot(x4,q5k_hi({{ sPre }}words.x,({{ sPre }}qh>>16u)&15u))+dot(x5,q5k_hi({{ sPre }}words.y,({{ sPre }}qh>>20u)&15u))+dot(x6,q5k_hi({{ sPre }}words.z,({{ sPre }}qh>>24u)&15u))+dot(x7,q5k_hi({{ sPre }}words.w,({{ sPre }}qh>>28u)&15u)));
{% elif sQuantBits == 8 and sIq %}let {{ sPre }}s_lo=f32({{ sScales }}[{{ sPre }}blk*2u]);let {{ sPre }}s_hi=f32({{ sScales }}[{{ sPre }}blk*2u+1u]);let {{ sPre }}w0={{ sBits }}[{{ sPre }}blk*2u];let {{ sPre }}w1={{ sBits }}[{{ sPre }}blk*2u+1u];{{ sAcc }}={{ sAcc }}+{{ sPre }}s_lo*(q8dot({{ sPre }}w0.x,x0)+q8dot({{ sPre }}w0.y,x1)+q8dot({{ sPre }}w0.z,x2)+q8dot({{ sPre }}w0.w,x3))+{{ sPre }}s_hi*(q8dot({{ sPre }}w1.x,x4)+q8dot({{ sPre }}w1.y,x5)+q8dot({{ sPre }}w1.z,x6)+q8dot({{ sPre }}w1.w,x7));
{% elif sLut2 %}{% filter indent(4, true) %}{{ lut2_block_scale(sPre ~ "blk", sPre ~ "2_", sPre ~ "s2", sPre ~ "s2lo", sPre ~ "s2hi", sPre ~ "db2", sLutc16, sMeta, sLutSfx, sLutId) }}{% endfilter %}let {{ sPre }}wa={{ sBits }}[{{ sPre }}blk*2u];let {{ sPre }}wb={{ sBits }}[{{ sPre }}blk*2u+1u];
{% if sLutc16 %}{{ sAcc }}={{ sAcc }}+{{ sPre }}s2lo*(dot(x0,q2q{{ sLutSfx }}({{ sPre }}wa,0u,{{ sPre }}db2[0]))+dot(x1,q2q{{ sLutSfx }}({{ sPre }}wa,8u,{{ sPre }}db2[0]))+dot(x2,q2q{{ sLutSfx }}({{ sPre }}wa,16u,{{ sPre }}db2[1]))+dot(x3,q2q{{ sLutSfx }}({{ sPre }}wa,24u,{{ sPre }}db2[1])))+{{ sPre }}s2hi*(dot(x4,q2q{{ sLutSfx }}({{ sPre }}wb,0u,{{ sPre }}db2[2]))+dot(x5,q2q{{ sLutSfx }}({{ sPre }}wb,8u,{{ sPre }}db2[2]))+dot(x6,q2q{{ sLutSfx }}({{ sPre }}wb,16u,{{ sPre }}db2[3]))+dot(x7,q2q{{ sLutSfx }}({{ sPre }}wb,24u,{{ sPre }}db2[3])));
{% else %}{{ sAcc }}={{ sAcc }}+{{ sPre }}s2*(q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,0u,x0,x4,{{ sPre }}db2[0],{{ sPre }}db2[2])+q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,8u,x1,x5,{{ sPre }}db2[0],{{ sPre }}db2[2])+q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,16u,x2,x6,{{ sPre }}db2[1],{{ sPre }}db2[3])+q2wp{{ sLutSfx }}({{ sPre }}wa,{{ sPre }}wb,24u,x3,x7,{{ sPre }}db2[1],{{ sPre }}db2[3]));
{% endif %}{% elif sLut16 or (sLutc and sLutc16) %}{% if sLutc and sLutc16 %}let {{ sPre }}mb=({{ sPre }}blk>>3u)*5u;let {{ sPre }}dl=bitcast<f32>({{ sMeta }}[{{ sPre }}mb+4u]);let {{ sPre }}cw={{ sMeta }}[{{ sPre }}mb+(({{ sPre }}blk&7u)>>1u)];let {{ sPre }}cs=({{ sPre }}blk&1u)*16u;let {{ sPre }}s_lo={{ sPre }}dl*lutScale{{ sLutSfx }}(({{ sPre }}cw>>{{ sPre }}cs)&255u);let {{ sPre }}s_hi={{ sPre }}dl*lutScale{{ sLutSfx }}(({{ sPre }}cw>>({{ sPre }}cs+8u))&255u);
{% else %}let {{ sPre }}s_lo=f32({{ sScales }}[{{ sPre }}blk*2u]);let {{ sPre }}s_hi=f32({{ sScales }}[{{ sPre }}blk*2u+1u]);
{% endif %}let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}s_lo*(dot(x0,q4_lo{{ sLutSfx }}({{ sPre }}w.x))+dot(x1,q4_lo{{ sLutSfx }}({{ sPre }}w.y))+dot(x2,q4_lo{{ sLutSfx }}({{ sPre }}w.z))+dot(x3,q4_lo{{ sLutSfx }}({{ sPre }}w.w)))+{{ sPre }}s_hi*(dot(x4,q4_hi{{ sLutSfx }}({{ sPre }}w.x))+dot(x5,q4_hi{{ sLutSfx }}({{ sPre }}w.y))+dot(x6,q4_hi{{ sLutSfx }}({{ sPre }}w.z))+dot(x7,q4_hi{{ sLutSfx }}({{ sPre }}w.w)));
{% elif sQ8Pair %}let {{ sPre }}s=q8pair_scale({{ sPre }}blk);let {{ sPre }}cb=q8pair_code_base({{ sPre }}blk);let {{ sPre }}w0={{ sBits }}[{{ sPre }}cb];let {{ sPre }}w1={{ sBits }}[{{ sPre }}cb+1u];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q8dot({{ sPre }}w0.x,x0)+q8dot({{ sPre }}w0.y,x1)+q8dot({{ sPre }}w0.z,x2)+q8dot({{ sPre }}w0.w,x3)+q8dot({{ sPre }}w1.x,x4)+q8dot({{ sPre }}w1.y,x5)+q8dot({{ sPre }}w1.z,x6)+q8dot({{ sPre }}w1.w,x7));
{% elif sQuantBits == 8 %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk]);let {{ sPre }}w0={{ sBits }}[{{ sPre }}blk*2u];let {{ sPre }}w1={{ sBits }}[{{ sPre }}blk*2u+1u];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q8dot({{ sPre }}w0.x,x0)+q8dot({{ sPre }}w0.y,x1)+q8dot({{ sPre }}w0.z,x2)+q8dot({{ sPre }}w0.w,x3)+q8dot({{ sPre }}w1.x,x4)+q8dot({{ sPre }}w1.y,x5)+q8dot({{ sPre }}w1.z,x6)+q8dot({{ sPre }}w1.w,x7));
{% else %}{% if sLutc %}let {{ sPre }}mb=({{ sPre }}blk>>3u)*3u;let {{ sPre }}s=bitcast<f32>({{ sMeta }}[{{ sPre }}mb+2u])*lutScale{{ sLutSfx }}(({{ sMeta }}[{{ sPre }}mb+(({{ sPre }}blk&7u)>>2u)]>>(({{ sPre }}blk&3u)*8u))&255u);
{% else %}let {{ sPre }}s=f32({{ sScales }}[{{ sPre }}blk]);
{% endif %}let {{ sPre }}w={{ sBits }}[{{ sPre }}blk];{{ sAcc }}={{ sAcc }}+{{ sPre }}s*(q4wp{{ sLutSfx }}({{ sPre }}w.x,x0,x4)+q4wp{{ sLutSfx }}({{ sPre }}w.y,x1,x5)+q4wp{{ sLutSfx }}({{ sPre }}w.z,x2,x6)+q4wp{{ sLutSfx }}({{ sPre }}w.w,x3,x7));
{% endif %}`],["ops/_shared/q4-dequant-helpers.wgsl.jinja",`{% if nibbleHelpers | default(true) %}{% if quantBits == 8 %}{% if emitQuantDotHelpers | default(true) %}fn q8dot(word:u32,xv:vec4<f32>)->f32{return dot(vec4<f32>(unpack4xI8(word)),xv);}
{% endif %}{% if q8Pair | default(false) %}fn q8pair_base(block:u32)->u32{return(block>>3u)*17u;}fn q8pair_code_base(block:u32)->u32{return q8pair_base(block)+(block&7u)*2u;}fn q8pair_scale(block:u32)->f32{return f32(q4_scales[(block>>3u)*136u+128u+(block&7u)]);}
{% endif %}{% else %}fn q4_lo(word:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(word&0x0F0F0F0Fu))-8.0;}fn q4_hi(word:u32)->vec4<f32>{return vec4<f32>(unpack4xU8((word>>4u)&0x0F0F0F0Fu))-8.0;}
{% if emitQuantDotHelpers | default(true) %}fn q4wp(word:u32,lo:vec4<f32>,hi:vec4<f32>)->f32{return dot(lo,q4_lo(word))+dot(hi,q4_hi(word));}
{% endif %}{% endif %}{% endif %}{% if q5 %}fn q5spread(h:u32)->u32{return(h*0x02040810u)&0x10101010u;}fn q5_lo(word:u32,h:u32)->vec4<f32>{return vec4<f32>(unpack4xU8((word&0x0F0F0F0Fu)|q5spread(h)))-16.0;}fn q5_hi(word:u32,h:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(((word>>4u)&0x0F0F0F0Fu)|q5spread(h)))-16.0;}fn q5wp(word:u32,qh:u32,w:u32,lo:vec4<f32>,hi:vec4<f32>)->f32{return dot(lo,q5_lo(word,(qh>>(4u*w))&15u))+dot(hi,q5_hi(word,(qh>>(4u*w+16u))&15u));}
{% endif %}{% if q2k %}fn q2w(cw:u32,w:u32)->vec4<f32>{return vec4<f32>(unpack4xU8((cw>>(2u*w))&0x03030303u));}
{% endif %}{% if q3k is defined and q3k %}fn q3w_lo(cw:u32,h:u32,w:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(((cw>>(2u*w))&0x03030303u)|(((h>>w)&0x01010101u)<<2u)))-4.0;}fn q3w_hi(cw:u32,h:u32,w:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(((cw>>(2u*w))&0x03030303u)|(((h>>(4u+w))&0x01010101u)<<2u)))-4.0;}
{% endif %}{% if q6k %}fn q6_lo(word:u32,hLo:u32,w:u32)->vec4<f32>{return vec4<f32>(unpack4xU8((word&0x0F0F0F0Fu)|(((hLo>>(2u*w))&0x03030303u)<<4u)))-32.0;}fn q6_hi(word:u32,hHi:u32,w:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(((word>>4u)&0x0F0F0F0Fu)|(((hHi>>(2u*w))&0x03030303u)<<4u)))-32.0;}
{% endif %}{% if q5k is defined and q5k %}fn q5kspread(h:u32)->u32{return(h*0x02040810u)&0x10101010u;}fn q5k_lo(word:u32,h:u32)->vec4<f32>{return vec4<f32>(unpack4xU8((word&0x0F0F0F0Fu)|q5kspread(h)));}fn q5k_hi(word:u32,h:u32)->vec4<f32>{return vec4<f32>(unpack4xU8(((word>>4u)&0x0F0F0F0Fu)|q5kspread(h)));}
{% endif %}`],["ops/_shared/q4-gate-up-accumulate-reduce.wgsl.jinja",`{% set gateUpIq1m = gateUpIq1m | default(iq1m | default(false)) %}{% set gateUpIq1s = gateUpIq1s | default(iq1s | default(false)) %}{% set gateUpIq1md = gateUpIq1md | default(iq1md | default(false)) %}{% set gateUpIq2xxs = gateUpIq2xxs | default(iq2xxs | default(false)) %}{% set gateUpIq3xxs = gateUpIq3xxs | default(iq3xxs | default(false)) %}{% set gateUpIq3s = gateUpIq3s | default(iq3s | default(false)) %}{% set gateUpIq2xs = gateUpIq2xs | default(iq2xs | default(false)) %}{% set gateUpIq2s = gateUpIq2s | default(iq2s | default(false)) %}{% set gateUpSuffixes = gateUpSuffixes | default([""]) %}{% set gateUpTiled = gateUpSuffixes | length > 1 %}{% set packMeta = packMeta | default("q4_scales") %}{% set lutcOn = lutc is defined and lutc %}{% set lut2On = lut2 is defined and lut2 %}{% set q2Split = q2Split | default(false) %}{% set mRows = mRows if mRows is defined else 1 %}{% set xOpen = xOpen | default("") %}{% set xClose = xClose | default("") %}{% set mStoreScalar = mStoreScalar | default("") %}{% set gateUpSplit = gateUpSplit | default(false) %}{% set upHasMin = upHasMin | default(hasMin) %}{% set upIq = upIq | default(iq | default(false)) %}{% set upQ6k = upQ6k | default(q6k | default(false)) %}{% set upQ5k = upQ5k | default(q5k | default(false)) %}{% set upQ3k = upQ3k | default(q3k | default(false)) %}{% set upQ2k = upQ2k | default(q2k | default(false)) %}{% set upQ4knc = upQ4knc | default(q4knCompact | default(false)) %}{% set upQuantBits = upQuantBits | default(quantBits) %}{% set upLut16 = upLut16 | default(lut16 | default(false)) %}{% set upLutc = upLutc | default(lutcOn) %}{% set upLut2 = upLut2 | default(lut2On) %}{% set upLutc16 = upLutc16 | default(lutc16 | default(false)) %}{% set gateUpNeedsSa = hasMin or (q4knCompact is defined and q4knCompact) or (q5k is defined and q5k)
      or (gateUpSplit and (upHasMin or upQ4knc or upQ5k)) %}{% set gateUpNeedsHalfSums = (q2k is defined and q2k) or (gateUpSplit and upQ2k) %}{% set pairGateBits = "gate_bits" if q2Split else "q4_bits" %}{% set pairGateScales = "gate_scales" if q2Split else "q4_scales" %}{% set pairUpBits = "up_bits" if (q2Split or gateUpSplit) else "q4_bits" %}{% set pairUpScales = "up_scales" if (q2Split or gateUpSplit) else "q4_scales" %}{% set pairView = not gateUpSplit %}{% set pairQ8 = (q8Pair | default(false)) and not gateUpSplit %}{% if gateUpOwned | default(false) %}{% set gateUpSumOpen = "subgroupAdd(" %}{% set gateUpSumClose = ")" %}{% else %}{% set gateUpSumOpen = "sg_safe_sum(" %}{% set gateUpSumClose = ", " ~ (gateUpFoldIndex | default("lid")) ~ ", " ~ (gateUpFoldSpan | default("SG_SAFE_WG")) ~ ", sgNative)" %}{% endif %}{% if gateUpIq1s or (upIq1s | default(false)) or gateUpIq1md or (upIq1md | default(false)) %}let dqK=b&7u;let dqMo=8u+(dqK>>1u);let dqSh=16u*(dqK&1u);
{% if gateUpIq1md or (upIq1md | default(false)) %}let dqCo=12u+(dqK>>2u);let dqCs=8u*(dqK&3u);
{% endif %}{% endif %}{% for m in range(mRows) %}{% set accG = "acc_g" if m == 0 else "accM" ~ m ~ "_g" %}{% set accU = "acc_u" if m == 0 else "accM" ~ m ~ "_u" %}{% if m > 0 %}{let hbase{{ m }}={{ m }}u*BPR*8u+hbase;
{% for i in range(8) %}let x{{ i }}={{ xOpen }}hbase{{ m }}{{ " + " ~ i ~ "u" if i else "" }}{{ xClose }};
{% endfor %}{% endif %}{% if gateUpNeedsSa %}let sa=dot(x0+x1+x2+x3+x4+x5+x6+x7,vec4<f32>(1.0));
{% endif %}{% if gateUpNeedsHalfSums %}let xsLo=dot(x0+x1+x2+x3,vec4<f32>(1.0));let xsHi=dot(x4+x5+x6+x7,vec4<f32>(1.0));
{% endif %}{% for r in gateUpSuffixes %}{% if gateUpTiled %}{
{% endif %}{% set sPre = "g" %}{% set sBlk = "gate_block_base" ~ r ~ " + b" %}{% set sAcc = accG ~ r %}{% include "ops/_shared/q4-block-dot-flags.wgsl.jinja" %}{% set sBits = pairGateBits %}{% set sScales = pairGateScales %}{% set sIq1m = gateUpIq1m %}{% set sIq1s = gateUpIq1s %}{% set sIq1md = gateUpIq1md %}{% set sIq2xxs = gateUpIq2xxs %}{% set sIq3xxs = gateUpIq3xxs %}{% set sIq3s = gateUpIq3s %}{% set sIq2xs = gateUpIq2xs %}{% set sIq2s = gateUpIq2s %}{% set sLutSfx = gateLutSfx | default("") %}{% set sPairView = pairView %}{% set sQ8Pair = pairQ8 %}{% include "ops/_shared/q4-block-dot.wgsl.jinja" %}{% set sPre = "u" %}{% set sBlk = "up_block_base" ~ r ~ " + b" %}{% set sAcc = accU ~ r %}{% set sBits = pairUpBits %}{% set sScales = pairUpScales %}{% if gateUpSplit %}{% set sMeta = "up_scales" %}{% set sQuantBits = upQuantBits %}{% set sHasMin = upHasMin %}{% set sIq = upIq %}{% set sQ6k = upQ6k %}{% set sQ5k = upQ5k %}{% set sQ3k = upQ3k %}{% set sQ2k = upQ2k %}{% set sQ4knc = upQ4knc %}{% set sLut16 = upLut16 %}{% set sLutc = upLutc %}{% set sLutc16 = upLutc16 %}{% set sLut2 = upLut2 %}{% set sIq1m = upIq1m | default(false) %}{% set sIq1s = upIq1s | default(false) %}{% set sIq1md = upIq1md | default(false) %}{% set sIq2xxs = upIq2xxs | default(false) %}{% set sIq3xxs = upIq3xxs | default(false) %}{% set sIq3s = upIq3s | default(false) %}{% set sIq2xs = upIq2xs | default(false) %}{% set sIq2s = upIq2s | default(false) %}{% set sLutId = upLutId | default(0) %}{% set sLutSfx = upLutSfx | default("") %}{% set sIqSfx = upIqSfx | default("") %}{% endif %}{% include "ops/_shared/q4-block-dot.wgsl.jinja" %}{% if gateUpTiled %}}
{% endif %}{% endfor %}{% if m > 0 %}}
{% endif %}{% endfor %}}
{% set gateUpRuntime = sgRuntime | default(false) %}{% include "ops/_shared/subgroup-runtime-fold.wgsl.jinja" %}{% if gateUpRuntime %}{% set gateUpFold = namespace(pairs=[]) %}{% for r in gateUpSuffixes %}{% set gateUpFold.pairs = gateUpFold.pairs + [["acc_g" ~ r, "g" ~ r], ["acc_u" ~ r, "u" ~ r]] %}{% endfor %}{% if gateUpReduceNorm %}{% set gateUpFold.pairs = gateUpFold.pairs + [["ss", "s"]] %}{% endif %}{% for m in range(1, mRows) %}{% for r in gateUpSuffixes %}{% set gateUpFold.pairs = gateUpFold.pairs + [["accM" ~ m ~ "_g" ~ r, "gM" ~ m ~ "_" ~ r], ["accM" ~ m ~ "_u" ~ r, "uM" ~ m ~ "_" ~ r]] %}{% endfor %}{% endfor %}{{ sg_fold_workgroup(gateUpFold.pairs) }}
{% elif gateUpTiled %}{% if useSubgroups %}{% for r in gateUpSuffixes %}let g{{ r }}={{ gateUpSumOpen }}acc_g{{ r }}{{ gateUpSumClose }};let u{{ r }}={{ gateUpSumOpen }}acc_u{{ r }}{{ gateUpSumClose }};
{% endfor %}{% if gateUpReduceNorm %}let s={{ gateUpSumOpen }}ss{{ gateUpSumClose }};
{% endif %}{% else %}{% for r in gateUpSuffixes %}partials_g{{ r }}[lid]=acc_g{{ r }};partials_u{{ r }}[lid]=acc_u{{ r }};
{% endfor %}{% if gateUpReduceNorm %}partials_ss[lid]=ss;
{% endif %}workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){
{% for r in gateUpSuffixes %}partials_g{{ r }}[lid]=partials_g{{ r }}[lid]+partials_g{{ r }}[lid+stride];partials_u{{ r }}[lid]=partials_u{{ r }}[lid]+partials_u{{ r }}[lid+stride];
{% endfor %}{% if gateUpReduceNorm %}partials_ss[lid]=partials_ss[lid]+partials_ss[lid+stride];
{% endif %}}workgroupBarrier();stride=stride/2u;}
{% for r in gateUpSuffixes %}let g{{ r }}=partials_g{{ r }}[0];let u{{ r }}=partials_u{{ r }}[0];
{% endfor %}{% if gateUpReduceNorm %}let s=partials_ss[0];
{% endif %}{% endif %}{% else %}{% if useSubgroups %}let g={{ gateUpSumOpen }}acc_g{{ gateUpSumClose }};let u={{ gateUpSumOpen }}acc_u{{ gateUpSumClose }};
{% if gateUpReduceNorm %}let s={{ gateUpSumOpen }}ss{{ gateUpSumClose }};
{% endif %}{% else %}partials_g[lid]=acc_g;partials_u[lid]=acc_u;
{% if gateUpReduceNorm %}partials_ss[lid]=ss;
{% endif %}workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){partials_g[lid]=partials_g[lid]+partials_g[lid+stride];partials_u[lid]=partials_u[lid]+partials_u[lid+stride];
{% if gateUpReduceNorm %}partials_ss[lid]=partials_ss[lid]+partials_ss[lid+stride];
{% endif %}}workgroupBarrier();stride=stride/2u;}let g=partials_g[0];let u=partials_u[0];
{% if gateUpReduceNorm %}let s=partials_ss[0];
{% endif %}{% endif %}{% endif %}{% if not useSubgroups and not gateUpRuntime and mRows > 1 %}{% endif %}{% for m in range(1, 1 if gateUpRuntime else mRows) %}{% if not useSubgroups %}workgroupBarrier();
{% for r in gateUpSuffixes %}partials_g{{ r }}[lid]=accM{{ m }}_g{{ r }};partials_u{{ r }}[lid]=accM{{ m }}_u{{ r }};
{% endfor %}workgroupBarrier();var strideM{{ m }}=WORKGROUP_SIZE/2u;loop{if(strideM{{ m }}==0u){break;}if(lid<strideM{{ m }}){
{% for r in gateUpSuffixes %}partials_g{{ r }}[lid]=partials_g{{ r }}[lid]+partials_g{{ r }}[lid+strideM{{ m }}];partials_u{{ r }}[lid]=partials_u{{ r }}[lid]+partials_u{{ r }}[lid+strideM{{ m }}];
{% endfor %}}workgroupBarrier();strideM{{ m }}=strideM{{ m }}/2u;}
{% for r in gateUpSuffixes %}let gM{{ m }}_{{ r }}=partials_g{{ r }}[0];let uM{{ m }}_{{ r }}=partials_u{{ r }}[0];
{% endfor %}{% else %}{% for r in gateUpSuffixes %}let gM{{ m }}_{{ r }}={{ gateUpSumOpen }}accM{{ m }}_g{{ r }}{{ gateUpSumClose }};let uM{{ m }}_{{ r }}={{ gateUpSumOpen }}accM{{ m }}_u{{ r }}{{ gateUpSumClose }};
{% endfor %}{% endif %}{% endfor %}{% if mRows > 1 %}if({{ gateUpLeader | default("lid") }}==0u){
{% for m in range(1, mRows) %}{% if gateUpTiled %}{% for r in gateUpSuffixes %}if(j0+{{ r }}u<INTERMEDIATE_SIZE){intermediate[{{ m }}u*INTERMEDIATE_SIZE+j0+{{ r }}u]={{ mStoreScalar }}(silu(gM{{ m }}_{{ r }})*uM{{ m }}_{{ r }});}
{% endfor %}{% else %}intermediate[{{ m }}u*INTERMEDIATE_SIZE+j]={{ mStoreScalar }}(silu(gM{{ m }}_)*uM{{ m }}_);
{% endif %}{% endfor %}}
{% endif %}`],["ops/_shared/q4nl-dequant-helpers.wgsl.jinja",`{% set lutSfx = lutSfx | default("") %}{% if quantBits == 8 %}{% if emitQuantDotHelpers | default(true) %}fn q8dot{{ lutSfx }}(word:u32,xv:vec4<f32>)->f32{return dot(vec4<f32>(unpack4xI8(word)),xv);}
{% endif %}{% else %}{% if lutId and lutId == 2 %}{% set gridRow0 = "8.0, 25.0, 43.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% set gridRow1 = "-8.0, -25.0, -43.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% elif lutId and lutId == 3 %}{% set gridRow0 = "1.0, 3.0, 5.0, 7.0, 9.0, 11.0, 13.0, 15.0" %}{% set gridRow1 = "-1.0, -3.0, -5.0, -7.0, -9.0, -11.0, -13.0, -15.0" %}{% elif lutId and lutId == 5 %}{% set gridRow0 = "4.0, 12.0, 20.0, 28.0, 36.0, 44.0, 52.0, 62.0" %}{% set gridRow1 = "-4.0, -12.0, -20.0, -28.0, -36.0, -44.0, -52.0, -62.0" %}{% elif lutId and lutId == 4 %}{% set gridRow0 = "-4.0, -3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0" %}{% set gridRow1 = "0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% elif lutId and lutId == 6 %}{% set gridRow0 = "0.0, 1.0, 2.0, 3.0, 4.0, 6.0, 8.0, 12.0" %}{% set gridRow1 = "0.0, -1.0, -2.0, -3.0, -4.0, -6.0, -8.0, -12.0" %}{% elif lutId and lutId in [8, 9] %}{% set gridRow0 = "-1.0, 0.0, 1.0, 2.0, 0.0, 0.0, 0.0, 0.0" %}{% set gridRow1 = "0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% elif lutId and lutId == 7 %}{% set gridRow0 = "-9.0, -1.0, 7.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% set gridRow1 = "-7.0, 1.0, 9.0, 0.0, 0.0, 0.0, 0.0, 0.0" %}{% else %}{% set gridRow0 = "-127.0, -104.0, -83.0, -65.0, -49.0, -35.0, -22.0, -10.0" %}{% set gridRow1 = "1.0, 13.0, 25.0, 38.0, 53.0, 69.0, 89.0, 113.0" %}{% endif %}{% if (emitQuantValueHelpers | default(true)) and (emitQuantDotHelpers | default(true)) %}{% endif %}const LUT_GRID{{ lutSfx }}=array<f32,16>({{ gridRow0 }},{{ gridRow1 }});
{% if emitQuantValueHelpers | default(true) %}fn q4nl{{ lutSfx }}(v:u32)->f32{return LUT_GRID{{ lutSfx }}[v&15u];}fn q4_lo{{ lutSfx }}(word:u32)->vec4<f32>{return vec4<f32>(q4nl{{ lutSfx }}(word),q4nl{{ lutSfx }}(word>>8u),q4nl{{ lutSfx }}(word>>16u),q4nl{{ lutSfx }}(word>>24u));}fn q4_hi{{ lutSfx }}(word:u32)->vec4<f32>{return vec4<f32>(q4nl{{ lutSfx }}(word>>4u),q4nl{{ lutSfx }}(word>>12u),q4nl{{ lutSfx }}(word>>20u),q4nl{{ lutSfx }}(word>>28u));}
{% endif %}{% if (emitQuantValueHelpers | default(true)) and (emitQuantDotHelpers | default(true)) %}fn q4wp{{ lutSfx }}(word:u32,lo:vec4<f32>,hi:vec4<f32>)->f32{return dot(lo,q4_lo{{ lutSfx }}(word))+dot(hi,q4_hi{{ lutSfx }}(word));}
{% endif %}{% if lut2 is defined and lut2 %}fn q2q{{ lutSfx }}(word:u32,sh:u32,db:u32)->vec4<f32>{let v=word>>sh;
{% if lutId in [8, 9] %}let base=LUT_GRID{{ lutSfx }}[db];let step=LUT_GRID{{ lutSfx }}[db+1u]-base;let codes=(vec4<u32>(v)>>vec4<u32>(0u,2u,4u,6u))&vec4<u32>(3u);return(bitcast<vec4<f32>>(codes|vec4<u32>(0x4b000000u))-vec4<f32>(8388608.0))*step+vec4<f32>(base);
{% else %}return vec4<f32>(LUT_GRID{{ lutSfx }}[(v&3u)+db],LUT_GRID{{ lutSfx }}[((v>>2u)&3u)+db],LUT_GRID{{ lutSfx }}[((v>>4u)&3u)+db],LUT_GRID{{ lutSfx }}[((v>>6u)&3u)+db]);
{% endif %}}
{% if emitLut2DotHelper | default(true) %}fn q2wp{{ lutSfx }}(w0:u32,w1:u32,sh:u32,lo:vec4<f32>,hi:vec4<f32>,dbLo:u32,dbHi:u32)->f32{return dot(lo,q2q{{ lutSfx }}(w0,sh,dbLo))+dot(hi,q2q{{ lutSfx }}(w1,sh,dbHi));}
{% endif %}{% endif %}{% if lutc is defined and lutc and lutId not in [8, 9] %}fn lutScale{{ lutSfx }}(c:u32)->f32{
{% if lutId == 1 or lutId == 4 %}return f32(c)-32.0;
{% elif lutId == 2 %}return(0.5+f32(c))*0.25;
{% elif lutId == 3 %}return 1.0+2.0*f32(c);
{% elif lutId == 5 %}return(0.5+f32(c))*0.5;
{% elif lutId == 6 and lutc16 %}if(c==0u||c==0x7fu){return 0.0;}let exp=(c>>3u)&0x0fu;let man=c&7u;if(exp==0u){return f32(man)/1024.0;}return bitcast<f32>(((exp+119u)<<23u)|(man<<20u));
{% elif lutId == 6 %}return bitcast<f32>(select((c-1u)<<23u,0x00200000u<<c,c<2u));
{% else %}return f32(c)*0.125;
{% endif %}}
{% endif %}{% endif %}`],["ops/_shared/silu-fn.wgsl.jinja","fn silu(x:f32)->f32{return x/(1.0+exp(-x));}"],["ops/_shared/subgroup-active-lanes.wgsl.jinja",`{% macro active_lanes(lane, prefix="active") %}let {{ prefix }}Mask=subgroupBallot(true);let {{ prefix }}Word={{ lane }}/32u;let {{ prefix }}Lower=(1u<<({{ lane }} %32u))-1u;let {{ prefix }}Count=max(countOneBits({{ prefix }}Mask.x)+countOneBits({{ prefix }}Mask.y)+countOneBits({{ prefix }}Mask.z)+countOneBits({{ prefix }}Mask.w),1u);var {{ prefix }}Rank=countOneBits({{ prefix }}Mask[{{ prefix }}Word]&{{ prefix }}Lower);for(var word=0u;word<{{ prefix }}Word;word++){ {{ prefix }}Rank+=countOneBits({{ prefix }}Mask[word]);}
{% endmacro %}`],["ops/_shared/subgroup-runtime-fold.wgsl.jinja",`{% macro sg_fold_workgroup(pairs) %}{% for p in pairs %}let {{ p[1] }}=sg_safe_sum({{ p[0] }},lid,SG_SAFE_WG,sgNative);
{% endfor %}{% endmacro %}{% macro sg_fold_cluster32(pairs) %}{% for p in pairs %}let {{ p[1] }}=sg_safe_sum({{ p[0] }},lid,32u,sgNative);
{% endfor %}{% endmacro %}`],["ops/_shared/subgroup-runtime-proof.wgsl.jinja",`const SG_SAFE_WG:u32={{ sgSafeStorageSize }}u;var<workgroup>sgSafeRejected:atomic<u32>;var<workgroup>sgSafeAccepted:u32;
{% if not (sgSafeExternalScratch | default(false)) %}var<workgroup>sgSafeScratch:array<f32,{{ sgSafeStorageSize }}>;
{% endif %}fn sg_native_layout(tid:u32,lane:u32,width:u32)->bool{let sgActiveMask=subgroupBallot(true);let population=countOneBits(sgActiveMask.x)+countOneBits(sgActiveMask.y)+countOneBits(sgActiveMask.z)+countOneBits(sgActiveMask.w);let first=subgroupBroadcastFirst(tid);if(width!=32u||population!=32u||lane!=(tid&31u)||first!=(tid&~31u)){atomicOr(&sgSafeRejected,1u);}workgroupBarrier();if(tid==0u){sgSafeAccepted=select(0u,1u,atomicLoad(&sgSafeRejected)==0u);}return workgroupUniformLoad(&sgSafeAccepted)!=0u;}fn sg_safe_sum(value:f32,tid:u32,span:u32,native:bool)->f32{if(native){let total=subgroupAdd(value);if(span==32u){return total;}if(subgroupElect()){ {{ sgSafeScratchName | default("sgSafeScratch") }}[tid>>5u]{{ sgSafeScratchComponent | default("") }}=total;}workgroupBarrier();var sum=0.0;for(var i=0u;i<(SG_SAFE_WG+31u)/32u;i=i+1u){sum=sum+{{ sgSafeScratchName | default("sgSafeScratch") }}[i]{{ sgSafeScratchComponent | default("") }};}workgroupBarrier();return sum;} {{ sgSafeScratchName | default("sgSafeScratch") }}[tid]{{ sgSafeScratchComponent | default("") }}=value;workgroupBarrier();let first=(tid/span)*span;let lane=tid-first;var live=span;loop{if(live<=1u){break;}let half=(live+1u)>>1u;if(lane<live-half){ {{ sgSafeScratchName | default("sgSafeScratch") }}[tid]{{ sgSafeScratchComponent | default("") }}={{ sgSafeScratchName | default("sgSafeScratch") }}[tid]{{ sgSafeScratchComponent | default("") }}+{{ sgSafeScratchName | default("sgSafeScratch") }}[tid+half]{{ sgSafeScratchComponent | default("") }};}workgroupBarrier();live=half;}let total={{ sgSafeScratchName | default("sgSafeScratch") }}[first]{{ sgSafeScratchComponent | default("") }};workgroupBarrier();return total;}
{% if sgSafeNeedsMax | default(false) %}fn sg_safe_max(value:f32,tid:u32,span:u32,native:bool)->f32{if(native){let total=subgroupMax(value);if(span==32u){return total;}if(subgroupElect()){ {{ sgSafeScratchName | default("sgSafeScratch") }}[tid>>5u]{{ sgSafeScratchComponent | default("") }}=total;}workgroupBarrier();var sum=-3.4028234663852886e38;for(var i=0u;i<(SG_SAFE_WG+31u)/32u;i=i+1u){sum=max(sum,{{ sgSafeScratchName | default("sgSafeScratch") }}[i]{{ sgSafeScratchComponent | default("") }});}workgroupBarrier();return sum;} {{ sgSafeScratchName | default("sgSafeScratch") }}[tid]{{ sgSafeScratchComponent | default("") }}=value;workgroupBarrier();let first=(tid/span)*span;let lane=tid-first;var live=span;loop{if(live<=1u){break;}let half=(live+1u)>>1u;if(lane<live-half){ {{ sgSafeScratchName | default("sgSafeScratch") }}[tid]{{ sgSafeScratchComponent | default("") }}=max({{ sgSafeScratchName | default("sgSafeScratch") }}[tid]{{ sgSafeScratchComponent | default("") }},{{ sgSafeScratchName | default("sgSafeScratch") }}[tid+half]{{ sgSafeScratchComponent | default("") }});}workgroupBarrier();live=half;}let total={{ sgSafeScratchName | default("sgSafeScratch") }}[first]{{ sgSafeScratchComponent | default("") }};workgroupBarrier();return total;}
{% endif %}`],["ops/_shared/argmax-wg-tree.wgsl.jinja","best_values[lid]=best_value;best_indices[lid]=best_token;workgroupBarrier();var am_stride=WORKGROUP_SIZE/2u;loop{if(lid<am_stride){let other_value=best_values[lid+am_stride];let other_index=best_indices[lid+am_stride];if(other_value>best_values[lid]||(other_value==best_values[lid]&&other_index<best_indices[lid])){best_values[lid]=other_value;best_indices[lid]=other_index;} }workgroupBarrier();if(am_stride==1u){break;}am_stride=am_stride/2u;}if(lid==0u){best_values_out[block]=best_values[0];best_ids_out[block]=best_indices[0];}"],["ops/_shared/matmul-argmax-hidden4.wgsl.jinja","fn hidden4(dim4:u32)->vec4<f32>{let base=dim4*4u;return vec4<f32>(hidden[base],hidden[base+1u],hidden[base+2u],hidden[base+3u]);}"],["ops/_shared/q4-decode-row-block-accumulate.wgsl.jinja",`{% set packMeta = packMeta | default("q4_metadata") %}{% set accPrefix = accPrefix | default("acc") %}{% set rowBase = rowBase | default("base0") %}{% include "ops/_shared/q4-block-dot-flags.wgsl.jinja" %}{% if sHasMin or sQ4knc or sQ5k %}let sa=dot(x0+x1+x2+x3+x4+x5+x6+x7,vec4<f32>(1.0));
{% endif %}{% if sQ2k %}let xsLo=dot(x0+x1+x2+x3,vec4<f32>(1.0));let xsHi=dot(x4+x5+x6+x7,vec4<f32>(1.0));
{% endif %}{% if sIq1m and (sIq1s or sIq1md) %}let dqK=b&7u;let dqMo=8u+(dqK>>1u);let dqSh=16u*(dqK&1u);
{% if sIq1md %}let dqCo=12u+(dqK>>2u);let dqCs=8u*(dqK&3u);
{% endif %}{% endif %}{% for r in range(regRows) %}{% set sPre = "r" ~ r %}{% set sBlk = rowBase ~ " + b + " ~ r ~ "u * BPR" %}{% set sAcc = accPrefix ~ r %}{% include "ops/_shared/q4-block-dot.wgsl.jinja" %}{% endfor %}`],["ops/_shared/quant-dot4-packed.wgsl.jinja","fn dot4_packed(a_word:u32,b_word:u32)->i32{return dot4I8Packed(a_word,b_word);}"],["ops/_shared/projection-multirow-reduce.wgsl.jinja",`{% set accPrefix = accPrefix | default("acc") %}{% set totalPrefix = totalPrefix | default("total") %}{% if (sgPath | default("")) == "general" %}{% for r in range(regRows) %}sgTotals[{{ sgTotalsSlot | default("") }}{{ r }}u]=subgroupAdd({{ accPrefix }}{{ r }});
{% endfor %}{% elif sgOwned | default(false) %}{% for r in range(regRows) %}let {{ totalPrefix }}{{ r }}=subgroupAdd({{ accPrefix }}{{ r }});
{% endfor %}{% elif useSubgroups %}{% for r in range(regRows) %}let {{ totalPrefix }}{{ r }}=sg_safe_sum({{ accPrefix }}{{ r }},lid,32u,sgNative);
{% endfor %}{% elif sgRuntime | default(false) %}{% set sgFold = namespace(pairs=[]) %}{% for r in range(regRows) %}{% set sgFold.pairs = sgFold.pairs + [[accPrefix ~ r, totalPrefix ~ r]] %}{% endfor %}{{ sg_fold_cluster32(sgFold.pairs) }}
{% elif partialsRowVector | default(false) %}  partials4[lid] = vec{{ regRows }}<f32>({% for r in range(regRows) %}{{ ", " if r else "" }}{{ accPrefix }}{{ r }}{% endfor %});
workgroupBarrier();var {{ totalPrefix }}Stride=16u;loop{if({{ totalPrefix }}Stride==0u){break;}if(lane<{{ totalPrefix }}Stride){partials4[lid]=partials4[lid]+partials4[lid+{{ totalPrefix }}Stride];}workgroupBarrier();{{ totalPrefix }}Stride={{ totalPrefix }}Stride/2u;}let {{ totalPrefix }}Rows=partials4[cluster*32u];
{% for r in range(regRows) %}let {{ totalPrefix }}{{ r }}={{ totalPrefix }}Rows.{{ "xyzw"[r] }};
{% endfor %}workgroupBarrier();
{% else %}{% for r in range(regRows) %}partials4[lid]={{ accPrefix }}{{ r }};workgroupBarrier();var {{ totalPrefix }}Stride{{ r }}=16u;loop{if({{ totalPrefix }}Stride{{ r }}==0u){break;}if(lane<{{ totalPrefix }}Stride{{ r }}){partials4[lid]=partials4[lid]+partials4[lid+{{ totalPrefix }}Stride{{ r }}];}workgroupBarrier();{{ totalPrefix }}Stride{{ r }}={{ totalPrefix }}Stride{{ r }}/2u;}let {{ totalPrefix }}{{ r }}=partials4[cluster*32u];workgroupBarrier();
{% endfor %}{% endif %}`],["ops/_shared/q1-tile4-dot-normed.wgsl.jinja","var acc0=0.0;var acc1=0.0;var acc2=0.0;var acc3=0.0;for(var word_index=lid;word_index<IN_WORDS;word_index=word_index+WORKGROUP_SIZE){let col_block=word_index/4u;let word_id=word_index%4u;let base4=word_index*8u;let n0=normed[base4];let n1=normed[base4+1u];let n2=normed[base4+2u];let n3=normed[base4+3u];let n4=normed[base4+4u];let n5=normed[base4+5u];let n6=normed[base4+6u];let n7=normed[base4+7u];let dot4=q1_dot_word_t4(block_offset,tile,BPR,col_block,word_id,n0,n1,n2,n3,n4,n5,n6,n7);acc0+=dot4.x;acc1+=dot4.y;acc2+=dot4.z;acc3+=dot4.w;}"],["ops/_shared/q1-tile4-reduce.wgsl.jinja",`{% if useSubgroups %}{% if q1ReduceVector %}let sum4=subgroupAdd(vec4<f32>(acc0,acc1,acc2,acc3));
{% for i in range(4) %}let sum{{ i }}=sum4[{{ i }}];
{% endfor %}{% else %}{% for i in range(4) %}let sum{{ i }}=subgroupAdd(acc{{ i }});
{% endfor %}{% endif %}let elected=subgroupElect();
{% for i in range(4) %}partials{{ i }}[lid]=select(0.0,sum{{ i }},elected);
{% endfor %}workgroupBarrier();if(lid==0u){
{% for i in range(4) %}var total{{ i }}=0.0;
{% endfor %}for(var i=0u;i<WORKGROUP_SIZE;i=i+1u){
{% for j in range(4) %}total{{ j }}=total{{ j }}+partials{{ j }}[i];
{% endfor %}}
{% for i in range(4) %}partials{{ i }}[0]=total{{ i }};
{% endfor %}}workgroupBarrier();
{% else %}partials0[lid]=acc0;partials1[lid]=acc1;partials2[lid]=acc2;partials3[lid]=acc3;workgroupBarrier();
{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ wgsl_tree_fold(["partials0", "partials1", "partials2", "partials3"], form="head", barrierFirst=true) }}
{% endif %}`],["ops/_shared/q4-normed-block-dot.wgsl.jinja",`{% set packMeta = packMeta | default("q4_scales") %}{% include "ops/_shared/q4-block-dot-flags.wgsl.jinja" %}var acc=0.0;
{% for m in range(1, mRows) %}var accM{{ m }}=0.0;
{% endfor %}{% for w in range(1, wRows) %}{% for m in range(mRows) %}var accW{{ w }}{{ "" if m == 0 else "M" ~ m }}=0.0;
{% endfor %}{% endfor %}for(var b=lid;b<{{ q4DotBound if q4DotBound is defined else "BPR" }};b=b+WORKGROUP_SIZE){
{% if wRows == 1 %}let blk=region_block_base+b;
{% endif %}let hbase=b*8u;
{% if q4NormedHbaseComment %}{% endif %}{% for m in range(mRows) %}{% set hbName = "hbase" if m == 0 else "hbase" ~ m %}{% if m > 0 %}{let {{ hbName }}={{ m }}u*BPR*8u+hbase;
{% endif %}{% for i in range(8) %}let x{{ i }}=vec4<f32>(normed[{{ hbName }}{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}{% if sHasMin or sQ4knc or sQ5k %}let sa=dot(x0+x1+x2+x3+x4+x5+x6+x7,vec4<f32>(1.0));
{% endif %}{% for w in range(wRows) %}{% if wRows > 1 %}{let blk=region_block_base+{{ w }}u*BPR+b;
{% endif %}{% if sIq1m and (sIq1s or sIq1md) %}let dqK=blk&7u;let dqMo=8u+(dqK>>1u);let dqSh=16u*(dqK&1u);
{% if sIq1md %}let dqCo=12u+(dqK>>2u);let dqCs=8u*(dqK&3u);
{% endif %}{% endif %}{% set sPre = "k" %}{% set sBlk = "blk" %}{% set sAcc = ("acc" if w == 0 else "accW" ~ w) ~ ("" if m == 0 else "M" ~ m) %}{% include "ops/_shared/q4-block-dot.wgsl.jinja" %}{% if wRows > 1 %}}
{% endif %}{% endfor %}{% if m > 0 %}}
{% endif %}{% endfor %}}`],["ops/_shared/q4-normed-projection-multirow.wgsl.jinja",`{% set packMeta = packMeta | default("q4_scales") %}{% set q5Route = true %}{% for r in range(regRows) %}var acc{{ r }}=0.0;
{% endfor %}for(var b=lane;b<BPR;b=b+32u){let hbase=b*8u;
{% for i in range(8) %}let x{{ i }}=vec4<f32>(hidden[hbase{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}{% include "ops/_shared/q4-decode-row-block-accumulate.wgsl.jinja" %}}`],["ops/_shared/q4-normed-projection-single-row.wgsl.jinja",`{% set packMeta = packMeta | default("q4_scales") %}var acc0=0.0;
{% if not preNormed %}var ss=0.0;
{% endif %}for(var b=lid;b<BPR;b=b+WORKGROUP_SIZE){let hbase=b*8u;
{% for i in range(8) %}let {{ "x" if preNormed else "h" }}{{ i }}=vec4<f32>(hidden[hbase{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}{% if not preNormed %}ss=ss+dot(h0,h0)+dot(h1,h1)+dot(h2,h2)+dot(h3,h3)+dot(h4,h4)+dot(h5,h5)+dot(h6,h6)+dot(h7,h7);let nb=W_NORM_VEC4+hbase;
{% for i in range(8) %}let x{{ i }}=h{{ i }}*vec4<f32>(norm_weight[nb{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}{% endif %}{% set regRows = 1 %}{% set rowBase = "row_block_base" %}{% set q5Route = true %}{% include "ops/_shared/q4-decode-row-block-accumulate.wgsl.jinja" %}}
{% if useSubgroups %}let acc_total=sg_safe_sum(acc0,lid,SG_SAFE_WG,sgNative);
{% if not preNormed %}let ss_total=sg_safe_sum(ss,lid,SG_SAFE_WG,sgNative);
{% endif %}{% elif sgRuntime | default(false) %}{{ sg_fold_workgroup([["acc0", "acc_total"]] if preNormed else [["acc0", "acc_total"], ["ss", "ss_total"]]) }}
{% else %}partials[lid]=acc0;
{% if not preNormed %}partials_ss[lid]=ss;
{% endif %}workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){partials[lid]=partials[lid]+partials[lid+stride];
{% if not preNormed %}partials_ss[lid]=partials_ss[lid]+partials_ss[lid+stride];
{% endif %}}workgroupBarrier();
{% if fuseConv is undefined %}if(stride==1u){break;}
{% endif %}stride=stride/2u;}let acc_total=partials[0];
{% if not preNormed %}let ss_total=partials_ss[0];
{% endif %}{% endif %}`],["ops/_shared/workgroup-reduce-scalar.wgsl.jinja",`{% set reduceAcc = reduceAcc | default("acc") %}{% if useSubgroups %}{% include "ops/_shared/workgroup-reduce-subgroup-arm.wgsl.jinja" %}{% else %}partials[lid]={{ reduceAcc }};workgroupBarrier();
{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ wgsl_tree_fold(["partials"], form="head", barrierFirst=true) }}
{% endif %}`],["ops/_shared/workgroup-reduce-subgroup-arm.wgsl.jinja",`{% set reduceAcc = reduceAcc | default("acc") %}let subgroup_sum=subgroupAdd({{ reduceAcc }});
{% if entrySubgroupIds | default(false) %}var reduce_total=subgroup_sum;if(sg_count!=1u){if(subgroupElect()){partials[sg_id]=subgroup_sum;}workgroupBarrier();reduce_total=partials[0];for(var i=1u;i<sg_count;i=i+1u){reduce_total=reduce_total+partials[i];}workgroupBarrier();}
{% else %}partials[lid]=select(0.0,subgroup_sum,subgroupElect());workgroupBarrier();if(lid==0u){var total=0.0;for(var i=0u;i<WORKGROUP_SIZE;i=i+1u){total=total+partials[i];}partials[0]=total;}workgroupBarrier();
{% endif %}`],["ops/_shared/subgroup-owned-clusters.wgsl.jinja",`{% include "ops/_shared/subgroup-active-lanes.wgsl.jinja" %}{% set sgClusterPaths = ["general", "full"] %}{% set sgClusterBuiltins = ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32, @builtin(subgroup_id) sg_id: u32, @builtin(num_subgroups) sg_count: u32" %}{% macro sg_cluster_open(path, clusters, totals) %}{% if path == "general" %}if(sg_size!=32u||sg_count!={{ clusters }}){if(sg_id==0u){ {{ active_lanes("sg_lid") }} let lane=activeRank;var sgTotals:array<f32,{{ totals }}>;for(var cluster=0u;cluster<{{ clusters }};cluster=cluster+1u){
{% else %}}else{let cluster=sg_id;let lane=sg_lid&31u;
{% endif %}{% endmacro %}{% macro sg_cluster_store(clusters) %}}for(var cluster=0u;cluster<{{ clusters }};cluster=cluster+1u){
{% endmacro %}{% macro sg_cluster_close(path) %}{% if path == "general" %}} }
{% else %}}
{% endif %}{% endmacro %}`],["ops/_shared/f16-bits-to-f32.wgsl.jinja","fn f16_bits_to_f32(h:u32)->f32{let sign=(h&0x8000u)<<16u;let exponent=(h>>10u)&0x1fu;let mantissa=h&0x3ffu;if(exponent==0u){return bitcast<f32>(sign|bitcast<u32>(f32(mantissa)*5.9604644775390625e-8));}if(exponent==31u){return bitcast<f32>(sign|0x7f800000u|(mantissa<<13u));}return bitcast<f32>(sign|((exponent+112u)<<23u)|(mantissa<<13u));}"],["ops/_shared/hadamard-block-stages.wgsl.jinja",`{% set shuffleSpanLimit = 16 if sgMinWidth > 32 else sgMaxWidth %}{% for span in [1, 2, 4, 8, 16, 32, 64, 128, 256] %}{% if span < workgroupSize and not (full and columnStages and span > 32) %}{% set shuffled = full and span < shuffleSpanLimit %}{% if shuffled %}{{ "if (" ~ span ~ "u < sgSize) {" if span >= sgMinWidth else "{" }}
{% for r in range(registers | int) %}let p{{ r }}=subgroupShuffleXor(v{{ r }},{{ span }}u);v{{ r }}=select(v{{ r }}+p{{ r }},p{{ r }}-v{{ r }},(lane&{{ span }}u)!=0u);
{% endfor %}{% endif %}{% if full and columnStages and span == 32 %}{
{% for r in range(registers | int) %}exchange[{{ r * workgroupSize }}u+tid]=v{{ r }};
{% endfor %}workgroupBarrier();
{% for r in range(registers | int) %}v{{ r }}=column_butterfly({{ r * workgroupSize }}u+(tid&31u),tid>>5u);
{% endfor %}{% elif not shuffled or span >= sgMinWidth %}{{ "} else {" if shuffled else "{" }}
{% for r in range(registers | int) %}exchange[{{ r * workgroupSize }}u+tid]=v{{ r }};
{% endfor %}workgroupBarrier();
{% for r in range(registers | int) %}{let p=exchange[{{ r * workgroupSize }}u+(tid^{{ span }}u)];v{{ r }}=select(v{{ r }}+p,p-v{{ r }},(tid&{{ span }}u)!=0u);}
{% endfor %}{% if span < workgroupSize / 2 %}workgroupBarrier();
{% endif %}{% endif %}}
{% endif %}{% endfor %}{% for span in [1, 2, 4, 8, 16, 32, 64] %}{% if span < registers %}{
{% for r in range(registers | int) %}{% if r % (2 * span) < span %}let a{{ r }}=v{{ r }};v{{ r }}=a{{ r }}+v{{ r + span }};v{{ r + span }}=a{{ r }}-v{{ r + span }};
{% endif %}{% endfor %}}
{% endif %}{% endfor %}`],["ops/_shared/hadamard-column-butterfly.wgsl.jinja",`{% set columnStages = useSubgroups and wave32Effective and workgroupSize == 256 %}{% if columnStages %}fn column_butterfly(c:u32,column:u32)->f32{
{% for k in range(8) %}let e{{ k }}=exchange[c{{ " + " ~ (32 * k) ~ "u" if k else "" }}];
{% endfor %}let t0=select(e0+e1,e0-e1,(column&1u)!=0u);let t1=select(e2+e3,e2-e3,(column&1u)!=0u);let t2=select(e4+e5,e4-e5,(column&1u)!=0u);let t3=select(e6+e7,e6-e7,(column&1u)!=0u);let u0=select(t0+t1,t0-t1,(column&2u)!=0u);let u1=select(t2+t3,t2-t3,(column&2u)!=0u);return select(u0+u1,u0-u1,(column&4u)!=0u);}
{% endif %}`],["ops/_shared/binary-lut4-matmul.wgsl.jinja",`{% set M_VECTOR_WIDTH = mVectorWidth if mVectorWidth is defined else 1 %}{% set outStride = outStride | default(0) %}{% set dstColStart = dstColStart | default(0) %}{% set scaleScalar = scaleScalar | default("f32") %}{{ env.wgsl.resourceDeclarations }} const IN_FEATURES:u32={{ inFeatures }}u;const OUT_FEATURES:u32={{ outFeatures }}u;const M_TILE:u32={{ tileM }}u;const N_TILE:u32={{ tileN }}u;const N_PER_THREAD:u32={{ nPerThread }}u;const IN_FEATURES_V4:u32=IN_FEATURES/4u;const NUM_GROUPS:u32=IN_FEATURES/128u;const WG_SIZE:u32=N_TILE/N_PER_THREAD;const LUT_M_ROWS:u32=M_TILE/{{ M_VECTOR_WIDTH }}u;const LUT_ENTRIES:u32=LUT_M_ROWS*32u*16u;const LUT_NIBBLES:u32=LUT_M_ROWS*32u;
{% if layout == "q1_row4" %}const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;
{% endif %}{% if M_VECTOR_WIDTH == 4 %}alias LutValue=vec4<{{ lutScalar }}>;alias AccValue=vec4<f32>;
{% else %}alias LutValue={{ lutScalar }};alias AccValue=f32;
{% endif %}var<workgroup>lut:array<LutValue,LUT_ENTRIES>;
{% if not assumeBiasNegHalfScale %}var<workgroup>sumA:array<AccValue,LUT_M_ROWS>;
{% endif %}fn lutIndex(m_row:u32,nibble:u32,mask:u32)->u32{return(m_row*32u+nibble)*16u+mask;}
{% if layout == "q1_row4" %}fn q1RowWord(vec_index:u32,row_lane:u32)->u32{return q1_bits[vec_index*4u+row_lane];}
{% include "ops/_shared/quant-q1-scale.wgsl.jinja" %}{% else %}{% if scaleBiasDtype is defined and scaleBiasDtype == "bfloat16" %}fn sbAt(i:u32)->f32{let word=scaleBias[i>>1u];return bitcast<f32>(select(word<<16u,word&0xFFFF0000u,(i&1u)!=0u));}
{% else %}fn sbAt(i:u32)->f32{return f32(scaleBias[i]);}
{% endif %}{% endif %}@compute@workgroup_size({{ workgroupSize }},1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let tid=lid.x;let n_block=wg.x*N_TILE;let m_block=(wg.y+wg.z*{{ DISPATCH_FOLD_WIDTH }}u)*M_TILE;
{% if layout == "q1_row4" and dynamicM is defined and dynamicM %}let active_m=min(real_len[0],params.m);if(m_block>=active_m){return;}
{% elif layout == "q1_row4" %}let active_m=params.m;
{% endif %}let n_base=n_block+tid;
{% for nn in range(nPerThread) %}let n{{ nn }}=n_base+{{ nn }}u*WG_SIZE;let n{{ nn }}_valid=n{{ nn }}<OUT_FEATURES;
{% endfor %}{% for nn in range(nPerThread) %}{% for m in range(0, tileM, M_VECTOR_WIDTH) %}var acc{{ nn }}_{{ m }}=AccValue(0.0);
{% endfor %}{% endfor %}for(var g=0u;g<NUM_GROUPS;g=g+1u){for(var i=tid;i<LUT_NIBBLES;i=i+WG_SIZE){let nibble=i&31u;let m_local=i>>5u;
{% if M_VECTOR_WIDTH == 4 %}let m=m_block+m_local*4u;var v0={{ aVec4 }}(0.0);var v1={{ aVec4 }}(0.0);var v2={{ aVec4 }}(0.0);var v3={{ aVec4 }}(0.0);if(m<active_m){v0=a[m*IN_FEATURES_V4+g*32u+nibble];}if(m+1u<active_m){v1=a[(m+1u)*IN_FEATURES_V4+g*32u+nibble];}if(m+2u<active_m){v2=a[(m+2u)*IN_FEATURES_V4+g*32u+nibble];}if(m+3u<active_m){v3=a[(m+3u)*IN_FEATURES_V4+g*32u+nibble];}let x=vec4<f32>(f32(v0.x),f32(v1.x),f32(v2.x),f32(v3.x));let yv=vec4<f32>(f32(v0.y),f32(v1.y),f32(v2.y),f32(v3.y));let z=vec4<f32>(f32(v0.z),f32(v1.z),f32(v2.z),f32(v3.z));let wv=vec4<f32>(f32(v0.w),f32(v1.w),f32(v2.w),f32(v3.w));
{% else %}let m=m_block+m_local;var v={{ aVec4 }}(0.0);
{% if layout == "q1_row4" %}if(m<active_m){
{% else %}if(m<params.M){
{% endif %}v=a[m*IN_FEATURES_V4+g*32u+nibble];}let x=f32(v.x);let yv=f32(v.y);let z=f32(v.z);let wv=f32(v.w);
{% endif %}let xy=x+yv;let xz=x+z;let yz=yv+z;let xyz=xy+z;let base=(m_local*32u+nibble)*16u;
{% if assumeBiasNegHalfScale %}let c=0.5*(x+yv+z+wv);lut[base]=LutValue(-c);lut[base+1u]=LutValue(x-c);lut[base+2u]=LutValue(yv-c);lut[base+3u]=LutValue(xy-c);lut[base+4u]=LutValue(z-c);lut[base+5u]=LutValue(xz-c);lut[base+6u]=LutValue(yz-c);lut[base+7u]=LutValue(xyz-c);lut[base+8u]=LutValue(wv-c);lut[base+9u]=LutValue(x+wv-c);lut[base+10u]=LutValue(yv+wv-c);lut[base+11u]=LutValue(xy+wv-c);lut[base+12u]=LutValue(z+wv-c);lut[base+13u]=LutValue(xz+wv-c);lut[base+14u]=LutValue(yz+wv-c);lut[base+15u]=LutValue(xyz+wv-c);
{% else %}lut[base]=LutValue(0.0);lut[base+1u]=LutValue(x);lut[base+2u]=LutValue(yv);lut[base+3u]=LutValue(xy);lut[base+4u]=LutValue(z);lut[base+5u]=LutValue(xz);lut[base+6u]=LutValue(yz);lut[base+7u]=LutValue(xyz);lut[base+8u]=LutValue(wv);lut[base+9u]=LutValue(x+wv);lut[base+10u]=LutValue(yv+wv);lut[base+11u]=LutValue(xy+wv);lut[base+12u]=LutValue(z+wv);lut[base+13u]=LutValue(xz+wv);lut[base+14u]=LutValue(yz+wv);lut[base+15u]=LutValue(xyz+wv);
{% endif %}}workgroupBarrier();
{% if not assumeBiasNegHalfScale %}for(var mi=tid;mi<LUT_M_ROWS;mi=mi+WG_SIZE){var s=AccValue(0.0);for(var nibble=0u;nibble<32u;nibble=nibble+1u){s=s+AccValue(lut[lutIndex(mi,nibble,15u)]);}sumA[mi]=s;}workgroupBarrier();
{% endif %}{% for nn in range(nPerThread) %}if(n{{ nn }}_valid){
{% if layout == "q1_row4" %}let row_tile{{ nn }}=n{{ nn }}/4u;let row_lane{{ nn }}=n{{ nn }}&3u;let group_base{{ nn }}=params.block_offset+(row_tile{{ nn }}*NUM_GROUPS+g)*4u;let p0{{ nn }}=q1RowWord(group_base{{ nn }},row_lane{{ nn }});let p1{{ nn }}=q1RowWord(group_base{{ nn }}+1u,row_lane{{ nn }});let p2{{ nn }}=q1RowWord(group_base{{ nn }}+2u,row_lane{{ nn }});let p3{{ nn }}=q1RowWord(group_base{{ nn }}+3u,row_lane{{ nn }});let scale{{ nn }}=2.0*q1Scale(group_base{{ nn }}+row_lane{{ nn }});
{% else %}let packed{{ nn }}=bits_buf[n{{ nn }}*NUM_GROUPS+g];let p0{{ nn }}=packed{{ nn }}.x;let p1{{ nn }}=packed{{ nn }}.y;let p2{{ nn }}=packed{{ nn }}.z;let p3{{ nn }}=packed{{ nn }}.w;
{% if not scaleOnly %}let sb{{ nn }}=(n{{ nn }}*NUM_GROUPS+g)*2u;
{% endif %}{% if assumeBiasNegHalfScale %}{% if scaleOnly %}let scale{{ nn }}=sbAt(n{{ nn }}*NUM_GROUPS+g);
{% else %}let scale{{ nn }}=sbAt(sb{{ nn }});
{% endif %}{% else %}let scale{{ nn }}=sbAt(sb{{ nn }});let bias{{ nn }}=sbAt(sb{{ nn }}+1u);
{% endif %}{% endif %}{% for m in range(0, tileM, M_VECTOR_WIDTH) %}var qa{{ nn }}_{{ m }}=AccValue(0.0);
{% endfor %}{% for w in range(4) %}{% for b in range(4) %}{let byte{{ nn }}_{{ w }}_{{ b }}=(p{{ w }}{{ nn }}>>{{ b * 8 }}u)&0xffu;let lo{{ nn }}_{{ w }}_{{ b }}=byte{{ nn }}_{{ w }}_{{ b }}&0x0fu;let hi{{ nn }}_{{ w }}_{{ b }}=byte{{ nn }}_{{ w }}_{{ b }}>>4u;
{% for m in range(0, tileM, M_VECTOR_WIDTH) %}qa{{ nn }}_{{ m }}=qa{{ nn }}_{{ m }}+AccValue(lut[lutIndex({{ m }}u/{{ M_VECTOR_WIDTH }}u,{{ w * 8 + b * 2 }}u,lo{{ nn }}_{{ w }}_{{ b }})])+AccValue(lut[lutIndex({{ m }}u/{{ M_VECTOR_WIDTH }}u,{{ w * 8 + b * 2 + 1 }}u,hi{{ nn }}_{{ w }}_{{ b }})]);
{% endfor %}}
{% endfor %}{% endfor %}{% for m in range(0, tileM, M_VECTOR_WIDTH) %}{% if assumeBiasNegHalfScale %}acc{{ nn }}_{{ m }}=acc{{ nn }}_{{ m }}+scale{{ nn }}*qa{{ nn }}_{{ m }};
{% else %}acc{{ nn }}_{{ m }}=acc{{ nn }}_{{ m }}+scale{{ nn }}*qa{{ nn }}_{{ m }}+bias{{ nn }}*sumA[{{ m }}u/{{ M_VECTOR_WIDTH }}u];
{% endif %}{% endfor %}}
{% endfor %}workgroupBarrier();}
{% for nn in range(nPerThread) %}if(n{{ nn }}_valid){
{% if M_VECTOR_WIDTH == 4 %}{% for m in range(0, tileM, 4) %}{% for lane in range(4) %}{let mr=m_block+{{ m + lane }}u;if(mr<active_m){y[mr*OUT_STRIDE+DST_COL+n{{ nn }}]={{ yScalar }}(acc{{ nn }}_{{ m }}[{{ lane }}u]);} }
{% endfor %}{% endfor %}{% else %}{% for m in range(tileM) %}{let mr=m_block+{{ m }}u;
{% if layout == "q1_row4" %}if(mr<active_m){y[mr*OUT_STRIDE+DST_COL+n{{ nn }}]={{ yScalar }}(acc{{ nn }}_{{ m }});}
{% else %}if(mr<params.M){y[mr*OUT_FEATURES+n{{ nn }}]={{ yScalar }}(acc{{ nn }}_{{ m }});}
{% endif %}}
{% endfor %}{% endif %}}
{% endfor %}}`],["ops/_shared/conv-activation.wgsl.jinja",`{% set hasActivation = hasActivation is defined and hasActivation %}{% set activation = activation | default("") %}{% set actAlpha = actAlpha | default(0.0) %}{% set actBeta = actBeta | default(0.0) %}{% if hasActivation %}{% include "ops/_shared/fused-activation.wgsl.jinja" %}{{ fused_act_fn(activation, actAlpha, actBeta) }}
{% endif %}`],["ops/_shared/dense-tiled-matmul.wgsl.jinja",`{% set usesF16 = tileDtype == "f16" %}{% set mPerThread = mPerThread if mPerThread is defined else 1 %}{% set rowVar = "m_row" if mPerThread > 1 else "m_local" %}{% if hasWeightOffset is not defined %}{% set hasWeightOffset = false %}{% endif %}{% set F32_OPEN = "vec4<f32>(" if usesF16 else "" %}{% set F32_CLOSE = ")" if usesF16 else "" %}{% set NARROW_OUTPUT = usesF16 and (outputDtype | default("float32")) == "float16" %}{% set F16_OPEN = "f16(" if NARROW_OUTPUT else "" %}{% set F16_CLOSE = ")" if NARROW_OUTPUT else "" %}{{ env.wgsl.resourceDeclarations }}
{% if mPerThread > 1 %}{% endif %}const K:u32={{ inFeatures }}u;const N:u32={{ outFeatures }}u;const K_V4:u32={{ kV4 }}u;const M_TILE:u32={{ mTile }}u;const N_TILE:u32={{ nTile }}u;const OUT_TILE:u32={{ outTile }}u;const K_TILE:u32={{ kTile }}u;const K_TILE_V4:u32={{ kTileV4 }}u;const WG:u32={{ workgroupSize }}u;
{% if mPerThread > 1 %}const M_PER_THREAD:u32={{ mPerThread }}u;
{% endif %}var<workgroup>aTile:array<vec4<{{ tileDtype }}>,{{ aTileElements }}>;var<workgroup>bTile:array<vec4<{{ tileDtype }}>,{{ bTileElements }}>;@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>){let tid=lid.x;let m_local=tid/N_TILE;let n_local=tid-m_local*N_TILE;
{% if mPerThread > 1 %}let m_row=m_local*M_PER_THREAD;
{% endif %}let n_tiles=(N+OUT_TILE-1u)/OUT_TILE;let m_tiles=(params.M+M_TILE-1u)/M_TILE;for(var wgx:u32=wg.x;wgx<n_tiles;wgx=wgx+nwg.x){let n0=wgx*OUT_TILE+n_local;for(var wgy:u32=wg.y;wgy<m_tiles;wgy=wgy+nwg.y){let m=wgy*M_TILE+{{ rowVar }};
{% for acc in range(mPerThread * nPerThread) %}var acc{{ acc }}:f32=0.0;
{% endfor %}for(var k_base:u32=0u;k_base<K;k_base=k_base+K_TILE){let k_base_v4=k_base/4u;for(var i:u32=tid;i<{{ aTileElements }}u;i=i+WG){let tm=i/K_TILE_V4;let kv=i-tm*K_TILE_V4;let gm=wgy*M_TILE+tm;if(gm<params.M){aTile[i]=a[gm*K_V4+k_base_v4+kv];}else{
            aTile[i] = vec4<{{ tileDtype }}>({% if usesF16 %}0.0h{% else %}0.0{% endif %});
} }for(var i:u32=tid;i<{{ bTileElements }}u;i=i+WG){let tn=i/K_TILE_V4;let kv=i-tn*K_TILE_V4;let gn=wgx*OUT_TILE+tn;if(gn<N){
            bTile[i] = w[{% if hasWeightOffset %}params.weightOffsetV4 + {% endif %}gn * K_V4 + k_base_v4 + kv];
}else{
            bTile[i] = vec4<{{ tileDtype }}>({% if usesF16 %}0.0h{% else %}0.0{% endif %});
} }workgroupBarrier();
{% if mPerThread > 1 %}if(m<params.M){for(var kv:u32=0u;kv<K_TILE_V4;kv=kv+1u){
{% for mm in range(mPerThread) %}let a_vec{{ mm }}=aTile[({{ rowVar }}+{{ mm }}u)*K_TILE_V4+kv];
{% endfor %}{% for nn in range(nPerThread) %}let b_vec{{ nn }}=bTile[(n_local+{{ nn * nTile }}u)*K_TILE_V4+kv];
{% endfor %}{% for mm in range(mPerThread) %}{% for nn in range(nPerThread) %}acc{{ mm * nPerThread + nn }}=acc{{ mm * nPerThread + nn }}+dot({{ F32_OPEN }}a_vec{{ mm }}{{ F32_CLOSE }},{{ F32_OPEN }}b_vec{{ nn }}{{ F32_CLOSE }});
{% endfor %}{% endfor %}} }
{% else %}if(m<params.M){for(var kv:u32=0u;kv<K_TILE_V4;kv=kv+1u){let a_idx=m_local*K_TILE_V4+kv;let a_vec=aTile[a_idx];
{% for nn in range(nPerThread) %}let b_idx{{ nn }}=(n_local+{{ nn * nTile }}u)*K_TILE_V4+kv;acc{{ nn }}=acc{{ nn }}+dot({{ F32_OPEN }}a_vec{{ F32_CLOSE }},{{ F32_OPEN }}bTile[b_idx{{ nn }}]{{ F32_CLOSE }});
{% endfor %}} }
{% endif %}workgroupBarrier();}
{% for mm in range(mPerThread) %}{% set mRow = "m" if mPerThread == 1 else "(m + " ~ mm ~ "u)" %}{% set aBase = mm * nPerThread %}if({{ mRow }}<params.M){
{% for nn in range(nPerThread) %}if(n0+{{ nn * nTile }}u<N){
{% if usesF16 %}          y[{{ mRow }} * N + n0 + {{ nn * nTile }}u] = {{ F16_OPEN }}acc{{ aBase + nn }}{% if hasBias %} + f32(bias[n0 + {{ nn * nTile }}u]){% endif %}{{ F16_CLOSE }};
{% else %}          y[{{ mRow }} * N + n0 + {{ nn * nTile }}u] = acc{{ aBase + nn }}{% if hasBias %} + bias[n0 + {{ nn * nTile }}u]{% endif %};
{% endif %}}
{% endfor %}}
{% endfor %}} } }`],["ops/_shared/flat-index-2d.wgsl.jinja",`{% macro flat_index_2d(workgroupSize, name="i", bound="params.count") %}{% set wgTerm = workgroupSize ~ "u" if workgroupSize is number else workgroupSize %}let {{ name }}=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ wgTerm }};
{% if bound != "" %}if({{ name }}>={{ bound }}){return;}
{% endif %}{% endmacro %}`],["ops/_shared/fused-activation.wgsl.jinja",`{% macro fused_act_fn(mode, alpha, beta) %}fn fused_act(v:f32)->f32{
{% if mode == "Relu" %}return max(v,0.0);
{% elif mode == "Clip" %}return clamp(v,f32({{ alpha }}),f32({{ beta }}));
{% elif mode == "LeakyRelu" %}return select(v*f32({{ alpha }}),v,v>=0.0);
{% elif mode == "Sigmoid" %}return 1.0/(1.0+exp(-v));
{% elif mode == "Tanh" %}return tanh(clamp(v,-10.0,10.0));
{% elif mode == "HardSigmoid" %}return clamp(f32({{ alpha }})*v+f32({{ beta }}),0.0,1.0);
{% elif mode == "QuickGelu" %}{% if alpha == 1 %}return v*(1.0/(1.0+exp(-v)));
{% else %}return v*(1.0/(1.0+exp(-(f32({{ alpha }})*v))));
{% endif %}{% elif mode == "Elu" %}return select(f32({{ alpha }})*(exp(v)-1.0),v,v>=0.0);
{% elif mode == "ThresholdedRelu" %}return select(0.0,v,v>f32({{ alpha }}));
{% elif mode == "Softplus" %}return max(v,0.0)+log(1.0+exp(-abs(v)));
{% elif mode == "Erf" %}let a=abs(v);let t=1.0/(1.0+0.3275911*a);let poly=((((1.061405429*t-1.453152027)*t+1.421413741)*t-0.284496736)*t+0.254829592)*t;return sign(v)*(1.0-poly*exp(-a*a));
{% elif mode == "Gelu" and alpha == 0 %}let u=v*0.70710678118654752;let a=abs(u);let t=1.0/(1.0+0.3275911*a);let poly=((((1.061405429*t-1.453152027)*t+1.421413741)*t-0.284496736)*t+0.254829592)*t;return 0.5*v*(1.0+sign(u)*(1.0-poly*exp(-a*a)));
{% elif mode == "Gelu" or mode == "FastGelu" %}let inner=v*(0.035677408136300125*v*v+0.7978845608028654);return v*(0.5+0.5*tanh(clamp(inner,-10.0,10.0)));
{% else %}return v*clamp(v*0.16666666666666666+0.5,0.0,1.0);
{% endif %}}{% endmacro %}`],["ops/_shared/gelu-erf.wgsl.jinja","fn erf_approx(x:f32)->f32{let ax=abs(x);if(ax<9.5367431640625e-7){return 1.1283791670955126*x;}let sign=select(-1.0,1.0,x>=0.0);let t=1.0/(1.0+0.3275911*ax);let y=1.0-(((((1.061405429*t-1.453152027)*t)+1.421413741)*t-0.284496736)*t+0.254829592)*t*exp(-(ax*ax));return sign*y;}"],["ops/_shared/gemm-activation.wgsl.jinja",'{% set actMode = gemmActivation | default("") %}{% set actA = gemmActAlpha | default(0.0) %}{% set actB = gemmActBeta | default(0.0) %}{% include "ops/_shared/fused-activation.wgsl.jinja" %}{{ fused_act_fn(actMode, actA, actB) }}'],["ops/_shared/gemm-subgroup-matrix.wgsl.jinja",`{% include "ops/_shared/subgroup-matrix-enable.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% set operandScalar = mmaScalar %}{% set operandZero = "0.0h" if operandScalar == "f16" else "0.0" %}{% set accScalar = "f32" %}{% set SPLIT_K = splitK if splitK is defined else 1 %}{% set kPerSplit = kPerSplit | default(0) %}{% set OUT = outputBuffer if outputBuffer is defined else "y" %}{% set OUT_SCALAR = outScalar if outScalar is defined else yScalar %}{% set tileMValue = tileM if tileM is defined else 32 %}{% set tileNValue = tileN if tileN is defined else 64 %}{% set tileKValue = tileK if tileK is defined else 32 %}{% set gemmEpi = gemmEpilogue if gemmEpilogue is defined else "none" %}{% set epiOpen = "gelu_tanh(" if gemmEpi == "fastgelu" else ("fused_act(" if gemmEpi == "activation" else ("gelu_erf(" if gemmEpi == "erfgelu" else "")) %}{% set epiClose = ")" if gemmEpi != "none" else "" %}{% set packedNTail = packedMatrixNTail is defined and packedMatrixNTail %}{% set packedKTail = packedMatrixKTail is defined and packedMatrixKTail %}{% set useDirectStore = directStore is defined and directStore and gemmEpi == "none" and not hasBias and not (hasResidual | default(false)) %}{% set useDirectAStorage = directAStorage is defined and directAStorage %}{% set useDoubleBufferedB = doubleBufferedB is defined and doubleBufferedB %}{% set useBalancedAccumulation = balancedMatrixAccumulation is defined and balancedMatrixAccumulation and useDoubleBufferedB and not useDirectStore and operandScalar == "f32" %}{% set matrixChunkTiles = compensatedMatrixChunkTiles if compensatedMatrixChunkTiles is defined else 0 %}{% set useChunkedAccumulation = matrixChunkTiles > 0 and useDirectAStorage and not useDoubleBufferedB and not useDirectStore and operandScalar == "f32" and SPLIT_K == 1 %}{% set aTransposed = aTransposedStorage if aTransposedStorage is defined else (matrixInputsTransposed is defined and matrixInputsTransposed) %}{% set bTransposed = bTransposedStorage if bTransposedStorage is defined else (matrixInputsTransposed is defined and matrixInputsTransposed) %}{% set fragmentMajorBStaging = (bf16BStaging | default(false)) and not (tailSafe | default(false)) and not bTransposed %}{% set needsInFeatures = (not aTransposed) or (not bTransposed) or (tailSafe is defined and tailSafe) or (kLoopEnd is not defined) %}{% set columnTileBaseValue = columnTileBase if columnTileBase is defined else 0 %}{% set rowTileBaseValue = rowTileBase if rowTileBase is defined else 0 %}{% set rowTileBaseRuntimeValue = rowTileBaseRuntime | default("") %}{% set runtimeRowTileBase = rowTileBaseRuntimeValue != "" %}{% set tileGroup = tileGroupRows if (tileGroupRows is defined and SPLIT_K == 1) else 1 %}{% set tileGroupColsValue = tileGroupCols | default(0) %}{% if matrixBlocks32 is defined and matrixBlocks32 %}{% set subgroupRows = (tileMValue / 32)|int %}{% set subgroupCols = (tileNValue / 32)|int %}{% set subgroupCount = subgroupRows * subgroupCols %}{% set subtileRows = 32 %}{% set subtileCols = 32 %}{% elif tileMValue > 32 %}{% set subgroupCount = 4 %}{% set subgroupRows = 4 %}{% set subgroupCols = 1 %}{% set subtileCols = tileNValue %}{% set subtileRows = (tileMValue / 4)|int %}{% else %}{% set subgroupCount = 4 %}{% set subgroupRows = 2 %}{% set subgroupCols = 2 %}{% set subtileCols = (tileNValue / 2)|int %}{% set subtileRows = 16 %}{% endif %}{% set aTilesPerSubgroup = (subtileRows / 8)|int %}{% set bTilesPerSubgroup = (subtileCols / 8)|int %}{% set workgroupThreads = 32 * subgroupCount %}{% set scratchBanks = subgroupCount * aTilesPerSubgroup * bTilesPerSubgroup %}{% set reuseB = reuseMatrixBForEpilogue is defined and reuseMatrixBForEpilogue and operandScalar == "f32" and not useBalancedAccumulation and not useChunkedAccumulation and useDoubleBufferedB and 2 * tileNValue * tileKValue >= scratchBanks * 64 %}{% macro matrixAccumulators(matrixSoftware) %}{% for m in range(aTilesPerSubgroup) %}{% for n in range(bTilesPerSubgroup) %}var matC{{ m }}{{ n }}:{{ matrixType(matrixSoftware, "result", accScalar) }};
{% if useBalancedAccumulation %}var matD{{ m }}{{ n }}:{{ matrixType(matrixSoftware, "result", accScalar) }};var matE{{ m }}{{ n }}:{{ matrixType(matrixSoftware, "result", accScalar) }};var matF{{ m }}{{ n }}:{{ matrixType(matrixSoftware, "result", accScalar) }};
{% endif %}{% endfor %}{% endfor %}{% endmacro %}{% macro storeMatrixAccumulators(matrixSoftware) %}{% for m in range(aTilesPerSubgroup) %}{% for n in range(bTilesPerSubgroup) %}{% if reuseB %}{{ matrixStore(matrixSoftware, "row_major", "&tile_B", "(bank + " ~ (m * bTilesPerSubgroup + n) ~ "u) * 64u", "matC" ~ (m) ~ (n), "8u") }}
{% else %}{{ matrixStore(matrixSoftware, "row_major", "&scratch[bank + " ~ (m * bTilesPerSubgroup + n) ~ "u]", "0u", "matC" ~ (m) ~ (n), "8u") }}
{% endif %}{% if useBalancedAccumulation %}{{ matrixStore(matrixSoftware, "row_major", "&tile_B", "(bank + " ~ (m * bTilesPerSubgroup + n) ~ "u) * 64u", "matD" ~ (m) ~ (n), "8u") }}
{% endif %}{% endfor %}{% endfor %}{% endmacro %}{% if gemmEpi == "erfgelu" %}{% include "ops/_shared/gelu-erf.wgsl.jinja" %}fn gelu_erf(v:f32)->f32{return 0.5*v*(1.0+erf_approx(v*0.7071067811865476));}
{% endif %}{% if gemmEpi == "fastgelu" %}{% include "ops/_shared/gelu-tanh.wgsl.jinja" %}{% elif gemmEpi == "activation" %}{% include "ops/_shared/gemm-activation.wgsl.jinja" %}{% endif %}{% if f16BStaging is defined and f16BStaging %}{% include "ops/_shared/f16-bits-to-f32.wgsl.jinja" %}{% endif %}{% if needsInFeatures %}const IN_F:       u32 = {{ inFeatures }}u;
{% endif %}const OUT_F:u32={{ outFeatures }}u;
{% if tailSafe is defined and tailSafe %}const K_LOOP:u32=((IN_F+{{ tileKValue - 1 }}u)/{{ tileKValue }}u)*{{ tileKValue }}u;
{% endif %}{% set kLoopEndValue = kLoopEnd if kLoopEnd is defined else ("K_LOOP" if tailSafe is defined and tailSafe else "IN_F") %}const TILE_COLS:u32={{ tileNValue }}u;const TILE_ROWS:u32={{ tileMValue }}u;const TILE_K:u32={{ tileKValue }}u;const SUB_COLS:u32={{ subtileCols }}u;const SUB_ROWS:u32={{ subtileRows }}u;const COLUMN_TILE_BASE:u32={{ columnTileBaseValue }}u;
{% if not runtimeRowTileBase %}const ROW_TILE_BASE:u32={{ rowTileBaseValue }}u;
{% endif %}{% if SPLIT_K > 1 %}const K_PER_SPLIT:u32={{ kPerSplit }}u;
{% endif %}{% if not useDirectAStorage %}var<workgroup>tile_A:array<{{ operandScalar }},{{ tileMValue }}*{{ tileKValue }}>;
{% endif %}var<workgroup>tile_B:array<{{ operandScalar }},{{ (2 if useDoubleBufferedB else 1) * tileNValue }}*{{ tileKValue }}>;
{% if not useDirectStore and not reuseB %}var<workgroup>scratch:array<array<{{ accScalar }},64>,{{ scratchBanks }}>;
{% endif %}{% if not useDirectAStorage %}fn loadSHMA(tile_base:u32,k_idx:u32,row:u32,c_idx:u32){
{% if aTransposed %}let local_idx=row*{{ (tileKValue / 8)|int }}u+c_idx;let vectors_per_k=TILE_ROWS/4u;let total_vectors=TILE_K*vectors_per_k;for(var vector_idx=local_idx;vector_idx<total_vectors;vector_idx+={{ workgroupThreads }}u){let k_local=vector_idx/vectors_per_k;let m4=(vector_idx-k_local*vectors_per_k)*4u;for(var component=0u;component<4u;component++){let m_local=m4+component;let global_m=tile_base+m_local;let global_k=k_idx+k_local;tile_A[k_local*TILE_ROWS+m_local]=x[global_k*params.M+global_m];} }
{% else %}let col:u32=c_idx*8u;for(var row_offset:u32=0u;row_offset<TILE_ROWS;row_offset+={{ (workgroupThreads * 8 / tileKValue)|int }}u){let r:u32=row+row_offset;let a_global:u32=min(tile_base+r,params.M-1u);for(var col_offset:u32=0u;col_offset<8u;col_offset++){let k:u32=k_idx+col+col_offset;
{% if (tailSafe is defined and tailSafe) or packedKTail %}if(k<IN_F){tile_A[r*TILE_K+col+col_offset]={{ operandScalar }}(x[a_global*IN_F+k]);}else{tile_A[r*TILE_K+col+col_offset]={{ operandZero }};}
{% else %}tile_A[r*TILE_K+col+col_offset]=x[a_global*IN_F+k];
{% endif %}} }
{% endif %}}
{% endif %}fn loadSHMB(tile_base:u32,k_idx:u32,row:u32,c_idx:u32,bank_offset:u32){
{% if bTransposed %}let local_idx=row*{{ (tileKValue / 16)|int }}u+c_idx;let vectors_per_k=TILE_COLS/4u;let total_vectors=TILE_K*vectors_per_k;for(var vector_idx=local_idx;vector_idx<total_vectors;vector_idx+={{ workgroupThreads }}u){let k_local=vector_idx/vectors_per_k;let n4=(vector_idx-k_local*vectors_per_k)*4u;for(var component=0u;component<4u;component++){let n_local=n4+component;let global_n=tile_base+n_local;let global_k=k_idx+k_local;tile_B[bank_offset+k_local*TILE_COLS+n_local]=w[global_k*OUT_F+global_n];} }
{% elif bf16BStaging is defined and bf16BStaging and not (tailSafe is defined and tailSafe) %}let col:u32=c_idx*16u;for(var row_offset:u32=0u;row_offset<TILE_COLS;row_offset+={{ (workgroupThreads * 16 / tileKValue)|int }}u){let b_row:u32=row+row_offset;if(b_row<TILE_COLS){
{% if packedNTail or packedKTail %}if(tile_base+b_row<OUT_F&&k_idx+col<IN_F){
{% endif %}let src:u32=((tile_base+b_row)*IN_F+k_idx+col)/8u;let dst:u32=bank_offset+((col/8u)*(TILE_COLS/8u)+b_row/8u)*64u+(b_row%8u)*8u;for(var i:u32=0u;i<2u;i++){let v=w[src+i];let o=dst+i*TILE_COLS*8u;tile_B[o]=bitcast<f32>(v.x<<16u);tile_B[o+1u]=bitcast<f32>(v.x&0xffff0000u);tile_B[o+2u]=bitcast<f32>(v.y<<16u);tile_B[o+3u]=bitcast<f32>(v.y&0xffff0000u);tile_B[o+4u]=bitcast<f32>(v.z<<16u);tile_B[o+5u]=bitcast<f32>(v.z&0xffff0000u);tile_B[o+6u]=bitcast<f32>(v.w<<16u);tile_B[o+7u]=bitcast<f32>(v.w&0xffff0000u);}
{% if packedNTail or packedKTail %}}else{for(var i=0u;i<16u;i++){tile_B[bank_offset+(((col+i)/8u)*(TILE_COLS/8u)+b_row/8u)*64u+(b_row%8u)*8u+(col+i)%8u]=0.0;} }
{% endif %}} }
{% elif f16BStaging is defined and f16BStaging and not (tailSafe is defined and tailSafe) %}let col:u32=c_idx*16u;for(var row_offset:u32=0u;row_offset<TILE_COLS;row_offset+={{ (workgroupThreads * 16 / tileKValue)|int }}u){let b_row:u32=row+row_offset;if(b_row<TILE_COLS){
{% if packedNTail or packedKTail %}if(tile_base+b_row<OUT_F&&k_idx+col<IN_F){
{% endif %}let src:u32=((tile_base+b_row)*IN_F+k_idx+col)/8u;let dst:u32=bank_offset+b_row*TILE_K+col;for(var i:u32=0u;i<2u;i++){let v=w[src+i];let o=dst+i*8u;tile_B[o]=f16_bits_to_f32(v.x);tile_B[o+1u]=f16_bits_to_f32(v.x>>16u);tile_B[o+2u]=f16_bits_to_f32(v.y);tile_B[o+3u]=f16_bits_to_f32(v.y>>16u);tile_B[o+4u]=f16_bits_to_f32(v.z);tile_B[o+5u]=f16_bits_to_f32(v.z>>16u);tile_B[o+6u]=f16_bits_to_f32(v.w);tile_B[o+7u]=f16_bits_to_f32(v.w>>16u);}
{% if packedNTail or packedKTail %}}else{for(var i=0u;i<16u;i++){tile_B[bank_offset+b_row*TILE_K+col+i]=0.0;} }
{% endif %}} }
{% elif q8BStaging is defined and q8BStaging and not (tailSafe is defined and tailSafe) %}let col:u32=c_idx*16u;for(var row_offset:u32=0u;row_offset<TILE_COLS;row_offset+={{ (workgroupThreads * 16 / tileKValue)|int }}u){let b_row:u32=row+row_offset;if(b_row<TILE_COLS){
{% if packedNTail or packedKTail %}if(tile_base+b_row<OUT_F&&k_idx+col<IN_F){
{% endif %}let block:u32=params.block_offset+(tile_base+b_row)*(IN_F/32u)+k_idx/32u;let scale:f32=f32(scales[block]);let v=w[block*2u+c_idx];let dst:u32=bank_offset+b_row*TILE_K+col;for(var i:u32=0u;i<4u;i++){let word=bitcast<i32>(v[i]);tile_B[dst+i*4u]=f32(extractBits(word,0u,8u))*scale;tile_B[dst+i*4u+1u]=f32(extractBits(word,8u,8u))*scale;tile_B[dst+i*4u+2u]=f32(extractBits(word,16u,8u))*scale;tile_B[dst+i*4u+3u]=f32(extractBits(word,24u,8u))*scale;}
{% if packedNTail or packedKTail %}}else{for(var i=0u;i<16u;i++){tile_B[bank_offset+b_row*TILE_K+col+i]=0.0;} }
{% endif %}} }
{% elif vectorBStaging is defined and vectorBStaging and not (tailSafe is defined and tailSafe) %}let col:u32=c_idx*16u;for(var row_offset:u32=0u;row_offset<TILE_COLS;row_offset+={{ (workgroupThreads * 16 / tileKValue)|int }}u){let b_row:u32=row+row_offset;if(b_row<TILE_COLS){let src:u32=((tile_base+b_row)*IN_F+k_idx+col)/4u;let dst:u32=bank_offset+b_row*TILE_K+col;for(var i:u32=0u;i<4u;i++){let v=w[src+i];tile_B[dst+i*4u]={{ operandScalar }}(v.x);tile_B[dst+i*4u+1u]={{ operandScalar }}(v.y);tile_B[dst+i*4u+2u]={{ operandScalar }}(v.z);tile_B[dst+i*4u+3u]={{ operandScalar }}(v.w);} } }
{% else %}let col:u32=c_idx*16u;for(var row_offset:u32=0u;row_offset<TILE_COLS;row_offset+={{ (workgroupThreads * 16 / tileKValue)|int }}u){let b_row:u32=row+row_offset;if(b_row<TILE_COLS){let w_global:u32=tile_base+b_row;for(var i:u32=0u;i<16u;i++){let k:u32=k_idx+col+i;
{% if tailSafe is defined and tailSafe %}if(w_global<OUT_F&&k<IN_F){tile_B[bank_offset+b_row*TILE_K+col+i]={{ operandScalar }}(w[w_global*IN_F+k]);}else{tile_B[bank_offset+b_row*TILE_K+col+i]={{ operandZero }};}
{% else %}tile_B[bank_offset+b_row*TILE_K+col+i]=w[w_global*IN_F+k];
{% endif %}} } }
{% endif %}}
{% if not useDirectStore %}{% set needsColBase = hasBias or (tailSafe is defined and tailSafe) or packedNTail %}{% set BIAS_1 = (" + " ~ accScalar ~ "(bias[col_base + col])") if hasBias else "" %}{% set BIAS_2 = (" + " ~ accScalar ~ "(bias[col_base + col2])") if hasBias else "" %}{% set ACC_1 = "(prior.x + (scratch[src_slot][row * 8u + col] + tile_B[src_slot * 64u + row * 8u + col]))" if useBalancedAccumulation else ("tile_B[src_slot * 64u + row * 8u + col]" if reuseB else "scratch[src_slot][row * 8u + col]") %}{% set ACC_2 = "(prior.y + (scratch[src_slot][row * 8u + col2] + tile_B[src_slot * 64u + row * 8u + col2]))" if useBalancedAccumulation else ("tile_B[src_slot * 64u + row * 8u + col2]" if reuseB else "scratch[src_slot][row * 8u + col2]") %}fn storeOutput(offset: u32{% if needsColBase %}, col_base: u32{% endif %}, row: u32, col: u32, src_slot: u32, row_limit: i32{% if useBalancedAccumulation %}, prior: vec2<f32>{% endif %}) {
if(row_limit>0&&row<u32(row_limit)){let col2:u32=col+1u;
{% if (tailSafe is defined and tailSafe) or packedNTail %}if(col_base+col<OUT_F){ {{ OUT }}[offset+row*OUT_F+col]={{ OUT_SCALAR }}({{ epiOpen }}({{ ACC_1 }}{{ BIAS_1 }}){{ " + residual[offset + row * OUT_F + col]" if hasResidual | default(false) else "" }}{{ epiClose }});}if(col_base+col2<OUT_F){ {{ OUT }}[offset+row*OUT_F+col2]={{ OUT_SCALAR }}({{ epiOpen }}({{ ACC_2 }}{{ BIAS_2 }}){{ " + residual[offset + row * OUT_F + col2]" if hasResidual | default(false) else "" }}{{ epiClose }});}
{% else %}{{ OUT }}[offset+row*OUT_F+col]={{ OUT_SCALAR }}({{ epiOpen }}({{ ACC_1 }}{{ BIAS_1 }}){{ " + residual[offset + row * OUT_F + col]" if hasResidual | default(false) else "" }}{{ epiClose }});{{ OUT }}[offset+row*OUT_F+col2]={{ OUT_SCALAR }}({{ epiOpen }}({{ ACC_2 }}{{ BIAS_2 }}){{ " + residual[offset + row * OUT_F + col2]" if hasResidual | default(false) else "" }}{{ epiClose }});
{% endif %}} }
{% endif %}{% include "ops/_shared/subgroup-matrix-topology.wgsl.jinja" %}{% include "ops/_shared/subgroup-matrix-software-8x8.wgsl.jinja" %}{% macro matrixKernelBody(matrixSoftware) %}{% if runtimeRowTileBase %}let ROW_TILE_BASE:u32={{ rowTileBaseRuntimeValue }};
{% endif %}{% if SPLIT_K > 1 %}let a_global_base:u32=(ROW_TILE_BASE+workgroup_id.y)*TILE_ROWS;let split_id:u32=workgroup_id.z;
{% elif tileGroup > 1 %}let tile_linear:u32=workgroup_id.x+workgroup_id.y*{{ tileGroupColsValue }}u;let tile_group:u32=tile_linear/{{ tileGroup * tileGroupColsValue }}u;let group_first_row:u32=tile_group*{{ tileGroup }}u;let group_rows:u32=min(nwg.y-group_first_row,{{ tileGroup }}u);let in_group:u32=tile_linear-tile_group*{{ tileGroup * tileGroupColsValue }}u;let a_global_base:u32=(ROW_TILE_BASE+group_first_row+in_group%group_rows)*TILE_ROWS;
{% else %}let a_global_base:u32=(ROW_TILE_BASE+workgroup_id.y+workgroup_id.z*nwg.y)*TILE_ROWS;
{% endif %}{% if tileGroup > 1 %}let w_global_base:u32=(COLUMN_TILE_BASE+in_group/group_rows)*TILE_COLS;
{% else %}let w_global_base:u32=(COLUMN_TILE_BASE+workgroup_id.x)*TILE_COLS;
{% endif %}let subtile_id:u32=matrixRole;
{% if subgroupCols == 1 %}let subtile_idx:u32=0u;let subtile_idy:u32=subtile_id;
{% else %}let subtile_idx:u32=subtile_id/{{ subgroupRows }}u;let subtile_idy:u32=subtile_id% {{ subgroupRows }}u;
{% endif %}let base_A:u32=subtile_idy*SUB_ROWS;let base_B:u32=subtile_idx*SUB_COLS;
{% if not useChunkedAccumulation %}{{ matrixAccumulators(matrixSoftware) }}
{% endif %}{% if useDoubleBufferedB %}var b_bank:u32=0u;loadSHMB(w_global_base,0u,local_idx/{{ (tileKValue / 16)|int }}u,local_idx% {{ (tileKValue / 16)|int }}u,0u);workgroupBarrier();for(var kidx:u32=0u;kidx<{{ kLoopEndValue }};kidx+=TILE_K){let b_bank_offset=b_bank*TILE_COLS*TILE_K;for(var step:u32=0u;step<{{ "min(TILE_K, IN_F - kidx)" if packedKTail else "TILE_K" }};step+={{ 32 if useBalancedAccumulation else 8 }}u){
{% if useBalancedAccumulation %}{% for stream in range(4) %}{
{% set matrixAccumulator = ["matC", "matD", "matE", "matF"][stream] %}{% set matrixStepOffset = stream * 8 %}{% include "ops/_shared/subgroup-matrix-dynamic-step.wgsl.jinja" %}}
{% endfor %}{% else %}{% include "ops/_shared/subgroup-matrix-dynamic-step.wgsl.jinja" %}{% endif %}}let next_kidx=kidx+TILE_K;if(next_kidx<{{ kLoopEndValue }}){let next_bank=1u-b_bank;loadSHMB(w_global_base,next_kidx,local_idx/{{ (tileKValue / 16)|int }}u,local_idx% {{ (tileKValue / 16)|int }}u,next_bank*TILE_COLS*TILE_K);workgroupBarrier();b_bank=next_bank;} }
{% else %}{% if useChunkedAccumulation %}let row=sg_id/4u;let col=(sg_id%4u)*2u;let bank=subtile_id*{{ aTilesPerSubgroup * bTilesPerSubgroup }}u;var chunkSums:array<vec2<f32>,{{ aTilesPerSubgroup * bTilesPerSubgroup }}>;var chunkErrors:array<vec2<f32>,{{ aTilesPerSubgroup * bTilesPerSubgroup }}>;for(var chunk_base=0u;chunk_base<IN_F;chunk_base+=TILE_K*{{ matrixChunkTiles }}u){ {{ matrixAccumulators(matrixSoftware) }} for(var kidx=chunk_base;kidx<min(chunk_base+TILE_K*{{ matrixChunkTiles }}u,IN_F);kidx+=TILE_K){
{% elif SPLIT_K > 1 %}let k_begin:u32=split_id*K_PER_SPLIT;let k_end:u32=min(k_begin+K_PER_SPLIT,{{ kLoopEndValue }});for(var kidx:u32=k_begin;kidx<k_end;kidx+=TILE_K){
{% else %}for(var kidx:u32=0u;kidx<{{ kLoopEndValue }};kidx+=TILE_K){
{% endif %}{% if not useDirectAStorage %}loadSHMA(a_global_base,kidx,local_idx/{{ (tileKValue / 8)|int }}u,local_idx% {{ (tileKValue / 8)|int }}u);
{% endif %}loadSHMB(w_global_base,kidx,local_idx/{{ (tileKValue / 16)|int }}u,local_idx% {{ (tileKValue / 16)|int }}u,0u);workgroupBarrier();for(var step:u32=0u;step<{{ "min(TILE_K, IN_F - kidx)" if packedKTail else "TILE_K" }};step+=8u){
{% include "ops/_shared/subgroup-matrix-dynamic-step.wgsl.jinja" %}}workgroupBarrier();}
{% if useChunkedAccumulation %}{{ storeMatrixAccumulators(matrixSoftware) }} workgroupBarrier();
{% for m in range(aTilesPerSubgroup) %}{% for n in range(bTilesPerSubgroup) %}{let partial=vec2<f32>(scratch[bank+{{ m * bTilesPerSubgroup + n }}u][row*8u+col],scratch[bank+{{ m * bTilesPerSubgroup + n }}u][row*8u+col+1u]);let corrected=partial-chunkErrors[{{ m * bTilesPerSubgroup + n }}];let next=chunkSums[{{ m * bTilesPerSubgroup + n }}]+corrected;chunkErrors[{{ m * bTilesPerSubgroup + n }}]=select(vec2<f32>(0.0),(next-chunkSums[{{ m * bTilesPerSubgroup + n }}])-corrected,abs(next)<=vec2<f32>(3.4028234663852886e38));chunkSums[{{ m * bTilesPerSubgroup + n }}]=next;}
{% endfor %}{% endfor %}workgroupBarrier();}
{% endif %}{% endif %}{% if useBalancedAccumulation %}workgroupBarrier();
{% endif %}  let matrix_c_offset: u32 = {% if SPLIT_K > 1 %}split_id * params.M * OUT_F + {% endif %}(a_global_base + base_A) * OUT_F + w_global_base + base_B;
{% if useDirectStore %}{% for m in range(aTilesPerSubgroup) %}{% for n in range(bTilesPerSubgroup) %}{% if packedNTail %}if(w_global_base+base_B+{{ n * 8 }}u<OUT_F){
{% endif %}{{ matrixStore(matrixSoftware, "row_major", "&" ~ (OUT), "matrix_c_offset + " ~ (m * 8) ~ "u * OUT_F + " ~ (n * 8) ~ "u", "matC" ~ (m) ~ (n), "OUT_F") }}
{% if packedNTail %}}
{% endif %}{% endfor %}{% endfor %}{% else %}{% if useChunkedAccumulation %}{% for m in range(aTilesPerSubgroup) %}{% for n in range(bTilesPerSubgroup) %}scratch[bank+{{ m * bTilesPerSubgroup + n }}u][row*8u+col]=chunkSums[{{ m * bTilesPerSubgroup + n }}].x;scratch[bank+{{ m * bTilesPerSubgroup + n }}u][row*8u+col+1u]=chunkSums[{{ m * bTilesPerSubgroup + n }}].y;
{% endfor %}{% endfor %}workgroupBarrier();
{% else %}let row:u32=sg_id/4u;let col:u32=(sg_id%4u)*2u;let bank:u32=subtile_id*{{ aTilesPerSubgroup * bTilesPerSubgroup }}u;
{% if reuseB %}workgroupBarrier();
{% endif %}{{ storeMatrixAccumulators(matrixSoftware) }} workgroupBarrier();
{% if useBalancedAccumulation %}var pairTotals:array<vec2<f32>,{{ aTilesPerSubgroup * bTilesPerSubgroup }}>;
{% for m in range(aTilesPerSubgroup) %}{% for n in range(bTilesPerSubgroup) %}pairTotals[{{ m * bTilesPerSubgroup + n }}]=vec2<f32>(scratch[bank+{{ m * bTilesPerSubgroup + n }}u][row*8u+col]+tile_B[(bank+{{ m * bTilesPerSubgroup + n }}u)*64u+row*8u+col],scratch[bank+{{ m * bTilesPerSubgroup + n }}u][row*8u+col+1u]+tile_B[(bank+{{ m * bTilesPerSubgroup + n }}u)*64u+row*8u+col+1u]);
{% endfor %}{% endfor %}workgroupBarrier();
{% for m in range(aTilesPerSubgroup) %}{% for n in range(bTilesPerSubgroup) %}{{ matrixStore(matrixSoftware, "row_major", "&scratch[bank + " ~ (m * bTilesPerSubgroup + n) ~ "u]", "0u", "matE" ~ (m) ~ (n), "8u") }} {{ matrixStore(matrixSoftware, "row_major", "&tile_B", "(bank + " ~ (m * bTilesPerSubgroup + n) ~ "u) * 64u", "matF" ~ (m) ~ (n), "8u") }}
{% endfor %}{% endfor %}workgroupBarrier();
{% endif %}{% endif %}{% for m in range(aTilesPerSubgroup) %}let row_limit_{{ m }}:i32=i32(params.M)-i32(a_global_base+base_A+{{ m * 8 }}u);
{% for n in range(bTilesPerSubgroup) %}storeOutput(
    matrix_c_offset + {{ m * 8 }}u * OUT_F + {{ n * 8 }}u{% if needsColBase %},
w_global_base+base_B+{{ n * 8 }}u,
{% else %},
{% endif %}row,col,bank+{{ m * bTilesPerSubgroup + n }}u,row_limit_{{ m }}{{ ", pairTotals[" ~ (m * bTilesPerSubgroup + n) ~ "]" if useBalancedAccumulation else "" }});
{% endfor %}{% endfor %}{% endif %}{% endmacro %}@compute@workgroup_size({{ workgroupThreads }},1,1){{ " @subgroup_size(32)" if pinSubgroupSize32 else "" }} fn main(@builtin(workgroup_id)workgroup_id:vec3<u32>,
{% if SPLIT_K == 1 %}@builtin(num_workgroups)nwg:vec3<u32>,
{% endif %}  @builtin(local_invocation_index) local_idx: u32{% if not useDirectStore %},
@builtin(subgroup_invocation_id)sg_id:u32,
{% else %},
{% endif %}@builtin(subgroup_size)sg_size:u32){let matrixRole=claimMatrixSubgroupRole(sg_size,32u);let matrixAcceptedHere=matrixTopologyAccepted(local_idx,{{ workgroupThreads }}u/32u);matrixSoftwareLane=local_idx%32u;if(matrixAcceptedHere){ {{ matrixKernelBody(false) }} }else{let matrixRole=local_idx/32u;
{% if not useDirectStore %}let sg_id=local_idx%32u;
{% endif %}{{ matrixKernelBody(true) }} } }`],["ops/_shared/matmul-reg-microtile-fma.wgsl.jinja",`{% set regAccumulator = regAccumulator | default("acc") %}{% set regColumnStride = regColumnStride | default(0) %}{% set regUnroll = regUnroll | default(0) %}{% macro regARead(row) %}{% if regCast %}vec4<{{ regT }}>(tileA[{{ row }}][kv]){% else %}tileA[{{ row }}][kv]{% endif %}{% endmacro %}{% macro regBRead(column) %}{% if regColumnStride and (regBHalfWords | default(false)) %}vec4<{{ regT }}>(tileB[2u * kv][{{ column }}], tileB[2u * kv + 1u][{{ column }}]){% else %}{% set regBWord = ("tileB[kv][" ~ column ~ "]") if regColumnStride else ("tileB[" ~ column ~ "][kv]") %}{% if regCast %}vec4<{{ regT }}>({{ regBWord }}){% else %}{{ regBWord }}{% endif %}{% endif %}{% endmacro %}let aRow=lid.y*TM;
let bCol = lid.x{% if not regColumnStride %} * TN{% endif %};
for(var kv:u32=0u;kv<BK/4u;kv=kv+1u){
{% if regUnroll %}{% for i in range(regUnroll) %}let a{{ i }}={{ regARead("aRow + " ~ i ~ "u") }};
{% endfor %}{% for j in range(regUnroll) %}let b{{ j }}={{ regBRead("bCol + " ~ (j * regColumnStride if regColumnStride else j) ~ "u") }};
{% endfor %}{% for i in range(regUnroll) %}{% for j in range(regUnroll) %}{% set regSlot = regAccumulator ~ "[" ~ (i * regUnroll + j) ~ "u]" %}{% if regSequentialK is defined and regSequentialK %}{% for component in range(4) %}{{ regSlot }}={{ regSlot }}+a{{ i }}[{{ component }}]*b{{ j }}[{{ component }}];
{% endfor %}{% else %}{{ regSlot }}={{ regSlot }}+dot(a{{ i }},b{{ j }});
{% endif %}{% endfor %}{% endfor %}{% else %}var av:array<vec4<{{ regT }}>,TM>;var bv:array<vec4<{{ regT }}>,TN>;for(var i:u32=0u;i<TM;i=i+1u){av[i]={{ regARead("aRow + i") }};}for(var j:u32=0u;j<TN;j=j+1u){bv[j]={{ regBRead(("bCol + j * " ~ regColumnStride ~ "u") if regColumnStride else "bCol + j") }};}for(var i:u32=0u;i<TM;i=i+1u){for(var j:u32=0u;j<TN;j=j+1u){
{% if regSequentialK is defined and regSequentialK %}{% for component in range(4) %}{{ regAccumulator }}[i*TN+j]={{ regAccumulator }}[i*TN+j]+av[i][{{ component }}]*bv[j][{{ component }}];
{% endfor %}{% else %}{{ regAccumulator }}[i*TN+j]={{ regAccumulator }}[i*TN+j]+dot(av[i],bv[j]);
{% endif %}} }
{% endif %}}`],["ops/_shared/matrix-shaped-convolution-io.wgsl.jinja",`{% macro spatialSuffix(dims, start) %}{% for axis in range(start, dims | length) %}{{ dims[axis] }}u * {% endfor %}1u{% endmacro %}{% macro shapedConvA(row, k, transposed=matrixConvTranspose | default(false), groups=matrixGroups | default(none), inputChannels=matrixInputChannelsPerGroup | default(none), kernelSpatial=matrixKernelSpatial | default(none)) %}{% if transposed %}a[((batch% {{ groups }}u)*{{ inputChannels }}u+{{ k }}/({{ spatialSuffix(kernelSpatial, 0) }}))*M*({{ spatialSuffix(kernelSpatial, 0) }})+{{ row }}*({{ spatialSuffix(kernelSpatial, 0) }})+{{ k }} %({{ spatialSuffix(kernelSpatial, 0) }})]
{% else %}a[((batch% {{ groups }}u)*M+{{ row }})*K+{{ k }}]
{% endif %}{% endmacro %}{% macro shapedConvY(row, col, groups=matrixGroups | default(none)) %}((batch / {{ groups }}u) * {{ groups }}u * M + (batch % {{ groups }}u) * M + {{ row }}) * N + {{ col }}{% endmacro %}{% macro shapedConvReaders(transposed, groups, inputChannels, kernelSpatial, inputSpatial, outputSpatial, pads, dilations, strides) %}fn shapedConvInputIndex(k: u32, col: u32, batch: u32) -> u32 {
let inputChannel=k/({{ spatialSuffix(kernelSpatial, 0) }});var index=(batch/{{ groups }}u)*{{ groups * inputChannels }}u+(batch% {{ groups }}u)*{{ inputChannels }}u+inputChannel;
{% for axis in range(inputSpatial | length) %}let output{{ axis }}=(col/({{ spatialSuffix(outputSpatial, axis + 1) }}))% {{ outputSpatial[axis] }}u;let kernel{{ axis }}=(k/({{ spatialSuffix(kernelSpatial, axis + 1) }}))% {{ kernelSpatial[axis] }}u;
{% if transposed %}let coordinate{{ axis }}=i32(output{{ axis }})+{{ pads[axis] }}i-i32(kernel{{ axis }}*{{ dilations[axis] }}u);if(coordinate{{ axis }}<0i||coordinate{{ axis }} % {{ strides[axis] }}i!=0i){return 0xffffffffu;}let input{{ axis }}=u32(coordinate{{ axis }})/{{ strides[axis] }}u;if(input{{ axis }}>={{ inputSpatial[axis] }}u){return 0xffffffffu;}
{% else %}let coordinate{{ axis }}=i32(output{{ axis }}*{{ strides[axis] }}u+kernel{{ axis }}*{{ dilations[axis] }}u)-{{ pads[axis] }}i;if(coordinate{{ axis }}<0i||coordinate{{ axis }}>={{ inputSpatial[axis] }}i){return 0xffffffffu;}let input{{ axis }}=u32(coordinate{{ axis }});
{% endif %}index=index*{{ inputSpatial[axis] }}u+input{{ axis }};
{% endfor %}return index;}fn shapedConvPairValid(k:u32,col:u32,batch:u32)->bool{return shapedConvInputIndex(k,col,batch)!=0xffffffffu;}fn readShapedConvB(k:u32,col:u32,batch:u32)->{{ operandScalar }} {let index=shapedConvInputIndex(k,col,batch);if(index==0xffffffffu){return {{ operandScalar }}(0);}return b[index];
}{% endmacro %}{{ shapedConvReaders(matrixConvTranspose | default(false), matrixGroups | default(none), matrixInputChannelsPerGroup | default(none), matrixKernelSpatial | default(none), matrixInputSpatial | default(none), matrixOutputSpatial | default(none), matrixPads | default(none), matrixDilations | default(none), matrixStrides | default(none)) }}`],["ops/_shared/matrix-shaped-projection-gemm.wgsl.jinja",`{% set pinSubgroupSize32 = false %}{% set CONVOLUTION = matrixConvolution | default(false) %}{% set WIDTH = matrixSubgroupWidth | default(32) %}{% set GROUPS = (128 / WIDTH) | int %}{% set OWNED = (WIDTH / 32) | int %}{% set STAGE_K = matrixStageK | default(matrixK) %}{% set CELLS_PER_THREAD = ((matrixM * 2 * matrixN * 2 + 127) / 128) | int %}{% include "ops/_shared/subgroup-matrix-enable.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const M:u32={{ matrixRows if matrixRows is defined else M }}u;const N:u32={{ N }}u;const K:u32={{ K }}u;const TILE_M:u32={{ matrixM * 2 }}u;const TILE_N:u32={{ matrixN * 2 }}u;const TILE_K:u32={{ STAGE_K }}u;var<workgroup>tileA:array<{{ operandScalar }},{{ matrixM * 2 * STAGE_K }}>;var<workgroup>tileB:array<{{ operandScalar }},{{ STAGE_K * matrixN * 2 }}>;var<workgroup>tileC:array<{{ accumulatorScalar }},{{ matrixM * 2 * matrixN * 2 }}>;
{% if CONVOLUTION %}var<workgroup>matrixNonFinite:atomic<u32>;var<workgroup>matrixOperandsFinite:bool;
{% endif %}{% include "ops/_shared/matrix-shaped-projection-io.wgsl.jinja" %}fn loadTiles(rowBase: u32, colBase: u32, kBase: u32, tid: u32{% if matrixBatched %}, batch: u32{% endif %}) {
for(var flat=tid;flat<TILE_M*TILE_K;flat+=128u){let row=rowBase+flat/TILE_K;let k=kBase+flat%TILE_K;var value={{ operandScalar }}(0);if(row<M&&k<K){value={{ operandScalar }}({{ shapedLoadA("row", "k") }});}tileA[flat]=value;}for(var flat=tid;flat<TILE_K*TILE_N;flat+=128u){let k=kBase+flat/TILE_N;let col=colBase+flat%TILE_N;var value={{ operandScalar }}(0);if(k<K&&col<N){value={{ operandScalar }}({{ shapedLoadB("k", "col") }});}tileB[flat]=value;} }
{% include "ops/_shared/subgroup-matrix-topology.wgsl.jinja" %}@compute@workgroup_size(128,1,1)fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_index)tid:u32,@builtin(subgroup_size)width:u32){
{% if CONVOLUTION %}let role=claimMatrixSubgroupRole(width,{{ WIDTH }}u);let provenTopology=matrixTopologyAccepted(tid,{{ GROUPS }}u);
{% endif %}{% if matrixBatched %}  let batch = wid.z;
{% endif %}  let rowBase = wid.y * TILE_M;
  let colBase = wid.x * TILE_N;{% if CONVOLUTION %}if(tid==0u){atomicStore(&matrixNonFinite,0u);}workgroupBarrier();for(var offset=0u;offset<TILE_M;offset+=1u){let row=rowBase+offset;if(row<M){for(var k=tid;k<K;k+=128u){let bits=bitcast<u32>(f32({{ shapedLoadA("row", "k") }}));if((bits&0x7f800000u)==0x7f800000u){atomicOr(&matrixNonFinite,1u);} } } }workgroupBarrier();if(tid==0u){matrixOperandsFinite=atomicLoad(&matrixNonFinite)==0u;}
  let operandsFinite = workgroupUniformLoad(&matrixOperandsFinite);{% endif %}{% if CONVOLUTION %}let nativeTopology=provenTopology&&operandsFinite;
{% else %}let role=claimMatrixSubgroupRole(width,{{ WIDTH }}u);let nativeTopology=matrixTopologyAccepted(tid,{{ GROUPS }}u);
{% endif %}if(nativeTopology){
{% for slot in range(OWNED) %}let tile{{ slot }}=role+{{ slot * GROUPS }}u;let row{{ slot }}=(tile{{ slot }}/2u)*{{ matrixM }}u;let col{{ slot }}=(tile{{ slot }} %2u)*{{ matrixN }}u;var acc{{ slot }}:subgroup_matrix_result<{{ accumulatorScalar }},{{ matrixN }},{{ matrixM }}>;
{% endfor %}for(var kk=0u;kk<K;kk+=TILE_K){
      loadTiles(rowBase, colBase, kk, tid{% if matrixBatched %}, batch{% endif %});
workgroupBarrier();for(var step=0u;step<TILE_K;step+={{ matrixK }}u){
{% for slot in range(OWNED) %}let left{{ slot }}=subgroupMatrixLoad<subgroup_matrix_left<{{ operandScalar }},{{ matrixK }},{{ matrixM }}>,row_major>(&tileA,row{{ slot }}*TILE_K+step,TILE_K);let right{{ slot }}=subgroupMatrixLoad<subgroup_matrix_right<{{ operandScalar }},{{ matrixN }},{{ matrixK }}>,row_major>(&tileB,step*TILE_N+col{{ slot }},TILE_N);acc{{ slot }}=subgroupMatrixMultiplyAccumulate(left{{ slot }},right{{ slot }},acc{{ slot }});
{% endfor %}}workgroupBarrier();}
{% for slot in range(OWNED) %}subgroupMatrixStore<row_major>(&tileC,row{{ slot }}*TILE_N+col{{ slot }},acc{{ slot }},TILE_N);
{% endfor %}workgroupBarrier();for(var flat=tid;flat<TILE_M*TILE_N;flat+=128u){let r=rowBase+flat/TILE_N;let c=colBase+flat%TILE_N;if(r<M&&c<N){ {{ shapedWrite("tileC[flat]", "r", "c") }} } } }else{var acc:array<{{ accumulatorScalar }},{{ CELLS_PER_THREAD }}>;for(var kk=0u;kk<K;kk+=TILE_K){
      loadTiles(rowBase, colBase, kk, tid{% if matrixBatched %}, batch{% endif %});
workgroupBarrier();for(var cell=0u;cell<{{ CELLS_PER_THREAD }}u;cell+=1u){let flat=tid+cell*128u;let row=flat/TILE_N;let col=flat%TILE_N;if(flat<TILE_M*TILE_N){for(var k=0u;k<TILE_K;k+=1u){
{% if CONVOLUTION %}            if (kk + k < K && colBase + col < N && shapedConvPairValid(kk + k, colBase + col, batch)) {
{% endif %}            acc[cell] = fma({{ accumulatorScalar }}(tileA[row * TILE_K + k]),
                            {{ accumulatorScalar }}(tileB[k * TILE_N + col]), acc[cell]);{% if CONVOLUTION %}            }{% endif %}} } }workgroupBarrier();}for(var cell=0u;cell<{{ CELLS_PER_THREAD }}u;cell+=1u){let flat=tid+cell*128u;let row=rowBase+flat/TILE_N;let col=colBase+flat%TILE_N;if(flat<TILE_M*TILE_N&&row<M&&col<N){ {{ shapedWrite("acc[cell]", "row", "col") }} } } } }`],["ops/_shared/matrix-shaped-projection-io.wgsl.jinja",`{% if matrixSupertonic | default(false) %}{% include "ops/_shared/matrix-shaped-supertonic-io.wgsl.jinja" %}{% endif %}{% if matrixQuantCode | default(false) %}{% include "ops/_shared/matrix-shaped-qat-io.wgsl.jinja" %}{% endif %}{% if (matrixDenseSrq | default(false)) or (matrixQuantCode | default(false)) %}{% include "ops/_shared/srq.wgsl.jinja" %}{% if matrixDensePle | default(false) %}{% include "ops/_shared/gelu-tanh.wgsl.jinja" %}{% include "ops/_shared/gemma4-gelu-grid.wgsl.jinja" %}{% endif %}{% endif %}{% if matrixEpilogue == "fastgelu" %}{% include "ops/_shared/gelu-tanh.wgsl.jinja" %}{% elif matrixEpilogue == "activation" and (hasActivation | default(false)) %}{% include "ops/_shared/gemm-activation.wgsl.jinja" %}{% elif matrixEpilogue == "conv_activation" %}{% include "ops/_shared/conv-activation.wgsl.jinja" %}{% endif %}{% if matrixConvolution | default(false) %}{% include "ops/_shared/matrix-shaped-convolution-io.wgsl.jinja" %}{% endif %}{% macro shapedLoadA(row, k, supertonicPartial=matrixSupertonicPartial | default(false), qatGrouped=matrixQatGrouped | default(false), aOffset=matrixAOffset | default(none), aRowStride=matrixARowStride | default(none), aKStride=matrixAKStride | default(none), batch1=matrixBatch1 | default(none), aBatch0Stride=matrixABatch0Stride | default(none), aBatch1Stride=matrixABatch1Stride | default(none)) %}{% if matrixSupertonic | default(false) %}readShapedSupertonicA({{ row }}, {{ k }}{% if supertonicPartial %}, batch{% endif %}){% elif matrixConvolution | default(false) %}{{ shapedConvA(row, k) }}{% else %}{% if matrixQuantCode | default(false) and not qatGrouped %}clamp(round(f32({% elif matrixDenseSrq | default(false) %}srq(f32({% endif %}a[{{ aOffset }}u + {{ row }} * {{ aRowStride }}u + {{ k }} * {{ aKStride }}u{% if matrixBatched %} + (batch / {{ batch1 }}u) * {{ aBatch0Stride }}u + (batch % {{ batch1 }}u) * {{ aBatch1Stride }}u{% endif %}]{% if matrixQuantCode | default(false) and not qatGrouped %}) * (1.0 / params.gridScale)), -128.0, 127.0){% elif matrixDenseSrq | default(false) %}), params.inScale){% endif %}{% endif %}{% endmacro %}{% macro shapedLoadB(k, col, bOffset=matrixBOffset | default(none), repeatB=matrixRepeatB | default(none), bKStride=matrixBKStride | default(none), bColStride=matrixBColStride | default(none), batch1=matrixBatch1 | default(none), bBatch0Stride=matrixBBatch0Stride | default(none), bBatch1Stride=matrixBBatch1Stride | default(none)) %}{% if matrixConvolution | default(false) %}readShapedConvB({{ k }}, {{ col }}, batch){% elif matrixQuantCode | default(false) %}readShapedQatB({{ k }}, {{ col }}){% else %}b[{{ bOffset }}u + {% if matrixEinsumRepeatB | default(false) %}({{ k }} / {{ repeatB }}u){% else %}{{ k }}{% endif %} * {{ bKStride }}u + {{ col }} * {{ bColStride }}u{% if matrixBatched %} + (batch / {{ batch1 }}u) * {{ bBatch0Stride }}u + (batch % {{ batch1 }}u) * {{ bBatch1Stride }}u{% endif %}]{% endif %}{% endmacro %}{% macro shapedWrite(value, row, col, beta=matrixBeta | default(none), groups=matrixGroups | default(none), qatGrouped=matrixQatGrouped | default(false), yStride=matrixYStride | default(none), yColumnOffset=matrixYColumnOffset | default(none), batch1=matrixBatch1 | default(none), yBatch1Stride=matrixYBatch1Stride | default(none), yBatchStride=matrixYBatchStride | default(none)) %}{
{% if matrixSupertonic | default(false) %}{{ shapedSupertonicWrite(value, row, col) }}
{% else %}var result=f32({{ matrixAlpha }})*f32({{ value }});
{% if matrixBiasMode != "none" %}  result += f32({{ beta }}) * f32(bias[{% if matrixBiasMode == "row" %}{{ col }}{% elif matrixBiasMode == "column" %}{% if matrixConvolution | default(false) %}(batch % {{ groups }}u) * M + {% endif %}{{ row }}{% elif matrixBiasMode == "matrix" %}{{ row }} * N + {{ col }}{% else %}0u{% endif %}]);
{% endif %}{% if matrixResidual | default(false) %}result+=f32(residual[{{ shapedConvY(row, col) }}]);
{% endif %}{% if matrixEpilogue == "fastgelu" %}result=gelu_tanh(result);
{% elif (matrixEpilogue == "activation" and (hasActivation | default(false))) or (matrixEpilogue == "conv_activation" and (hasActivation | default(false))) %}result=fused_act(result);
{% endif %}{% if matrixQuantCode | default(false) %}{% if not qatGrouped %}result=scale[{{ col }}]*(params.gridScale*result);
{% endif %}result=srq(result,params.outScale);
{% endif %}{% if matrixDenseSrq | default(false) %}result=srq(result,params.outScale);
{% if matrixDensePle | default(false) %}let projectedStored={{ outputScalar }}(result);result=gelu_grid(f32(projectedStored),params.outScale)*ple[params.pleOffset+{{ row }}*params.pleStride+{{ col }}];
{% endif %}{% endif %}  y[{% if matrixConvolution | default(false) %}{{ shapedConvY(row, col) }}{% else %}{{ row }} * {{ yStride }}u + {{ col }} + {{ yColumnOffset }}u{% if matrixBatched %} + {% if matrixYBatch0Stride is defined %}(batch / {{ batch1 }}u) * {{ matrixYBatch0Stride }}u + (batch % {{ batch1 }}u) * {{ yBatch1Stride }}u{% else %}batch * {{ yBatchStride }}u{% endif %}{% endif %}{% endif %}] = {{ outputScalar }}(result);
{% endif %}}
{% endmacro %}`],["ops/_shared/matrix-shaped-qat-io.wgsl.jinja",`{% macro shapedQatReader(bits, zero, grouped, groups) %}fn readShapedQatB(k: u32, col: u32) -> f32 {
let packed=b[col*{{ ((K * bits) / 32) | int }}u+k/{{ (32 / bits) | int }}u];let code=f32((packed>>((k% {{ (32 / bits) | int }}u)*{{ bits }}u))&{{ (2 ** bits) - 1 }}u);let centered=code-f32({{ zero }});
{% if grouped %}return centered*scale[col*{{ groups }}u+k/32u];
{% else %}return centered;
{% endif %}}{% endmacro %}{{ shapedQatReader(matrixQatBits | default(none), matrixQatZero | default(none), matrixQatGrouped | default(false), matrixQatGroups | default(none)) }}`],["ops/_shared/matrix-shaped-supertonic-io.wgsl.jinja",`{% if (matrixSupertonicFromPartials | default(false)) or (not (matrixSupertonicPartial | default(false)) and (geluEpilogue | default(false))) %}{% include "ops/_shared/gelu-erf.wgsl.jinja" %}{% endif %}{% macro shapedSupertonicReader(partial, logicalRows, fromPartials, splitCount, inputFeatures, inputGroups, inputWidth) %}fn readShapedSupertonicA(row: u32, k: u32{% if partial %}, batch: u32{% endif %}) -> f32 {
{% if partial %}if(row>={{ logicalRows }}u){return 0.0;}let inputK=k+batch*K;
{% else %}let inputK=k;
{% endif %}{% if fromPartials %}var value=xb[inputK];for(var slice=0u;slice<{{ splitCount }}u;slice+=1u){value+=xpart[(slice*M+row)*{{ inputFeatures }}u+inputK];}return 0.5*value*(1.0+erf_approx(value*0.7071067811865476));
{% elif inputGroups > 1 %}return a[((inputK/{{ inputWidth }}u)*M+row)*{{ inputWidth }}u+inputK% {{ inputWidth }}u];
{% else %}return a[row*{{ inputFeatures }}u+inputK];
{% endif %}}
{% endmacro %}{{ shapedSupertonicReader(matrixSupertonicPartial | default(false), matrixLogicalM | default(none), matrixSupertonicFromPartials | default(false), splitKIn | default(none), inFeatures | default(none), matrixSupertonicInputGroups | default(none), matrixSupertonicInputWidth | default(none)) }}{% macro shapedSupertonicWrite(value, row, col, partial=matrixSupertonicPartial | default(false), logicalRows=matrixLogicalM | default(none), gelu=geluEpilogue | default(false), residual=hasResidual | default(false), rowVector=hasRowVec | default(false), mask=hasMask | default(false), outputGroups=matrixSupertonicOutputGroups | default(none), outputWidth=matrixSupertonicOutputWidth | default(none)) %}{% if partial %}var result=f32({{ value }});if({{ row }}>={{ logicalRows }}u){result=0.0;}y[(batch*M+{{ row }})*N+{{ col }}]=result;
{% else %}var result=f32({{ value }})+bias[{{ col }}];
{% if gelu %}result=0.5*result*(1.0+erf_approx(result*0.7071067811865476));
{% endif %}{% if residual %}result+=residual[{{ row }}*N+{{ col }}];
{% endif %}{% if rowVector %}result+=rowVector[{{ col }}];
{% endif %}{% if mask %}result*=mask[{{ row }}];
{% endif %}  y[{% if outputGroups > 1 %}(({{ col }} / {{ outputWidth }}u) * M + {{ row }}) * {{ outputWidth }}u + {{ col }} % {{ outputWidth }}u{% else %}{{ row }} * N + {{ col }}{% endif %}] = result;
{% endif %}{% endmacro %}`],["ops/_shared/parallel-row-dot-storage.wgsl.jinja",`{% set entrySubgroupIds = (useSubgroups or sgRuntime) and (subgroupIdBuiltins | default(false)) %}{% set rowDotEntryBuiltins = ", @builtin(subgroup_id) sg_id: u32, @builtin(num_subgroups) sg_count: u32" if entrySubgroupIds else (", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "") %}{% if entrySubgroupIds %}var<workgroup>partials:array<f32,{{ (lanes / 4) | int }}>;
{% elif useSubgroups or sgRuntime %}{% set sgSafeStorageSize = lanes %}{% include "ops/_shared/subgroup-runtime-proof.wgsl.jinja" %}{% else %}var<workgroup>scratch:array<f32,{{ lanes }}>;
{% endif %}`],["ops/_shared/parallel-row-dot.wgsl.jinja",`{% if (useSubgroups or sgRuntime) and not entrySubgroupIds %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}let wb={{ rowDotBase }};var acc=0.0;for(var v=lid;v<{{ rowDotLen4 }}u;v+={{ lanes }}u){acc+={{ rowDotTerm }};}
{% if entrySubgroupIds %}{% include "ops/_shared/workgroup-reduce-subgroup-arm.wgsl.jinja" %}if(lid==0u){ {{ row_dot_store("reduce_total") }} }
{% elif useSubgroups or sgRuntime %}let total=sg_safe_sum(acc,lid,SG_SAFE_WG,sgNative);if(lid==0u){ {{ row_dot_store("total") }} }
{% else %}scratch[lid]=acc;workgroupBarrier();for(var stride={{ (lanes / 2) | int }}u;stride>0u;stride>>=1u){if(lid<stride){scratch[lid]+=scratch[lid+stride];}workgroupBarrier();}if(lid==0u){ {{ row_dot_store("scratch[0]") }} }
{% endif %}`],["ops/_shared/quant-q1-scale.wgsl.jinja",`fn q1Scale(index:u32)->f32{
{% if scaleScalar == "f32" %}return q1_scales[index];
{% else %}return f32(q1_scales[index]);
{% endif %}}`],["ops/_shared/srq.wgsl.jinja","fn srq(x:f32,s:f32)->f32{if(s==0.0){return x;}return clamp(round(x/s),-128.0,127.0)*s;}"],["ops/_shared/subgroup-matrix-32x64-staged-kloop.wgsl.jinja",`for(var kidx:u32=0u;kidx<IN_F;kidx+=TILE_K){loadSHMA(a_global_base,kidx,local_idx/4u,local_idx%4u);loadSHMB(w_global_base,kidx,local_idx/2u,local_idx%2u);workgroupBarrier();for(var step:u32=0u;step<TILE_K;step+=8u){
{% include "ops/_shared/subgroup-matrix-8mma-step.wgsl.jinja" %}}workgroupBarrier();}`],["ops/_shared/subgroup-matrix-8mma-step.wgsl.jinja",`{% set directInputs = directMatrixInputs is defined and directMatrixInputs %}      let matrix_a_offset = {% if directInputs %}(a_global_base + subtile_idy * SUB_ROWS) * K + kidx + step{% else %}subtile_idy * SUB_ROWS * TILE_K + step{% endif %};
{% for r in range(2) %}var matA{{ r }}:{{ matrixType(matrixSoftware, "left", operandScalar) }}={{ matrixLoad(matrixSoftware, "left", operandScalar, "row_major", "&" ~ ("w" if directInputs else "tile_A"), ("matrix_a_offset" ~ (" + 8u * " ~ ("K" if directInputs else "TILE_K") if r > 0 else "")), "K" if directInputs else "TILE_K") }};
{% endfor %}      let matrix_b_offset = {% if directInputs %}b_base + (kidx + step) * N + b_global_base + subtile_idx * SUB_COLS{% else %}subtile_idx * SUB_COLS * TILE_K + step{% endif %};
{% for c in range(4) %}      var matB{{ c }}: {{ matrixType(matrixSoftware, "right", operandScalar) }} = {% set stepBPointer %}{% if useBufferView is defined and useBufferView and not directInputs %}tile_B{% else %}&{{ "xm" if directInputs else "tile_B" }}{% endif %}{% endset %}{{ matrixLoad(matrixSoftware, "right", operandScalar, ("row_major" if directInputs else "col_major"), stepBPointer, ("matrix_b_offset" ~ (" + " ~ (c * 8) ~ "u" ~ (" * TILE_K" if not directInputs else "") if c > 0 else "")), "N" if directInputs else "TILE_K") }};
{% endfor %}matC00={{ matrixMma(matrixSoftware, accScalar, "matA0", "matB0", "matC00") }};matC01={{ matrixMma(matrixSoftware, accScalar, "matA0", "matB1", "matC01") }};matC02={{ matrixMma(matrixSoftware, accScalar, "matA0", "matB2", "matC02") }};matC03={{ matrixMma(matrixSoftware, accScalar, "matA0", "matB3", "matC03") }};matC10={{ matrixMma(matrixSoftware, accScalar, "matA1", "matB0", "matC10") }};matC11={{ matrixMma(matrixSoftware, accScalar, "matA1", "matB1", "matC11") }};matC12={{ matrixMma(matrixSoftware, accScalar, "matA1", "matB2", "matC12") }};matC13={{ matrixMma(matrixSoftware, accScalar, "matA1", "matB3", "matC13") }};`],["ops/_shared/subgroup-matrix-8x8-accumulators.wgsl.jinja",`  var matC00: {% set matrixTypeScalar447 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar447) }};
  var matC01: {% set matrixTypeScalar448 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar448) }};
  var matC02: {% set matrixTypeScalar449 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar449) }};
  var matC03: {% set matrixTypeScalar450 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar450) }};
  var matC10: {% set matrixTypeScalar451 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar451) }};
  var matC11: {% set matrixTypeScalar452 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar452) }};
  var matC12: {% set matrixTypeScalar453 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar453) }};
  var matC13: {% set matrixTypeScalar454 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar454) }};`],["ops/_shared/subgroup-matrix-dynamic-step.wgsl.jinja",`{% set dynamicATiles = aTilesPerSubgroup if aTilesPerSubgroup is defined else 2 %}{% set matrixAccumulator = matrixAccumulator | default("matC") %}{% set matrixStepOffset = matrixStepOffset | default(0) %}{% set matrixStep = "step" if matrixStepOffset == 0 else "(step + " ~ matrixStepOffset ~ "u)" %}{% set aTransposed = aTransposedStorage if aTransposedStorage is defined else (matrixInputsTransposed is defined and matrixInputsTransposed) %}{% set bTransposed = bTransposedStorage if bTransposedStorage is defined else (matrixInputsTransposed is defined and matrixInputsTransposed) %}{% set directA = directAStorage is defined and directAStorage %}{% set B_BANK = "b_bank_offset + " if doubleBufferedB is defined and doubleBufferedB else "" %}{% if directA %}{% set matrixAStorage = matrixAStorage | default("x") %}{% set matrixAStride = matrixAStride | default("IN_F") %}let matrix_a_offset=(a_global_base+base_A)*{{ matrixAStride }}+kidx+{{ matrixStep }};
{% elif aTransposed %}let matrix_a_offset={{ matrixStep }}*TILE_ROWS+subtile_idy*SUB_ROWS;
{% else %}let matrix_a_offset=subtile_idy*SUB_ROWS*TILE_K+{{ matrixStep }};
{% endif %}{% for m in range(dynamicATiles) %}{% if directA %}var matA{{ m }}:{{ matrixType(matrixSoftware, "left", operandScalar) }}={{ matrixLoad(matrixSoftware, "left", operandScalar, "row_major", "&" ~ (matrixAStorage), ("matrix_a_offset" ~ (" + " ~ (m * 8) ~ "u * " ~ matrixAStride if m > 0 else "")), matrixAStride) }};
{% elif aTransposed %}var matA{{ m }}:{{ matrixType(matrixSoftware, "left", operandScalar) }}={{ matrixLoad(matrixSoftware, "left", operandScalar, "col_major", "&tile_A", "matrix_a_offset + " ~ (m * 8) ~ "u", "TILE_ROWS") }};
{% else %}var matA{{ m }}:{{ matrixType(matrixSoftware, "left", operandScalar) }}={{ matrixLoad(matrixSoftware, "left", operandScalar, "row_major", "&tile_A", ("matrix_a_offset" ~ (" + " ~ (m * 8) ~ "u * TILE_K" if m > 0 else "")), "TILE_K") }};
{% endif %}{% endfor %}{% if bTransposed %}let matrix_b_offset={{ B_BANK }}{{ matrixStep }}*TILE_COLS+subtile_idx*SUB_COLS;
{% elif fragmentMajorBStaging | default(false) %}let matrix_b_offset={{ B_BANK }}({{ matrixStep }}/8u)*TILE_COLS*8u+subtile_idx*SUB_COLS*8u;
{% else %}let matrix_b_offset={{ B_BANK }}subtile_idx*SUB_COLS*TILE_K+{{ matrixStep }};
{% endif %}{% for n in range(bTilesPerSubgroup) %}{% if bTransposed %}var matB{{ n }}:{{ matrixType(matrixSoftware, "right", operandScalar) }}={{ matrixLoad(matrixSoftware, "right", operandScalar, "row_major", "&tile_B", "matrix_b_offset + " ~ (n * 8) ~ "u", "TILE_COLS") }};
{% elif fragmentMajorBStaging | default(false) %}var matB{{ n }}:{{ matrixType(matrixSoftware, "right", operandScalar) }}={{ matrixLoad(matrixSoftware, "right", operandScalar, "col_major", "&tile_B", "matrix_b_offset + " ~ (n * 64) ~ "u", "8u") }};
{% else %}var matB{{ n }}:{{ matrixType(matrixSoftware, "right", operandScalar) }}={{ matrixLoad(matrixSoftware, "right", operandScalar, "col_major", "&tile_B", "matrix_b_offset + " ~ (n * 8) ~ "u * TILE_K", "TILE_K") }};
{% endif %}{% endfor %}{% for m in range(dynamicATiles) %}{% for n in range(bTilesPerSubgroup) %}{{ matrixAccumulator }}{{ m }}{{ n }}={{ matrixMma(matrixSoftware, accScalar, "matA" ~ (m), "matB" ~ (n), (matrixAccumulator) ~ (m) ~ (n)) }};
{% endfor %}{% endfor %}`],["ops/_shared/subgroup-matrix-topology-entry.wgsl.jinja",`{% macro matrix_topology_entry(workgroupSize, bankCount, workgroupName, laneName) %}@compute@workgroup_size({{ workgroupSize }},1,1){{ " @subgroup_size(32)" if pinSubgroupSize32 else "" }} fn main(@builtin(workgroup_id){{ workgroupName }}:vec3<u32>,@builtin(local_invocation_index)local_idx:u32,@builtin(subgroup_size)matrixPhysicalWidth:u32,@builtin(subgroup_invocation_id)matrixPhysicalLane:u32){let matrixRole=claimMatrixSubgroupRole(matrixPhysicalWidth,32u);let matrixAcceptedHere=matrixTopologyAccepted(local_idx,{{ bankCount }});matrixSoftwareLane=local_idx%32u;if(matrixAcceptedHere){let {{ laneName }}=matrixPhysicalLane;{{ matrixKernelBody(false) }} }else{let matrixRole=local_idx/32u;let {{ laneName }}=local_idx%32u;{{ matrixKernelBody(true) }} } }
{% endmacro %}`]]),tf=new Map([["co.huggingface.GatedMlpActivation",{manifest:{domain:"co.huggingface",name:"GatedMlpActivation",sinceVersion:1,inputs:{xT:{onnx:"X",dtype:"X",rank:2,optional:!0},gateT:{onnx:"Gate",dtype:"T",rank:2,optional:!0},upT:{onnx:"Up",dtype:"T",rank:2,optional:!0},geluLutT:{onnx:"GeluLut",dtype:"L",rank:1,optional:!0}},outputs:{yT:{onnx:"Y",dtype:"Y",rank:2,shape:["args.rows","args.dstStride if args.dstStride is defined else args.mlpInner"]}},args:{rows:{kind:"u32"},mlpInner:{kind:"u32"},srcStride:{kind:"u32",required:!1},srcStart:{kind:"u32",required:!1},dstStride:{kind:"u32",required:!1},dstStart:{kind:"u32",required:!1},activation:{kind:"string",required:!1,oneOf:["silu","fast_silu","gelu_tanh"]},split:{kind:"string",required:!1,oneOf:["chunk","interleaved"]},clampLimit:{kind:"f32",required:!1},alpha:{kind:"f32",required:!1},upBias:{kind:"f32",required:!1},gateScale:{kind:"f32",onnx:"input_scale",required:!1}},typeConstraints:{X:["float32","float16"],Y:["float32","float16"],T:["float32"],L:["float32"]},derive:{packedSource:'present.xT and not present.gateT and not present.upT and not present.geluLutT and args.srcStride is not defined and args.srcStart is not defined and args.dstStride is not defined and args.dstStart is not defined and args.gateScale is not defined and (args.activation is not defined or args.activation == "silu" or args.activation == "fast_silu")',stridedDestinationOk:"(args.dstStride is not defined and args.dstStart is not defined) or (args.dstStride is defined and args.dstStart is defined and args.dstStart + args.mlpInner <= args.dstStride)",stridedSource:'present.xT and not present.gateT and not present.upT and not present.geluLutT and args.srcStride is defined and args.srcStart is defined and args.gateScale is not defined and (args.activation is not defined or args.activation == "silu") and args.split is not defined and args.clampLimit is not defined and args.alpha is not defined and args.upBias is not defined',separateSource:'not present.xT and present.gateT and present.upT and present.geluLutT and args.srcStride is not defined and args.srcStart is not defined and args.dstStride is not defined and args.dstStart is not defined and args.activation == "gelu_tanh" and args.split is not defined and args.clampLimit is not defined and args.alpha is not defined and args.upBias is not defined',interleavedSplit:'args.split == "interleaved"',gluAlpha:"args.alpha if args.alpha is defined else 1",gluUpBias:"args.upBias if args.upBias is defined else 0",gluClamp:"args.clampLimit if args.clampLimit is defined else 0",hasClampArg:"args.clampLimit is defined",generalizedGlu:"interleavedSplit or hasClampArg or (gluAlpha != 1) or (gluUpBias != 0)"},when:["ranks.yT == 2","dim(shapes.yT, 0) == args.rows","dim(shapes.yT, 1) == (args.dstStride if args.dstStride is defined else args.mlpInner)"],bindings:{x:{arg:"xT",elementType:"$xElement"},y:{arg:"yT",elementType:"$yElement"},params:{struct:[{name:"rows",type:"u32"}]}},variants:[{id:"scalar",priority:0,when:["packedSource","ranks.xT == 2","args.rows == dim(shapes.xT, 0)","dim(shapes.xT, 1) == args.mlpInner * 2","f16Ok(dtypes.X) and f16Ok(dtypes.Y)"],tunables:{WG:{default:256}},derive:{vectorized:!1,fastSilu:'args.activation == "fast_silu"',interleaved:"interleavedSplit",generalized:"generalizedGlu",hasClamp:"hasClampArg",clampLimit:"gluClamp",alpha:"gluAlpha",upBias:"gluUpBias",xElement:"dtypes.X",yElement:"dtypes.Y",yScalar:"dtypes.Y",mlpInner:"args.mlpInner",wg:"tunables.WG"},passes:[{id:"main",name:"GatedMlpActivation",shader:"gated-mlp-activation.wgsl.jinja",bindings:["x","y","params"],dispatch:{x:"min(args.rows, DISPATCH_FOLD_WIDTH)",y:"ceil(args.mlpInner / tunables.WG)",z:"ceilDiv(args.rows, DISPATCH_FOLD_WIDTH)"}}]},{id:"vec4",priority:10,when:["packedSource","dtypes.X == dtypes.Y","f16Ok(dtypes.X)","args.mlpInner % 4 == 0","ranks.xT == 2","args.rows == dim(shapes.xT, 0)","dim(shapes.xT, 1) == args.mlpInner * 2"],tunables:{WG:{default:256}},derive:{vectorized:!0,fastSilu:'args.activation == "fast_silu"',generalized:"generalizedGlu",hasClamp:"hasClampArg",clampLimit:"gluClamp",alpha:"gluAlpha",upBias:"gluUpBias",xElement:'"vec4<" ~ dtypes.X ~ ">"',yElement:'"vec4<" ~ dtypes.Y ~ ">"',yScalar:"dtypes.Y",mlpInnerV4:"args.mlpInner / 4",wg:"tunables.WG",interleaved:"interleavedSplit"},passes:[{id:"main",name:"GatedMlpActivation",shader:"gated-mlp-activation.wgsl.jinja",bindings:["x","y","params"],dispatch:{x:"min(args.rows, DISPATCH_FOLD_WIDTH)",y:"ceil((args.mlpInner / 4) / tunables.WG)",z:"ceilDiv(args.rows, DISPATCH_FOLD_WIDTH)"}}]},{cases:[{source:"strided"}],variants:[{id:"{source}_vec4_f16",priority:10,when:["stridedSource","args.rows > 0","args.mlpInner > 0","ranks.xT == 2","dim(shapes.xT, 0) >= args.rows","dim(shapes.xT, 1) == args.srcStride","args.srcStart + 2 * args.mlpInner <= args.srcStride","stridedDestinationOk","(args.dstStride is not defined or (args.dstStride % 4 == 0 and args.dstStart % 4 == 0))",'dtypes.X == "f16"','dtypes.Y == "f16"','device.features.has("shader-f16")',"args.srcStride % 4 == 0","(args.srcStart % 4 == 0 or (args.srcStart % 4 == 2 and numel(shapes.xT) >= (args.rows - 1) * args.srcStride + args.srcStart + 2 * args.mlpInner + 2))","args.mlpInner % 4 == 0"],derive:{vectorized:!0,misalignedStart2:"args.srcStart % 4 == 2",workgroupSize:64},passes:[{id:"main",name:"SwiGluColumns",shader:"swiglu-columns.wgsl.jinja",bindings:[{arg:"xT",name:"x",elementType:"vec4<f16>"},{arg:"yT",name:"y",elementType:"vec4<f16>"},{name:"p",struct:[{name:"rows",type:"u32"},{name:"srcStride",type:"u32",value:"args.srcStride / 4"},{name:"srcStart",type:"u32",value:"args.srcStart / 4 if args.srcStart % 4 == 0 else (args.srcStart - 2) / 4"},{name:"mlpInner",type:"u32",value:"args.mlpInner / 4"},{name:"dstStride",type:"u32",value:"(args.dstStride if args.dstStride is defined else args.mlpInner) / 4"},{name:"dstStart",type:"u32",value:"(args.dstStart if args.dstStart is defined else 0) / 4"}]}],dispatch:{x:"min(args.rows, DISPATCH_FOLD_WIDTH)",y:"ceil((args.mlpInner / 4) / workgroupSize)",z:"ceilDiv(args.rows, DISPATCH_FOLD_WIDTH)"}}]},{id:"{source}_scalar",priority:0,when:["stridedSource","args.rows > 0","args.mlpInner > 0","ranks.xT == 2","dim(shapes.xT, 0) >= args.rows","dim(shapes.xT, 1) == args.srcStride","args.srcStart + 2 * args.mlpInner <= args.srcStride","stridedDestinationOk","f16Ok(dtypes.X)","f16Ok(dtypes.Y)"],derive:{vectorized:!1,workgroupSize:64},passes:[{id:"main",name:"SwiGluColumns",shader:"swiglu-columns.wgsl.jinja",bindings:[{arg:"xT",name:"x"},{arg:"yT",name:"y"},{name:"p",struct:[{name:"rows",type:"u32"},{name:"srcStride",type:"u32"},{name:"srcStart",type:"u32"},{name:"mlpInner",type:"u32"},{name:"dstStride",type:"u32",value:"args.dstStride if args.dstStride is defined else args.mlpInner"},{name:"dstStart",type:"u32",value:"args.dstStart if args.dstStart is defined else 0"}]}],dispatch:{x:"min(args.rows, DISPATCH_FOLD_WIDTH)",y:"ceil(args.mlpInner / workgroupSize)",z:"ceilDiv(args.rows, DISPATCH_FOLD_WIDTH)"}}]}]},{cases:[{source:"separate"}],variants:[{id:"{source}_gelu_lut",when:["separateSource","args.rows > 0","args.mlpInner > 0","ranks.gateT == 2","ranks.upT == 2","ranks.geluLutT == 1","dim(shapes.gateT, 0) == args.rows","dim(shapes.gateT, 1) == args.mlpInner","dim(shapes.upT, 0) == args.rows","dim(shapes.upT, 1) == args.mlpInner","dim(shapes.geluLutT, 0) >= 256",'dtypes.T == "f32"','dtypes.L == "f32"','dtypes.Y == "f32"'],derive:{workgroupSize:256},passes:[{id:"main",name:"Gemma4GeluMul",shader:"gemma4-gelu-mul.wgsl.jinja",bindings:[{arg:"gateT",name:"gate",elementType:"f32"},{arg:"upT",name:"up",elementType:"f32"},{arg:"geluLutT",name:"gelu_lut",elementType:"f32",length:256},{arg:"yT",name:"out",elementType:"f32"},{name:"params",struct:[{name:"count",type:"u32",value:"args.rows * args.mlpInner"},{name:"gateScale",type:"f32",value:"args.gateScale if args.gateScale else 0.0"}]}],dispatch:{x:"min(ceilDiv(args.rows * args.mlpInner, workgroupSize), device.limits.maxComputeWorkgroupsPerDimension)",y:1,z:1}}]}]}]},assets:[["gated-mlp-activation.wgsl.jinja",`{% include "ops/_shared/swiglu-value.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% if vectorized %}const MLP_V4:u32={{ mlpInnerV4 }}u;
{% else %}const MLP:u32={{ mlpInner }}u;
{% endif %}const WG:u32={{ wg }}u;@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let r=wg.x+wg.z*{{ DISPATCH_FOLD_WIDTH }}u;if(r>=params.rows){return;}let i0=wg.y*WG+lid.x;
{% if vectorized %}if(i0>=MLP_V4){return;}let row_base=r*2u*MLP_V4;
{% if interleaved %}let pair_a=vec4<f32>(x[row_base+2u*i0]);let pair_b=vec4<f32>(x[row_base+2u*i0+1u]);let x1=vec4<f32>(pair_a.x,pair_a.z,pair_b.x,pair_b.z);let x2=vec4<f32>(pair_a.y,pair_a.w,pair_b.y,pair_b.w);
{% else %}let x1=vec4<f32>(x[row_base+i0]);let x2=vec4<f32>(x[row_base+MLP_V4+i0]);
{% endif %}{% if generalized %}{% if hasClamp %}let g=min(x1,vec4<f32>({{ clampLimit }}));let u=clamp(x2,vec4<f32>(-{{ clampLimit }}),vec4<f32>({{ clampLimit }}));
{% else %}let g=x1;let u=x2;
{% endif %}let glu={{ swiglu_gate("g", "vec4<f32>(" ~ alpha ~ ")", "vec4<f32>(1.0)") }};y[r*MLP_V4+i0]={{ yElement }}((u+vec4<f32>({{ upBias }}))*glu);
{% elif fastSilu %}let sigmoid=vec4<f32>(0.5)+vec4<f32>(0.5)*x1/(vec4<f32>(1.0)+abs(x1));y[r*MLP_V4+i0]={{ yElement }}((x1*sigmoid)*x2);
{% else %}y[r*MLP_V4+i0]={{ yElement }}(({{ swiglu_gate("x1", "vec4<f32>(1.0)", "vec4<f32>(1.0)") }})*x2);
{% endif %}{% else %}if(i0>=MLP){return;}
{% if interleaved %}let x1=f32(x[r*2u*MLP+2u*i0]);let x2=f32(x[r*2u*MLP+2u*i0+1u]);
{% else %}let x1=f32(x[r*2u*MLP+i0]);let x2=f32(x[r*2u*MLP+MLP+i0]);
{% endif %}{% if generalized %}{% if hasClamp %}let g=min(x1,f32({{ clampLimit }}));let u=clamp(x2,f32(-{{ clampLimit }}),f32({{ clampLimit }}));
{% else %}let g=x1;let u=x2;
{% endif %}let glu={{ swiglu_gate("g", "f32(" ~ alpha ~ ")", "1.0") }};y[r*MLP+i0]={{ yScalar }}((u+f32({{ upBias }}))*glu);
{% elif fastSilu %}let sigmoid=0.5+0.5*x1/(1.0+abs(x1));y[r*MLP+i0]={{ yScalar }}((x1*sigmoid)*x2);
{% else %}y[r*MLP+i0]={{ yScalar }}(({{ swiglu_gate("x1", "1.0", "1.0") }})*x2);
{% endif %}{% endif %}}`],["gemma4-gelu-mul.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/gelu-tanh.wgsl.jinja" %}{% include "ops/_shared/gemma4-gelu-grid.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }},1,1)fn main(@builtin(global_invocation_id)gid:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>){let stride=nwg.x*{{ workgroupSize }}u;var i=gid.x;loop{if(i>=params.count){break;}out[i]=gelu_grid(gate[i],params.gateScale)*up[i];i=i+stride;} }`],["swiglu-columns.wgsl.jinja",`{% include "ops/_shared/swiglu-value.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}@compute@workgroup_size({{ workgroupSize }},1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let r=wg.x+wg.z*{{ DISPATCH_FOLD_WIDTH }}u;if(r>=p.rows){return;}let i=wg.y*{{ workgroupSize }}u+lid.x;if(i>=p.mlpInner){return;}let base=r*p.srcStride+p.srcStart+i;
{% if vectorized %}{% if misalignedStart2 %}let x1Lo=vec4<f32>(x[base]);let x1Hi=vec4<f32>(x[base+1u]);let x2Lo=vec4<f32>(x[base+p.mlpInner]);let x2Hi=vec4<f32>(x[base+p.mlpInner+1u]);let x1=vec4<f32>(x1Lo.z,x1Lo.w,x1Hi.x,x1Hi.y);let x2=vec4<f32>(x2Lo.z,x2Lo.w,x2Hi.x,x2Hi.y);
{% else %}let x1=vec4<f32>(x[base]);let x2=vec4<f32>(x[base+p.mlpInner]);
{% endif %}y[r*p.dstStride+p.dstStart+i]=vec4<f16>(({{ swiglu_gate("x1", "vec4<f32>(1.0)", "vec4<f32>(1.0)") }})*x2);
{% else %}let x1=f32(x[base]);let x2=f32(x[base+p.mlpInner]);let s=({{ swiglu_gate("x1", "1.0", "1.0") }})*x2;y[r*p.dstStride+p.dstStart+i]={{ "f16(s)" if Y == "f16" else "s" }};
{% endif %}}`]]}],["co.huggingface.RMSNorm",{manifest:{domain:"co.huggingface",name:"RMSNorm",sinceVersion:1,inputs:{xT:{onnx:"X",dtype:"X"},wT:{onnx:"W",dtype:"W",rank:1,optional:!0}},outputs:{yT:{onnx:"Y",dtype:"Y",rank:"ranks.xT",shape:"shapes.xT"}},args:{rows:{kind:"u32"},dim:{kind:"u32"},eps:{kind:"f32",required:!1},weightBaseOffset:{kind:"u32",onnx:"weight_base_offset",required:!1},weightOffset:{kind:"u32",onnx:"weight_offset",required:!1},castBeforeScale:{kind:"bool",onnx:"cast_before_scale",required:!1},exact:{kind:"u32",onnx:"exact_reference_order",required:!1}},typeConstraints:{X:["float32","float16"],W:["float32","float16"],Y:["float32","float16"]},tunables:{WG_CAP:{default:1024},ROW_WG:{default:64}},derive:{subgroupsWide32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and device.adapterInfo.subgroupMinSize >= 32',normWgCap:"tunables.WG_CAP if subgroupsWide32 else min(128, tunables.WG_CAP)",normWg:"min(normWgCap, max(32, pow2ceil(args.dim / 8)))",hasSubgroupId:'device.wgslLanguageFeatures.has("subgroup_id")',weightPlusOne:"args.weightOffset == 1",applyCastBeforeScale:'(args.castBeforeScale == true) and present.wT and (dtypes.Y != "f32")',arenaMode:"args.weightBaseOffset is defined and (not args.exact) and (not args.weightOffset) and (not args.castBeforeScale) and present.wT and args.dim > 0",arenaDecodeContract:'arenaMode and ranks.xT == 1 and ranks.wT == 1 and ranks.yT == 1 and tensorDtypes.xT == "float32" and (tensorDtypes.wT == "float32" or tensorDtypes.wT == "float16") and f16Ok(dtypes.W) and f16Ok(dtypes.Y) and args.rows == 1 and args.dim % 4 == 0 and args.weightBaseOffset % 4 == 0 and dim(shapes.xT, 0) == args.dim and dim(shapes.yT, 0) == args.dim and dim(shapes.wT, 0) >= args.weightBaseOffset + args.dim',arenaRowsContract:'arenaMode and ranks.xT == 2 and ranks.wT == 1 and ranks.yT == 2 and (tensorDtypes.xT == "float32" or tensorDtypes.xT == "float16") and (tensorDtypes.wT == "float32" or tensorDtypes.wT == "float16") and f16Ok(dtypes.X) and f16Ok(dtypes.W) and f16Ok(dtypes.Y) and args.rows > 0 and numel(shapes.xT) >= args.rows * args.dim and numel(shapes.wT) >= args.weightBaseOffset + args.dim and numel(shapes.yT) >= args.rows * args.dim'},bindings:{x:{arg:"xT",elementType:"$xScalar"},y:{arg:"yT",elementType:"$yScalar"},x_x_t:{arg:"xT",name:"x",elementType:"vec4<f32>"},w:{arg:"wT",elementType:"vec4<f32>",length:"$W_VEC4_LEN"},y_y_t:{arg:"yT",name:"y",elementType:"vec4<f32>"},params_rows:{name:"params",struct:[{name:"rows",type:"u32"}]},params_main:{name:"params",struct:[{name:"rows",type:"u32"},{name:"rowStride",type:"u32",value:"min(args.rows, device.limits.maxComputeWorkgroupsPerDimension)"}]},w_w_t:{arg:"wT",name:"w",elementType:"$wScalar",length:"$W_SCALAR_LEN"}},variants:[{id:"arena_decode",priority:40,when:["arenaDecodeContract"],derive:{weightVec4:'"vec4<f16>" if tensorDtypes.wT == "float16" else "vec4<f32>"',normedVec4:'"vec4<f16>" if tensorDtypes.yT == "float16" else "vec4<f32>"',hiddenSize:"args.dim",hiddenSizeVec4:"args.dim / 4",rmsEps:"args.eps if args.eps is defined else 0.000001",normOffsetVec4:"args.weightBaseOffset / 4",HIDDEN_VEC4_LEN:"args.dim / 4",ARENA_W_VEC4_LEN:"(args.weightBaseOffset + args.dim) / 4",workgroupSize:256},passes:[{id:"main",name:"LlamaDecodeRmsNorm",shader:"rms-norm-arena-decode.wgsl.jinja",bindings:[{arg:"xT",name:"hidden",elementType:"vec4<f32>",length:"$HIDDEN_VEC4_LEN"},{arg:"wT",name:"weights",elementType:"$weightVec4",length:"$ARENA_W_VEC4_LEN"},{arg:"yT",name:"normed",elementType:"$normedVec4",length:"$HIDDEN_VEC4_LEN"}],dispatch:{x:1}}]},{id:"arena_rows",priority:40,when:["arenaRowsContract"],derive:{xScalar:"dtypes.X",wScalar:"dtypes.W",yScalar:"dtypes.Y",dimSize:"args.dim",workgroupSize:256},passes:[{id:"main",name:"LlamaPrefillRmsNorm",shader:"rms-norm-arena-rows.wgsl.jinja",bindings:["x",{arg:"wT",name:"weights",elementType:"$wScalar"},"y",{name:"params",struct:[{name:"norm_offset",type:"u32",value:"args.weightBaseOffset"},{name:"rows",type:"u32"},{name:"eps",type:"f32",value:"args.eps if args.eps is defined else 0.000001"}]}],dispatch:{workgroups:"args.rows"}}]},{id:"weighted_vec4",priority:14,when:["args.weightBaseOffset is undefined","not args.exact","present.wT","ranks.wT == 1","dim(shapes.wT, 0) == args.dim","args.rows >= 0","args.dim > 0","args.dim % 4 == 0","numel(shapes.xT) >= args.rows * args.dim","numel(shapes.yT) >= args.rows * args.dim","f16Ok(dtypes.X) and f16Ok(dtypes.W) and f16Ok(dtypes.Y)"],derive:{xVec4:'"vec4<f16>" if dtypes.X == "f16" else "vec4<f32>"',wVec4:'"vec4<f16>" if dtypes.W == "f16" else "vec4<f32>"',yVec4:'"vec4<f16>" if dtypes.Y == "f16" else "vec4<f32>"',SCALE_VEC4_LEN:"args.dim / 4"},passes:[{id:"main",name:"RMSNorm",shader:"ops/_shared/norm-row-stats.wgsl.jinja",derive:{modeSpec:'"rms"',vec4:!0,scalar:"dtypes.Y",hidden:"args.dim",hiddenVec:"args.dim / 4",wg:"normWg",epsilon:"args.eps if args.eps is defined else 0.000001",vecType:'"vec4<" ~ dtypes.Y ~ ">"',combineSubgroups:"subgroupsWide32 and hasSubgroupId",rmsWeightOffset:"weightPlusOne",rmsScaleAfterCast:"applyCastBeforeScale"},bindings:[{arg:"xT",name:"x",elementType:"$xVec4"},{arg:"wT",name:"scale",elementType:"$wVec4",length:"$SCALE_VEC4_LEN"},{arg:"yT",name:"y",elementType:"$yVec4"},{name:"params",struct:[{name:"rows",type:"u32"},{name:"rowStride",type:"u32",value:"max(1, min(args.rows, device.limits.maxComputeWorkgroupsPerDimension))"}]}],dispatch:{workgroups:"args.rows"},subgroupCollectivesWidth:"portable"}]},{cases:[{id:"exact_weighted",priority:30,contract:['args.weightBaseOffset is undefined and args.exact and present.wT and ranks.wT == 1 and dim(shapes.wT, 0) == args.dim and tensorDtypes.wT == "float32" and args.rows >= 0 and args.dim > 0 and args.dim % 4 == 0 and args.dim < 4194304 and numel(shapes.xT) >= args.rows * args.dim and numel(shapes.yT) >= args.rows * args.dim and tensorDtypes.xT == "float32" and tensorDtypes.yT == "float32"'],passName:"RMSNormExactW",bindings:["x_x_t","w","y_y_t","params_rows"]},{id:"exact_unweighted",priority:20,contract:['args.weightBaseOffset is undefined and args.exact and (not present.wT) and args.rows >= 0 and args.dim > 0 and args.dim % 4 == 0 and args.dim < 4194304 and numel(shapes.xT) >= args.rows * args.dim and numel(shapes.yT) >= args.rows * args.dim and tensorDtypes.xT == "float32" and tensorDtypes.yT == "float32"'],passName:"RMSNormExact",bindings:["x_x_t","y_y_t","params_rows"]}],variants:[{id:"{id}",priority:"{priority}",when:"{contract}",derive:{hasWeight:"present.wT",dimSize:"args.dim",W_VEC4_LEN:"args.dim / 4",INV_DIM:"1.0 / args.dim",eps:"args.eps if args.eps is defined else 0.000001"},passes:[{id:"main",name:"{passName}",shader:"rms-norm-exact.wgsl.jinja",derive:{workgroupSize:64},bindings:"{bindings}",dispatch:{threads:"args.rows",workgroupSize:"workgroupSize"}}]}]},{cases:[{id:"unweighted",priority:0,contract:["args.weightBaseOffset is undefined and (not args.exact) and (not present.wT and args.rows >= 0 and args.dim > 0 and numel(shapes.xT) >= args.rows * args.dim and numel(shapes.yT) >= args.rows * args.dim and f16Ok(dtypes.X) and f16Ok(dtypes.Y))"],bindings:["x","y","params_main"],derive:{hasWeight:!1,weightPlusOne:!1,castBeforeScale:!1,xScalar:"dtypes.X",yScalar:"dtypes.Y",dimSize:"args.dim",eps:"args.eps if args.eps is defined else 0.000001",rowWg:"tunables.ROW_WG"}},{id:"weighted",priority:10,contract:["args.weightBaseOffset is undefined and (not args.exact) and (present.wT and ranks.wT == 1 and args.rows >= 0 and args.dim > 0 and numel(shapes.xT) >= args.rows * args.dim and dim(shapes.wT, 0) == args.dim and numel(shapes.yT) >= args.rows * args.dim and f16Ok(dtypes.X) and f16Ok(dtypes.W) and f16Ok(dtypes.Y))"],bindings:["x","w_w_t","y","params_main"],derive:{hasWeight:!0,castBeforeScale:"applyCastBeforeScale",xScalar:"dtypes.X",wScalar:"dtypes.W",yScalar:"dtypes.Y",dimSize:"args.dim",W_SCALAR_LEN:"args.dim",eps:"args.eps if args.eps is defined else 0.000001",rowWg:"tunables.ROW_WG"}}],variants:[{id:"{id}",priority:"{priority}",when:"{contract}",derive:"{derive}",passes:[{id:"main",name:"RMSNorm",shader:"rms-norm.wgsl.jinja",bindings:"{bindings}",dispatch:{workgroups:"args.rows"}}]}]}]},assets:[["rms-norm-arena-decode.wgsl.jinja",'{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const RMS_EPS:f32={{ rmsEps }};const W_NORM_VEC4:u32={{ normOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>partials:array<f32,{{ workgroupSize }}>;fn sum4(value:vec4<f32>)->f32{return value.x+value.y+value.z+value.w;}fn load_weight(value:{{ weightVec4 }})->vec4<f32>{return vec4<f32>(value);}fn store_normed(value:vec4<f32>)->{{ normedVec4 }} {return {{ normedVec4 }}(value);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(local_invocation_id)lid3:vec3<u32>){let lid=lid3.x;var sum=0.0;for(var dim=lid;dim<HIDDEN_SIZE_VEC4;dim=dim+WORKGROUP_SIZE){let value=hidden[dim];sum=sum+sum4(value*value);}partials[lid]=sum;workgroupBarrier();{{ wgsl_tree_fold(["partials"]) }} let variance=partials[0];let scale=inverseSqrt(variance/f32(HIDDEN_SIZE)+RMS_EPS);for(var dim=lid;dim<HIDDEN_SIZE_VEC4;dim=dim+WORKGROUP_SIZE){normed[dim]=store_normed(hidden[dim]*vec4<f32>(scale)*load_weight(weights[W_NORM_VEC4+dim]));} }'],["rms-norm-arena-rows.wgsl.jinja",'{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const DIM:u32={{ dimSize }}u;const WG:u32={{ workgroupSize }}u;var<workgroup>partials:array<f32,{{ workgroupSize }}>;@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let row=wid.x+wid.y*{{ DISPATCH_FOLD_WIDTH }}u;if(row>=params.rows){return;}let lid=lid3.x;let base=row*DIM;var sum=0.0;for(var d=lid;d<DIM;d=d+WG){let v=f32(x[base+d]);sum=sum+v*v;}partials[lid]=sum;workgroupBarrier();{{ wgsl_tree_fold(["partials"], wg="WG") }} let scale=inverseSqrt(partials[0]/f32(DIM)+params.eps);for(var d=lid;d<DIM;d=d+WG){y[base+d]={{ yScalar }}(f32(x[base+d])*scale*f32(weights[params.norm_offset+d]));} }'],["rms-norm-exact.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const DIM:u32={{ dimSize }}u;const DIM_F:f32={{ dimSize }}.0;const INV_DIM:f32={{ INV_DIM }};const EPS:f32={{ eps }};const VEC_SIZE:u32=DIM/4u;const SIZE_ILP:u32=VEC_SIZE/4u;const WG:u32={{ workgroupSize }}u;
{% include "ops/_shared/div-exact.wgsl.jinja" %}{% include "ops/_shared/float-reciprocal-exact.wgsl.jinja" %}fn sqrt_exact(d:f32)->f32{var y=inverseSqrt(d);var x=d*y;var w=0.5*y;y=fma(-x,w,0.5);x=fma(x,y,x);w=fma(w,y,w);y=fma(-x,w,1.5);w=w+w;w=w*y;x=w*d;y=fma(w,d,-x);var z=fma(-w,x,1.0);z=fma(-w,y,z);w=0.5*x;w=fma(w,z,y);w=w+x;return w;}
{% macro accumulate_vectors() -%}let v0=vec4<f32>(x[vb]);
{% for stream in range(1, 4) -%}let v{{ stream }}=vec4<f32>(x[vb+{{ stream }}u]);
{% endfor -%}{% for stream in range(4) -%}a0_{{ stream }}=fma(fma(v{{ stream }},v{{ stream }},vec4<f32>(0.0)),vec4<f32>(1.0),a0_{{ stream }});
{% endfor -%}{% endmacro %}{% macro promote_streams(level) -%}{% for stream in range(4) -%}a{{ level + 1 }}_{{ stream }}=fma(a{{ level }}_{{ stream }},vec4<f32>(1.0),a{{ level + 1 }}_{{ stream }});
{% endfor -%}{% for stream in range(4) -%}a{{ level }}_{{ stream }}=vec4<f32>(0.0);
{% endfor -%}{% endmacro %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let wgFlat=wg.x+wg.y*{{ DISPATCH_FOLD_WIDTH }}u;let row=wgFlat*WG+lid.x;if(row>=params.rows){return;}let base=row*VEC_SIZE;
{% for level in range(4) %}{% for stream in range(4) %}  var a{{ level }}_{{ stream }} = vec4<f32>(0.0);
{% endfor %}{% endfor %}var i:u32=0u;loop{if(i+16u>SIZE_ILP){break;}for(var b:u32=0u;b<16u;b=b+1u){let vb=base+(i+b)*4u;{{ accumulate_vectors() | indent(6, true) }} }i=i+16u;{{ promote_streams(0) | indent(4, true) }} if((i&0xF0u)==0u){ {{ promote_streams(1) | indent(6, true) }} if((i&0xF00u)==0u){ {{ promote_streams(2) | indent(8, true) }} } } }loop{if(i>=SIZE_ILP){break;}let vb=base+i*4u;{{ accumulate_vectors() | indent(4, true) }} i=i+1u;}
{% for level in range(1, 4) %}{% for stream in range(4) %}  a0_{{ stream }} = fma(a{{ level }}_{{ stream }}, vec4<f32>(1.0), a0_{{ stream }});
{% endfor %}{% endfor %}var vi:u32=SIZE_ILP*4u;loop{if(vi>=VEC_SIZE){break;}let v=vec4<f32>(x[base+vi]);a0_0=fma(fma(v,v,vec4<f32>(0.0)),vec4<f32>(1.0),a0_0);vi=vi+1u;}
{% for stream in range(1, 4) %}  a0_0 = fma(a0_{{ stream }}, vec4<f32>(1.0), a0_0);
{% endfor %}var s:f32=a0_0.x;
{% for lane in ["y", "z", "w"] %}  s = fma(a0_0.{{ lane }}, 1.0, s);
{% endfor %}let ms=div_exact(s,DIM_F,INV_DIM)+EPS;let rinv=recip_exact(sqrt_exact(ms));for(var v:u32=0u;v<VEC_SIZE;v=v+1u){let xv=vec4<f32>(x[base+v]);let n=xv*vec4<f32>(rinv);
{% if hasWeight %}{% if weightPlusOne %}let wv=vec4<f32>(1.0)+vec4<f32>(w[v]);
{% else %}let wv=vec4<f32>(w[v]);
{% endif %}y[base+v]=n*wv;
{% else %}y[base+v]=n;
{% endif %}} }`],["rms-norm.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const DIM:u32={{ dimSize }}u;const EPS:f32={{ eps }};const WG:u32={{ rowWg }}u;var<workgroup>partial:array<f32,WG>;
{% include "ops/_shared/reduce-sum.wgsl.jinja" %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let rowStride=select(params.rowStride,params.rows,params.rowStride==0u);let row=wg.x+wg.y*rowStride;if(row>=params.rows){return;}let tid=lid.x;let base=row*DIM;var acc:f32=0.0;var i:u32=tid;loop{if(i>=DIM){break;}let v=f32(x[base+i]);acc=acc+v*v;i=i+WG;}let scale=inverseSqrt(reduce_sum(acc,tid)/f32(DIM)+EPS);var j:u32=tid;loop{if(j>=DIM){break;}let xv=f32(x[base+j]);
{% if hasWeight %}{% if weightPlusOne %}let wv=1.0+f32(w[j]);
{% else %}let wv=f32(w[j]);
{% endif %}{% if castBeforeScale %}y[base+j]={{ yScalar }}(xv*scale)*{{ yScalar }}(wv);
{% else %}y[base+j]={{ yScalar }}(xv*scale*wv);
{% endif %}{% else %}y[base+j]={{ yScalar }}(xv*scale);
{% endif %}j=j+WG;} }`]]}],["com.xenova.Axpy",{manifest:{domain:"com.xenova",name:"Axpy",sinceVersion:1,inputs:{xT:{onnx:"X",dtype:"X"},yT:{onnx:"Y",dtype:"Y"},gateT:{onnx:"gate",dtype:"G",optional:!0}},outputs:{xT:{onnx:"X",dtype:"X",rank:"ranks.xT",shape:"shapes.xT"}},args:{count:{kind:"u32",onnx:"kernel.count"},alpha:{kind:"f32"},rowLength:{kind:"u32",required:!1},gateLayout:{kind:"string",required:!1,oneOf:["row","column"]},gateOffset:{kind:"u32",required:!1}},attributes:{alpha:{default:1}},typeConstraints:{X:["float32","float16"],Y:["float32","float16"],G:["float32","float16"]},tunables:{WORKGROUP_SIZE:{default:64}},derive:{gated:"present.gateT",columnGate:'gated and args.gateLayout == "column"',rowLength:"args.rowLength if args.rowLength is defined else 0",gateOffset:"args.gateOffset if args.gateOffset is defined else 0",sizeOk:"numel(shapes.xT) >= args.count and numel(shapes.yT) >= args.count",dtypeOk:"f16Ok(dtypes.X) and f16Ok(dtypes.Y)",gateContract:"gated and f16Ok(dtypes.G) and rowLength > 0"},when:["args.count >= 0","sizeOk","dtypeOk"],bindings:{x:{arg:"xT",elementType:"$X"},y:{arg:"yT",elementType:"$Y"}},variants:[{id:"gated_scalar",when:["gateContract","not columnGate","gateOffset == 0","args.count % rowLength == 0 and numel(shapes.gateT) >= args.count / rowLength"],derive:{hasGate:!0},passes:[{id:"main",name:"gated_scalar",shader:"axpy.wgsl.jinja",bindings:["x","y",{arg:"gateT",name:"g"},{name:"params",struct:[{name:"count",type:"u32"},{name:"alpha",type:"f32"},{name:"wgY",type:"u32",value:"min(ceil(args.count / tunables.WORKGROUP_SIZE), device.limits.maxComputeWorkgroupsPerDimension)"},{name:"rowLength",type:"u32",value:"rowLength"}]}],dispatch:{threads:"args.count",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"gated_column",when:["gateContract","columnGate","numel(shapes.gateT) >= gateOffset + rowLength"],derive:{hasGate:!0,vectorized:'dtypes.X == "f16" and dtypes.Y == "f16" and dtypes.G == "f32" and device.features.has("shader-f16") and args.count % 4 == 0 and rowLength % 4 == 0 and gateOffset % 4 == 0',width:"4 if vectorized else 1",xElement:'"vec4<f16>" if vectorized else dtypes.X',yElement:'"vec4<f16>" if vectorized else dtypes.Y',gateElement:'"vec4<f32>" if vectorized else dtypes.G',GATE_LEN:"(gateOffset + rowLength) / width"},passes:[{id:"main",name:"gated_column",shader:"axpy.wgsl.jinja",bindings:[{arg:"xT",name:"x",elementType:"$xElement"},{arg:"yT",name:"y",elementType:"$yElement"},{arg:"gateT",name:"g",elementType:"$gateElement",length:"$GATE_LEN"},{name:"params",struct:[{name:"count",type:"u32",value:"args.count / width"},{name:"alpha",type:"f32"},{name:"wgY",type:"u32",value:"min(ceil((args.count / width) / tunables.WORKGROUP_SIZE), device.limits.maxComputeWorkgroupsPerDimension)"},{name:"rowLength",type:"u32",value:"rowLength / width"},{name:"gateOffset",type:"u32",value:"gateOffset / width"}]}],dispatch:{threads:"args.count / width",workgroupSize:"tunables.WORKGROUP_SIZE"}}]},{id:"scalar",when:["not gated"],derive:{hasGate:!1},passes:[{id:"main",name:"scalar",shader:"axpy.wgsl.jinja",bindings:["x","y",{name:"params",struct:[{name:"count",type:"u32"},{name:"alpha",type:"f32"},{name:"wgY",type:"u32",value:"min(ceil(args.count / tunables.WORKGROUP_SIZE), device.limits.maxComputeWorkgroupsPerDimension)"}]}],dispatch:{threads:"args.count",workgroupSize:"tunables.WORKGROUP_SIZE"}}]}]},assets:[["axpy.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const WG:u32={{ tunables.WORKGROUP_SIZE }}u;@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let wg_idx=wg.x+wg.y*params.wgY;let i=wg_idx*WG+lid.x;if(i>=params.count){return;}
{% if columnGate %}{% elif hasGate %}{% endif %}{% if columnGate and vectorized %}let gv=g[params.gateOffset+i%params.rowLength];x[i]=vec4<f16>(vec4<f32>(x[i])+params.alpha*gv*vec4<f32>(y[i]));
{% elif columnGate or hasGate %}let gv=f32(g[{{ "params.gateOffset + i % params.rowLength" if columnGate else "i / params.rowLength" }}]);x[i]={{ "f16(f32(x[i]) + params.alpha * gv * f32(y[i]))" if dtypes.X == "f16" else "f32(x[i]) + params.alpha * gv * f32(y[i])" }};
{% else %}x[i]={{ "f16(f32(x[i]) + params.alpha * f32(y[i]))" if dtypes.X == "f16" else "f32(x[i]) + params.alpha * f32(y[i])" }};
{% endif %}}`]]}],["com.xenova.CachedDecodeAttention",{manifest:{domain:"com.xenova",name:"CachedDecodeAttention",sinceVersion:1,inputs:{qT:{onnx:"Q",dtype:"Q",rank:1,optional:!0},qkvT:{onnx:"Qkv",dtype:"Q",rank:2,optional:!0},gateT:{onnx:"Gate",dtype:"O",optional:!0},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1}},outputs:{attnT:{onnx:"Attn",dtype:"O",rank:"2 if present.qkvT else 1",shape:"[args.seqLen, args.numHeads * args.headDim] if present.qkvT else [args.numHeads * args.headDim]"},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1,optional:!0,shape:"shapes.cacheKeysT"},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1,optional:!0,shape:"shapes.cacheValuesT"}},args:{layer:{kind:"u32"},cacheLen:{kind:"u32",onnx:"cache_len"},pastLen:{kind:"u32",onnx:"past_len",required:!1},seqLen:{kind:"u32",onnx:"seq_len",required:!1},numHeads:{kind:"u32",onnx:"num_heads"},numKvHeads:{kind:"u32",onnx:"num_kv_heads"},headDim:{kind:"u32",onnx:"head_dim"},scale:{kind:"f32"},workgroupSize:{kind:"u32",onnx:"workgroup_size",required:!1},tileSize:{kind:"u32",onnx:"tile_size",required:!1},numKvChunks:{kind:"u32",onnx:"num_kv_chunks",required:!1},minKeysPerChunk:{kind:"u32",onnx:"min_keys_per_chunk",required:!1},qOffset:{kind:"u32",onnx:"q_offset",required:!1},kOffset:{kind:"u32",onnx:"k_offset",required:!1},vOffset:{kind:"u32",onnx:"v_offset",required:!1}},typeConstraints:{Q:["float32","float16"],C:["float32","float16"],O:["float32","float16"]},tunables:{DECODE_SCORE_TILE:{default:1024},WORKGROUP_SIZE:{default:128},MAX_SCORE_TILE:{default:4096}},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWide32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and device.adapterInfo.subgroupMinSize >= 32',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32Effective:"wave32Adapter or pinSubgroupSize32",subgroupMatrixTopologyBytes:"3 * 16",cacheWriteArgsPresent:"args.qOffset is defined and args.kOffset is defined and args.vOffset is defined",cacheWriteArgsAbsent:"args.qOffset is undefined and args.kOffset is undefined and args.vOffset is undefined",splitArgsPresent:"args.workgroupSize is defined and args.tileSize is defined and args.numKvChunks is defined and args.minKeysPerChunk is defined",splitArgsAbsent:"args.tileSize is undefined and args.numKvChunks is undefined and args.minKeysPerChunk is undefined",splitPartialOutBytes:"args.numHeads * args.numKvChunks * args.headDim * 4 if splitArgsPresent else 0",splitPartialMetaBytes:"args.numHeads * args.numKvChunks * 2 * 4 if splitArgsPresent else 0",splitScratchFits:"splitPartialOutBytes <= device.limits.maxStorageBufferBindingSize and splitPartialOutBytes <= device.limits.maxBufferSize and splitPartialMetaBytes <= device.limits.maxStorageBufferBindingSize and splitPartialMetaBytes <= device.limits.maxBufferSize",decodeMode:"present.qT and not present.qkvT and args.seqLen is undefined",prefillMode:"present.qkvT and not present.qT and args.seqLen is defined and cacheWriteArgsAbsent and splitArgsAbsent",coreContract:'decodeMode and cacheWriteArgsAbsent and ranks.qT == 1 and (ranks.cacheKeysT == 1 or ranks.cacheKeysT == 5) and (ranks.cacheValuesT == 1 or ranks.cacheValuesT == 5) and ranks.attnT == 1 and (tensorDtypes.qT == "float32" or tensorDtypes.qT == "float16") and tensorDtypes.cacheKeysT == tensorDtypes.cacheValuesT and (tensorDtypes.cacheKeysT == "float32" or tensorDtypes.cacheKeysT == "float16") and f16Ok(dtypes.Q) and f16Ok(dtypes.C) and f16Ok(dtypes.O) and args.cacheLen > 0 and args.numHeads > 0 and args.numKvHeads > 0 and args.numHeads % args.numKvHeads == 0 and args.headDim > 0 and numel(shapes.qT) >= args.numHeads * args.headDim and dim(shapes.attnT, 0) == args.numHeads * args.headDim and numel(shapes.cacheKeysT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim and numel(shapes.cacheValuesT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim',qBytes:'numel(shapes.qT) * (2 if tensorDtypes.qT == "float16" else 4) if present.qT else 0',cacheKeysBytes:'numel(shapes.cacheKeysT) * (2 if tensorDtypes.cacheKeysT == "float16" else 4)',cacheValuesBytes:'numel(shapes.cacheValuesT) * (2 if tensorDtypes.cacheValuesT == "float16" else 4)',attnBytes:'numel(shapes.attnT) * (2 if tensorDtypes.attnT == "float16" else 4)',buffersFit:"qBytes <= device.limits.maxStorageBufferBindingSize and qBytes <= device.limits.maxBufferSize and cacheKeysBytes <= device.limits.maxStorageBufferBindingSize and cacheKeysBytes <= device.limits.maxBufferSize and cacheValuesBytes <= device.limits.maxStorageBufferBindingSize and cacheValuesBytes <= device.limits.maxBufferSize and attnBytes <= device.limits.maxStorageBufferBindingSize and attnBytes <= device.limits.maxBufferSize",workgroupLimit:"min(256, device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",autoWorkgroupSize:"256 if args.headDim > 128 and workgroupLimit >= 256 else 128 if args.headDim > 64 and workgroupLimit >= 128 else 64 if args.headDim > 32 and workgroupLimit >= 64 else 32 if args.headDim > 16 and workgroupLimit >= 32 else 16 if args.headDim > 8 and workgroupLimit >= 16 else 8 if args.headDim > 4 and workgroupLimit >= 8 else 4 if workgroupLimit >= 4 else 0",decodeWorkgroupSize:"args.workgroupSize if args.workgroupSize else autoWorkgroupSize",attentionStorageBytes:"(args.cacheLen + decodeWorkgroupSize) * 4",workgroupSupported:"(decodeWorkgroupSize == 4 or decodeWorkgroupSize == 8 or decodeWorkgroupSize == 16 or decodeWorkgroupSize == 32 or decodeWorkgroupSize == 64 or decodeWorkgroupSize == 128 or decodeWorkgroupSize == 256)",resourcesFitCore:"workgroupSupported and decodeWorkgroupSize <= deviceWorkgroupCap",resourcesFit:"resourcesFitCore and attentionStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",dispatchFits:"args.numHeads <= device.limits.maxComputeWorkgroupsPerDimension",baseOk:"coreContract and buffersFit and resourcesFit and dispatchFits",baseOkCore:"coreContract and buffersFit and resourcesFitCore and dispatchFits",onlineScoreTile:"tunables.DECODE_SCORE_TILE",onlineStorageBytes:"(onlineScoreTile + decodeWorkgroupSize) * 4",onlineResourcesFit:"onlineScoreTile > 0 and pow2ceil(onlineScoreTile) == onlineScoreTile and onlineStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",sharedScoresCrowded:"attentionStorageBytes * 2 > device.limits.maxComputeWorkgroupStorageSize",onlineOk:"baseOkCore and onlineResourcesFit and sharedScoresCrowded",subgroupRangeKnown:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize > 0 and device.adapterInfo.subgroupMinSize <= device.adapterInfo.subgroupMaxSize',shortSubgroupOk:"baseOk and subgroupRangeKnown and device.adapterInfo.subgroupMaxSize <= decodeWorkgroupSize and decodeWorkgroupSize % device.adapterInfo.subgroupMinSize == 0 and decodeWorkgroupSize % device.adapterInfo.subgroupMaxSize == 0 and attentionStorageBytes + 8 <= device.limits.maxComputeWorkgroupStorageSize",sharedSplitUseSubgroups:"(subgroupRangeKnown and device.adapterInfo.subgroupMaxSize <= args.workgroupSize and args.workgroupSize % device.adapterInfo.subgroupMinSize == 0 and args.workgroupSize % device.adapterInfo.subgroupMaxSize == 0) if splitArgsPresent else false",sharedSplitStorageBytes:'((args.headDim) * 8 + (args.workgroupSize) * 12 + (16 * (args.workgroupSize) if (args.workgroupSize) % ((args.headDim) / 4) == 0 and (args.workgroupSize) / ((args.headDim) / 4) >= 2 else 0) + (4 * (args.workgroupSize) if (sharedSplitUseSubgroups) and ((args.headDim) / 4) >= 8 and not ((tensorDtypes.qT == "float16" or tensorDtypes.cacheKeysT == "float16") and ((args.headDim) / 4) <= 32) and (true) else 0) + (ceilDiv((args.workgroupSize), 32) * 4 + 8 if (sharedSplitUseSubgroups) else 0) if (args.headDim) >= 4 and (args.workgroupSize) > 0 else 0) if splitArgsPresent else 0',sharedSplitResourcesFit:"((args.headDim) >= 4 and (args.workgroupSize) > 0 and (args.workgroupSize) <= deviceWorkgroupCap and (pow2ceil(args.workgroupSize) == (args.workgroupSize) or (sharedSplitUseSubgroups)) and (sharedSplitStorageBytes) <= device.limits.maxComputeWorkgroupStorageSize) if splitArgsPresent else false",scalarWorkgroupLimit:"min(tunables.WORKGROUP_SIZE, deviceWorkgroupCap)",scalarWorkgroupSize:"max(1, pow2ceil(scalarWorkgroupLimit + 1) / 2)",scalarScoreTileCapacity:"floor((device.limits.maxComputeWorkgroupStorageSize - scalarWorkgroupSize * 4 - 8) / 4)",scalarScoreTile:"max(1, min(args.cacheLen, tunables.MAX_SCORE_TILE, scalarScoreTileCapacity))",scalarDeviceOk:"tunables.WORKGROUP_SIZE > 0 and tunables.MAX_SCORE_TILE > 0 and scalarWorkgroupLimit > 0 and scalarScoreTileCapacity > 0",fusedWriteContract:'decodeMode and cacheWriteArgsPresent and splitArgsAbsent and not present.gateT and ranks.qT == 1 and ranks.cacheKeysT == 1 and ranks.cacheValuesT == 1 and ranks.attnT == 1 and tensorDtypes.qT == "float32" and tensorDtypes.cacheKeysT == "float32" and tensorDtypes.cacheValuesT == "float32" and tensorDtypes.attnT == "float32" and args.cacheLen > 0 and args.numHeads > 0 and args.numKvHeads > 0 and args.numHeads == args.numKvHeads and args.headDim > 0 and dim(shapes.qT, 0) >= args.qOffset + args.numHeads * args.headDim and dim(shapes.qT, 0) >= args.kOffset + args.numKvHeads * args.headDim and dim(shapes.qT, 0) >= args.vOffset + args.numKvHeads * args.headDim and numel(shapes.cacheKeysT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim and numel(shapes.cacheValuesT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim and dim(shapes.attnT, 0) == args.numHeads * args.headDim',fusedWriteOk:"fusedWriteContract and buffersFit and resourcesFit and dispatchFits",fusedShortSubgroupOk:"fusedWriteOk and subgroupRangeKnown and device.adapterInfo.subgroupMaxSize <= decodeWorkgroupSize and decodeWorkgroupSize % device.adapterInfo.subgroupMinSize == 0 and decodeWorkgroupSize % device.adapterInfo.subgroupMaxSize == 0 and attentionStorageBytes + 8 <= device.limits.maxComputeWorkgroupStorageSize",prefillQkvStride:"(args.numHeads + 2 * args.numKvHeads) * args.headDim",prefillQHidden:"args.numHeads * args.headDim",prefillCacheElements:"(args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim",prefillEffectivePastLen:"args.pastLen if args.pastLen else 0",prefillBaseContract:'prefillMode and ranks.qkvT == 2 and (ranks.cacheKeysT == 1 or ranks.cacheKeysT == 5) and (ranks.cacheValuesT == 1 or ranks.cacheValuesT == 5) and ranks.attnT == 2 and tensorDtypes.cacheKeysT == tensorDtypes.cacheValuesT and (tensorDtypes.qkvT == "float32" or tensorDtypes.qkvT == "float16") and (tensorDtypes.cacheKeysT == "float32" or tensorDtypes.cacheKeysT == "float16") and f16Ok(tensorDtypes.qkvT) and f16Ok(tensorDtypes.cacheKeysT) and f16Ok(tensorDtypes.attnT) and args.cacheLen > 0 and args.seqLen > 0 and args.numHeads > 0 and args.numKvHeads > 0 and args.numHeads % args.numKvHeads == 0 and args.headDim > 0 and dim(shapes.qkvT, 1) == prefillQkvStride and numel(shapes.qkvT) >= args.seqLen * prefillQkvStride and dim(shapes.attnT, 1) == prefillQHidden and numel(shapes.attnT) >= args.seqLen * prefillQHidden and numel(shapes.cacheKeysT) >= prefillCacheElements and numel(shapes.cacheValuesT) >= prefillCacheElements',prefillGateContract:"present.gateT and ranks.gateT == 2 and tensorDtypes.gateT == tensorDtypes.attnT and f16Ok(tensorDtypes.gateT) and numel(shapes.gateT) >= args.seqLen * prefillQHidden",prefillFlashContract:"prefillBaseContract and args.headDim % 16 == 0",prefillFlashDispatchFits:"ceilDiv(args.seqLen, 16) <= device.limits.maxComputeWorkgroupsPerDimension and args.numHeads <= device.limits.maxComputeWorkgroupsPerDimension",prefillFlashUsesF16:'tensorDtypes.qkvT == "float16" or tensorDtypes.cacheKeysT == "float16" or tensorDtypes.attnT == "float16"',prefillFlashLpq:"8 if args.headDim % 32 == 0 else 4",prefillFlashTileK:"16 if subgroupsWide32 else (15 if not prefillFlashUsesF16 else 16)",prefillFlashWorkgroupStorageBytes:"2 * prefillFlashTileK * args.headDim * (2 if prefillFlashUsesF16 else 4) + (0 if subgroupsWide32 else 16 * prefillFlashLpq * 4)",headDim:"args.headDim",layer:"args.layer",cacheLen:"args.cacheLen",numKvHeads:"args.numKvHeads",numKvGroups:"args.numHeads / args.numKvHeads",f32Min:'"-3.4028234663852886e38"',qNumHeads:"args.numHeads",kvNumHeads:"args.numKvHeads",headDimV4:"args.headDim / 4",hasBias:!1,hasWindow:!1,cacheVec4:'"vec4<f16>" if tensorDtypes.cacheKeysT == "float16" else "vec4<f32>"',scaling:"args.scale",numHeads:"args.numHeads",scale:"args.scale",qScalar:"dtypes.Q",cacheScalar:"dtypes.C",outScalar:"dtypes.O"},bindings:{q:{arg:"qT",elementType:"f32"},attn_out:{arg:"attnT",elementType:"f32"},params:{arg:"paramsT",struct:[{name:"past_len",type:"u32",value:"args.pastLen if args.pastLen else 0"}]},key:{arg:"cacheKeysT",buffer:"read-only-storage",elementType:"$cacheVec4"},value:{arg:"cacheValuesT",buffer:"read-only-storage",elementType:"$cacheVec4"},partial_out:{scratch:"partialOut",elementType:"vec4<f32>"},partial_out_merge:{scratch:"partialOut",name:"partial_out",buffer:"read-only-storage",elementType:"vec4<f32>"},q_q_t:{arg:"qT",name:"q",elementType:"$qScalar"},cache_keys:{arg:"cacheKeysT",buffer:"read-only-storage",elementType:"$cacheScalar"},cache_values:{arg:"cacheValuesT",buffer:"read-only-storage",elementType:"$cacheScalar"},attn_out_main:{arg:"attnT",name:"attn_out",elementType:"$outScalar"},qkv:{arg:"qkvT",elementType:"$qElement"},cache_keys_main:{arg:"cacheKeysT",name:"cache_keys",buffer:"read-only-storage",elementType:"$cacheElement"},cache_values_main:{arg:"cacheValuesT",name:"cache_values",buffer:"read-only-storage",elementType:"$cacheElement"},attn_out_attn_t:{arg:"attnT",name:"attn_out",elementType:"$outElement"},params_main:{name:"params",struct:[{name:"past_len",type:"u32",value:"args.pastLen if args.pastLen else 0"},{name:"cache_len",type:"u32",value:"args.cacheLen",required:!0,internal:!0},{name:"seq_len",type:"u32",value:"args.seqLen"}]},gate_main:{arg:"gateT",name:"gate",elementType:"$outElement"},cache_keys_rw:{arg:"cacheKeysT",name:"cache_keys",elementType:"f32"},cache_values_rw:{arg:"cacheValuesT",name:"cache_values",elementType:"f32"},query:{arg:"qT",elementType:"$qVec4"},partial_meta:{scratch:"partialMeta",name:"partial_stats",elementType:"vec2<f32>"},partial_meta_ro:{scratch:"partialMeta",name:"partial_stats",buffer:"read-only-storage",elementType:"vec2<f32>"},output:{arg:"attnT",elementType:"$outVec4"},query_f32:{arg:"qT",name:"query",elementType:"vec4<f32>"},partial_stats:{scratch:"partialStats",elementType:"vec2<f32>"},partial_stats_ro:{scratch:"partialStats",name:"partial_stats",buffer:"read-only-storage",elementType:"vec2<f32>"},gate_vec4:{arg:"gateT",name:"gate",elementType:"vec4<f32>"},output_f32:{arg:"attnT",name:"output",elementType:"vec4<f32>"},gate_f32:{arg:"gateT",name:"gate",elementType:"f32"}},variants:[{id:"cache_write_adaptive_f32",priority:30,when:["fusedWriteOk"],derive:{useSubgroupShortPath:"fusedShortSubgroupOk",workgroupSize:"decodeWorkgroupSize"},passes:[{id:"main",name:"CachedDecodeAttention.CacheWriteAdaptive",shader:"decode-cache-attention.wgsl.jinja",derive:{qOffset:"args.qOffset",writeCache:!0,kOffset:"args.kOffset",vOffset:"args.vOffset"},bindings:["q","cache_keys_rw","cache_values_rw","attn_out","params"],dispatch:{x:"args.numHeads"}}]},{id:"llama_splitk",priority:20,when:["not present.gateT","splitArgsPresent","coreContract","resourcesFitCore","dispatchFits","splitScratchFits","args.headDim % 4 == 0","args.headDim <= args.workgroupSize or args.headDim == args.workgroupSize * 2","args.tileSize > 0","args.workgroupSize >= args.tileSize","args.numKvChunks > 0","args.minKeysPerChunk > 0","args.headDim <= device.limits.maxComputeInvocationsPerWorkgroup","args.headDim <= device.limits.maxComputeWorkgroupSizeX","device.limits.maxComputeWorkgroupsPerDimension is undefined or args.numKvChunks <= device.limits.maxComputeWorkgroupsPerDimension","(args.workgroupSize + args.tileSize + args.headDim) * 4 <= device.limits.maxComputeWorkgroupStorageSize","sharedSplitResourcesFit"],derive:{qVec4:'"vec4<f16>" if tensorDtypes.qT == "float16" else "vec4<f32>"',outVec4:'"vec4<f16>" if tensorDtypes.attnT == "float16" else "vec4<f32>"',scalar:"dtypes.O",numSplits:"args.numKvChunks",useSubgroups:"sharedSplitUseSubgroups"},intermediates:[{id:"partialOut",dtype:"float32",shape:"[args.numHeads * args.numKvChunks * args.headDim]"},{id:"partialMeta",dtype:"float32",shape:"[args.numHeads * args.numKvChunks * 2]"}],passes:[{id:"split",name:"LlamaDecodeAttentionSplitK",shader:"ops/_shared/attn-flash-decode-splitk.wgsl.jinja",derive:{usesF16:'tensorDtypes.qT == "float16" or tensorDtypes.cacheKeysT == "float16"',layout:'"layer_cache"',workgroupSizeSpec:"args.workgroupSize",minKeysPerChunk:"args.minKeysPerChunk"},bindings:["query","key","value","partial_out","partial_meta","params"],dispatch:{x:"args.numKvChunks",y:"args.numHeads",z:1}},{id:"merge",name:"LlamaDecodeAttentionSplitKMerge",shader:"ops/_shared/attn-flash-decode-splitk-merge.wgsl.jinja",derive:{layout:'"layer_cache"'},bindings:["partial_out_merge","partial_meta_ro","output"],dispatch:{x:1,y:"args.numHeads",z:1}}]},{cases:[{splitId:"qwen_splitk",splitPriority:20,splitCount:8,splitWhen:[],splitDemote:null},{splitId:"qwen_splitk_parallel",splitPriority:25,splitCount:32,splitWhen:["args.numHeads == 8","args.numKvHeads == 2","args.headDim == 256",'tensorDtypes.cacheKeysT == "float32"'],splitDemote:null},{splitId:"qwen_splitk_mid_heads",splitPriority:25,splitCount:16,splitWhen:["args.numHeads == 16","args.numKvHeads == 4","args.headDim == 256",'tensorDtypes.cacheKeysT == "float32"'],splitDemote:null}],variants:[{id:"{splitId}",priority:"{splitPriority}",when:["decodeMode","cacheWriteArgsAbsent","present.gateT","ranks.qT == 1","ranks.gateT == 1","(ranks.cacheKeysT == 1 or ranks.cacheKeysT == 5)","(ranks.cacheValuesT == 1 or ranks.cacheValuesT == 5)","ranks.attnT == 1",'tensorDtypes.qT == "float32"','tensorDtypes.gateT == "float32"','tensorDtypes.attnT == "float32"',"tensorDtypes.cacheKeysT == tensorDtypes.cacheValuesT",'(tensorDtypes.cacheKeysT == "float32" or tensorDtypes.cacheKeysT == "float16")',"f16Ok(tensorDtypes.cacheKeysT)","args.cacheLen > 0","(args.pastLen if args.pastLen else 0) < args.cacheLen","args.numHeads > 0","args.numKvHeads > 0","args.numHeads % args.numKvHeads == 0","args.headDim > 0","scalarDeviceOk","args.headDim % 4 == 0","args.headDim <= 1024","dim(shapes.qT, 0) == args.numHeads * args.headDim","dim(shapes.gateT, 0) == args.numHeads * args.headDim","dim(shapes.attnT, 0) == args.numHeads * args.headDim","numel(shapes.cacheKeysT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim","numel(shapes.cacheValuesT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim","(args.headDim) >= 4 and (tunables.WORKGROUP_SIZE) > 0 and (tunables.WORKGROUP_SIZE) <= deviceWorkgroupCap and (pow2ceil(tunables.WORKGROUP_SIZE) == (tunables.WORKGROUP_SIZE) or (false)) and ((args.headDim) * 8 + (tunables.WORKGROUP_SIZE) * 12 + (16 * (tunables.WORKGROUP_SIZE) if (tunables.WORKGROUP_SIZE) % ((args.headDim) / 4) == 0 and (tunables.WORKGROUP_SIZE) / ((args.headDim) / 4) >= 2 else 0) + (4 * (tunables.WORKGROUP_SIZE) if (false) and ((args.headDim) / 4) >= 8 and not ((false) and ((args.headDim) / 4) <= 32) and (true) else 0) + (ceilDiv((tunables.WORKGROUP_SIZE), 32) * 4 + 8 if (false) else 0) if (args.headDim) >= 4 and (tunables.WORKGROUP_SIZE) > 0 else 0) <= device.limits.maxComputeWorkgroupStorageSize","{splitWhen}"],derive:{numSplits:"{splitCount}",usesF16:'tensorDtypes.cacheKeysT == "float16"',scalar:'"f32"',hasGate:!0,useSubgroups:!1},intermediates:[{id:"partialOut",dtype:"float32",shape:"[args.numHeads * numSplits * args.headDim]"},{id:"partialStats",dtype:"float32",shape:"[2 * args.numHeads * numSplits]"}],passes:[{id:"partial",name:"Qwen35DecodeFullAttention.SplitK",shader:"ops/_shared/attn-flash-decode-splitk.wgsl.jinja",derive:{layout:'"layer_cache"'},bindings:["query_f32","key","value","partial_out","partial_stats","params"],dispatch:{x:"numSplits",y:"args.numHeads",z:1}},{id:"merge",name:"Qwen35DecodeFullAttention.SplitKMerge",shader:"ops/_shared/attn-flash-decode-splitk-merge.wgsl.jinja",derive:{layout:'"layer_cache"'},bindings:["partial_out_merge","partial_stats_ro","gate_vec4","output_f32"],dispatch:{x:1,y:"args.numHeads",z:1}}],demoteWhen:"{splitDemote}"}]},{id:"llama_online_tiled",priority:1,when:["not present.gateT","splitArgsAbsent","onlineOk"],derive:{workgroupSize:"decodeWorkgroupSize",scoreTile:"onlineScoreTile"},passes:[{id:"main",name:"LlamaDecodeAttentionOnlineTiled",shader:"decode-cache-attention-online.wgsl.jinja",bindings:["q_q_t","cache_keys","cache_values","attn_out_main","params"],dispatch:{x:"args.numHeads"}}]},{id:"llama_adaptive",priority:0,when:["not present.gateT","splitArgsAbsent","baseOk"],derive:{useSubgroupShortPath:"shortSubgroupOk",workgroupSize:"decodeWorkgroupSize"},passes:[{id:"main",name:"LlamaDecodeAttention",shader:"decode-cache-attention.wgsl.jinja",bindings:["q_q_t","cache_keys","cache_values","attn_out_main","params"],dispatch:{x:"args.numHeads"}}]},{id:"qwen_scalar",priority:0,when:["decodeMode","cacheWriteArgsAbsent","present.gateT","ranks.gateT == 1","ranks.attnT == 1",'tensorDtypes.gateT == "float32"','tensorDtypes.attnT == "float32"',"ranks.qT == 1","(ranks.cacheKeysT == 1 or ranks.cacheKeysT == 5)","(ranks.cacheValuesT == 1 or ranks.cacheValuesT == 5)",'tensorDtypes.qT == "float32"',"tensorDtypes.cacheKeysT == tensorDtypes.cacheValuesT",'(tensorDtypes.cacheKeysT == "float32" or tensorDtypes.cacheKeysT == "float16")',"f16Ok(tensorDtypes.cacheKeysT)","args.cacheLen > 0","(args.pastLen if args.pastLen else 0) < args.cacheLen","args.numHeads > 0","args.numKvHeads > 0","args.numHeads % args.numKvHeads == 0","args.headDim > 0","dim(shapes.qT, 0) == args.numHeads * args.headDim","dim(shapes.gateT, 0) == args.numHeads * args.headDim","dim(shapes.attnT, 0) == args.numHeads * args.headDim","numel(shapes.cacheKeysT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim","numel(shapes.cacheValuesT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim"],derive:{workgroupSize:"scalarWorkgroupSize",scoreTile:"scalarScoreTile",hasGate:!0},passes:[{id:"main",name:"Qwen35DecodeFullAttention",shader:"decode-cache-attention-online.wgsl.jinja",bindings:["q","gate_f32","cache_keys","cache_values","attn_out","params"],dispatch:{x:"args.numHeads"}}]},{cases:[{gated:"",gateGuard:"not present.gateT",hasOutputGate:!1,gate:[]},{gated:"_gated",gateGuard:"prefillGateContract",hasOutputGate:!0,gate:["gate_main"]}],variants:[{id:"prefill_scalar{gated}",priority:0,when:["prefillBaseContract","{gateGuard}","args.workgroupSize == 8 or args.workgroupSize == 16 or args.workgroupSize == 32 or args.workgroupSize == 64 or args.workgroupSize == 128 or args.workgroupSize == 256"],derive:{qElement:"dtypes.Q",cacheElement:"dtypes.C",outElement:"dtypes.O",qHidden:"prefillQHidden",qkvStride:"prefillQkvStride",workgroupSize:"args.workgroupSize",hasOutputGate:"{hasOutputGate}"},passes:[{id:"main",name:"LlamaPrefillAttention",shader:"prefill-attention.wgsl.jinja",bindings:["qkv","cache_keys_main","cache_values_main","attn_out_attn_t","params_main","{gate}"],dispatch:{x:"args.numHeads",y:"min(args.seqLen, device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{cases:[{gated:"",gateGuard:"not present.gateT",hasOutputGate:!1,gate:[],flashPriority:1,flashTileQ:16,flashTileK:"prefillFlashTileK",flashLpq:"prefillFlashLpq",flashWhen:["prefillFlashDispatchFits","prefillFlashWorkgroupStorageBytes <= device.limits.maxComputeWorkgroupStorageSize"],flashRequires:{},flashDemote:null,validateClusterLayout:!1,clusterCollectivesWidth:null},{gated:"_gated",gateGuard:"prefillGateContract",hasOutputGate:!0,gate:["gate_main"],flashPriority:1,flashTileQ:16,flashTileK:"prefillFlashTileK",flashLpq:"prefillFlashLpq",flashWhen:["prefillFlashDispatchFits","prefillFlashWorkgroupStorageBytes <= device.limits.maxComputeWorkgroupStorageSize"],flashRequires:{},flashDemote:null,validateClusterLayout:!1,clusterCollectivesWidth:null},{gated:"_parallel_gated",gateGuard:"prefillGateContract",hasOutputGate:!0,gate:["gate_main"],flashPriority:2,flashTileQ:32,flashTileK:16,flashLpq:16,flashWhen:["subgroupsWide32","args.headDim == 256","args.seqLen >= 128 and args.seqLen <= 512","(args.numHeads == 8 and args.numKvHeads == 2) or (args.numHeads == 16 and args.numKvHeads == 4) or (args.numHeads == 24 and args.numKvHeads == 4)","not prefillFlashUsesF16","ceilDiv(args.seqLen, 32) <= device.limits.maxComputeWorkgroupsPerDimension","args.numHeads <= device.limits.maxComputeWorkgroupsPerDimension","256 <= device.limits.maxComputeWorkgroupSizeX","256 <= device.limits.maxComputeInvocationsPerWorkgroup","32768 <= device.limits.maxComputeWorkgroupStorageSize"],flashRequires:{features:["subgroups"]},flashDemote:null,validateClusterLayout:!0,clusterCollectivesWidth:"portable"}],variants:[{id:"prefill_flash{gated}",priority:"{flashPriority}",when:["prefillFlashContract","{gateGuard}","args.layer == 0","prefillEffectivePastLen + args.seqLen <= args.cacheLen","{flashWhen}"],derive:{usesF16:"prefillFlashUsesF16",scalar:"dtypes.O",qHiddenV4:"prefillQHidden / 4",kvHiddenV4:"args.numKvHeads * args.headDim / 4",qkvStrideV4:"prefillQkvStride / 4",hasCausal:!0,causalRightAlign:!0,TILE_Q:"{flashTileQ}",TILE_K:"{flashTileK}",LPQ:"{flashLpq}",qElement:'"vec4<f16>" if tensorDtypes.qkvT == "float16" else "vec4<f32>"',cacheElement:'"vec4<f16>" if tensorDtypes.cacheKeysT == "float16" else "vec4<f32>"',outElement:'"vec4<f16>" if tensorDtypes.attnT == "float16" else "vec4<f32>"',useSubgroups:"subgroupsWide32",batchNoSgReduction:!1},passes:[{id:"main",name:"LlamaPrefillAttention",shader:"ops/_shared/attn-flash-prefill-cluster.wgsl.jinja",derive:{layout:'"bsh"',sourceProfile:1,outputGate:"{hasOutputGate}",validateClusterLayout:"{validateClusterLayout}"},bindings:["qkv","cache_keys_main","cache_values_main","attn_out_attn_t","params_main","{gate}"],dispatch:{x:"ceilDiv(args.seqLen, TILE_Q)",y:"args.numHeads"},subgroupCollectivesWidth:"{clusterCollectivesWidth}"}],requires:"{flashRequires}",demoteWhen:"{flashDemote}"}]},{cases:[{gated:"",gateGuard:"not present.gateT",hasOutputGate:!1,gate:[]},{gated:"_gated",gateGuard:"prefillGateContract",hasOutputGate:!0,gate:["gate_main"]}],variants:[{id:"prefill_flash_sgmat{gated}",priority:3,when:["prefillFlashContract","{gateGuard}","args.layer == 0","prefillEffectivePastLen + args.seqLen <= args.cacheLen",'tensorDtypes.qkvT == "float32"','tensorDtypes.cacheKeysT == "float32"','tensorDtypes.attnT == "float32"',"args.headDim == 128 or args.headDim == 256","args.seqLen >= 256","wave32Effective","args.numHeads % args.numKvHeads == 0","(args.numHeads / args.numKvHeads) == 2 or (args.numHeads / args.numKvHeads) == 4 or (args.numHeads / args.numKvHeads) == 6","(args.numHeads / args.numKvHeads) * 32 <= deviceWorkgroupCap","ceilDiv(args.seqLen, 8) <= device.limits.maxComputeWorkgroupsPerDimension","args.numKvHeads <= device.limits.maxComputeWorkgroupsPerDimension","(args.numHeads / args.numKvHeads) * 3744 + 32 + subgroupMatrixTopologyBytes <= device.limits.maxComputeWorkgroupStorageSize"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{qElement:'"f32"',cacheElement:'"f32"',outElement:'"f32"',sgHeadDim:"args.headDim",sgQHeads:"args.numHeads",sgKvHeads:"args.numKvHeads",sgGroup:"args.numHeads / args.numKvHeads",hasOutputGate:"{hasOutputGate}"},passes:[{id:"main",name:"LlamaPrefillAttention.Sgmat",shader:"prefill-flash-sgmat.wgsl.jinja",bindings:["qkv","cache_keys_main","cache_values_main","attn_out_attn_t","params_main","{gate}"],dispatch:{x:"ceilDiv(args.seqLen, 8)",y:"args.numKvHeads"},subgroupCollectivesWidth:"portable"}]}]}]},assets:[["decode-cache-attention-online.wgsl.jinja",`{% set ACC_SLOTS = ((headDim + workgroupSize - 1) / workgroupSize)|int %}{% set gatedOutput = hasGate if hasGate is defined else false %}{{ env.wgsl.resourceDeclarations }} const LAYER:u32={{ layer }}u;const CACHE_LEN:u32={{ cacheLen }}u;const NUM_HEADS:u32={{ numHeads }}u;const NUM_KV_HEADS:u32={{ numKvHeads }}u;const NUM_KV_GROUPS:u32={{ numKvGroups }}u;const HEAD_DIM:u32={{ headDim }}u;const ATTN_SCALING:f32={{ scaling }};const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const Q_OFFSET:u32=0u;const SCORE_TILE:u32={{ scoreTile }}u;var<workgroup>scores:array<f32,{{ scoreTile }}>;var<workgroup>partials:array<f32,{{ workgroupSize }}>;
{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{% include "./decode-cache-io.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let head=wid.x;let lid=lid3.x;if(head>=NUM_HEADS){return;}if(params.past_len>=CACHE_LEN){for(var d=lid;d<HEAD_DIM;d=d+WORKGROUP_SIZE){attn_out[head*HEAD_DIM+d]=store_attn(0.0);}return;}let kv_head=head/NUM_KV_GROUPS;let q_base=head*HEAD_DIM;let total_len=params.past_len+1u;
{% for slot in range(ACC_SLOTS) %}var acc{{ slot }}=0.0;
{% endfor %}var running_max={{ f32Min }};var running_denom=0.0;for(var tile_base=0u;tile_base<total_len;tile_base=tile_base+SCORE_TILE){let tile_len=min(SCORE_TILE,total_len-tile_base);var local_max={{ f32Min }};for(var i=lid;i<tile_len;i=i+WORKGROUP_SIZE){var score=0.0;for(var d=0u;d<HEAD_DIM;d=d+1u){score=fma(load_q(q[Q_OFFSET+q_base+d]),load_cache(cache_keys[cache_index(tile_base+i,kv_head,d)]),score);}score=score*ATTN_SCALING;scores[i]=score;local_max=max(local_max,score);}partials[lid]=local_max;workgroupBarrier();{{ wgsl_tree_fold(["partials"], op="max", svar="max_stride") }} let tile_max=partials[0];workgroupBarrier();let new_max=max(running_max,tile_max);let rescale=exp(running_max-new_max);var local_denom=0.0;for(var i=lid;i<tile_len;i=i+WORKGROUP_SIZE){let unnormalized_prob=exp(scores[i]-new_max);scores[i]=unnormalized_prob;local_denom=local_denom+unnormalized_prob;}partials[lid]=local_denom;workgroupBarrier();{{ wgsl_tree_fold(["partials"], op="add", svar="denom_stride") }} let tile_denom=partials[0];workgroupBarrier();running_denom=running_denom*rescale+tile_denom;
{% for slot in range(ACC_SLOTS) %}{let d{{ slot }}=lid+{{ slot * workgroupSize }}u;if(d{{ slot }}<HEAD_DIM){var value=0.0;for(var i=0u;i<tile_len;i=i+1u){value=fma(scores[i],load_cache(cache_values[cache_index(tile_base+i,kv_head,d{{ slot }})]),value);}acc{{ slot }}=acc{{ slot }}*rescale+value;} }
{% endfor %}running_max=new_max;workgroupBarrier();}
{% for slot in range(ACC_SLOTS) %}{let d{{ slot }}=lid+{{ slot * workgroupSize }}u;if(d{{ slot }}<HEAD_DIM){let normalized=acc{{ slot }}/running_denom;
{% if gatedOutput %}attn_out[q_base+d{{ slot }}]=store_attn(normalized*(1.0/(1.0+exp(-gate[q_base+d{{ slot }}]))));
{% else %}attn_out[q_base+d{{ slot }}]=store_attn(normalized);
{% endif %}} }
{% endfor %}}`],["decode-cache-attention.wgsl.jinja",`{% set qOffsetValue = qOffset if qOffset is defined else 0 %}{% set writeCacheValue = writeCache if writeCache is defined else false %}{{ env.wgsl.resourceDeclarations }} const LAYER:u32={{ layer }}u;const CACHE_LEN:u32={{ cacheLen }}u;const NUM_HEADS:u32={{ numHeads }}u;const NUM_KV_HEADS:u32={{ numKvHeads }}u;const NUM_KV_GROUPS:u32={{ numKvGroups }}u;const HEAD_DIM:u32={{ headDim }}u;const ATTN_SCALING:f32={{ scaling }};const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const Q_OFFSET:u32={{ qOffsetValue }}u;
{% if writeCacheValue %}const K_OFFSET:u32={{ kOffset }}u;const V_OFFSET:u32={{ vOffset }}u;
{% endif %}var<workgroup>scores:array<f32,{{ cacheLen }}>;var<workgroup>partials:array<f32,{{ workgroupSize }}>;
{% include "./decode-cache-io.wgsl.jinja" %}{% if useSubgroupShortPath %}{% include "ops/_shared/subgroup-local-layout-proof.wgsl.jinja" %}{% endif %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,
  @builtin(local_invocation_id) lid3: vec3<u32>{% if useSubgroupShortPath %},
@builtin(subgroup_size)subgroup_size:u32,@builtin(subgroup_invocation_id)physical_lane:u32
{% endif %}){let head=wid.x;let lid=lid3.x;if(head>=NUM_HEADS){return;}if(params.past_len>=CACHE_LEN){for(var d=lid;d<HEAD_DIM;d=d+WORKGROUP_SIZE){attn_out[head*HEAD_DIM+d]=store_attn(0.0);}return;}let kv_head=head/NUM_KV_GROUPS;let q_base=head*HEAD_DIM;
{% if writeCacheValue %}for(var d=lid;d<HEAD_DIM;d=d+WORKGROUP_SIZE){let cache_offset=cache_index(params.past_len,kv_head,d);let src_offset=kv_head*HEAD_DIM+d;cache_keys[cache_offset]={{ cacheScalar }}(load_q(q[K_OFFSET+src_offset]));cache_values[cache_offset]={{ cacheScalar }}(load_q(q[V_OFFSET+src_offset]));}storageBarrier();workgroupBarrier();
{% endif %}{% if useSubgroupShortPath %}let population=subgroupBallot(true);let populationCount=countOneBits(population.x)+countOneBits(population.y)+countOneBits(population.z)+countOneBits(population.w);let firstLocal=subgroupBroadcastFirst(lid);let nativeShortLayout=subgroupLocalLayoutAccepted(lid,populationCount==subgroup_size&&physical_lane==lid%subgroup_size&&firstLocal==lid-physical_lane);let total_len=params.past_len+1u;if(nativeShortLayout&&subgroup_size>0u&&subgroup_size<=WORKGROUP_SIZE&&WORKGROUP_SIZE%subgroup_size==0u&&total_len<=subgroup_size){let subgroup_lane=lid%subgroup_size;let subgroup_index=lid/subgroup_size;var score={{ f32Min }};if(subgroup_index==0u&&subgroup_lane<total_len){var acc=0.0;for(var d=0u;d<HEAD_DIM;d=d+1u){acc=fma(load_q(q[Q_OFFSET+q_base+d]),load_cache(cache_keys[cache_index(subgroup_lane,kv_head,d)]),acc);}score=acc*ATTN_SCALING;}let max_score=subgroupMax(score);let weight=select(0.0,exp(score-max_score),subgroup_index==0u&&subgroup_lane<total_len);let denom=subgroupAdd(weight);if(subgroup_index==0u&&subgroup_lane<total_len){scores[subgroup_lane]=weight;}if(lid==0u){partials[0]=denom;}workgroupBarrier();for(var d=lid;d<HEAD_DIM;d=d+WORKGROUP_SIZE){var value=0.0;for(var src=0u;src<total_len;src=src+1u){value=fma(scores[src]/partials[0],load_cache(cache_values[cache_index(src,kv_head,d)]),value);}attn_out[q_base+d]=store_attn(value);}return;}
{% endif %}var local_max={{ f32Min }};for(var src=lid;src<=params.past_len;src=src+WORKGROUP_SIZE){var score=0.0;for(var d=0u;d<HEAD_DIM;d=d+1u){score=fma(load_q(q[Q_OFFSET+q_base+d]),load_cache(cache_keys[cache_index(src,kv_head,d)]),score);}score=score*ATTN_SCALING;scores[src]=score;local_max=max(local_max,score);}partials[lid]=local_max;workgroupBarrier();
{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ wgsl_tree_fold(["partials"], op="max") }} let max_score=partials[0];workgroupBarrier();var local_denom=0.0;for(var src=lid;src<=params.past_len;src=src+WORKGROUP_SIZE){let unnormalized_prob=exp(scores[src]-max_score);scores[src]=unnormalized_prob;local_denom=local_denom+unnormalized_prob;}partials[lid]=local_denom;workgroupBarrier();{{ wgsl_tree_fold(["partials"], reuse=true) }} let denom=partials[0];workgroupBarrier();for(var d=lid;d<HEAD_DIM;d=d+WORKGROUP_SIZE){var value=0.0;for(var src=0u;src<=params.past_len;src=src+1u){let prob=scores[src]/denom;value=fma(prob,load_cache(cache_values[cache_index(src,kv_head,d)]),value);}attn_out[q_base+d]=store_attn(value);} }`],["decode-cache-io.wgsl.jinja",'{% include "ops/_shared/kv-cache-index.wgsl.jinja" %}fn load_q(value:{{ qScalar }})->f32{return f32(value);}fn load_cache(value:{{ cacheScalar }})->f32{return f32(value);}fn store_attn(value:f32)->{{ outScalar }} {return {{ outScalar }}(value);}'],["prefill-attention.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const LAYER:u32={{ layer }}u;const CACHE_LEN:u32={{ cacheLen }}u;const NUM_HEADS:u32={{ numHeads }}u;const NUM_KV_HEADS:u32={{ numKvHeads }}u;const NUM_KV_GROUPS:u32={{ numKvGroups }}u;const HEAD_DIM:u32={{ headDim }}u;const Q_HIDDEN:u32={{ qHidden }}u;const QKV_STRIDE:u32={{ qkvStride }}u;const ATTN_SCALING:f32={{ scaling }};const WG:u32={{ workgroupSize }}u;var<workgroup>partial:array<f32,WG>;var<workgroup>running_max:f32;var<workgroup>running_denom:f32;var<workgroup>running_out:array<f32,HEAD_DIM>;var<workgroup>previous_scale:f32;var<workgroup>current_score:f32;
{% include "ops/_shared/kv-cache-index.wgsl.jinja" %}{% include "ops/_shared/reduce-sum.wgsl.jinja" %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(num_workgroups)ngroups:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let head=wg.x;if(head>=NUM_HEADS){return;}let kv_head=head/NUM_KV_GROUPS;let tid=lid.x;for(var query_token:u32=wg.y;query_token<params.seq_len;query_token=query_token+ngroups.y){let abs_pos=params.past_len+query_token;if(abs_pos>=CACHE_LEN){continue;}if(tid==0u){running_max=-3.4028234663852886e38;running_denom=0.0;}for(var d:u32=tid;d<HEAD_DIM;d=d+WG){running_out[d]=0.0;}workgroupBarrier();let q_row=query_token*QKV_STRIDE+head*HEAD_DIM;for(var key_token:u32=0u;key_token<=abs_pos;key_token=key_token+1u){var partial_dot=0.0;for(var d:u32=tid;d<HEAD_DIM;d=d+WG){partial_dot=partial_dot+f32(qkv[q_row+d])*f32(cache_keys[cache_index(key_token,kv_head,d)]);}let score=reduce_sum(partial_dot,tid)*ATTN_SCALING;if(tid==0u){current_score=score;let next_max=max(running_max,score);let old_factor=exp(running_max-next_max);let new_factor=exp(score-next_max);previous_scale=old_factor;running_max=next_max;running_denom=running_denom*old_factor+new_factor;}workgroupBarrier();let probability_numerator=exp(current_score-running_max);for(var d:u32=tid;d<HEAD_DIM;d=d+WG){running_out[d]=running_out[d]*previous_scale+probability_numerator*f32(cache_values[cache_index(key_token,kv_head,d)]);}workgroupBarrier();}let inv_denom=1.0/running_denom;for(var d:u32=tid;d<HEAD_DIM;d=d+WG){let out_index=query_token*Q_HIDDEN+head*HEAD_DIM+d;
{% if hasOutputGate %}let rounded=f32({{ outScalar }}(running_out[d]*inv_denom));let gate_value=1.0/(1.0+exp(-f32(gate[out_index])));attn_out[out_index]={{ outScalar }}(rounded*gate_value);
{% else %}attn_out[out_index]={{ outScalar }}(running_out[d]*inv_denom);
{% endif %}}workgroupBarrier();} }`],["prefill-flash-sgmat.wgsl.jinja",`{% include "ops/_shared/subgroup-matrix-enable.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% set DH = (sgHeadDim / 2) | int %}{% set DF = (DH / 8) | int %}{% set CHUNKS = (DH / 32) | int %}{% set TEAMS = (sgGroup / 2) | int %}const HEAD_DIM:u32={{ sgHeadDim }}u;const GROUP:u32={{ sgGroup }}u;const DH:u32={{ DH }}u;const QKV_STRIDE:u32={{ (sgQHeads + 2 * sgKvHeads) * sgHeadDim }}u;const KV_STRIDE:u32={{ sgKvHeads * sgHeadDim }}u;const OUT_STRIDE:u32={{ sgQHeads * sgHeadDim }}u;const NK:u32=32u;const PART:u32=512u;const DIAG:u32=16u;const SC:u32=32u;const TAU:f32=8.0;const NEG_INF:f32=-3.4028234663852886e38;const SCALE:f32={{ scaling }};var<workgroup>scores:array<f32,{{ TEAMS * 2 * 512 }}>;var<workgroup>probs:array<f32,{{ TEAMS * 512 }}>;var<workgroup>diag:array<f32,{{ sgGroup * 8 * 16 }}>;var<workgroup>rowScale:array<f32,{{ sgGroup * 8 }}>;var<workgroup>red:array<f32,{{ TEAMS * 64 }}>;var<workgroup>rescaleFlag:array<atomic<u32>,2>;var<workgroup>rescaleSeen:u32;
{% include "ops/_shared/subgroup-matrix-topology.wgsl.jinja" %}{% include "ops/_shared/subgroup-matrix-software-8x8.wgsl.jinja" %}fn addf(a:f32,b:f32)->f32{return a+b;}
{% macro row_reduce(sw, op, value) %}{% if sw %}red[teamLane]={{ value }};workgroupBarrier();{{ value }}={{ op }}({{ op }}(red[teamRow],red[teamRow+1u]),{{ op }}(red[teamRow+2u],red[teamRow+3u]));workgroupBarrier();
{% else %}{{ value }}={{ op }}({{ value }},subgroupShuffleXor({{ value }},1u));{{ value }}={{ op }}({{ value }},subgroupShuffleXor({{ value }},2u));
{% endif %}{% endmacro %}{% macro stage_out(sw, r, c) %}{% for n in range(4) %}{{ matrixStore(sw, "row_major", "&scores", "stageAt + " ~ (n * 8) ~ "u", "o" ~ r ~ "_" ~ (c * 4 + n), "SC") }}
{% endfor %}{% endmacro %}{% macro stage_in(sw, r, c) %}{% for n in range(4) %}o{{ r }}_{{ c * 4 + n }}={{ matrixLoad(sw, "result", "f32", "row_major", "&scores", "stageAt + " ~ (n * 8) ~ "u", "SC") }};
{% endfor %}{% endmacro %}{% macro kernel_body(sw) %}let team=matrixRole/2u;let h=matrixRole%2u;let kvHead=workgroup_id.y;let headBase=kvHead*GROUP+team*2u;let seq=params.seq_len;let fullFrags=seq/8u;let q0=select(workgroup_id.x*8u,seq-8u,workgroup_id.x==fullFrags);let qFresh=select(0u,fullFrags*8u-q0,workgroup_id.x==fullFrags);let r0=params.past_len+q0;let mainEnd=r0&~7u;let diagCount=r0+8u-mainEnd;let lane={{ "matrixSoftwareLane" if sw else "subgroupLane" }};let teamIndex=h*32u+lane;let srow=teamIndex/4u;let slane=teamIndex%4u;
{% if sw %}let teamLane=team*64u+teamIndex;let teamRow=team*64u+srow*4u;
{% endif %}let rowSlot=(team*2u+srow/8u)*8u+srow%8u;let scoreBase=team*2u*PART;let probBase=team*PART;let stageAt=scoreBase+h*PART;let kvBase=kvHead*HEAD_DIM;let dBase=h*DH;for(var p=teamIndex;p<2u*8u*DIAG;p+=64u){let hr=p/(8u*DIAG);let rr=(p/DIAG)%8u;let kk=p%DIAG;let key=mainEnd+kk;var s=NEG_INF;if(kk<diagCount&&key<=r0+rr){var acc=0.0;let qAt=(q0+rr)*QKV_STRIDE+(headBase+hr)*HEAD_DIM;let kAt=kvBase+key*KV_STRIDE;for(var d=0u;d<HEAD_DIM;d++){acc=fma(qkv[qAt+d],cache_keys[kAt+d],acc);}s=acc*SCALE;}diag[(team*2u+hr)*8u*DIAG+rr*DIAG+kk]=s;}workgroupBarrier();var mUsed=NEG_INF;var l=0.0;{let at=rowSlot*DIAG+slane;let d0=diag[at];let d1=diag[at+4u];let d2=diag[at+8u];let d3=diag[at+12u];var mx=max(max(d0,d1),max(d2,d3));{{ row_reduce(sw, "max", "mx") }} mUsed=mx;let p0=exp(d0-mUsed);let p1=exp(d1-mUsed);let p2=exp(d2-mUsed);let p3=exp(d3-mUsed);var sum=(p0+p1)+(p2+p3);{{ row_reduce(sw, "addf", "sum") }} l=sum;diag[at]=p0;diag[at+4u]=p1;diag[at+8u]=p2;diag[at+12u]=p3;}workgroupBarrier();
{% for r in range(2) %}{% for n in range(DF) %}var o{{ r }}_{{ n }}:{{ matrixType(sw, "result", "f32") }};
{% endfor %}{% endfor %}{% for r in range(2) %}{% for c in range(CHUNKS) %}for(var e=lane;e<8u*SC;e+=32u){let rr=e/SC;let d=dBase+{{ c * 32 }}u+e%SC;var acc=0.0;for(var kk=0u;kk<diagCount;kk++){let key=mainEnd+kk;if(key<=r0+rr){acc=fma(diag[(team*2u+{{ r }}u)*8u*DIAG+rr*DIAG+kk],cache_values[kvBase+key*KV_STRIDE+d],acc);} }scores[stageAt+e]=acc;}workgroupBarrier();{{ stage_in(sw, r, c) }} workgroupBarrier();
{% endfor %}{% endfor %}let fullTiles=mainEnd/NK;let tileCount=(mainEnd+NK-1u)/NK;let slide=mainEnd>=NK;for(var t=0u;t<tileCount;t++){let tail=t==fullTiles&&slide;let k0=select(t*NK,mainEnd-NK,tail);let kFresh=select(0u,fullTiles*NK-k0,tail);let kf=select(min(4u,(mainEnd-k0)/8u),4u,slide);
{% for r in range(2) %}{% for j in range(4) %}var s{{ r }}_{{ j }}:{{ matrixType(sw, "result", "f32") }};
{% endfor %}{% endfor %}{% for guarded in [false, true] %}{{ "if (slide) {" if not guarded else "} else {" }} for(var k=dBase;k<dBase+DH;k+=8u){
{% for r in range(2) %}let a{{ r }}={{ matrixLoad(sw, "left", "f32", "row_major", "&qkv", "q0 * QKV_STRIDE + (headBase + " ~ r ~ "u) * HEAD_DIM + k", "QKV_STRIDE") }};
{% endfor %}{% for j in range(4) %}{% if guarded and j > 0 %}        if (kf > {{ j }}u) {
{% endif %}{let b={{ matrixLoad(sw, "right", "f32", "col_major", "&cache_keys", "kvBase + (k0 + " ~ (j * 8) ~ "u) * KV_STRIDE + k", "KV_STRIDE") }};
{% for r in range(2) %}s{{ r }}_{{ j }}={{ matrixMma(sw, "f32", "a" ~ r, "b", "s" ~ r ~ "_" ~ j) }};
{% endfor %}}
{% if guarded and j > 0 %}        }
{% endif %}{% endfor %}}
{% endfor %}}
{% for r in range(2) %}{% for j in range(4) %}{{ matrixStore(sw, "row_major", "&scores", "stageAt + " ~ (r * 256 + j * 8) ~ "u", "s" ~ r ~ "_" ~ j, "NK") }}
{% endfor %}{% endfor %}workgroupBarrier();var sv:array<f32,8>;var mx=NEG_INF;for(var i=0u;i<8u;i++){let j=slane+4u*i;let at=scoreBase+srow*NK+j;sv[i]=select(NEG_INF,(scores[at]+scores[at+PART])*SCALE,j>=kFresh&&j<kf*8u);mx=max(mx,sv[i]);} {{ row_reduce(sw, "max", "mx") }} var alpha=1.0;if(mx>mUsed+TAU){alpha=exp(mUsed-mx);mUsed=mx;atomicOr(&rescaleFlag[t&1u],1u);}var sum=0.0;for(var i=0u;i<8u;i++){let pr=exp(sv[i]-mUsed);probs[probBase+srow*NK+slane+4u*i]=pr;sum+=pr;} {{ row_reduce(sw, "addf", "sum") }} l=l*alpha+sum;if(slane==0u){rowScale[rowSlot]=alpha;}
{% if not sw %}workgroupBarrier();
{% endif %}if(local_idx==0u){rescaleSeen=atomicLoad(&rescaleFlag[t&1u]);atomicStore(&rescaleFlag[(t+1u)&1u],0u);}if(workgroupUniformLoad(&rescaleSeen)!=0u){
{% for r in range(2) %}{% for c in range(CHUNKS) %}{{ stage_out(sw, r, c) }} workgroupBarrier();for(var e=lane;e<8u*SC;e+=32u){scores[stageAt+e]*=rowScale[(team*2u+{{ r }}u)*8u+e/SC];}workgroupBarrier();{{ stage_in(sw, r, c) }} workgroupBarrier();
{% endfor %}{% endfor %}}
{% for kk in range(4) %}{% if kk > 0 %}    if (slide || kf > {{ kk }}u) {
{% endif %}{
{% for r in range(2) %}let p{{ r }}={{ matrixLoad(sw, "left", "f32", "row_major", "&probs", "probBase + " ~ (r * 256 + kk * 8) ~ "u", "NK") }};
{% endfor %}let vAt=kvBase+(k0+{{ kk * 8 }}u)*KV_STRIDE+dBase;
{% for n in range(DF) %}{let v={{ matrixLoad(sw, "right", "f32", "row_major", "&cache_values", "vAt + " ~ (n * 8) ~ "u", "KV_STRIDE") }};
{% for r in range(2) %}o{{ r }}_{{ n }}={{ matrixMma(sw, "f32", "p" ~ r, "v", "o" ~ r ~ "_" ~ n) }};
{% endfor %}}
{% endfor %}}
{% if kk > 0 %}    }
{% endif %}{% endfor %}}if(slane==0u){rowScale[rowSlot]=select(1.0/l,0.0,l==0.0);}workgroupBarrier();
{% for r in range(2) %}{% for c in range(CHUNKS) %}{{ stage_out(sw, r, c) }} workgroupBarrier();for(var e=lane;e<8u*SC;e+=32u){let rr=e/SC;if(rr>=qFresh){let at=(q0+rr)*OUT_STRIDE+(headBase+{{ r }}u)*HEAD_DIM+dBase+{{ c * 32 }}u+e%SC;let value=scores[stageAt+e]*rowScale[(team*2u+{{ r }}u)*8u+rr];
{% if hasOutputGate %}attn_out[at]=value*(1.0/(1.0+exp(-gate[at])));
{% else %}attn_out[at]=value;
{% endif %}} }workgroupBarrier();
{% endfor %}{% endfor %}{% endmacro %}@compute@workgroup_size({{ sgGroup * 32 }},1,1){{ " @subgroup_size(32)" if pinSubgroupSize32 else "" }} fn main(@builtin(workgroup_id)workgroup_id:vec3<u32>,@builtin(local_invocation_index)local_idx:u32,@builtin(subgroup_size)sg_size:u32,@builtin(subgroup_invocation_id)subgroupLane:u32){let matrixRole=claimMatrixSubgroupRole(sg_size,32u);let matrixAcceptedHere=matrixTopologyAccepted(local_idx,GROUP);matrixSoftwareLane=local_idx%32u;if(matrixAcceptedHere){ {{ kernel_body(false) }} }else{let matrixRole=local_idx/32u;{{ kernel_body(true) }} } }`]]}],["com.xenova.CandidateArgMax",{manifest:{domain:"com.xenova",name:"CandidateArgMax",sinceVersion:1,inputs:{valuesT:{onnx:"CandidateValues",dtype:"float32",rank:1},indicesT:{onnx:"CandidateIndices",dtype:"uint32",rank:1}},outputs:{outT:{onnx:"Output",dtype:"uint32",rank:1}},args:{count:{kind:"u32",onnx:"candidate_count"},outputOffset:{kind:"u32",onnx:"output_offset",required:!1}},tunables:{WORKGROUP_SIZE:{default:256},CANDIDATES_PER_SEGMENT:{default:4096},SPLIT_MIN_COUNT:{default:16384}},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",outputOffset:"args.outputOffset if args.outputOffset else 0",workgroupSize:"min(tunables.WORKGROUP_SIZE, deviceWorkgroupCap)",powerOfTwoWorkgroup:"workgroupSize == 1 or workgroupSize == 2 or workgroupSize == 4 or workgroupSize == 8 or workgroupSize == 16 or workgroupSize == 32 or workgroupSize == 64 or workgroupSize == 128 or workgroupSize == 256",subgroupReduction:'device.features.has("subgroups")',portableWorkgroupStorageOk:"workgroupSize * 12 <= device.limits.maxComputeWorkgroupStorageSize",subgroupWorkgroupStorageOk:"workgroupSize * 12 <= device.limits.maxComputeWorkgroupStorageSize",reductionStorageOk:"subgroupWorkgroupStorageOk if subgroupReduction else portableWorkgroupStorageOk",packedSplit:"ceilDiv(args.count, max(1, tunables.CANDIDATES_PER_SEGMENT))"},when:["ranks.valuesT == 1","ranks.indicesT == 1","ranks.outT == 1",'tensorDtypes.valuesT == "float32"','tensorDtypes.indicesT == "uint32"','tensorDtypes.outT == "uint32"',"args.count > 0","dim(shapes.valuesT, 0) >= args.count","dim(shapes.indicesT, 0) >= args.count","dim(shapes.outT, 0) > outputOffset","powerOfTwoWorkgroup"],variants:[{id:"packed_atomic",priority:10,requires:{features:["atomic-vec2u-min-max","subgroups"]},when:["workgroupSize >= 4","packedSplit <= device.limits.maxComputeWorkgroupsPerDimension","args.count >= tunables.SPLIT_MIN_COUNT"],derive:{scalar:'"f32"',modeSpec:'"max"',selectLastIndex:!1,vec4:"args.count % 4 == 0 and dim(shapes.valuesT, 0) % 4 == 0 and dim(shapes.indicesT, 0) % 4 == 0",chunkCount:"args.count / 4 if vec4 else args.count",scoreElement:'"vec4<f32>" if vec4 else "f32"',idElement:'"vec4<u32>" if vec4 else "u32"',TOKEN_OUT_LEN:"outputOffset + 1"},intermediates:[{id:"winner",dtype:"uint32",shape:[2],zeroInit:!0}],passes:[{id:"reduce",name:"CandidateArgMax.PackedAtomic",shader:"candidate-argmax-atomic.wgsl.jinja",subgroupCollectivesWidth:"portable",bindings:[{arg:"valuesT",name:"values",elementType:"$scoreElement"},{arg:"indicesT",name:"indices",elementType:"$idElement"},{scratch:"winner",name:"winner",buffer:"storage",elementType:"atomic<vec2<u32>>",length:1},{name:"params",struct:[{name:"chunkCount",type:"u32",value:"chunkCount"}]}],dispatch:{x:"packedSplit"}},{id:"decode",name:"CandidateArgMax.Decode",shader:"candidate-argmax-atomic-decode.wgsl.jinja",bindings:[{scratch:"winner",name:"winner",buffer:"read-only-storage",elementType:"vec2<u32>",length:1},{arg:"outT",name:"token_out",length:"$TOKEN_OUT_LEN"}],dispatch:{x:1}}]},{id:"split_typed",priority:5,when:["packedSplit <= device.limits.maxComputeWorkgroupsPerDimension","reductionStorageOk","args.count >= tunables.SPLIT_MIN_COUNT"],derive:{vec4:"args.count % 4 == 0 and dim(shapes.valuesT, 0) % 4 == 0 and dim(shapes.indicesT, 0) % 4 == 0",chunkCount:"args.count / 4 if vec4 else args.count",scoreElement:'"vec4<f32>" if vec4 else "f32"',idElement:'"vec4<u32>" if vec4 else "u32"',TOKEN_OUT_LEN:"outputOffset + 1",numBlocks:"packedSplit",useSubgroups:"subgroupReduction"},intermediates:[{id:"partial_scores",dtype:"float32",shape:"[packedSplit]"},{id:"partial_ids",dtype:"uint32",shape:"[packedSplit]"},{id:"partial_live",dtype:"uint32",shape:"[packedSplit]"}],passes:[{id:"partials",name:"CandidateArgMax.SplitTyped",shader:"candidate-argmax-split.wgsl.jinja",subgroupCollectivesWidth:"portable",bindings:[{arg:"valuesT",name:"values",elementType:"$scoreElement"},{arg:"indicesT",name:"indices",elementType:"$idElement"},{scratch:"partial_scores",name:"scores_out",buffer:"storage"},{scratch:"partial_ids",name:"ids_out",buffer:"storage"},{scratch:"partial_live",name:"valid_out",buffer:"storage"},{name:"params",struct:[{name:"chunkCount",type:"u32",value:"chunkCount"}]}],dispatch:{x:"packedSplit"}},{id:"finalize",name:"CandidateArgMax.SplitFinalize",shader:"candidate-argmax-split-finalize.wgsl.jinja",subgroupCollectivesWidth:"portable",bindings:[{scratch:"partial_scores",name:"best_values_in",buffer:"read-only-storage"},{scratch:"partial_ids",name:"best_ids_in",buffer:"read-only-storage"},{scratch:"partial_live",name:"valid_in",buffer:"read-only-storage"},{arg:"outT",name:"token_out",length:"$TOKEN_OUT_LEN"}],dispatch:{x:1}}]},{id:"reduce",priority:0,when:["reductionStorageOk"],derive:{numBlocks:"args.count",useSubgroups:"subgroupReduction",trackValidity:!0,TOKEN_OUT_LEN:"outputOffset + 1"},passes:[{id:"main",name:"CandidateArgMax",shader:"ops/_shared/matmul-argmax-reduce.wgsl.jinja",bindings:[{arg:"valuesT",name:"best_values_in"},{arg:"indicesT",name:"best_ids_in"},{arg:"outT",name:"token_out",length:"$TOKEN_OUT_LEN"}],dispatch:{x:1},subgroupCollectivesWidth:"portable"}]}]},assets:[["candidate-argmax-atomic-decode.wgsl.jinja","{{ env.wgsl.resourceDeclarations }}@compute@workgroup_size(1)fn main(){token_out[{{ outputOffset }}u]=~winner[0].x;}"],["candidate-argmax-atomic.wgsl.jinja",`enable atomic_vec2u_min_max;enable subgroups;{{ env.wgsl.resourceDeclarations }} const WG:u32={{ workgroupSize }}u;const SPLIT:u32={{ packedSplit }}u;
{% include "ops/_shared/arg-reduction-packed-key.wgsl.jinja" %}{% include "ops/_shared/arg-reduction-packed-key-compare.wgsl.jinja" %}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_index)tid:u32){let quotient=params.chunkCount/SPLIT;let remainder=params.chunkCount%SPLIT;let begin=wid.x*quotient+min(wid.x,remainder);let end=begin+quotient+select(0u,1u,wid.x<remainder);let length=end-begin;var best=vec2<u32>(0u);for(var offset=tid;offset<length;){let chunk=begin+offset;
{% if vec4 %}let value4=values[chunk];let index4=indices[chunk];
{% for component in ["x", "y", "z", "w"] %}{let key=candidate_key(value4.{{ component }},index4.{{ component }});if(better_key(key,best)){best=key;} }
{% endfor %}{% else %}let key=candidate_key(values[chunk],indices[chunk]);if(better_key(key,best)){best=key;}
{% endif %}if(length-offset<=WG){break;}offset+=WG;}let high=subgroupMax(best.y);let low=subgroupMax(select(0u,best.x,best.y==high));if(subgroupElect()&&high!=0u){atomicStoreMax(&winner[0],vec2<u32>(low,high));} }`],["candidate-argmax-split-finalize.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const NUM_BLOCKS:u32={{ numBlocks }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const OUTPUT_OFFSET:u32={{ outputOffset }}u;const INVALID_INDEX:u32=0xffffffffu;
{% if useSubgroups %}{% include "ops/_shared/float-neg-inf.wgsl.jinja" %}{% endif %}{% if useSubgroups %}var<workgroup>partial_values:array<f32,{{ workgroupSize }}>;var<workgroup>partial_ids:array<u32,{{ workgroupSize }}>;var<workgroup>partial_valid:array<u32,{{ workgroupSize }}>;
{% else %}var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;var<workgroup>best_valid:array<u32,{{ workgroupSize }}>;
{% endif %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(local_invocation_id)lid3:vec3<u32>){let lid=lid3.x;var value=0.0;var index=INVALID_INDEX;var has_best=false;for(var i=lid;i<NUM_BLOCKS;i=i+WORKGROUP_SIZE){let v=best_values_in[i];let id=best_ids_in[i];if(valid_in[i]!=0u&&(!has_best||v>value||(v==value&&id<index))){value=v;index=id;has_best=true;} }
{% if useSubgroups %}let sg_best=subgroupMax(select(negative_infinity(),value,has_best));let sg_token=subgroupMin(select(INVALID_INDEX,index,has_best&&value==sg_best));let sg_valid=subgroupMax(select(0u,1u,has_best));let elected=subgroupElect();partial_values[lid]=select(negative_infinity(),sg_best,elected);partial_ids[lid]=select(INVALID_INDEX,sg_token,elected);partial_valid[lid]=select(0u,sg_valid,elected);workgroupBarrier();if(lid==0u){var best_value=0.0;var best_index=INVALID_INDEX;var has_valid=false;for(var i=0u;i<WORKGROUP_SIZE;i=i+1u){let v=partial_values[i];let id=partial_ids[i];if(partial_valid[i]!=0u&&(!has_valid||v>best_value||(v==best_value&&id<best_index))){best_value=v;best_index=id;has_valid=true;} }token_out[OUTPUT_OFFSET]=best_index;}
{% else %}best_values[lid]=value;best_indices[lid]=index;best_valid[lid]=select(0u,1u,has_best);workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){let other_valid=best_valid[lid+stride];let other_value=best_values[lid+stride];let other_index=best_indices[lid+stride];if(other_valid!=0u&&(best_valid[lid]==0u||other_value>best_values[lid]||(other_value==best_values[lid]&&other_index<best_indices[lid]))){best_values[lid]=other_value;best_indices[lid]=other_index;best_valid[lid]=1u;} }workgroupBarrier();stride=stride/2u;}if(lid==0u){token_out[OUTPUT_OFFSET]=best_indices[0];}
{% endif %}}`],["candidate-argmax-split.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% include "ops/_shared/float-is-nan.wgsl.jinja" %}const INVALID_INDEX:u32=0xffffffffu;
{% if useSubgroups %}{% include "ops/_shared/float-neg-inf.wgsl.jinja" %}{% endif %}{% if useSubgroups %}var<workgroup>partial_values:array<f32,{{ workgroupSize }}>;var<workgroup>partial_ids:array<u32,{{ workgroupSize }}>;var<workgroup>partial_valid:array<u32,{{ workgroupSize }}>;
{% else %}var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;var<workgroup>best_valid:array<u32,{{ workgroupSize }}>;
{% endif %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let lid=lid3.x;var value=0.0;var index=INVALID_INDEX;var has_best=false;let quotient=params.chunkCount/{{ packedSplit }}u;let remainder=params.chunkCount% {{ packedSplit }}u;let begin=wid.x*quotient+min(wid.x,remainder);let end=begin+quotient+select(0u,1u,wid.x<remainder);let length=end-begin;for(var offset=lid;offset<length;){let chunk=begin+offset;
{% if vec4 %}let value4=values[chunk];let index4=indices[chunk];
{% for component in ["x", "y", "z", "w"] %}{let v=value4.{{ component }};let id=index4.{{ component }};if(!is_nan_f32(v)&&(!has_best||v>value||(v==value&&id<index))){value=v;index=id;has_best=true;} }
{% endfor %}{% else %}let v=values[chunk];let id=indices[chunk];if(!is_nan_f32(v)&&(!has_best||v>value||(v==value&&id<index))){value=v;index=id;has_best=true;}
{% endif %}if(length-offset<=WORKGROUP_SIZE){break;}offset+=WORKGROUP_SIZE;}
{% if useSubgroups %}let sg_best=subgroupMax(select(negative_infinity(),value,has_best));let sg_token=subgroupMin(select(INVALID_INDEX,index,has_best&&value==sg_best));let sg_valid=subgroupMax(select(0u,1u,has_best));let elected=subgroupElect();partial_values[lid]=select(negative_infinity(),sg_best,elected);partial_ids[lid]=select(INVALID_INDEX,sg_token,elected);partial_valid[lid]=select(0u,sg_valid,elected);workgroupBarrier();if(lid==0u){var best_value=0.0;var best_index=INVALID_INDEX;var has_valid=false;for(var i=0u;i<WORKGROUP_SIZE;i=i+1u){let v=partial_values[i];let id=partial_ids[i];if(partial_valid[i]!=0u&&(!has_valid||v>best_value||(v==best_value&&id<best_index))){best_value=v;best_index=id;has_valid=true;} }scores_out[wid.x]=best_value;ids_out[wid.x]=best_index;valid_out[wid.x]=select(0u,1u,has_valid);}
{% else %}best_values[lid]=value;best_indices[lid]=index;best_valid[lid]=select(0u,1u,has_best);workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){let other_valid=best_valid[lid+stride];let other_value=best_values[lid+stride];let other_index=best_indices[lid+stride];if(other_valid!=0u&&(best_valid[lid]==0u||other_value>best_values[lid]||(other_value==best_values[lid]&&other_index<best_indices[lid]))){best_values[lid]=other_value;best_indices[lid]=other_index;best_valid[lid]=1u;} }workgroupBarrier();stride=stride/2u;}if(lid==0u){scores_out[wid.x]=best_values[0];ids_out[wid.x]=best_indices[0];valid_out[wid.x]=best_valid[0];}
{% endif %}}`]]}],["com.xenova.LlamaDecodeGateUp",{manifest:{domain:"com.xenova",name:"LlamaDecodeGateUp",sinceVersion:1,inputs:{normedT:{onnx:"Normed",dtype:"N",rank:1,optional:!0},hiddenT:{onnx:"Hidden",dtype:"H",rank:1,optional:!0},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},NormWeights:{dtype:"W",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},upBitsT:{onnx:"UpBits",dtype:"uint32",rank:1,optional:!0},upScalesT:{onnx:"UpScales",dtype:"U",rank:1,optional:!0},gridT:{onnx:"Grid",dtype:"uint32",rank:1,optional:!0},upGridT:{onnx:"UpGrid",dtype:"uint32",rank:1,optional:!0}},outputs:{intermediateT:{onnx:"Intermediate",dtype:"I",rank:1,shape:["tokenRows * args.intermediateSize"]}},args:{hiddenSize:{kind:"u32",onnx:"hidden_size",required:!0},intermediateSize:{kind:"u32",onnx:"intermediate_size",required:!0},gateOffset:{kind:"u32",onnx:"gate_proj_offset",required:!0},upOffset:{kind:"u32",onnx:"up_proj_offset",required:!0},rmsEps:{kind:"f32",onnx:"rms_eps",required:!1},normOffset:{kind:"u32",onnx:"input_norm_offset",required:!1},activation:{kind:"string",required:!1,oneOf:["silu","fast_silu"]},strategy:{kind:"string",onnx:"schedule_strategy",required:!1,oneOf:["generic","latency"]},format:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","lut2_32","lut2_16","q3_k","q2_k","q6_k","q5_k","q1_0"]},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1},upFormat:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","lut2_32","lut2_16","q3_k","q2_k","q6_k","q5_k"]},upLut:{kind:"u32",onnx:"dequant_lut_id",required:!1},iqRaw:{kind:"u32",onnx:"iq_raw_pack_format",required:!1},tokens:{kind:"u32",required:!1},upIqRaw:{kind:"u32",onnx:"iq_raw_pack_format",required:!1}},typeConstraints:{N:["float32","float16"],H:["float32","float16"],S:["float32","float16","uint32"],W:["float32","float16"],I:["float32","float16"],U:["float32","float16","uint32"]},derive:{tokenRows:"args.tokens if args.tokens is defined else 1",fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',lutcWords:"1 if (lut2 == 1 and args.lut == 9) else (5 if lutc16 == 1 else 3)",q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWave32:'device.features.has("subgroups") and wave32Adapter',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32SubgroupsUsable:"subgroupsWave32 or pinSubgroupSize32",subgroupIdBuiltins:'device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id")',deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",subgroupCoveringWorkgroup:'min(deviceWorkgroupCap, max(32, device.adapterInfo.subgroupMinSize)) if (device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize")) else 32',sgRuntime:'device.features.has("subgroups") and not wave32SubgroupsUsable',iqRawSel:"args.iqRaw if args.iqRaw else 0",upIqRawSel:"args.upIqRaw if args.upIqRaw else 0",gateUpMaxBlocks:"max(args.gateOffset + args.intermediateSize * (args.hiddenSize / 32), args.upOffset + args.intermediateSize * (args.hiddenSize / 32)) if (args.gateOffset is defined and args.upOffset is defined and args.intermediateSize is defined and args.hiddenSize is defined) else 0",gateUpSuperblocks:"ceilDiv(gateUpMaxBlocks, 8)",gateMaxBlocks:"args.gateOffset + args.intermediateSize * (args.hiddenSize / 32) if (args.gateOffset is defined and args.intermediateSize is defined and args.hiddenSize is defined) else 0",upMaxBlocks:"args.upOffset + args.intermediateSize * (args.hiddenSize / 32) if (args.upOffset is defined and args.intermediateSize is defined and args.hiddenSize is defined) else 0",gateSuperblocks:"ceilDiv(gateMaxBlocks, 8)",upSuperblocks:"ceilDiv(upMaxBlocks, 8)",q4kncSel:'1 if fmt == "q4_k_native_compact" else 0',upFmt:"args.upFormat if args.upFormat else fmt",upQuantBits:'8 if (upFmt == "q8_0" or upFmt == "iq") else 4',upHasMin:'1 if upFmt == "q4_k" else 0',upIq:'1 if upFmt == "iq" else 0',upQ6k:'1 if upFmt == "q6_k" else 0',upQ5k:'1 if upFmt == "q5_k" else 0',upQ3k:'1 if upFmt == "q3_k" else 0',upQ2k:'1 if upFmt == "q2_k" else 0',upQ4knc:'1 if upFmt == "q4_k_native_compact" else 0',upLut16:'1 if upFmt == "lut16" else 0',upLut2:'1 if (upFmt == "lut2_32" or upFmt == "lut2_16") else 0',upLutc:'1 if (upFmt == "lut32_compact" or upFmt == "lut16_compact" or upLut2 == 1) else 0',upLutc16:'1 if (upFmt == "lut16_compact" or upFmt == "lut2_16") else 0',upLutcWords:"1 if (upLut2 == 1 and args.upLut == 9) else (5 if upLutc16 == 1 else 3)",gateBitsNeeded:"(gateSuperblocks * 14 if iqRawSel == 8 else (gateSuperblocks * 12 if iqRawSel == 7 else (gateSuperblocks * 27 if iqRawSel == 4 else (gateSuperblocks * 18 if iqRawSel == 6 else (gateSuperblocks * 20 if iqRawSel == 5 else (gateMaxBlocks * 2 if (iqRawSel == 1 or iqRawSel == 2) else gateMaxBlocks * 3)))))) if iqRawSel else (gateSuperblocks * 52 if q6k else (gateSuperblocks * 44 if q5k else (gateSuperblocks * 28 if q3k else (gateSuperblocks * 20 if q2k else (gateMaxBlocks * (2 if lut2 else 4) if (lutc or q4kncSel) else gateMaxBlocks * quantBits)))))",gateScalesNeeded:"gateSuperblocks if iqRawSel else (gateSuperblocks if (q6k or q3k) else (gateSuperblocks * 2 if (q5k or q2k) else (gateSuperblocks * lutcWords if lutc else (gateSuperblocks * 5 if q4kncSel else gateMaxBlocks * (2 if (hasMin or iq or lut16) else 1)))))",upBitsNeeded:"(upSuperblocks * 14 if upIqRawSel == 8 else (upSuperblocks * 12 if upIqRawSel == 7 else (upSuperblocks * 27 if upIqRawSel == 4 else (upSuperblocks * 18 if upIqRawSel == 6 else (upSuperblocks * 20 if upIqRawSel == 5 else (upMaxBlocks * 2 if (upIqRawSel == 1 or upIqRawSel == 2) else upMaxBlocks * 3)))))) if upIqRawSel else (upSuperblocks * 52 if upQ6k else (upSuperblocks * 44 if upQ5k else (upSuperblocks * 28 if upQ3k else (upSuperblocks * 20 if upQ2k else (upMaxBlocks * (2 if upLut2 else 4) if (upLutc or upQ4knc) else upMaxBlocks * upQuantBits)))))",upScalesNeeded:"upSuperblocks if upIqRawSel else (upSuperblocks if (upQ6k or upQ3k) else (upSuperblocks * 2 if (upQ5k or upQ2k) else (upSuperblocks * upLutcWords if upLutc else (upSuperblocks * 5 if upQ4knc else upMaxBlocks * (2 if (upHasMin or upIq or upLut16) else 1)))))",gateScalesDtypeOk:'tensorDtypes.scalesT == "uint32" if (lutc or q4kncSel) else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',upScalesDtypeOk:'tensorDtypes.upScalesT == "uint32" if (upLutc or upQ4knc) else (tensorDtypes.upScalesT == "float32" or tensorDtypes.upScalesT == "float16")',gateNibbleFam:'0 if iqRawSel else (1 if (fmt == "q4_0" or fmt == "q4_k" or fmt == "q4_k_native_compact") else 0)',upNibbleFam:'0 if upIqRawSel else (1 if (upFmt == "q4_0" or upFmt == "q4_k" or upFmt == "q4_k_native_compact") else 0)',splitRoute:'present.bitsT and present.scalesT and present.upBitsT and present.upScalesT and args.upFormat is defined and fmt != "q1_0" and upFmt != "q1_0" and fmt != "q5_0" and upFmt != "q5_0" and fmt != "q8_0_pair" and upFmt != "q8_0_pair" and (iqRawSel == 0 or iqRawSel == 1 or iqRawSel == 2 or iqRawSel == 3 or iqRawSel == 4 or iqRawSel == 5 or iqRawSel == 6 or iqRawSel == 7 or iqRawSel == 8) and (upIqRawSel == 0 or upIqRawSel == 1 or upIqRawSel == 2 or upIqRawSel == 3 or upIqRawSel == 4 or upIqRawSel == 5 or upIqRawSel == 6 or upIqRawSel == 7 or upIqRawSel == 8)',mixedRawPair:"iqRawSel != 0 and upIqRawSel != 0 and iqRawSel != upIqRawSel",firstRawSel:"iqRawSel if iqRawSel else upIqRawSel",gridElemsNeeded:"1024 if firstRawSel == 6 else (4096 if (firstRawSel == 1 or firstRawSel == 7 or firstRawSel == 8) else (2048 if firstRawSel == 5 else (512 if firstRawSel == 4 else (512 if firstRawSel == 2 else 256))))",upGridElemsNeeded:"1024 if upIqRawSel == 6 else (4096 if (upIqRawSel == 1 or upIqRawSel == 7 or upIqRawSel == 8) else (2048 if upIqRawSel == 5 else (512 if upIqRawSel == 4 else (512 if upIqRawSel == 2 else 256))))",splitPackContract:'splitRoute and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.upBitsT == 1 and ranks.upScalesT == 1 and tensorDtypes.bitsT == "uint32" and tensorDtypes.upBitsT == "uint32" and gateScalesDtypeOk and upScalesDtypeOk and (args.hiddenSize % 256 == 0 if (lutc or q4kncSel or upLutc or upQ4knc or q2k or upQ2k) else args.hiddenSize % 32 == 0) and (args.gateOffset % 8 == 0 if (q6k or q5k or q3k or q2k or lutc or q4kncSel) else true) and (args.upOffset % 8 == 0 if (upQ6k or upQ5k or upQ3k or upQ2k or upLutc or upQ4knc) else true) and (args.lut is defined and args.lut != 0 if lutc else true) and (args.upLut is defined and args.upLut != 0 if upLutc else true) and (quantBits == 4 or quantBits == 8) and (upQuantBits == 4 or upQuantBits == 8) and (dim(shapes.bitsT, 0) % (2 if lut2 else 4) == 0 if iqRawSel == 0 else true) and (dim(shapes.upBitsT, 0) % (2 if upLut2 else 4) == 0 if upIqRawSel == 0 else true) and dim(shapes.bitsT, 0) >= gateBitsNeeded and dim(shapes.scalesT, 0) >= gateScalesNeeded and dim(shapes.upBitsT, 0) >= upBitsNeeded and dim(shapes.upScalesT, 0) >= upScalesNeeded',preNormalizedMode:"present.normedT and not present.hiddenT and args.rmsEps is undefined and args.normOffset is undefined",fusedRmsNormMode:"present.hiddenT and not present.normedT and present.weightsT and args.rmsEps is defined and args.normOffset is defined",denseRoute:"present.weightsT and not present.bitsT and not present.scalesT and not present.upBitsT and not present.upScalesT",q2Route:'present.bitsT and present.scalesT and present.upBitsT and present.upScalesT and fmt == "q2_k" and iqRawSel == 0',q4Route:'present.bitsT and present.scalesT and not present.upBitsT and not present.upScalesT and fmt != "q1_0" and fmt != "q4_k_native_compact" and fmt != "q5_0" and fmt != "q8_0_pair" and lutc == 0 and iqRawSel == 0',q4knCompactRoute:'present.bitsT and present.scalesT and not present.upBitsT and not present.upScalesT and fmt == "q4_k_native_compact" and iqRawSel == 0',lutcRoute:"present.bitsT and present.scalesT and not present.upBitsT and not present.upScalesT and lutc == 1 and iqRawSel == 0",q1Route:'present.bitsT and present.scalesT and not present.upBitsT and not present.upScalesT and fmt == "q1_0" and iqRawSel == 0',q2PackContract:'q2Route and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.upBitsT == 1 and ranks.upScalesT == 1 and tensorDtypes.bitsT == "uint32" and tensorDtypes.upBitsT == "uint32" and tensorDtypes.scalesT == tensorDtypes.upScalesT and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and args.hiddenSize % 256 == 0 and args.gateOffset % 8 == 0 and args.upOffset % 8 == 0 and dim(shapes.bitsT, 0) >= gateSuperblocks * 20 and dim(shapes.scalesT, 0) >= gateSuperblocks * 2 and dim(shapes.upBitsT, 0) >= upSuperblocks * 20 and dim(shapes.upScalesT, 0) >= upSuperblocks * 2',q4PackContract:'q4Route and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and args.hiddenSize % 32 == 0 and (quantBits == 4 or quantBits == 8) and (not (q6k or q5k or q3k or q2k) or (args.gateOffset % 8 == 0 and args.upOffset % 8 == 0)) and dim(shapes.bitsT, 0) >= (gateUpSuperblocks * 52 if q6k else (gateUpSuperblocks * 44 if q5k else (gateUpSuperblocks * 28 if q3k else (gateUpSuperblocks * 20 if q2k else gateUpMaxBlocks * quantBits)))) and dim(shapes.scalesT, 0) >= (gateUpSuperblocks if (q6k or q3k) else (gateUpSuperblocks * 2 if (q5k or q2k) else gateUpMaxBlocks * (2 if (hasMin or iq or lut16) else 1)))',q4knCompactPackContract:'q4knCompactRoute and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and args.hiddenSize % 256 == 0 and args.gateOffset % 8 == 0 and args.upOffset % 8 == 0 and dim(shapes.bitsT, 0) % 4 == 0 and dim(shapes.bitsT, 0) >= gateUpMaxBlocks * 4 and dim(shapes.scalesT, 0) >= gateUpSuperblocks * 5',lutcPackContract:'lutcRoute and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and args.lut is defined and args.lut != 0 and args.hiddenSize % 256 == 0 and args.gateOffset % 8 == 0 and args.upOffset % 8 == 0 and dim(shapes.bitsT, 0) % (2 if lut2 else 4) == 0 and dim(shapes.bitsT, 0) >= gateUpMaxBlocks * (2 if lut2 else 4) and dim(shapes.scalesT, 0) >= gateUpSuperblocks * lutcWords',rawBitsNeeded:"gateUpMaxBlocks * (3 if iqRawSel == 3 else 2) if iqRawSel <= 3 else gateUpSuperblocks * (27 if iqRawSel == 4 else (20 if iqRawSel == 5 else (18 if iqRawSel == 6 else (12 if iqRawSel == 7 else 14))))",rawPackContract:'iqRawSel >= 1 and iqRawSel <= 8 and present.bitsT and present.scalesT and not present.upBitsT and not present.upScalesT and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and args.hiddenSize % 256 == 0 and args.gateOffset % 8 == 0 and args.upOffset % 8 == 0 and dim(shapes.bitsT, 0) >= rawBitsNeeded and dim(shapes.scalesT, 0) >= gateUpSuperblocks',preNormalizedContract:"preNormalizedMode and ranks.normedT == 1 and ranks.intermediateT == 1 and args.hiddenSize > 0 and args.intermediateSize > 0 and dim(shapes.normedT, 0) == tokenRows * args.hiddenSize and dim(shapes.intermediateT, 0) == tokenRows * args.intermediateSize and tokenRows >= 1 and tokenRows <= 4",fusedRmsNormContract:'fusedRmsNormMode and ranks.hiddenT == 1 and ranks.weightsT == 1 and ranks.intermediateT == 1 and (tensorDtypes.hiddenT == "float32" or tensorDtypes.hiddenT == "float16") and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and args.hiddenSize > 0 and args.intermediateSize > 0 and args.normOffset % 4 == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.intermediateT, 0) == args.intermediateSize',intermediateScalar:"dtypes.I",intermediateSize:"args.intermediateSize",bitsElem:'"u32" if lut2 else "vec4<u32>"'},bindings:{normed:{arg:"normedT",elementType:"$normedVec4"},weights:{arg:"weightsT",elementType:"$weightVec4"},intermediate:{arg:"intermediateT",elementType:"$intermediateScalar"},q4_bits:{arg:"bitsT",elementType:"$bitsElem"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},q4_bits_main:{arg:"bitsT",name:"q4_bits",elementType:"$splitBitsElem"},up_bits:{arg:"upBitsT",elementType:"$splitUpBitsElem"},up_scales:{arg:"upScalesT",elementType:"$upScaleScalar"},iq_grid_data:{arg:"gridT",elementType:"vec2<u32>"},q1_bits:{arg:"bitsT",elementType:"vec4<u32>"},q1_scales:{arg:"scalesT",elementType:"$scaleScalar"},hidden:{arg:"hiddenT",elementType:"$hiddenVec4"},norm_weight:{arg:"weightsT",elementType:"$normWeightVec4"},q4_bits_raw:{arg:"bitsT",name:"q4_bits",elementType:"$rawBitsElem"}},variants:[{id:"dense",priority:0,when:["preNormalizedContract","denseRoute","ranks.weightsT == 1",'(tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")',"f16Ok(tensorDtypes.normedT) and f16Ok(tensorDtypes.weightsT) and f16Ok(tensorDtypes.intermediateT)","args.hiddenSize % 4 == 0","args.gateOffset % 4 == 0","args.upOffset % 4 == 0","dim(shapes.weightsT, 0) >= max(args.gateOffset + args.intermediateSize * args.hiddenSize, args.upOffset + args.intermediateSize * args.hiddenSize)","tokenRows == 1"],derive:{normedVec4:'"vec4<f16>" if tensorDtypes.normedT == "float16" else "vec4<f32>"',weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',weightScalar:"dtypes.W",hiddenSizeVec4:"args.hiddenSize / 4",gateOffsetVec4:"args.gateOffset / 4",upOffsetVec4:"args.upOffset / 4",workgroupSize:64},passes:[{id:"main",name:"LlamaDecodeGateUp",shader:"decode-gate-up.wgsl.jinja",bindings:["normed","weights","intermediate"],dispatch:{x:"args.intermediateSize"}}]},{id:"q4",priority:0,when:["preNormalizedContract","q4PackContract","f16Ok(tensorDtypes.normedT) and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.intermediateT)","(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"],tunables:{GATE_UP_TILE_ROWS:{default:4},GATE_UP_TILE_ROWS_NARROW:{default:1},GATE_UP_WIDE_HIDDEN:{default:2048},GATE_UP_TILE_ROWS_KQ4:{default:8},GATE_UP_TILE_ROWS_KQ4_NARROW:{default:1},GATE_UP_KQ4_WIDE_HIDDEN:{default:2048},GATE_UP_TILE_ROWS_Q2K_SHARED:{default:1}},derive:{tileRows:'(tunables.GATE_UP_TILE_ROWS_Q2K_SHARED if q2k else ((tunables.GATE_UP_TILE_ROWS_KQ4 if args.hiddenSize >= tunables.GATE_UP_KQ4_WIDE_HIDDEN else tunables.GATE_UP_TILE_ROWS_KQ4_NARROW) if hasMin else (tunables.GATE_UP_TILE_ROWS if args.hiddenSize >= tunables.GATE_UP_WIDE_HIDDEN else tunables.GATE_UP_TILE_ROWS_NARROW))) if (has(device.adapterInfo, "vendor") and device.adapterInfo.vendor == "apple") else tunables.GATE_UP_TILE_ROWS',scaleScalar:"dtypes.S",normedVec4:'"vec4<f16>" if tensorDtypes.normedT == "float16" else "vec4<f32>"',lutId:"args.lut if args.lut else 0",preNormed:"1",mRows:"tokenRows",useSubgroups:"wave32SubgroupsUsable",workgroupSize:"subgroupCoveringWorkgroup",blocksPerRow:"args.hiddenSize / 32",gateOffset:"args.gateOffset",upOffset:"args.upOffset"},passes:[{id:"main",name:"LlamaDecodeGateUpQ4",shader:"decode-gate-up-q4.wgsl.jinja",bindings:["normed","q4_bits","q4_scales","intermediate"],dispatch:{x:"ceil(args.intermediateSize / tileRows)"},subgroupCollectivesWidth:"portable"}]},{cases:[{id:"q4_split_grid",passes_when:["present.gridT","ranks.gridT == 1",'tensorDtypes.gridT == "uint32"',"dim(shapes.gridT, 0) >= gridElemsNeeded","not mixedRawPair"],passes_iqDot:"iqRawSel != 0 or upIqRawSel != 0",passes_splitBitsElem:'"u32" if (iqRawSel or lut2) else "vec4<u32>"',passes_splitUpBitsElem:'"u32" if (upIqRawSel or upLut2) else "vec4<u32>"',passes_iqGridBound:!0,main_name:"LlamaDecodeGateUpQ4SplitGrid",main_bindings:["iq_grid_data"],sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{id:"q4_split_grid_pair",passes_when:["mixedRawPair","present.gridT","ranks.gridT == 1",'tensorDtypes.gridT == "uint32"',"dim(shapes.gridT, 0) >= gridElemsNeeded","present.upGridT","ranks.upGridT == 1",'tensorDtypes.upGridT == "uint32"',"dim(shapes.upGridT, 0) >= upGridElemsNeeded"],passes_iqDot:!0,passes_splitBitsElem:'"u32"',passes_splitUpBitsElem:'"u32"',passes_iqGridBound:!0,main_name:"LlamaDecodeGateUpQ4SplitGridPair",main_bindings:["iq_grid_data",{arg:"upGridT",name:"iq_grid_data_u",elementType:"vec2<u32>"}],sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{id:"q4_split",passes_when:[],passes_iqDot:"iqRawSel != 0 or upIqRawSel != 0",passes_splitBitsElem:'"u32" if (iqRawSel or lut2) else "vec4<u32>"',passes_splitUpBitsElem:'"u32" if (upIqRawSel or upLut2) else "vec4<u32>"',passes_iqGridBound:null,main_name:"LlamaDecodeGateUpQ4Split",main_bindings:[],sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"{id}",priority:2,when:["preNormalizedContract","splitPackContract","f16Ok(tensorDtypes.normedT) and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.upScalesT) and f16Ok(tensorDtypes.intermediateT)","{passes_when}","{sgFoldStorageGate0}"],tunables:{GATE_UP_TILE_ROWS_SPLIT:{default:1},GATE_UP_TILE_ROWS:{default:4},GATE_UP_TILE_ROWS_NARROW:{default:1},GATE_UP_WIDE_HIDDEN:{default:2048},GATE_UP_TILE_ROWS_KQ4:{default:8},GATE_UP_TILE_ROWS_KQ4_NARROW:{default:1},GATE_UP_KQ4_WIDE_HIDDEN:{default:2048},GATE_UP_TILE_ROWS_Q2K_SHARED:{default:1}},derive:{tileRows:'tunables.GATE_UP_TILE_ROWS_SPLIT if (has(device.adapterInfo, "vendor") and device.adapterInfo.vendor == "apple") else tunables.GATE_UP_TILE_ROWS',scaleScalar:'"u32" if (lutc or q4kncSel) else dtypes.S',upScaleScalar:'"u32" if (upLutc or upQ4knc) else dtypes.U',normedVec4:'"vec4<f16>" if tensorDtypes.normedT == "float16" else "vec4<f32>"',gateUpSplit:!0,q4knCompact:"q4kncSel",lutId:"args.lut if args.lut else 0",upLutId:"args.upLut if args.upLut else 0",preNormed:1,mRows:"tokenRows",gateUpIq1m:"iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8",upIq1m:"upIqRawSel == 1 or upIqRawSel == 7 or upIqRawSel == 8",gateUpIq2xxs:"iqRawSel == 2",upIq2xxs:"upIqRawSel == 2",gateUpIq2xs:"iqRawSel == 6",upIq2xs:"upIqRawSel == 6",gateUpIq3xxs:"iqRawSel == 3",upIq3xxs:"upIqRawSel == 3",gateUpIq3s:"iqRawSel == 4",gateUpIq2s:"iqRawSel == 5",upIq3s:"upIqRawSel == 4",upIq2s:"upIqRawSel == 5",iqDot:"{passes_iqDot}",splitBitsElem:"{passes_splitBitsElem}",splitUpBitsElem:"{passes_splitUpBitsElem}",iqGridBound:"{passes_iqGridBound}",gateUpIq1s:"iqRawSel == 7",gateUpIq1md:"iqRawSel == 8",upIq1s:"upIqRawSel == 7",upIq1md:"upIqRawSel == 8",useSubgroups:"wave32SubgroupsUsable",workgroupSize:"subgroupCoveringWorkgroup",blocksPerRow:"args.hiddenSize / 32",gateOffset:"args.gateOffset",upOffset:"args.upOffset"},passes:[{id:"main",name:"{main_name}",shader:"decode-gate-up-q4.wgsl.jinja",bindings:["normed","q4_bits_main","q4_scales","up_bits","up_scales","intermediate","{main_bindings}"],dispatch:{x:"ceil(args.intermediateSize / tileRows)"},subgroupCollectivesWidth:"portable"}]}]},{cases:[{id:"q4_lut32",passes_when:["q4PackContract",'fmt == "lut32"',"args.lut is defined and args.lut != 0",'args.strategy is defined and args.strategy == "latency"',"f16Ok(tensorDtypes.normedT) and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.intermediateT)"],passes_tileRows:'tunables.GATE_UP_TILE_ROWS_LUT32 if tunables.GATE_UP_TILE_ROWS_LUT32 > 0 else (1 if (has(device.adapterInfo, "vendor") and device.adapterInfo.vendor == "apple") else 4)',passes_scaleScalar:"dtypes.S",passes_normedVec4:'"vec4<f16>" if tensorDtypes.normedT == "float16" else "vec4<f32>"',passes_lutId:"args.lut",passes_q4knCompact:null,passes_lutc:null,main_name:"LlamaDecodeGateUpLut32",routeTunables:{GATE_UP_TILE_ROWS_LUT32:{default:0}},sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{id:"q4kn_compact",passes_when:["q4knCompactPackContract",'tensorDtypes.normedT == "float32" and tensorDtypes.intermediateT == "float32"'],passes_tileRows:'tunables.GATE_UP_TILE_ROWS_Q4KN if tunables.GATE_UP_TILE_ROWS_Q4KN > 0 else (1 if (has(device.adapterInfo, "vendor") and device.adapterInfo.vendor == "apple") else 4)',passes_scaleScalar:'"u32"',passes_normedVec4:'"vec4<f32>"',passes_lutId:0,passes_q4knCompact:!0,passes_lutc:null,main_name:"LlamaDecodeGateUpQ4KNativeCompact",routeTunables:{GATE_UP_TILE_ROWS_Q4KN:{default:0}},sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{id:"lut_compact",passes_when:["lutcPackContract","f16Ok(tensorDtypes.normedT) and f16Ok(tensorDtypes.intermediateT)"],passes_tileRows:'tunables.GATE_UP_TILE_ROWS_LUTC if tunables.GATE_UP_TILE_ROWS_LUTC > 0 else (1 if (has(device.adapterInfo, "vendor") and device.adapterInfo.vendor == "apple") else 4)',passes_scaleScalar:'"u32"',passes_normedVec4:'"vec4<f16>" if tensorDtypes.normedT == "float16" else "vec4<f32>"',passes_lutId:"args.lut",passes_q4knCompact:null,passes_lutc:!0,main_name:"LlamaDecodeGateUpLutCompact",routeTunables:{GATE_UP_TILE_ROWS_LUTC:{default:0}},sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"{id}",priority:1,when:["preNormalizedContract","{passes_when}","{sgFoldStorageGate0}"],tunables:"{routeTunables}",derive:{tileRows:"{passes_tileRows}",scaleScalar:"{passes_scaleScalar}",normedVec4:"{passes_normedVec4}",lutId:"{passes_lutId}",preNormed:1,mRows:"tokenRows",q4knCompact:"{passes_q4knCompact}",sgSum32:"{passes_q4knCompact}",lutc:"{passes_lutc}",quantBits:4,hasMin:!1,iq:!1,q6k:!1,q5k:!1,lut16:!1,useSubgroups:"wave32SubgroupsUsable",workgroupSize:"subgroupCoveringWorkgroup",blocksPerRow:"args.hiddenSize / 32",gateOffset:"args.gateOffset",upOffset:"args.upOffset"},passes:[{id:"main",name:"{main_name}",shader:"decode-gate-up-q4.wgsl.jinja",bindings:["normed","q4_bits","q4_scales","intermediate"],dispatch:{x:"ceil(args.intermediateSize / tileRows)"},subgroupCollectivesWidth:"portable"}]}]},{id:"q2",priority:1,when:["preNormalizedContract","q2PackContract","f16Ok(tensorDtypes.normedT) and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.intermediateT)","(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"],tunables:{GATE_UP_TILE_ROWS_Q2K:{default:1}},derive:{tileRows:"tunables.GATE_UP_TILE_ROWS_Q2K",scaleScalar:"dtypes.S",normedVec4:'"vec4<f16>" if tensorDtypes.normedT == "float16" else "vec4<f32>"',q2k:!0,q2Split:!0,lutId:0,preNormed:1,mRows:"tokenRows",quantBits:4,hasMin:!1,iq:!1,q6k:!1,q5k:!1,lut16:!1,useSubgroups:"wave32SubgroupsUsable",workgroupSize:"subgroupCoveringWorkgroup",blocksPerRow:"args.hiddenSize / 32",gateOffset:"args.gateOffset",upOffset:"args.upOffset"},passes:[{id:"main",name:"LlamaDecodeGateUpQ2K",shader:"decode-gate-up-q4.wgsl.jinja",bindings:["normed",{arg:"bitsT",name:"gate_bits",elementType:"vec4<u32>"},{arg:"scalesT",name:"gate_scales",elementType:"$scaleScalar"},{arg:"upBitsT",name:"up_bits",elementType:"vec4<u32>"},{arg:"upScalesT",name:"up_scales",elementType:"$scaleScalar"},"intermediate"],dispatch:{x:"ceil(args.intermediateSize / tileRows)"},subgroupCollectivesWidth:"portable"}]},{id:"q1",priority:0,when:["preNormalizedContract","q1Route","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"f16Ok(tensorDtypes.normedT) and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.intermediateT)","args.hiddenSize % 128 == 0","args.intermediateSize % 4 == 0","dim(shapes.bitsT, 0) >= max(args.gateOffset + args.intermediateSize * (args.hiddenSize / 128), args.upOffset + args.intermediateSize * (args.hiddenSize / 128)) * 4","dim(shapes.scalesT, 0) >= max(args.gateOffset + args.intermediateSize * (args.hiddenSize / 128), args.upOffset + args.intermediateSize * (args.hiddenSize / 128))","tokenRows == 1"],derive:{useSubgroups:'device.features.has("subgroups")',fastSilu:'args.activation == "fast_silu"',normedVec4:'"vec4<f16>" if tensorDtypes.normedT == "float16" else "vec4<f32>"',scaleScalar:"dtypes.S",hiddenSizeQ1Words:"args.hiddenSize / 32",blocksPerRow:"args.hiddenSize / 128",workgroupSize:64,gateOffset:"args.gateOffset",upOffset:"args.upOffset"},passes:[{id:"main",name:"LlamaDecodeGateUpQ1",shader:"decode-gate-up-q1.wgsl.jinja",bindings:["normed","q1_bits","q1_scales","intermediate"],dispatch:{x:"ceil(args.intermediateSize / 4)"}}]},{id:"dense_norm",priority:0,when:["fusedRmsNormContract","denseRoute","args.activation is undefined","f16Ok(dtypes.H) and f16Ok(dtypes.W) and f16Ok(dtypes.I)","args.hiddenSize % 4 == 0","args.gateOffset % 4 == 0","args.upOffset % 4 == 0","dim(shapes.weightsT, 0) >= max(args.normOffset + args.hiddenSize, max(args.gateOffset + args.intermediateSize * args.hiddenSize, args.upOffset + args.intermediateSize * args.hiddenSize))","tokenRows == 1"],derive:{weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',weightScalar:"dtypes.W",hiddenSizeVec4:"args.hiddenSize / 4",gateOffsetVec4:"args.gateOffset / 4",upOffsetVec4:"args.upOffset / 4",workgroupSize:64,hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",rmsEps:"args.rmsEps",normOffsetVec4:"args.normOffset / 4"},passes:[{id:"main",name:"LlamaDecodeGateUpNorm",shader:"decode-gate-up-norm.wgsl.jinja",bindings:["hidden","weights","intermediate"],dispatch:{x:"args.intermediateSize"}}]},{id:"q4_norm",priority:0,when:["fusedRmsNormContract","q4PackContract","args.activation is undefined","f16Ok(dtypes.H) and f16Ok(dtypes.S) and f16Ok(dtypes.I) and f16Ok(dtypes.W)","dim(shapes.weightsT, 0) >= args.normOffset + args.hiddenSize","tokenRows == 1","(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"],tunables:{GATE_UP_TILE_ROWS:{default:4},GATE_UP_TILE_ROWS_NARROW:{default:1},GATE_UP_WIDE_HIDDEN:{default:2048},GATE_UP_TILE_ROWS_KQ4:{default:8},GATE_UP_TILE_ROWS_KQ4_NARROW:{default:1},GATE_UP_KQ4_WIDE_HIDDEN:{default:2048}},derive:{tileRows:'((tunables.GATE_UP_TILE_ROWS_KQ4 if args.hiddenSize >= tunables.GATE_UP_KQ4_WIDE_HIDDEN else tunables.GATE_UP_TILE_ROWS_KQ4_NARROW) if hasMin else (tunables.GATE_UP_TILE_ROWS if args.hiddenSize >= tunables.GATE_UP_WIDE_HIDDEN else tunables.GATE_UP_TILE_ROWS_NARROW)) if (has(device.adapterInfo, "vendor") and device.adapterInfo.vendor == "apple") else tunables.GATE_UP_TILE_ROWS',scaleScalar:"dtypes.S",normWeightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',lutId:"args.lut if args.lut else 0",preNormed:"0",useSubgroups:"wave32SubgroupsUsable",workgroupSize:"subgroupCoveringWorkgroup",blocksPerRow:"args.hiddenSize / 32",hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",rmsEps:"args.rmsEps",normOffsetVec4:"args.normOffset / 4",gateOffset:"args.gateOffset",upOffset:"args.upOffset"},passes:[{id:"main",name:"LlamaDecodeGateUpNormQ4",shader:"decode-gate-up-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","intermediate","norm_weight"],dispatch:{x:"ceil(args.intermediateSize / tileRows)"},subgroupCollectivesWidth:"portable"}]},{cases:[{id:"q4kn_compact_norm",passes_when:"q4knCompactPackContract",passes_tileRows:'tunables.GATE_UP_TILE_ROWS_Q4KN_NORM if tunables.GATE_UP_TILE_ROWS_Q4KN_NORM > 0 else (1 if (has(device.adapterInfo, "vendor") and device.adapterInfo.vendor == "apple") else 4)',passes_q4knCompact:!0,passes_lutId:0,passes_lutc:null,main_name:"LlamaDecodeGateUpNormQ4KNativeCompact",routeTunables:{GATE_UP_TILE_ROWS_Q4KN_NORM:{default:0}},sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{id:"lut_compact_norm",passes_when:"lutcPackContract",passes_tileRows:'tunables.GATE_UP_TILE_ROWS_LUTC_NORM if tunables.GATE_UP_TILE_ROWS_LUTC_NORM > 0 else (1 if (has(device.adapterInfo, "vendor") and device.adapterInfo.vendor == "apple") else 4)',passes_q4knCompact:null,passes_lutId:"args.lut",passes_lutc:!0,main_name:"LlamaDecodeGateUpNormLutCompact",routeTunables:{GATE_UP_TILE_ROWS_LUTC_NORM:{default:0}},sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"{id}",priority:1,when:["fusedRmsNormContract","{passes_when}","args.activation is undefined","f16Ok(dtypes.H) and f16Ok(dtypes.I) and f16Ok(dtypes.W)","dim(shapes.weightsT, 0) >= args.normOffset + args.hiddenSize","tokenRows == 1","{sgFoldStorageGate0}"],tunables:"{routeTunables}",derive:{tileRows:"{passes_tileRows}",scaleScalar:'"u32"',normWeightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',q4knCompact:"{passes_q4knCompact}",lutId:"{passes_lutId}",preNormed:0,lutc:"{passes_lutc}",quantBits:4,hasMin:!1,iq:!1,q6k:!1,q5k:!1,lut16:!1,useSubgroups:"wave32SubgroupsUsable",workgroupSize:"subgroupCoveringWorkgroup",blocksPerRow:"args.hiddenSize / 32",hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",rmsEps:"args.rmsEps",normOffsetVec4:"args.normOffset / 4",gateOffset:"args.gateOffset",upOffset:"args.upOffset"},passes:[{id:"main",name:"{main_name}",shader:"decode-gate-up-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","intermediate","norm_weight"],dispatch:{x:"ceil(args.intermediateSize / tileRows)"},subgroupCollectivesWidth:"portable"}]}]},{cases:[{suffix:"",gridName:"",prio:1,gridGuard:[],gridBound:null,gridBind:[],sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize",sgFoldStorageGate1:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"_grid",gridName:"Grid",prio:10,gridGuard:["present.gridT","ranks.gridT == 1",'tensorDtypes.gridT == "uint32"',"dim(shapes.gridT, 0) >= gridElemsNeeded"],gridBound:!0,gridBind:["iq_grid_data"],sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize",sgFoldStorageGate1:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (subgroupCoveringWorkgroup) + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"iq_raw{suffix}",priority:"{prio}",when:["preNormalizedContract","rawPackContract","f16Ok(tensorDtypes.normedT) and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.intermediateT)","{gridGuard}","{sgFoldStorageGate0}"],tunables:{GATE_UP_TILE_ROWS_LUTC:{default:0}},derive:{tileRows:'tunables.GATE_UP_TILE_ROWS_LUTC if tunables.GATE_UP_TILE_ROWS_LUTC > 0 else (1 if (has(device.adapterInfo, "vendor") and device.adapterInfo.vendor == "apple") else 4)',scaleScalar:"dtypes.S",normedVec4:'"vec4<f16>" if tensorDtypes.normedT == "float16" else "vec4<f32>"',lutId:0,lutc:!1,preNormed:1,iqDot:!0,iqGridBound:"{gridBound}",mRows:"tokenRows",rawBitsElem:'"vec2<u32>" if (iqRawSel == 1 or iqRawSel == 2) else "u32"',quantBits:4,hasMin:!1,iq:!1,q6k:!1,q5k:!1,lut16:!1,useSubgroups:"wave32SubgroupsUsable",workgroupSize:"subgroupCoveringWorkgroup",blocksPerRow:"args.hiddenSize / 32",gateOffset:"args.gateOffset",upOffset:"args.upOffset",iq1m:"iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8",iq1s:"iqRawSel == 7",iq1md:"iqRawSel == 8",iq2xxs:"iqRawSel == 2",iq2xs:"iqRawSel == 6",iq2s:"iqRawSel == 5",iq3xxs:"iqRawSel == 3",iq3s:"iqRawSel == 4"},passes:[{id:"main",name:"LlamaDecodeGateUpIqRaw{gridName}",shader:"decode-gate-up-q4.wgsl.jinja",bindings:["normed","q4_bits_raw","q4_scales","intermediate","{gridBind}"],dispatch:{x:"ceil(args.intermediateSize / tileRows)"},subgroupCollectivesWidth:"portable"}]},{id:"iq_raw_norm{suffix}",priority:"{prio}",when:["fusedRmsNormContract","rawPackContract","args.activation is undefined","f16Ok(dtypes.H) and f16Ok(dtypes.S) and f16Ok(dtypes.I) and f16Ok(dtypes.W)","dim(shapes.weightsT, 0) >= args.normOffset + args.hiddenSize","{gridGuard}","tokenRows == 1","{sgFoldStorageGate1}"],tunables:{GATE_UP_TILE_ROWS_LUTC_NORM:{default:0}},derive:{tileRows:'tunables.GATE_UP_TILE_ROWS_LUTC_NORM if tunables.GATE_UP_TILE_ROWS_LUTC_NORM > 0 else (1 if (has(device.adapterInfo, "vendor") and device.adapterInfo.vendor == "apple") else 4)',scaleScalar:"dtypes.S",normWeightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',lutId:0,lutc:!1,preNormed:0,iqDot:!0,iqGridBound:"{gridBound}",rawBitsElem:'"vec2<u32>" if (iqRawSel == 1 or iqRawSel == 2) else "u32"',quantBits:4,hasMin:!1,iq:!1,q6k:!1,q5k:!1,lut16:!1,useSubgroups:"wave32SubgroupsUsable",workgroupSize:"subgroupCoveringWorkgroup",blocksPerRow:"args.hiddenSize / 32",hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",rmsEps:"args.rmsEps",normOffsetVec4:"args.normOffset / 4",gateOffset:"args.gateOffset",upOffset:"args.upOffset",iq1m:"iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8",iq1s:"iqRawSel == 7",iq1md:"iqRawSel == 8",iq2xxs:"iqRawSel == 2",iq2xs:"iqRawSel == 6",iq2s:"iqRawSel == 5",iq3xxs:"iqRawSel == 3",iq3s:"iqRawSel == 4"},passes:[{id:"main",name:"LlamaDecodeGateUpIqRawNorm{gridName}",shader:"decode-gate-up-q4.wgsl.jinja",bindings:["hidden","q4_bits_raw","q4_scales","intermediate","norm_weight","{gridBind}"],dispatch:{x:"ceil(args.intermediateSize / tileRows)"},subgroupCollectivesWidth:"portable"}]}]},{id:"q1_norm",priority:0,when:["fusedRmsNormContract","q1Route","args.activation is undefined","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"f16Ok(dtypes.H) and f16Ok(dtypes.S) and f16Ok(dtypes.I) and f16Ok(dtypes.W)","args.hiddenSize % 128 == 0","args.intermediateSize % 4 == 0","dim(shapes.weightsT, 0) >= args.normOffset + args.hiddenSize","dim(shapes.bitsT, 0) >= max(args.gateOffset + args.intermediateSize * (args.hiddenSize / 128), args.upOffset + args.intermediateSize * (args.hiddenSize / 128)) * 4","dim(shapes.scalesT, 0) >= max(args.gateOffset + args.intermediateSize * (args.hiddenSize / 128), args.upOffset + args.intermediateSize * (args.hiddenSize / 128))","tokenRows == 1"],tunables:{GATE_UP_Q1_TILE_GROUPS:{default:4}},derive:{useSubgroups:'device.features.has("subgroups")',normWeightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',scaleScalar:"dtypes.S",workgroupSize:64,tileGroups:"min(tunables.GATE_UP_Q1_TILE_GROUPS, floor((device.limits.maxComputeWorkgroupStorageSize - 4 * workgroupSize) / (32 * workgroupSize)))",hiddenSizeQ1Words:"args.hiddenSize / 32",blocksPerRow:"args.hiddenSize / 128",hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",rmsEps:"args.rmsEps",normOffsetVec4:"args.normOffset / 4",gateOffset:"args.gateOffset",upOffset:"args.upOffset"},passes:[{id:"main",name:"LlamaDecodeGateUpNormQ1",shader:"decode-gate-up-norm-q1.wgsl.jinja",bindings:["hidden","q1_bits","q1_scales","intermediate","norm_weight"],dispatch:{x:"ceil(args.intermediateSize / (4 * tileGroups))"}}]}]},assets:[["decode-gate-up-norm-q1.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const HIDDEN_SIZE_Q1_WORDS:u32={{ hiddenSizeQ1Words }}u;const BPR:u32={{ blocksPerRow }}u;const INTERMEDIATE_SIZE:u32={{ intermediateSize }}u;const RMS_EPS:f32={{ rmsEps }};const W_NORM_VEC4:u32={{ normOffsetVec4 }}u;const W_GATE_Q1:u32={{ gateOffset }}u;const W_UP_Q1:u32={{ upOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% set groups = tileGroups | default(1) %}{% set rows = groups * 4 %}{% set lanes = ["x", "y", "z", "w"] %}const TILE_GROUPS:u32={{ groups }}u;const TILE_ROWS:u32={{ rows }}u;
{% for t in range(groups) %}var<workgroup>partials_g{{ t }}:array<vec4<f32>,{{ workgroupSize }}>;var<workgroup>partials_u{{ t }}:array<vec4<f32>,{{ workgroupSize }}>;
{% endfor %}var<workgroup>partials_ss:array<f32,{{ workgroupSize }}>;
{% include "ops/_shared/silu-fn.wgsl.jinja" %}{% include "ops/_shared/q1-dot-helpers.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})
{% include "ops/_shared/decode-entry-point.wgsl.jinja" %}let j0=wid.x*TILE_ROWS;let lid=lid3.x;if(j0>=INTERMEDIATE_SIZE){return;}let tile0=wid.x*TILE_GROUPS;
{% for t in range(groups) %}var acc_g{{ t }}=vec4<f32>(0.0);var acc_u{{ t }}=vec4<f32>(0.0);
{% endfor %}var ss=0.0;for(var word_index=lid;word_index<HIDDEN_SIZE_Q1_WORDS;word_index=word_index+WORKGROUP_SIZE){
{% include "ops/_shared/q1-hidden-rmsnorm-load.wgsl.jinja" %}{% for t in range(groups) %}acc_g{{ t }}+=q1_dot_word_t4(W_GATE_Q1,tile0+{{ t }}u,BPR,col_block,word_id,n0,n1,n2,n3,n4,n5,n6,n7);acc_u{{ t }}+=q1_dot_word_t4(W_UP_Q1,tile0+{{ t }}u,BPR,col_block,word_id,n0,n1,n2,n3,n4,n5,n6,n7);
{% endfor %}}
{% if useSubgroups %}{% for t in range(groups) %}let sgG{{ t }}=subgroupAdd(acc_g{{ t }});let sgU{{ t }}=subgroupAdd(acc_u{{ t }});
{% endfor %}let sgSS=subgroupAdd(ss);let elected=subgroupElect();
{% for t in range(groups) %}partials_g{{ t }}[lid]=select(vec4<f32>(0.0),sgG{{ t }},elected);partials_u{{ t }}[lid]=select(vec4<f32>(0.0),sgU{{ t }},elected);
{% endfor %}partials_ss[lid]=select(0.0,sgSS,elected);workgroupBarrier();if(lid==0u){
{% for t in range(groups) %}var totalG{{ t }}=vec4<f32>(0.0);var totalU{{ t }}=vec4<f32>(0.0);
{% endfor %}var totalSS=0.0;for(var i=0u;i<WORKGROUP_SIZE;i=i+1u){
{% for t in range(groups) %}totalG{{ t }}=totalG{{ t }}+partials_g{{ t }}[i];totalU{{ t }}=totalU{{ t }}+partials_u{{ t }}[i];
{% endfor %}totalSS=totalSS+partials_ss[i];}
{% for t in range(groups) %}partials_g{{ t }}[0]=totalG{{ t }};partials_u{{ t }}[0]=totalU{{ t }};
{% endfor %}partials_ss[0]=totalSS;}workgroupBarrier();
{% else %}{% for t in range(groups) %}partials_g{{ t }}[lid]=acc_g{{ t }};partials_u{{ t }}[lid]=acc_u{{ t }};
{% endfor %}partials_ss[lid]=ss;workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(stride==0u){break;}if(lid<stride){
{% for t in range(groups) %}{{ wgsl_tree_fold_stmt("partials_g" ~ t, "add", "lid", "stride") }} {{ wgsl_tree_fold_stmt("partials_u" ~ t, "add", "lid", "stride") }}
{% endfor %}{{ wgsl_tree_fold_stmt("partials_ss", "add", "lid", "stride") }} }stride=stride/2u;workgroupBarrier();}
{% endif %}if(lid==0u){let scale=inverseSqrt(partials_ss[0]/f32(HIDDEN_SIZE)+RMS_EPS);
{% for t in range(groups) %}let g{{ t }}=partials_g{{ t }}[0];let u{{ t }}=partials_u{{ t }}[0];
{% for c in lanes %}if(j0+{{ t * 4 + loop.index0 }}u<INTERMEDIATE_SIZE){intermediate[j0+{{ t * 4 + loop.index0 }}u]={{ intermediateScalar }}(silu(g{{ t }}.{{ c }}*scale)*(u{{ t }}.{{ c }}*scale));}
{% endfor %}{% endfor %}} }`],["decode-gate-up-norm.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const INTERMEDIATE_SIZE:u32={{ intermediateSize }}u;const RMS_EPS:f32={{ rmsEps }};const W_NORM_VEC4:u32={{ normOffsetVec4 }}u;const W_GATE_VEC4:u32={{ gateOffsetVec4 }}u;const W_UP_VEC4:u32={{ upOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>partials0:array<f32,{{ workgroupSize }}>;var<workgroup>partials1:array<f32,{{ workgroupSize }}>;var<workgroup>partials_ss:array<f32,{{ workgroupSize }}>;
{% include "ops/_shared/silu-fn.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let j=wid.x;let lid=lid3.x;if(j>=INTERMEDIATE_SIZE){return;}let gate_base=W_GATE_VEC4+j*HIDDEN_SIZE_VEC4;let up_base=W_UP_VEC4+j*HIDDEN_SIZE_VEC4;var acc0=0.0;var acc1=0.0;var ss=0.0;for(var dim=lid;dim<HIDDEN_SIZE_VEC4;dim=dim+WORKGROUP_SIZE){let h=vec4<f32>(hidden[dim]);ss=ss+dot(h,h);let n=h*vec4<f32>(weights[W_NORM_VEC4+dim]);
{% if weightScalar == "f16" %}let nh=vec4<f16>(n);acc0=acc0+f32(dot(nh,weights[gate_base+dim]));acc1=acc1+f32(dot(nh,weights[up_base+dim]));
{% else %}acc0=acc0+dot(n,weights[gate_base+dim]);acc1=acc1+dot(n,weights[up_base+dim]);
{% endif %}}partials0[lid]=acc0;partials1[lid]=acc1;partials_ss[lid]=ss;workgroupBarrier();{{ wgsl_tree_fold(["partials0", "partials1", "partials_ss"]) }} if(lid==0u){let scale=inverseSqrt(partials_ss[0]/f32(HIDDEN_SIZE)+RMS_EPS);intermediate[j]={{ intermediateScalar }}(silu(partials0[0]*scale)*(partials1[0]*scale));} }`],["decode-gate-up-q1.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE_Q1_WORDS:u32={{ hiddenSizeQ1Words }}u;const BPR:u32={{ blocksPerRow }}u;const INTERMEDIATE_SIZE:u32={{ intermediateSize }}u;const W_GATE_Q1:u32={{ gateOffset }}u;const W_UP_Q1:u32={{ upOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const TILE_ROWS:u32=4u;var<workgroup>partials_gate:array<vec4<f32>,{{ workgroupSize }}>;var<workgroup>partials_up:array<vec4<f32>,{{ workgroupSize }}>;fn silu(x:f32)->f32{
{% if fastSilu %}let sigmoid=0.5+0.5*x/(1.0+abs(x));return x*sigmoid;
{% else %}return x/(1.0+exp(-x));
{% endif %}}
{% include "ops/_shared/q1-dot-helpers.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})
{% include "ops/_shared/decode-entry-point.wgsl.jinja" %}let j0=wid.x*TILE_ROWS;let lid=lid3.x;if(j0>=INTERMEDIATE_SIZE){return;}let tile=j0/TILE_ROWS;var acc0=0.0;var acc1=0.0;var acc2=0.0;var acc3=0.0;var acc_u0=0.0;var acc_u1=0.0;var acc_u2=0.0;var acc_u3=0.0;for(var word_index=lid;word_index<HIDDEN_SIZE_Q1_WORDS;word_index=word_index+WORKGROUP_SIZE){let col_block=word_index/4u;let word_id=word_index%4u;let base4=word_index*8u;
{% for i in range(8) %}let n{{ i }}={{ "vec4<f32>(" if normedVec4 != "vec4<f32>" else "" }}normed[base4{{ " + " ~ i ~ "u" if i else "" }}]{{ ")" if normedVec4 != "vec4<f32>" else "" }};
{% endfor %}let gate4=q1_dot_word_t4(W_GATE_Q1,tile,BPR,col_block,word_id,n0,n1,n2,n3,n4,n5,n6,n7);let up4=q1_dot_word_t4(W_UP_Q1,tile,BPR,col_block,word_id,n0,n1,n2,n3,n4,n5,n6,n7);acc0+=gate4.x;acc1+=gate4.y;acc2+=gate4.z;acc3+=gate4.w;acc_u0+=up4.x;acc_u1+=up4.y;acc_u2+=up4.z;acc_u3+=up4.w;}
{% if useSubgroups %}let subgroupTotal0=subgroupAdd(vec4<f32>(acc0,acc1,acc2,acc3));let subgroupTotal1=subgroupAdd(vec4<f32>(acc_u0,acc_u1,acc_u2,acc_u3));let elected=subgroupElect();partials_gate[lid]=select(vec4<f32>(0.0),subgroupTotal0,elected);partials_up[lid]=select(vec4<f32>(0.0),subgroupTotal1,elected);workgroupBarrier();if(lid==0u){var total0=vec4<f32>(0.0);var total1=vec4<f32>(0.0);for(var i=0u;i<WORKGROUP_SIZE;i=i+1u){total0=total0+partials_gate[i];total1=total1+partials_up[i];}partials_gate[0]=total0;partials_up[0]=total1;}workgroupBarrier();
{% else %}partials_gate[lid]=vec4<f32>(acc0,acc1,acc2,acc3);partials_up[lid]=vec4<f32>(acc_u0,acc_u1,acc_u2,acc_u3);workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(lid<stride){partials_gate[lid]+=partials_gate[lid+stride];partials_up[lid]+=partials_up[lid+stride];}workgroupBarrier();if(stride==1u){break;}stride=stride/2u;}
{% endif %}if(lid==0u){let gate=partials_gate[0];let up=partials_up[0];
{% if intermediateScalar == "f32" %}intermediate[j0+0u]=silu(gate.x)*up.x;intermediate[j0+1u]=silu(gate.y)*up.y;intermediate[j0+2u]=silu(gate.z)*up.z;intermediate[j0+3u]=silu(gate.w)*up.w;
{% else %}intermediate[j0+0u]={{ intermediateScalar }}(silu(gate.x)*up.x);intermediate[j0+1u]={{ intermediateScalar }}(silu(gate.y)*up.y);intermediate[j0+2u]={{ intermediateScalar }}(silu(gate.z)*up.z);intermediate[j0+3u]={{ intermediateScalar }}(silu(gate.w)*up.w);
{% endif %}} }`],["decode-gate-up-q4.wgsl.jinja",`{% set emitQuantDotHelpers = not (
not(gateUpSplit|default(false))and(lutId|default(0))and((lut16|default(false))or((lutc|default(false))and(lutc16|default(false))))
    ) %}{% set packed4x8CompatUnpackI8 = (emitQuantDotHelpers and (quantBits == 8 or ((gateUpSplit | default(false)) and upQuantBits == 8))) or q3k or q6k or ((gateUpSplit | default(false)) and (upQ3k or upQ6k)) or (iq1m | default(false)) or (gateUpIq1m | default(false)) or (upIq1m | default(false)) %}{% set packed4x8CompatUnpackU8 = (iq2s | default(false)) or (gateUpIq2s | default(false)) or (upIq2s | default(false)) or ((iqGridBound | default(false)) and ((iq2xxs | default(false)) or (iq2xs | default(false)) or (iq3xxs | default(false)) or (iq3s | default(false)) or (gateUpIq2xxs | default(false)) or (upIq2xxs | default(false)) or (gateUpIq2xs | default(false)) or (upIq2xs | default(false)) or (gateUpIq3xxs | default(false)) or (upIq3xxs | default(false)) or (gateUpIq3s | default(false)) or (upIq3s | default(false)))) or ((gateUpSplit | default(false)) and (gateNibbleFam or upNibbleFam or q2k or q3k or q5k or q6k or upQ2k or upQ3k or upQ5k or upQ6k)) or (not (gateUpSplit | default(false)) and not ((iq1m | default(false)) or (iq2xxs | default(false)) or (iq2xs | default(false)) or (iq2s | default(false)) or (iq3xxs | default(false)) or (iq3s | default(false))) and not lutId and quantBits != 8) %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% set tileRows = tileRows | default(1) %}{% set gateUpSuffixes = [""] if tileRows == 1 else range(tileRows) | list %}{% set mRows = mRows if mRows is defined else 1 %}{% set xOpen = "vec4<f32>(normed[" %}{% set xClose = "])" %}{% set mStoreScalar = intermediateScalar %}{% set gateUpTiled = tileRows > 1 %}{% if not preNormed %}const HIDDEN_SIZE:u32={{ hiddenSize }}u;const RMS_EPS:f32={{ rmsEps }};const W_NORM_VEC4:u32={{ normOffsetVec4 }}u;
{% endif %}const BPR:u32={{ blocksPerRow }}u;const INTERMEDIATE_SIZE:u32={{ intermediateSize }}u;const W_GATE_Q4:u32={{ gateOffset }}u;const W_UP_Q4:u32={{ upOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% if gateUpTiled %}const TILE_ROWS:u32={{ tileRows }}u;
{% endif %}{% macro act_loads() %}{% if preNormed %}{% for i in range(8) %}let x{{ i }}=vec4<f32>(normed[hbase{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}{% else %}{% for i in range(8) %}let h{{ i }}=vec4<f32>(hidden[hbase{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}ss=ss+dot(h0,h0)+dot(h1,h1)+dot(h2,h2)+dot(h3,h3)+dot(h4,h4)+dot(h5,h5)+dot(h6,h6)+dot(h7,h7);let nb=W_NORM_VEC4+hbase;
{% for i in range(8) %}let x{{ i }}=h{{ i }}*vec4<f32>(norm_weight[nb{{ " + " ~ i ~ "u" if i else "" }}]);
{% endfor %}{% endif %}{% endmacro %}{% include "ops/_shared/subgroup-runtime-fold.wgsl.jinja" %}{% set gateUpOwned = useSubgroups and not sgRuntime and workgroupSize == 32 and (subgroupIdBuiltins | default(false)) %}{% if gateUpOwned %}{% include "ops/_shared/subgroup-active-lanes.wgsl.jinja" %}{% elif useSubgroups or sgRuntime %}{% set sgSafeStorageSize = workgroupSize %}{% include "ops/_shared/subgroup-runtime-proof.wgsl.jinja" %}{% endif %}{% if not useSubgroups and not sgRuntime %}{% for r in gateUpSuffixes %}var<workgroup>partials_g{{ r }}:array<f32,{{ workgroupSize }}>;var<workgroup>partials_u{{ r }}:array<f32,{{ workgroupSize }}>;
{% endfor %}{% if not preNormed %}var<workgroup>partials_ss:array<f32,{{ workgroupSize }}>;
{% endif %}{% endif %}{% include "ops/_shared/silu-fn.wgsl.jinja" %}{% if gateUpSplit is defined and gateUpSplit %}{% set gateSaveQuantBits = quantBits %}{% set gateSaveQ5k = q5k %}{% set gateSaveQ6k = q6k %}{% set gateSaveQ3k = q3k %}{% set gateSaveQ2k = q2k %}{% set gateSaveLutId = lutId %}{% set gateSaveLutc = lutc %}{% set quantBits = 8 if (quantBits == 8 or upQuantBits == 8) else 4 %}{% set q5k = q5k or upQ5k %}{% set q6k = q6k or upQ6k %}{% set q3k = q3k or upQ3k %}{% set q2k = q2k or upQ2k %}{% if lutId or upLutId %}{% set mixedLutNibble = (lutId and upNibbleFam) or (upLutId and gateNibbleFam) %}{% set gateLutSfx = "_g" if (lutId and upNibbleFam) else "" %}{% set upLutSfx = "_u" if (upLutId and ((lutId and lutId != upLutId) or gateNibbleFam)) else "" %}{% set gateSaveLutc16 = lutc16 | default(false) %}{% set lutSfx = gateLutSfx if lutId else upLutSfx %}{% set lutId = lutId if lutId else upLutId %}{% set lutc = lutc or upLutc %}{% set gateSaveLut2 = lut2 | default(false) %}{% set lut2 = gateSaveLut2 or upLut2 %}{% set emitQuantValueHelpers = (gateSaveLutId and not gateSaveLut2 and gateLutSfx == lutSfx) or (upLutId and not upLut2 and upLutSfx == lutSfx) %}{% set emitLut2DotHelper = (gateSaveLut2 and not (lutc16 | default(false))) or (upLut2 and not upLutc16) %}{% set gateSaveUnionQuantBits = quantBits %}{% set quantBits = 4 %}{% set gatePer32Lut = gateSaveLutId and not (lut16 | default(false)) and not gateSaveLutc16 and not gateSaveLut2 %}{% set upPer32Lut = (upLutId | default(0)) and not upLut16 and not upLutc16 and not upLut2 %}{% set emitQuantDotHelpers = (gatePer32Lut and gateLutSfx == lutSfx) or (upPer32Lut and upLutSfx == lutSfx) %}{% include "ops/_shared/q4nl-dequant-helpers.wgsl.jinja" %}{% set emitQuantValueHelpers = true %}{% if gateSaveLutId and upLutId and gateSaveLutId != upLutId %}{% set lutSfx = upLutSfx %}{% set lutId = upLutId %}{% set lutc = upLutc %}{% set lutc16 = upLutc16 %}{% set lut2 = upLut2 %}{% set emitQuantValueHelpers = not upLut2 %}{% set emitLut2DotHelper = upLut2 and not upLutc16 %}{% set emitQuantDotHelpers = upPer32Lut %}{% include "ops/_shared/q4nl-dequant-helpers.wgsl.jinja" %}{% set emitQuantValueHelpers = true %}{% set lutc16 = gateSaveLutc16 %}{% endif %}{% set emitQuantDotHelpers = true %}{% set lut2 = gateSaveLut2 %}{% set quantBits = gateSaveUnionQuantBits %}{% set lutSfx = "" %}{% if mixedLutNibble %}{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% else %}{% set nibbleHelpers = quantBits == 8 %}{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% set nibbleHelpers = true %}{% endif %}{% else %}{% set nibbleHelpers = gateNibbleFam or upNibbleFam or quantBits == 8 %}{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% if quantBits == 8 and (gateNibbleFam or upNibbleFam) %}{% set quantBits = 4 %}{% set q2k = false %}{% set q3k = false %}{% set q5k = false %}{% set q6k = false %}{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% endif %}{% set nibbleHelpers = true %}{% endif %}{% set quantBits = gateSaveQuantBits %}{% set q5k = gateSaveQ5k %}{% set q6k = gateSaveQ6k %}{% set q3k = gateSaveQ3k %}{% set q2k = gateSaveQ2k %}{% set lutId = gateSaveLutId %}{% set lutc = gateSaveLutc %}{% set iqInline = true %}{% set gateRawTag = "iq1" if gateUpIq1m else ("iq2xxs" if gateUpIq2xxs else ("iq2xs" if gateUpIq2xs else ("iq2s" if gateUpIq2s else ("iq3s" if gateUpIq3s else ("iq3xxs" if gateUpIq3xxs else ""))))) %}{% set upRawTag = "iq1" if upIq1m else ("iq2xxs" if upIq2xxs else ("iq2xs" if upIq2xs else ("iq2s" if upIq2s else ("iq3s" if upIq3s else ("iq3xxs" if upIq3xxs else ""))))) %}{% set upIqSfx = "_u" if (gateRawTag and upRawTag and gateRawTag != upRawTag) else "" %}{% if gateRawTag or upRawTag %}{% set iqPrimitiveMode = "parity8" if (gateUpIq2xxs or upIq2xxs or gateUpIq2xs or upIq2xs or gateUpIq3xxs or upIq3xxs) else ("explicit8" if (gateUpIq2s or upIq2s) else ("explicit4" if (gateUpIq3s or upIq3s) else "none")) %}{% set iq1m = gateUpIq1m if gateRawTag else upIq1m %}{% set iq1s = ((gateUpIq1s | default(false)) or ((upIq1s | default(false)) and not upIqSfx)) if gateRawTag else (upIq1s | default(false)) %}{% set iq1mLive = ((gateUpIq1m | default(false)) and not (gateUpIq1s | default(false))) or ((not upIqSfx) and (upIq1m | default(false)) and not (upIq1s | default(false))) if gateRawTag else ((upIq1m | default(false)) and not (upIq1s | default(false))) %}{% set iq2xxs = gateUpIq2xxs if gateRawTag else upIq2xxs %}{% set iq2xs = gateUpIq2xs if gateRawTag else upIq2xs %}{% set iq3xxs = gateUpIq3xxs if gateRawTag else upIq3xxs %}{% set iq3s = gateUpIq3s if gateRawTag else upIq3s %}{% set iq2s = gateUpIq2s if gateRawTag else upIq2s %}{% include "ops/_shared/iq-grid.wgsl.jinja" %}{% include "ops/_shared/iqx-raw-dequant-helpers.wgsl.jinja" %}{% if upIqSfx %}{% set iqSfx = upIqSfx %}{% set iqPrimitives = false %}{% set iqGridBinding = "iq_grid_data_u" %}{% set iq1m = upIq1m %}{% set iq1s = upIq1s | default(false) %}{% set iq1mLive = (upIq1m | default(false)) and not (upIq1s | default(false)) %}{% set iq2xxs = upIq2xxs %}{% set iq2xs = upIq2xs %}{% set iq3xxs = upIq3xxs %}{% set iq3s = upIq3s %}{% set iq2s = upIq2s %}{% include "ops/_shared/iq-grid.wgsl.jinja" %}{% include "ops/_shared/iqx-raw-dequant-helpers.wgsl.jinja" %}{% set iqSfx = "" %}{% set iqPrimitives = true %}{% endif %}{% endif %}{% elif (iq1m is defined and iq1m) or (iq2xxs is defined and iq2xxs) or (iq2xs is defined and iq2xs) or (iq3xxs is defined and iq3xxs) or (iq3s is defined and iq3s) or (iq2s is defined and iq2s) %}{% set iqInline = true %}{% include "ops/_shared/iq-grid.wgsl.jinja" %}{% include "ops/_shared/iqx-raw-dequant-helpers.wgsl.jinja" %}{% elif lutId %}{% set emitQuantValueHelpers = not (lut2 | default(false)) %}{% set emitLut2DotHelper = not ((lut2 | default(false)) and (lutc16 | default(false))) %}{% include "ops/_shared/q4nl-dequant-helpers.wgsl.jinja" %}{% set emitQuantValueHelpers = true %}{% else %}{% set nibbleHelpers = not (q2k or q3k or q5k or q6k) %}{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% endif %}{% if useSubgroups and sgSum32 is defined and sgSum32 %}{% endif %}@compute@workgroup_size({{ workgroupSize }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_id) sg_id: u32, @builtin(num_subgroups) sg_count: u32" if gateUpOwned else (", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "") }}){
{% for sgPath in (["general", "full"] if gateUpOwned else [""]) %}{% set gateUpStride = "activeCount" if sgPath == "general" else "WORKGROUP_SIZE" %}{% set gateUpLeader = "activeRank" if sgPath == "general" else "lid" %}{% if sgPath == "general" %}if(sg_count!=1u){if(sg_id==0u){ {{ active_lanes("sg_lid") }}
{% elif sgPath == "full" %}}else{
{% endif %}{% if gateUpTiled %}let j0=wid.x*TILE_ROWS;let lid={{ "activeRank" if sgPath == "general" else "lid3.x" }};
{% if (useSubgroups or sgRuntime) and not gateUpOwned %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}if(j0>=INTERMEDIATE_SIZE){return;}
{% for r in gateUpSuffixes %}let row{{ r }}=min(j0+{{ r }}u,INTERMEDIATE_SIZE-1u);let gate_block_base{{ r }}=W_GATE_Q4+row{{ r }}*BPR;let up_block_base{{ r }}=W_UP_Q4+row{{ r }}*BPR;var acc_g{{ r }}=0.0;var acc_u{{ r }}=0.0;
{% endfor %}{% else %}let j=wid.x;let lid={{ "activeRank" if sgPath == "general" else "lid3.x" }};
{% if (useSubgroups or sgRuntime) and not gateUpOwned %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}if(j>=INTERMEDIATE_SIZE){return;}let gate_block_base=W_GATE_Q4+j*BPR;let up_block_base=W_UP_Q4+j*BPR;var acc_g=0.0;var acc_u=0.0;
{% endif %}{% for m in range(1, mRows) %}{% for r in gateUpSuffixes %}var accM{{ m }}_g{{ r }}=0.0;var accM{{ m }}_u{{ r }}=0.0;
{% endfor %}{% endfor %}{% if not preNormed %}var ss=0.0;
{% endif %}{% if ((iq1s | default(false)) or (iq1md | default(false))) and not (gateUpSplit | default(false)) and mRows == 1 %}let dqPu=((BPR/2u)/{{ gateUpStride }})*{{ gateUpStride }};for(var dqP=lid;dqP<dqPu;dqP=dqP+{{ gateUpStride }}){let b=2u*dqP;let hbase=b*8u;{{ act_loads() | trim }} let dqK=b&7u;let dqMo=8u+(dqK>>1u);
{% if iq1md | default(false) %}let dqCo=12u+(dqK>>2u);let dqCs=8u*(dqK&3u);
{% endif %}{% for r in gateUpSuffixes | default([""]) %}let dpGs{{ r }}=(gate_block_base{{ r }}+b)>>3u;
{% if iq1md | default(false) %}let dpGw{{ r }}=dpGs{{ r }}*14u;let dpGm{{ r }}=q4_bits[dpGw{{ r }}+dqMo];let dpGc{{ r }}=q4_bits[dpGw{{ r }}+dqCo]>>dqCs;let dpGd{{ r }}=f32(q4_scales[dpGs{{ r }}]);{{ iq1_dense_dot("ga" ~ r, "acc_g" ~ r, true, "q4_bits[dpGw" ~ r ~ " + dqK]", "(dpGm" ~ r ~ " & 0xFFFFu) | ((dpGc" ~ r ~ " & 0x3Fu) << 16u)", "dpGd" ~ r) | trim }}
{% else %}let dpGw{{ r }}=dpGs{{ r }}*12u;let dpGm{{ r }}=q4_bits[dpGw{{ r }}+dqMo];let dpGd{{ r }}=f32(q4_scales[dpGs{{ r }}]);{{ iq1_dense_dot("ga" ~ r, "acc_g" ~ r, false, "q4_bits[dpGw" ~ r ~ " + dqK]", "dpGm" ~ r ~ " & 0xFFFFu", "dpGd" ~ r) | trim }}
{% endif %}let dpUs{{ r }}=(up_block_base{{ r }}+b)>>3u;
{% if iq1md | default(false) %}let dpUw{{ r }}=dpUs{{ r }}*14u;let dpUm{{ r }}=q4_bits[dpUw{{ r }}+dqMo];let dpUc{{ r }}=q4_bits[dpUw{{ r }}+dqCo]>>dqCs;let dpUd{{ r }}=f32(q4_scales[dpUs{{ r }}]);{{ iq1_dense_dot("ua" ~ r, "acc_u" ~ r, true, "q4_bits[dpUw" ~ r ~ " + dqK]", "(dpUm" ~ r ~ " & 0xFFFFu) | ((dpUc" ~ r ~ " & 0x3Fu) << 16u)", "dpUd" ~ r) | trim }}
{% else %}let dpUw{{ r }}=dpUs{{ r }}*12u;let dpUm{{ r }}=q4_bits[dpUw{{ r }}+dqMo];let dpUd{{ r }}=f32(q4_scales[dpUs{{ r }}]);{{ iq1_dense_dot("ua" ~ r, "acc_u" ~ r, false, "q4_bits[dpUw" ~ r ~ " + dqK]", "dpUm" ~ r ~ " & 0xFFFFu", "dpUd" ~ r) | trim }}
{% endif %}{% endfor %}{let hbase=b*8u+8u;{{ act_loads() | trim }}
{% for r in gateUpSuffixes | default([""]) %}{% if iq1md | default(false) %}{{ iq1_dense_dot("gb" ~ r, "acc_g" ~ r, true, "q4_bits[dpGw" ~ r ~ " + dqK + 1u]", "(dpGm" ~ r ~ " >> 16u) | (((dpGc" ~ r ~ " >> 8u) & 0x3Fu) << 16u)", "dpGd" ~ r) | trim }} {{ iq1_dense_dot("ub" ~ r, "acc_u" ~ r, true, "q4_bits[dpUw" ~ r ~ " + dqK + 1u]", "(dpUm" ~ r ~ " >> 16u) | (((dpUc" ~ r ~ " >> 8u) & 0x3Fu) << 16u)", "dpUd" ~ r) | trim }}
{% else %}{{ iq1_dense_dot("gb" ~ r, "acc_g" ~ r, false, "q4_bits[dpGw" ~ r ~ " + dqK + 1u]", "dpGm" ~ r ~ " >> 16u", "dpGd" ~ r) | trim }} {{ iq1_dense_dot("ub" ~ r, "acc_u" ~ r, false, "q4_bits[dpUw" ~ r ~ " + dqK + 1u]", "dpUm" ~ r ~ " >> 16u", "dpUd" ~ r) | trim }}
{% endif %}{% endfor %}} }for(var b=2u*dqPu+lid;b<BPR;b=b+{{ gateUpStride }}){
{% else %}for(var b=lid;b<BPR;b=b+{{ gateUpStride }}){
{% endif %}let hbase=b*8u;{{ act_loads() | trim }}
{% set gateUpReduceNorm = not preNormed %}{% include "ops/_shared/q4-gate-up-accumulate-reduce.wgsl.jinja" %}if({{ gateUpLeader }}==0u){
{% if gateUpTiled %}{% if not preNormed %}let scale=inverseSqrt(s/f32(HIDDEN_SIZE)+RMS_EPS);
{% endif %}{% for r in gateUpSuffixes %}if(j0+{{ r }}u<INTERMEDIATE_SIZE){
{% if preNormed %}intermediate[j0+{{ r }}u]={{ intermediateScalar }}(silu(g{{ r }})*u{{ r }});
{% else %}intermediate[j0+{{ r }}u]={{ intermediateScalar }}(silu(g{{ r }}*scale)*(u{{ r }}*scale));
{% endif %}}
{% endfor %}{% else %}{% if preNormed %}intermediate[j]={{ intermediateScalar }}(silu(g)*u);
{% else %}let scale=inverseSqrt(s/f32(HIDDEN_SIZE)+RMS_EPS);intermediate[j]={{ intermediateScalar }}(silu(g*scale)*(u*scale));
{% endif %}{% endif %}}
{% if sgPath %}}
{% endif %}{% endfor %}}`],["decode-gate-up.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const INTERMEDIATE_SIZE:u32={{ intermediateSize }}u;const W_GATE_VEC4:u32={{ gateOffsetVec4 }}u;const W_UP_VEC4:u32={{ upOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>partials0:array<f32,{{ workgroupSize }}>;var<workgroup>partials1:array<f32,{{ workgroupSize }}>;
{% include "ops/_shared/silu-fn.wgsl.jinja" %}fn load_normed(value:{{ normedVec4 }})->vec4<f32>{return vec4<f32>(value);}fn store_intermediate(value:f32)->{{ intermediateScalar }} {return {{ intermediateScalar }}(value);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let j=wid.x;let lid=lid3.x;if(j>=INTERMEDIATE_SIZE){return;}let gate_base=W_GATE_VEC4+j*HIDDEN_SIZE_VEC4;let up_base=W_UP_VEC4+j*HIDDEN_SIZE_VEC4;var acc0=0.0;var acc1=0.0;for(var dim=lid;dim<HIDDEN_SIZE_VEC4;dim=dim+WORKGROUP_SIZE){let n=load_normed(normed[dim]);
{% if weightScalar == "f16" %}acc0=acc0+f32(dot(vec4<f16>(n),weights[gate_base+dim]));acc1=acc1+f32(dot(vec4<f16>(n),weights[up_base+dim]));
{% else %}acc0=acc0+dot(n,weights[gate_base+dim]);acc1=acc1+dot(n,weights[up_base+dim]);
{% endif %}}partials0[lid]=acc0;partials1[lid]=acc1;workgroupBarrier();{{ wgsl_tree_fold(["partials0", "partials1"]) }} if(lid==0u){intermediate[j]=store_intermediate(silu(partials0[0])*partials1[0]);} }`]]}],["com.xenova.LlamaDecodeLmHeadArgmax",{manifest:{domain:"com.xenova",name:"LlamaDecodeLmHeadArgmax",sinceVersion:1,inputs:{hiddenT:{onnx:"Hidden",dtype:"float32",optional:!0},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},normedT:{onnx:"Normed",dtype:"N",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},normQ8T:{onnx:"NormQ8",dtype:"uint32",rank:1,optional:!0},normScaleT:{onnx:"NormScale",dtype:"float32",rank:1,optional:!0}},outputs:{bestValuesT:{onnx:"BestValues",dtype:"float32",rank:1,shape:["mRowsCount * effectiveNumBlocks * effectiveTopK"]},bestIdsT:{onnx:"BestIds",dtype:"uint32",rank:1,shape:["mRowsCount * effectiveNumBlocks * effectiveTopK"]}},args:{hiddenSize:{kind:"u32",onnx:"hidden_size"},vocabSize:{kind:"u32",onnx:"vocab_size"},numBlocks:{kind:"u32",onnx:"num_blocks"},rmsEps:{kind:"f32",onnx:"rms_epsilon",required:!1},finalNormOffset:{kind:"u32",onnx:"final_norm_offset",required:!1},lmHeadOffset:{kind:"u32",onnx:"lm_head_block_offset",required:!1},topK:{kind:"u32",onnx:"candidates_per_block",required:!1},mRows:{kind:"u32",onnx:"hidden_rows",required:!1},quantBits:{kind:"u32",onnx:"packed_quant_bits",required:!1},colMajor:{kind:"u32",onnx:"block_col_major_pack",required:!1},format:{kind:"string",required:!1,oneOf:["q4_0","q1_0","q8_rows","q3_k","q2_k","q6_k","q5_k"]}},typeConstraints:{W:["float32","float16"],N:["float32","float16"],S:["float32","float16","uint32"]},tunables:{COL_MAJOR:{default:-1},WORKGROUP_SIZE:{default:256},COOPERATIVE_WORKGROUP_SIZE:{default:256},COOPERATIVE_SUBGROUP_SIZE:{default:32},COOPERATIVE_MIN_HIDDEN_SIZE:{default:2048}},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',packedPresent:"present.bitsT and present.scalesT",effectiveQuantBits:"args.quantBits if args.quantBits else 4",effectiveTopK:"args.topK if args.topK else 1",effectiveNumBlocks:"args.numBlocks",mRowsRequested:"args.mRows is defined",mRowsCount:"args.mRows if args.mRows is defined else 1",mRowsShapeOk:"mRowsRequested and args.mRows >= 2 and args.mRows <= 16 and present.hiddenT and ranks.hiddenT == 2 and dim(shapes.hiddenT, 0) == args.mRows and dim(shapes.hiddenT, 1) == args.hiddenSize and dim(shapes.bestValuesT, 0) >= args.mRows * args.numBlocks * effectiveTopK and dim(shapes.bestIdsT, 0) >= args.mRows * args.numBlocks * effectiveTopK",densePresent:"present.hiddenT and present.weightsT and not packedPresent and not present.normedT and not present.normQ8T and not present.normScaleT",q1Present:'present.normedT and packedPresent and fmt == "q1_0" and not present.hiddenT and not present.weightsT',q4Present:'present.hiddenT and present.weightsT and packedPresent and fmt == "q4_0"',kSuperFmt:"1 if (q6k or q5k or q3k or q2k) else 0",kSuperPresent:"present.hiddenT and present.weightsT and packedPresent and kSuperFmt == 1",kSuperSupers:"args.vocabSize * (args.hiddenSize / 256)",kSuperBitsWords:"(kSuperSupers * 52) if q6k else ((kSuperSupers * 44) if q5k else ((kSuperSupers * 28) if q3k else (kSuperSupers * 20)))",kSuperScaleWords:"(kSuperSupers * 2) if (q2k or q5k) else kSuperSupers",q8Present:'present.normQ8T and present.normScaleT and packedPresent and fmt == "q8_rows" and not present.hiddenT and not present.weightsT and not present.normedT',q8NormPresent:'present.hiddenT and present.weightsT and packedPresent and fmt == "q8_rows" and not present.normQ8T',baseContract:'ranks.bestValuesT == 1 and ranks.bestIdsT == 1 and tensorDtypes.bestValuesT == "float32" and tensorDtypes.bestIdsT == "uint32" and args.hiddenSize > 0 and args.vocabSize > 0 and args.numBlocks > 0 and args.numBlocks <= device.limits.maxComputeWorkgroupsPerDimension and dim(shapes.bestValuesT, 0) >= args.numBlocks * effectiveTopK and dim(shapes.bestIdsT, 0) >= args.numBlocks * effectiveTopK and (not mRowsRequested or mRowsShapeOk)',denseContract:'densePresent and effectiveTopK == 1 and args.rmsEps is defined and args.finalNormOffset is defined and args.lmHeadOffset is defined and ranks.hiddenT == 1 and ranks.weightsT == 1 and tensorDtypes.hiddenT == "float32" and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and f16Ok(dtypes.W) and args.hiddenSize % 4 == 0 and args.finalNormOffset % 4 == 0 and args.lmHeadOffset % 4 == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.weightsT, 0) >= max(args.finalNormOffset + args.hiddenSize, args.lmHeadOffset + args.vocabSize * args.hiddenSize)',q1StorageBytes:"16 * (args.hiddenSize / 4) + 2048 if q1Present else 0",q1Contract:'q1Present and effectiveTopK == 1 and args.lmHeadOffset is defined and ranks.normedT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and (tensorDtypes.normedT == "float32" or tensorDtypes.normedT == "float16") and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(dtypes.N) and f16Ok(dtypes.S) and args.hiddenSize % 128 == 0 and dim(shapes.normedT, 0) == args.hiddenSize and dim(shapes.bitsT, 0) >= (args.lmHeadOffset + args.vocabSize * (args.hiddenSize / 128)) * 4 and dim(shapes.scalesT, 0) >= args.lmHeadOffset + args.vocabSize * (args.hiddenSize / 128) and q1StorageBytes <= device.limits.maxComputeWorkgroupStorageSize',q4Contract:'q4Present and args.lmHeadOffset is defined and args.finalNormOffset is defined and ranks.hiddenT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.weightsT == 1 and tensorDtypes.hiddenT == "float32" and tensorDtypes.bitsT == "uint32" and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and f16Ok(dtypes.W) and args.hiddenSize % 32 == 0 and args.finalNormOffset % 4 == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.weightsT, 0) >= args.finalNormOffset + args.hiddenSize and dim(shapes.bitsT, 0) >= (args.lmHeadOffset + args.vocabSize * (args.hiddenSize / 32)) * effectiveQuantBits and dim(shapes.scalesT, 0) >= args.lmHeadOffset + args.vocabSize * (args.hiddenSize / 32)',q8Contract:'q8Present and effectiveTopK == 1 and ranks.normQ8T == 1 and ranks.normScaleT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.normQ8T == "uint32" and tensorDtypes.normScaleT == "float32" and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "float32" and args.hiddenSize % 4 == 0 and dim(shapes.normQ8T, 0) == args.hiddenSize / 4 and dim(shapes.normScaleT, 0) == 1 and dim(shapes.bitsT, 0) >= args.vocabSize * (args.hiddenSize / 4) and dim(shapes.scalesT, 0) >= args.vocabSize',q8NormContract:'q8NormPresent and effectiveTopK == 1 and args.rmsEps is defined and args.finalNormOffset is defined and ranks.hiddenT == 1 and ranks.weightsT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.hiddenT == "float32" and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "float32" and f16Ok(dtypes.W) and args.hiddenSize % 4 == 0 and args.finalNormOffset % 4 == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.weightsT, 0) >= args.finalNormOffset + args.hiddenSize and dim(shapes.bitsT, 0) >= args.vocabSize * (args.hiddenSize / 4) and dim(shapes.scalesT, 0) >= args.vocabSize',workgroupSize:"tunables.WORKGROUP_SIZE",cooperativeWorkgroupSize:"tunables.COOPERATIVE_WORKGROUP_SIZE",workgroupStorageBytes:"args.hiddenSize * 4 + workgroupSize * 8 + (args.hiddenSize / 8 if effectiveQuantBits == 2 else 0) if q4Present else 0",geometryOk:"q4Present and workgroupSize > 0 and pow2ceil(workgroupSize) == workgroupSize and workgroupSize <= deviceWorkgroupCap and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension and workgroupStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",cooperativeNumSubgroups:"cooperativeWorkgroupSize / max(1, tunables.COOPERATIVE_SUBGROUP_SIZE)",cooperativeStorageBytes:"args.hiddenSize * 4 + cooperativeWorkgroupSize * 8 + 8 if q4Present else 0",fixedCooperativeSubgroup:'q4Present and device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id") and tunables.COOPERATIVE_SUBGROUP_SIZE > 0 and cooperativeWorkgroupSize % max(1, tunables.COOPERATIVE_SUBGROUP_SIZE) == 0 and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == tunables.COOPERATIVE_SUBGROUP_SIZE and device.adapterInfo.subgroupMaxSize == tunables.COOPERATIVE_SUBGROUP_SIZE',cooperativeGeometryOk:"q4Present and fixedCooperativeSubgroup and cooperativeWorkgroupSize > 0 and cooperativeWorkgroupSize <= deviceWorkgroupCap and cooperativeNumSubgroups > 0 and workgroupSize % cooperativeNumSubgroups == 0 and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension and cooperativeStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",scalesPk2:'q4Present and effectiveQuantBits == 4 and tensorDtypes.scalesT == "float16" and (args.hiddenSize / 32) % 2 == 0 and args.lmHeadOffset is defined and args.lmHeadOffset % 2 == 0',q4CooperativeContract:'q4Contract and effectiveQuantBits == 4 and effectiveTopK == 1 and not args.colMajor and cooperativeGeometryOk and args.hiddenSize >= tunables.COOPERATIVE_MIN_HIDDEN_SIZE and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(dtypes.S) and geometryOk',q4TiledContract:'q4Contract and (((effectiveQuantBits == 4 and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(dtypes.S) and effectiveTopK <= 8 and (effectiveTopK == 1 or args.vocabSize <= workgroupSize * effectiveNumBlocks)) or (effectiveQuantBits == 2 and tensorDtypes.scalesT == "uint32" and effectiveTopK > 1 and effectiveTopK <= 16 and args.vocabSize <= workgroupSize * effectiveNumBlocks and (args.hiddenSize / 32) % 2 == 0 and args.lmHeadOffset % 2 == 0 and not args.colMajor)) and geometryOk and (effectiveQuantBits != 4 or (not args.colMajor) or args.lmHeadOffset == 0))',headBlocksPerRow:"args.hiddenSize / 32",mRowsBytesPerBlock:"mRowsCount * (132 if effectiveQuantBits == 2 else 128)",mRowsLdsBudget:"device.limits.maxComputeWorkgroupStorageSize - workgroupSize * 8",mRowsChunkBlocks:"pick([[headBlocksPerRow % 64 == 0 and 64 * mRowsBytesPerBlock <= mRowsLdsBudget, 64], [headBlocksPerRow % 32 == 0 and 32 * mRowsBytesPerBlock <= mRowsLdsBudget, 32], [headBlocksPerRow % 16 == 0 and 16 * mRowsBytesPerBlock <= mRowsLdsBudget, 16], [headBlocksPerRow % 8 == 0 and 8 * mRowsBytesPerBlock <= mRowsLdsBudget, 8], [headBlocksPerRow % 4 == 0 and 4 * mRowsBytesPerBlock <= mRowsLdsBudget, 4], [headBlocksPerRow % 2 == 0 and 2 * mRowsBytesPerBlock <= mRowsLdsBudget, 2]], 0)",mRowsStorageBytes:"mRowsChunkBlocks * mRowsBytesPerBlock + workgroupSize * 8",mRowsWidthOk:'(effectiveQuantBits == 4 and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(dtypes.S) and effectiveTopK <= 8) or (effectiveQuantBits == 2 and tensorDtypes.scalesT == "uint32" and effectiveTopK > 1 and effectiveTopK <= 16 and args.lmHeadOffset is defined and args.lmHeadOffset % 2 == 0)',mRowsGeometryOk:"mRowsChunkBlocks > 0 and mRowsStorageBytes <= device.limits.maxComputeWorkgroupStorageSize and workgroupSize > 0 and pow2ceil(workgroupSize) == workgroupSize and workgroupSize <= deviceWorkgroupCap and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension",q4MRowsContract:'q4Present and mRowsShapeOk and args.lmHeadOffset is defined and args.finalNormOffset is defined and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.weightsT == 1 and tensorDtypes.hiddenT == "float32" and tensorDtypes.bitsT == "uint32" and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and f16Ok(dtypes.W) and args.hiddenSize % 64 == 0 and args.finalNormOffset % 4 == 0 and dim(shapes.weightsT, 0) >= args.finalNormOffset + args.hiddenSize and dim(shapes.bitsT, 0) >= (args.lmHeadOffset + args.vocabSize * headBlocksPerRow) * effectiveQuantBits and dim(shapes.scalesT, 0) >= args.lmHeadOffset + args.vocabSize * headBlocksPerRow and not args.colMajor and args.vocabSize <= workgroupSize * effectiveNumBlocks and mRowsWidthOk and mRowsGeometryOk',kSuperStorageBytes:"args.hiddenSize * 4 + workgroupSize * 8 if kSuperPresent else 0",kSuperGeometryOk:"kSuperPresent and workgroupSize > 0 and pow2ceil(workgroupSize) == workgroupSize and workgroupSize <= deviceWorkgroupCap and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension and kSuperStorageBytes <= device.limits.maxComputeWorkgroupStorageSize",kSuperMRowsGeometryOk:"kSuperPresent and mRowsChunkBlocks > 0 and mRowsStorageBytes <= device.limits.maxComputeWorkgroupStorageSize and workgroupSize > 0 and pow2ceil(workgroupSize) == workgroupSize and workgroupSize <= deviceWorkgroupCap and effectiveNumBlocks > 0 and effectiveNumBlocks <= device.limits.maxComputeWorkgroupsPerDimension",kSuperMRowsContract:'kSuperPresent and mRowsShapeOk and effectiveTopK == 1 and not args.colMajor and args.finalNormOffset is defined and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.weightsT == 1 and tensorDtypes.hiddenT == "float32" and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "float32" and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and f16Ok(dtypes.W) and args.hiddenSize % 256 == 0 and args.finalNormOffset % 4 == 0 and (args.lmHeadOffset if args.lmHeadOffset else 0) == 0 and dim(shapes.weightsT, 0) >= args.finalNormOffset + args.hiddenSize and dim(shapes.bitsT, 0) >= kSuperBitsWords and dim(shapes.scalesT, 0) >= kSuperScaleWords and args.vocabSize <= workgroupSize * effectiveNumBlocks and kSuperMRowsGeometryOk',kSuperContract:'kSuperPresent and effectiveTopK == 1 and not args.colMajor and args.finalNormOffset is defined and ranks.hiddenT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.weightsT == 1 and tensorDtypes.hiddenT == "float32" and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "float32" and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and f16Ok(dtypes.W) and args.hiddenSize % 256 == 0 and args.finalNormOffset % 4 == 0 and (args.lmHeadOffset if args.lmHeadOffset else 0) == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.weightsT, 0) >= args.finalNormOffset + args.hiddenSize and dim(shapes.bitsT, 0) >= kSuperBitsWords and dim(shapes.scalesT, 0) >= kSuperScaleWords and kSuperGeometryOk'},when:["baseContract"],bindings:{lm_head_q8:{arg:"bitsT",elementType:"u32"},lm_head_scales:{arg:"scalesT",elementType:"f32"},best_values_out:{arg:"bestValuesT",elementType:"f32"},best_ids_out:{arg:"bestIdsT",elementType:"u32"},hidden_q4:{arg:"hiddenT",name:"hidden",elementType:"f32"},q4_bits:{arg:"bitsT",elementType:"vec4<u32>"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},norm_weight:{arg:"weightsT",elementType:"$normWeightVec4"},weights_main:{arg:"weightsT",name:"weights",elementType:"$weightVec4"},norm_q8_q8:{arg:"normQ8T",name:"norm_q8",elementType:"u32"},norm_scale_q8:{arg:"normScaleT",name:"norm_scale",elementType:"f32"},normed:{arg:"normedT",elementType:"$normedVec4"},q1_bits:{arg:"bitsT",elementType:"vec4<u32>"},q1_scales:{arg:"scalesT",elementType:"$scaleScalar"},hidden:{arg:"hiddenT",elementType:"vec4<f32>"},weights:{arg:"weightsT",elementType:"$wVec4"},norm_q8:{scratch:"normQ8",elementType:"u32"},norm_scale_out:{scratch:"normScale",elementType:"f32"},norm_q8_in:{scratch:"normQ8",name:"norm_q8",buffer:"read-only-storage",elementType:"u32"},norm_scale:{scratch:"normScale",buffer:"read-only-storage",elementType:"f32"}},variants:[{id:"q8_norm",priority:1,when:["q8NormPresent","q8NormContract"],derive:{wVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",rmsEps:"args.rmsEps",wFinalNormVec4:"args.finalNormOffset / 4",weightLoad:'"vec4<f32>(weights[W_FINAL_NORM_VEC4 + dim])" if tensorDtypes.weightsT == "float16" else "weights[W_FINAL_NORM_VEC4 + dim]"',workgroupSize:256,hiddenSizeVec4:"args.hiddenSize / 4",vocabSize:"args.vocabSize",numBlocks:"effectiveNumBlocks",f32Min:'"-3.4028234663852886e38"'},intermediates:[{id:"normQ8",dtype:"uint32",shape:"[args.hiddenSize / 4]"},{id:"normScale",dtype:"float32",shape:"[1]"}],passes:[{id:"quantize",name:"LlamaDecodeFinalNormQuantizeQ8",shader:"decode-final-norm-quantize-q8.wgsl.jinja",bindings:["hidden","weights","norm_q8","norm_scale_out"],dispatch:{x:1}},{id:"head",name:"LlamaDecodeLmHeadArgmax",shader:"decode-lm-head-q8-argmax-tiled.wgsl.jinja",bindings:["norm_q8_in","norm_scale","lm_head_q8","lm_head_scales","best_values_out","best_ids_out"],dispatch:{x:"effectiveNumBlocks"}}]},{id:"row_cooperative_subgroup32",priority:2,when:["q4Present","q4CooperativeContract"],requires:{features:["subgroups"]},derive:{normWeightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',scaleScalar:'"u32" if scalesPk2 else dtypes.S',hiddenSizeBlocks:"args.hiddenSize / 32",lmHeadOffset:"args.lmHeadOffset",finalNormOffsetVec4:"args.finalNormOffset / 4",workgroupSize:"cooperativeWorkgroupSize",vocabTileSize:"workgroupSize",subgroupSize:"tunables.COOPERATIVE_SUBGROUP_SIZE",numSubgroups:"cooperativeNumSubgroups",scalesPacked2:"1 if scalesPk2 else 0",hiddenSizeVec4:"args.hiddenSize / 4",vocabSize:"args.vocabSize",numBlocks:"effectiveNumBlocks",f32Min:'"-3.4028234663852886e38"'},passes:[{id:"main",name:"LlamaDecodeLmHeadArgmax",shader:"decode-lm-head-q4-argmax-row-cooperative.wgsl.jinja",bindings:["hidden_q4","q4_bits","q4_scales","norm_weight","best_values_out","best_ids_out"],dispatch:{x:"effectiveNumBlocks"}}]},{id:"q4_mrows",priority:1,when:["q4Present","mRowsShapeOk","q4MRowsContract"],derive:{normWeightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',scaleScalar:'"u32" if effectiveQuantBits == 2 or scalesPk2 else dtypes.S',hiddenSizeBlocks:"headBlocksPerRow",lmHeadOffset:"args.lmHeadOffset",finalNormOffsetVec4:"args.finalNormOffset / 4",topK:"effectiveTopK",scalesPacked2:"1 if scalesPk2 else 0",quantBits:"effectiveQuantBits",mRows:"mRowsCount",chunkBlocks:"mRowsChunkBlocks",hiddenSizeVec4:"args.hiddenSize / 4",vocabSize:"args.vocabSize",numBlocks:"effectiveNumBlocks",f32Min:'"-3.4028234663852886e38"'},passes:[{id:"main",name:"LlamaDecodeLmHeadArgmax",shader:"decode-lm-head-q4-argmax-mrows-tiled.wgsl.jinja",bindings:["hidden_q4","q4_bits","q4_scales","norm_weight","best_values_out","best_ids_out"],dispatch:{x:"effectiveNumBlocks"}}]},{cases:[{id:"dense",priority:0,formatWhen:"densePresent",contract:"denseContract",bindings:["hidden_q4","weights_main","best_values_out","best_ids_out"],shader:"decode-final-norm-lm-head-argmax-tiled.wgsl.jinja",derive:{weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',weightScalar:"dtypes.W",hiddenSize:"args.hiddenSize",rmsEps:"args.rmsEps",finalNormOffsetVec4:"args.finalNormOffset / 4",lmHeadOffsetVec4:"args.lmHeadOffset / 4",unroll4:"(args.hiddenSize / 4) % 4 == 0",workgroupSize:256}},{id:"q8",priority:0,formatWhen:"q8Present",contract:"q8Contract",bindings:["norm_q8_q8","norm_scale_q8","lm_head_q8","lm_head_scales","best_values_out","best_ids_out"],shader:"decode-lm-head-q8-argmax-tiled.wgsl.jinja",derive:{workgroupSize:256}},{id:"q1",priority:0,formatWhen:"q1Present",contract:"q1Contract",bindings:["normed","q1_bits","q1_scales","best_values_out","best_ids_out"],shader:"decode-lm-head-q1-argmax-tiled.wgsl.jinja",derive:{normedVec4:'"vec4<f16>" if tensorDtypes.normedT == "float16" else "vec4<f32>"',scaleScalar:"dtypes.S",hiddenSizeBlocks:"args.hiddenSize / 128",lmHeadOffset:"args.lmHeadOffset",colMajor:"1 if (tunables.COL_MAJOR if tunables.COL_MAJOR >= 0 else args.colMajor) else 0",workgroupSize:256}},{id:"q4",priority:1,formatWhen:"q4Present",contract:"q4TiledContract",bindings:["hidden_q4","q4_bits","q4_scales","norm_weight","best_values_out","best_ids_out"],shader:"decode-lm-head-q4-argmax-tiled.wgsl.jinja",derive:{normWeightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',scaleScalar:'"u32" if effectiveQuantBits == 2 or scalesPk2 else dtypes.S',hiddenSizeBlocks:"args.hiddenSize / 32",lmHeadOffset:"args.lmHeadOffset",finalNormOffsetVec4:"args.finalNormOffset / 4",topK:"effectiveTopK",scalesPacked2:"1 if scalesPk2 else 0",quantBits:"effectiveQuantBits",colMajor:"1 if effectiveQuantBits == 4 and args.colMajor else 0"}},{id:"ksuper",priority:1,formatWhen:"kSuperPresent",contract:"kSuperContract",bindings:["hidden_q4","q4_bits","q4_scales","norm_weight","best_values_out","best_ids_out"],shader:"decode-lm-head-ksuper-argmax-tiled.wgsl.jinja",derive:{normWeightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',scaleScalar:"dtypes.S",hiddenSizeBlocks:"args.hiddenSize / 32",lmHeadOffset:"args.lmHeadOffset if args.lmHeadOffset else 0",finalNormOffsetVec4:"args.finalNormOffset / 4",quantBits:"4"}},{id:"ksuper_mrows",priority:1,formatWhen:"kSuperPresent",contract:"kSuperMRowsContract",bindings:["hidden_q4","q4_bits","q4_scales","norm_weight","best_values_out","best_ids_out"],shader:"decode-lm-head-ksuper-argmax-mrows-tiled.wgsl.jinja",derive:{normWeightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',scaleScalar:"dtypes.S",hiddenSizeBlocks:"args.hiddenSize / 32",lmHeadOffset:"args.lmHeadOffset if args.lmHeadOffset else 0",finalNormOffsetVec4:"args.finalNormOffset / 4",quantBits:"4",mRows:"mRowsCount",chunkBlocks:"mRowsChunkBlocks"}}],variants:[{id:"{id}",priority:"{priority}",when:["{formatWhen}","{contract}"],derive:"{derive}",passes:[{id:"main",name:"LlamaDecodeLmHeadArgmax",shader:"{shader}",derive:{hiddenSizeVec4:"args.hiddenSize / 4",vocabSize:"args.vocabSize",numBlocks:"effectiveNumBlocks",f32Min:'"-3.4028234663852886e38"'},bindings:"{bindings}",dispatch:{x:"effectiveNumBlocks"}}]}]}]},assets:[["decode-final-norm-lm-head-argmax-tiled.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const VOCAB_SIZE:u32={{ vocabSize }}u;const HIDDEN_SIZE:u32={{ hiddenSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const RMS_EPS:f32={{ rmsEps }};const W_FINAL_NORM_VEC4:u32={{ finalNormOffsetVec4 }}u;const W_LM_HEAD_VEC4:u32={{ lmHeadOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const NUM_BLOCKS:u32={{ numBlocks }}u;const F32_MIN:f32={{ f32Min }};var<workgroup>norm_partials:array<f32,{{ workgroupSize }}>;var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;
{% include "ops/_shared/matmul-argmax-hidden4.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let block=wid.x;let lid=lid3.x;var sum=0.0;for(var dim=lid;dim<HIDDEN_SIZE;dim=dim+WORKGROUP_SIZE){let value=hidden[dim];sum=fma(value,value,sum);}norm_partials[lid]=sum;workgroupBarrier();{{ wgsl_tree_fold(["norm_partials"]) }} let scale=inverseSqrt(norm_partials[0]/f32(HIDDEN_SIZE)+RMS_EPS);workgroupBarrier();var best_token=0u;var best_value=F32_MIN;for(var vocab=block*WORKGROUP_SIZE+lid;vocab<VOCAB_SIZE;vocab=vocab+WORKGROUP_SIZE*NUM_BLOCKS){let weight_base=W_LM_HEAD_VEC4+vocab*HIDDEN_SIZE_VEC4;var acc=0.0;
{% if unroll4 %}for(var dim=0u;dim<HIDDEN_SIZE_VEC4;dim=dim+4u){acc=acc+vocab_dot_part(dim,scale,weight_base);acc=acc+vocab_dot_part(dim+1u,scale,weight_base);acc=acc+vocab_dot_part(dim+2u,scale,weight_base);acc=acc+vocab_dot_part(dim+3u,scale,weight_base);}
{% else %}for(var dim=0u;dim<HIDDEN_SIZE_VEC4;dim=dim+1u){acc=acc+vocab_dot_part(dim,scale,weight_base);}
{% endif %}if(acc>best_value){best_value=acc;best_token=vocab;} }
{% include "ops/_shared/argmax-wg-tree.wgsl.jinja" %}}fn vocab_dot_part(dim:u32,scale:f32,weight_base:u32)->f32{
{% if weightScalar == "f16" %}let normed=vec4<f16>(hidden4(dim)*vec4<f32>(scale))*weights[W_FINAL_NORM_VEC4+dim];return f32(dot(normed,weights[weight_base+dim]));
{% else %}let normed=hidden4(dim)*vec4<f32>(scale)*weights[W_FINAL_NORM_VEC4+dim];return dot(normed,weights[weight_base+dim]);
{% endif %}}`],["decode-final-norm-quantize-q8.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{% set packed4x8CompatPackI8 = true %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const HIDDEN_SIZE:u32={{ hiddenSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const RMS_EPS:f32={{ rmsEps }};const W_FINAL_NORM_VEC4:u32={{ wFinalNormVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>partials:array<f32,{{ workgroupSize }}>;fn sum4(value:vec4<f32>)->f32{return value.x+value.y+value.z+value.w;}fn max4(value:vec4<f32>)->f32{return max(max(abs(value.x),abs(value.y)),max(abs(value.z),abs(value.w)));}fn quantize4(value:vec4<f32>,scale:f32)->u32{let q=vec4<i32>(round(clamp(value/vec4<f32>(scale),vec4<f32>(-127.0),vec4<f32>(127.0))));return pack4xI8(q);}fn reduce_sum(value:f32,lid:u32)->f32{partials[lid]=value;workgroupBarrier();{{ wgsl_tree_fold(["partials"]) }} return partials[0];}fn reduce_max(value:f32,lid:u32)->f32{partials[lid]=value;workgroupBarrier();{{ wgsl_tree_fold(["partials"], op="max") }} return partials[0];}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(local_invocation_id)lid3:vec3<u32>){let lid=lid3.x;var sum=0.0;for(var dim=lid;dim<HIDDEN_SIZE_VEC4;dim=dim+WORKGROUP_SIZE){let value=hidden[dim];sum=sum+sum4(value*value);}let total_sum=reduce_sum(sum,lid);let rms_scale=inverseSqrt(total_sum/f32(HIDDEN_SIZE)+RMS_EPS);workgroupBarrier();var local_max=0.0;for(var dim=lid;dim<HIDDEN_SIZE_VEC4;dim=dim+WORKGROUP_SIZE){let normed=hidden[dim]*vec4<f32>(rms_scale)*{{ weightLoad }};local_max=max(local_max,max4(normed));}let max_abs=reduce_max(local_max,lid);let q_scale=max(max_abs/127.0,1.0e-12);if(lid==0u){norm_scale_out[0]=q_scale;}workgroupBarrier();for(var dim=lid;dim<HIDDEN_SIZE_VEC4;dim=dim+WORKGROUP_SIZE){let normed=hidden[dim]*vec4<f32>(rms_scale)*{{ weightLoad }};norm_q8[dim]=quantize4(normed,q_scale);} }`],["decode-lm-head-ksuper-argmax-mrows-tiled.wgsl.jinja",`{% set nibbleHelpers = false %}{% set packed4x8CompatUnpackI8 = q3k or q6k %}{% set packed4x8CompatUnpackU8 = q2k or q3k or q5k or q6k %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const VOCAB_SIZE:u32={{ vocabSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const HIDDEN_SIZE_BLOCKS:u32={{ hiddenSizeBlocks }}u;const BPR:u32={{ hiddenSizeBlocks }}u;const W_LM_HEAD:u32={{ lmHeadOffset }}u;const W_NORM_VEC4:u32={{ finalNormOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const NUM_BLOCKS:u32={{ numBlocks }}u;const M_ROWS:u32={{ mRows }}u;const CHUNK_BLOCKS:u32={{ chunkBlocks }}u;const CHUNK_VEC4:u32={{ chunkBlocks * 8 }}u;const F32_MIN:f32={{ f32Min }};var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;var<workgroup>shared_acts:array<vec4<f32>,{{ mRows * chunkBlocks * 8 }}>;
{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% include "ops/_shared/matmul-argmax-hidden4.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let block=wid.x;let lid=lid3.x;let vocab=block*WORKGROUP_SIZE+lid;let in_range=vocab<VOCAB_SIZE;let base0=W_LM_HEAD+vocab*HIDDEN_SIZE_BLOCKS;var accs:array<f32,M_ROWS>;for(var r=0u;r<M_ROWS;r=r+1u){accs[r]=0.0;}for(var chunk=0u;chunk<HIDDEN_SIZE_BLOCKS;chunk=chunk+CHUNK_BLOCKS){workgroupBarrier();for(var i=lid;i<M_ROWS*CHUNK_VEC4;i=i+WORKGROUP_SIZE){let r=i/CHUNK_VEC4;let h=chunk*8u+(i-r*CHUNK_VEC4);shared_acts[i]=hidden4(r*HIDDEN_SIZE_VEC4+h)*vec4<f32>(norm_weight[W_NORM_VEC4+h]);}workgroupBarrier();if(in_range){for(var r=0u;r<M_ROWS;r=r+1u){let row_acts=r*CHUNK_VEC4;var acc0=accs[r];for(var b=chunk;b<chunk+CHUNK_BLOCKS;b=b+1u){let act=row_acts+(b-chunk)*8u;
{% for i in range(8) %}let x{{ i }}=shared_acts[act{{ " + " ~ i ~ "u" if i else "" }}];
{% endfor %}{% set regRows = 1 %}{% set packMeta = "q4_scales" %}{% include "ops/_shared/q4-decode-row-block-accumulate.wgsl.jinja" %}}accs[r]=acc0;} } }
{% for r in range(mRows) %}{var best_value=F32_MIN;var best_token=0u;if(in_range&&accs[{{ r }}u]>F32_MIN){best_value=accs[{{ r }}u];best_token=vocab;}best_values[lid]=best_value;best_indices[lid]=best_token;workgroupBarrier();var am_stride=WORKGROUP_SIZE/2u;loop{if(lid<am_stride){let other_value=best_values[lid+am_stride];let other_index=best_indices[lid+am_stride];if(other_value>best_values[lid]||(other_value==best_values[lid]&&other_index<best_indices[lid])){best_values[lid]=other_value;best_indices[lid]=other_index;} }workgroupBarrier();if(am_stride==1u){break;}am_stride=am_stride/2u;}if(lid==0u){best_values_out[{{ r }}u*NUM_BLOCKS+block]=best_values[0];best_ids_out[{{ r }}u*NUM_BLOCKS+block]=best_indices[0];}workgroupBarrier();}
{% endfor %}}`],["decode-lm-head-ksuper-argmax-tiled.wgsl.jinja",`{% set nibbleHelpers = false %}{% set packed4x8CompatUnpackI8 = q3k or q6k %}{% set packed4x8CompatUnpackU8 = q2k or q3k or q5k or q6k %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const VOCAB_SIZE:u32={{ vocabSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const HIDDEN_SIZE_BLOCKS:u32={{ hiddenSizeBlocks }}u;const BPR:u32={{ hiddenSizeBlocks }}u;const W_LM_HEAD:u32={{ lmHeadOffset }}u;const W_NORM_VEC4:u32={{ finalNormOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const NUM_BLOCKS:u32={{ numBlocks }}u;const F32_MIN:f32={{ f32Min }};var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;var<workgroup>shared_acts:array<vec4<f32>,{{ hiddenSizeVec4 }}>;
{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% include "ops/_shared/matmul-argmax-hidden4.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let block=wid.x;let lid=lid3.x;for(var i=lid;i<HIDDEN_SIZE_VEC4;i=i+WORKGROUP_SIZE){shared_acts[i]=hidden4(i)*vec4<f32>(norm_weight[W_NORM_VEC4+i]);}workgroupBarrier();var best_token=0u;var best_value=F32_MIN;for(var vocab=block*WORKGROUP_SIZE+lid;vocab<VOCAB_SIZE;vocab=vocab+WORKGROUP_SIZE*NUM_BLOCKS){var acc0=0.0;let base0=W_LM_HEAD+vocab*HIDDEN_SIZE_BLOCKS;for(var b=0u;b<HIDDEN_SIZE_BLOCKS;b=b+1u){let act=b*8u;
{% for i in range(8) %}let x{{ i }}=shared_acts[act{{ " + " ~ i ~ "u" if i else "" }}];
{% endfor %}{% set regRows = 1 %}{% set packMeta = "q4_scales" %}{% include "ops/_shared/q4-decode-row-block-accumulate.wgsl.jinja" %}}let acc=acc0;if(acc>best_value){best_value=acc;best_token=vocab;} }
{% include "ops/_shared/argmax-wg-tree.wgsl.jinja" %}}`],["decode-lm-head-q1-argmax-tiled.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const VOCAB_SIZE:u32={{ vocabSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const HIDDEN_SIZE_BLOCKS:u32={{ hiddenSizeBlocks }}u;const W_LM_HEAD_Q1:u32={{ lmHeadOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const NUM_BLOCKS:u32={{ numBlocks }}u;const F32_MIN:f32={{ f32Min }};var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;var<workgroup>shared_acts:array<vec4<f32>,{{ hiddenSizeVec4 }}>;fn q1_scale(block:u32)->f32{return f32(q1_scales[block]);}
{% include "ops/_shared/q1-signs.wgsl.jinja" %}fn load_normed(value:{{ normedVec4 }})->vec4<f32>{return vec4<f32>(value);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let block=wid.x;let lid=lid3.x;for(var i=lid;i<HIDDEN_SIZE_VEC4;i=i+WORKGROUP_SIZE){shared_acts[i]=load_normed(normed[i]);}workgroupBarrier();var best_token=0u;var best_value=F32_MIN;for(var vocab=block*WORKGROUP_SIZE+lid;vocab<VOCAB_SIZE;vocab=vocab+WORKGROUP_SIZE*NUM_BLOCKS){var acc=0.0;
{% if not colMajor %}let row_block_base=W_LM_HEAD_Q1+vocab*HIDDEN_SIZE_BLOCKS;
{% endif %}for(var cb=0u;cb<HIDDEN_SIZE_BLOCKS;cb=cb+1u){let block_idx={{ "W_LM_HEAD_Q1 + cb * VOCAB_SIZE + vocab" if colMajor else "row_block_base + cb" }};let bw=q1_bits[block_idx];let s=q1_scale(block_idx);let act_base=cb*32u;let unscaled=dot(shared_acts[act_base+0u],q1_signs4_unit(bw.x,0u))+dot(shared_acts[act_base+1u],q1_signs4_unit(bw.x,4u))+dot(shared_acts[act_base+2u],q1_signs4_unit(bw.x,8u))+dot(shared_acts[act_base+3u],q1_signs4_unit(bw.x,12u))+dot(shared_acts[act_base+4u],q1_signs4_unit(bw.x,16u))+dot(shared_acts[act_base+5u],q1_signs4_unit(bw.x,20u))+dot(shared_acts[act_base+6u],q1_signs4_unit(bw.x,24u))+dot(shared_acts[act_base+7u],q1_signs4_unit(bw.x,28u))+dot(shared_acts[act_base+8u],q1_signs4_unit(bw.y,0u))+dot(shared_acts[act_base+9u],q1_signs4_unit(bw.y,4u))+dot(shared_acts[act_base+10u],q1_signs4_unit(bw.y,8u))+dot(shared_acts[act_base+11u],q1_signs4_unit(bw.y,12u))+dot(shared_acts[act_base+12u],q1_signs4_unit(bw.y,16u))+dot(shared_acts[act_base+13u],q1_signs4_unit(bw.y,20u))+dot(shared_acts[act_base+14u],q1_signs4_unit(bw.y,24u))+dot(shared_acts[act_base+15u],q1_signs4_unit(bw.y,28u))+dot(shared_acts[act_base+16u],q1_signs4_unit(bw.z,0u))+dot(shared_acts[act_base+17u],q1_signs4_unit(bw.z,4u))+dot(shared_acts[act_base+18u],q1_signs4_unit(bw.z,8u))+dot(shared_acts[act_base+19u],q1_signs4_unit(bw.z,12u))+dot(shared_acts[act_base+20u],q1_signs4_unit(bw.z,16u))+dot(shared_acts[act_base+21u],q1_signs4_unit(bw.z,20u))+dot(shared_acts[act_base+22u],q1_signs4_unit(bw.z,24u))+dot(shared_acts[act_base+23u],q1_signs4_unit(bw.z,28u))+dot(shared_acts[act_base+24u],q1_signs4_unit(bw.w,0u))+dot(shared_acts[act_base+25u],q1_signs4_unit(bw.w,4u))+dot(shared_acts[act_base+26u],q1_signs4_unit(bw.w,8u))+dot(shared_acts[act_base+27u],q1_signs4_unit(bw.w,12u))+dot(shared_acts[act_base+28u],q1_signs4_unit(bw.w,16u))+dot(shared_acts[act_base+29u],q1_signs4_unit(bw.w,20u))+dot(shared_acts[act_base+30u],q1_signs4_unit(bw.w,24u))+dot(shared_acts[act_base+31u],q1_signs4_unit(bw.w,28u));acc=acc+s*unscaled;}if(acc>best_value){best_value=acc;best_token=vocab;} }
{% include "ops/_shared/argmax-wg-tree.wgsl.jinja" %}}`],["decode-lm-head-q4-argmax-mrows-tiled.wgsl.jinja",`{% set nibbleHelpers = quantBits != 2 %}{% set packed4x8CompatUnpackU8 = quantBits != 2 %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const VOCAB_SIZE:u32={{ vocabSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const HIDDEN_SIZE_BLOCKS:u32={{ hiddenSizeBlocks }}u;const W_LM_HEAD_Q4:u32={{ lmHeadOffset }}u;const W_NORM_VEC4:u32={{ finalNormOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const NUM_BLOCKS:u32={{ numBlocks }}u;const M_ROWS:u32={{ mRows }}u;const CHUNK_BLOCKS:u32={{ chunkBlocks }}u;const CHUNK_VEC4:u32={{ chunkBlocks * 8 }}u;const F32_MIN:f32={{ f32Min }};var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;var<workgroup>shared_acts:array<vec4<f32>,{{ mRows * chunkBlocks * 8 }}>;
{% if quantBits == 2 %}var<workgroup>shared_asum:array<f32,{{ mRows * chunkBlocks }}>;
{% endif %}{% include "./lm-head-scan-helpers.wgsl.jinja" %}{% include "ops/_shared/matmul-argmax-hidden4.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let block=wid.x;let lid=lid3.x;let vocab=block*WORKGROUP_SIZE+lid;let in_range=vocab<VOCAB_SIZE;let row_block_base=W_LM_HEAD_Q4+vocab*HIDDEN_SIZE_BLOCKS;var accs:array<f32,M_ROWS>;for(var r=0u;r<M_ROWS;r=r+1u){accs[r]=0.0;}for(var chunk=0u;chunk<HIDDEN_SIZE_BLOCKS;chunk=chunk+CHUNK_BLOCKS){workgroupBarrier();for(var i=lid;i<M_ROWS*CHUNK_VEC4;i=i+WORKGROUP_SIZE){let r=i/CHUNK_VEC4;let h=chunk*8u+(i-r*CHUNK_VEC4);shared_acts[i]=hidden4(r*HIDDEN_SIZE_VEC4+h)*vec4<f32>(norm_weight[W_NORM_VEC4+h]);}workgroupBarrier();
{% if quantBits == 2 %}for(var i=lid;i<M_ROWS*CHUNK_BLOCKS;i=i+WORKGROUP_SIZE){let r=i/CHUNK_BLOCKS;let b8=r*CHUNK_VEC4+(i-r*CHUNK_BLOCKS)*8u;var sv=vec4<f32>(0.0);for(var k=0u;k<8u;k=k+1u){sv=sv+shared_acts[b8+k];}shared_asum[i]=dot(sv,vec4<f32>(1.0));}workgroupBarrier();
{% endif %}if(in_range){for(var r=0u;r<M_ROWS;r=r+1u){let row_acts=r*CHUNK_VEC4;
{% if quantBits == 2 %}let row_sums=r*CHUNK_BLOCKS;
{% endif %}var acc=accs[r];
{% if quantBits == 2 %}let row_q2_base=row_block_base/2u;for(var cb2=chunk/2u;cb2<(chunk+CHUNK_BLOCKS)/2u;cb2=cb2+1u){let w=q4_bits[row_q2_base+cb2];let s0=unpack2x16float(q4_scales[row_block_base+cb2*2u]);let s1=unpack2x16float(q4_scales[row_block_base+cb2*2u+1u]);let a0=row_acts+(cb2*16u-chunk*8u);let b0=row_sums+(cb2*2u-chunk);acc=acc+s0.x*(q2wp(w.x,shared_acts[a0+0u],shared_acts[a0+1u],shared_acts[a0+2u],shared_acts[a0+3u])+q2wp(w.y,shared_acts[a0+4u],shared_acts[a0+5u],shared_acts[a0+6u],shared_acts[a0+7u]))+s0.y*shared_asum[b0];acc=acc+s1.x*(q2wp(w.z,shared_acts[a0+8u],shared_acts[a0+9u],shared_acts[a0+10u],shared_acts[a0+11u])+q2wp(w.w,shared_acts[a0+12u],shared_acts[a0+13u],shared_acts[a0+14u],shared_acts[a0+15u]))+s1.y*shared_asum[b0+1u];}
{% elif scalesPacked2 %}for(var cb=chunk;cb<chunk+CHUNK_BLOCKS;cb=cb+2u){let bw0=q4_bits[row_block_base+cb];let bw1=q4_bits[row_block_base+cb+1u];let sp=unpack2x16float(q4_scales[(row_block_base+cb)>>1u]);let act0=row_acts+(cb-chunk)*8u;acc=acc+sp.x*({{ q4_block_dot("bw0", "act0") | indent(12) }});let act1=act0+8u;acc=acc+sp.y*({{ q4_block_dot("bw1", "act1") | indent(12) }});}
{% else %}for(var cb=chunk;cb<chunk+CHUNK_BLOCKS;cb=cb+1u){let bw=q4_bits[row_block_base+cb];let s=f32(q4_scales[row_block_base+cb]);let act_base=row_acts+(cb-chunk)*8u;acc=acc+s*({{ q4_block_dot("bw", "act_base") | indent(12) }});}
{% endif %}accs[r]=acc;} } }
{% for r in range(mRows) %}{var masked_value=F32_MIN;var best_token=0u;if(in_range&&accs[{{ r }}u]>F32_MIN){masked_value=accs[{{ r }}u];best_token=vocab;}
{% set topKStripe = "(" ~ r ~ "u * NUM_BLOCKS + block)" %}{% filter indent(4, true) %}{% include "./lm-head-topk-tree.wgsl.jinja" %}{% endfilter %}}
{% endfor %}}`],["decode-lm-head-q4-argmax-row-cooperative.wgsl.jinja",`{% set packed4x8CompatUnpackU8 = true %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const VOCAB_SIZE:u32={{ vocabSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const HIDDEN_SIZE_BLOCKS:u32={{ hiddenSizeBlocks }}u;const W_LM_HEAD_Q4:u32={{ lmHeadOffset }}u;const W_NORM_VEC4:u32={{ finalNormOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const VOCAB_TILE_SIZE:u32={{ vocabTileSize }}u;const SUBGROUP_SIZE:u32={{ subgroupSize }}u;const NUM_SUBGROUPS:u32={{ numSubgroups }}u;const NUM_BLOCKS:u32={{ numBlocks }}u;const F32_MIN:f32={{ f32Min }};var<workgroup>shared_acts:array<vec4<f32>,{{ hiddenSizeVec4 }}>;var<workgroup>subgroup_values:array<f32,{{ workgroupSize }}>;var<workgroup>subgroup_ids:array<u32,{{ workgroupSize }}>;
{% include "ops/_shared/subgroup-local-layout-proof.wgsl.jinja" %}{% include "./lm-head-scan-helpers.wgsl.jinja" %}{% include "ops/_shared/matmul-argmax-hidden4.wgsl.jinja" %}fn q4_scale(block:u32)->f32{
{% if scalesPacked2 %}let pair=unpack2x16float(q4_scales[block>>1u]);return select(pair.x,pair.y,(block&1u)!=0u);
{% else %}return f32(q4_scales[block]);
{% endif %}}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>,@builtin(subgroup_invocation_id)subgroup_lane:u32,@builtin(subgroup_id)subgroup_id:u32,@builtin(subgroup_size)actual_width:u32,@builtin(num_subgroups)actual_groups:u32,){let block=wid.x;let lid=lid3.x;let population_mask=subgroupBallot(true);let population=countOneBits(population_mask.x)+countOneBits(population_mask.y)+countOneBits(population_mask.z)+countOneBits(population_mask.w);let nativeLayout=subgroupLocalLayoutAccepted(lid,actual_width==SUBGROUP_SIZE&&population==SUBGROUP_SIZE&&actual_groups==NUM_SUBGROUPS);for(var i=lid;i<HIDDEN_SIZE_VEC4;i=i+WORKGROUP_SIZE){shared_acts[i]=hidden4(i)*vec4<f32>(norm_weight[W_NORM_VEC4+i]);}workgroupBarrier();var best_token=0u;var best_value=F32_MIN;if(nativeLayout){for(var tile_base=block*VOCAB_TILE_SIZE;tile_base<VOCAB_SIZE;tile_base=tile_base+VOCAB_TILE_SIZE*NUM_BLOCKS){for(var round=0u;round<VOCAB_TILE_SIZE/NUM_SUBGROUPS;round=round+1u){let vocab=tile_base+round*NUM_SUBGROUPS+subgroup_id;var partial=0.0;if(vocab<VOCAB_SIZE){let row_block_base=W_LM_HEAD_Q4+vocab*HIDDEN_SIZE_BLOCKS;for(var cb=subgroup_lane;cb<HIDDEN_SIZE_BLOCKS;cb=cb+SUBGROUP_SIZE){let bw=q4_bits[row_block_base+cb];let scale=q4_scale(row_block_base+cb);let act_base=cb*8u;partial=partial+scale*({{ q4_block_dot("bw", "act_base") | indent(12) }});} }let value=subgroupAdd(partial);if(vocab<VOCAB_SIZE&&(value>best_value||(value==best_value&&vocab<best_token))){best_value=value;best_token=vocab;} } }if(subgroupElect()){subgroup_values[subgroup_id]=best_value;subgroup_ids[subgroup_id]=best_token;} }else{for(var tile_base=block*VOCAB_TILE_SIZE;tile_base<VOCAB_SIZE;tile_base+=VOCAB_TILE_SIZE*NUM_BLOCKS){for(var offset=lid;offset<VOCAB_TILE_SIZE;offset+=WORKGROUP_SIZE){let vocab=tile_base+offset;var value=0.0;if(vocab<VOCAB_SIZE){let row_block_base=W_LM_HEAD_Q4+vocab*HIDDEN_SIZE_BLOCKS;for(var cb=0u;cb<HIDDEN_SIZE_BLOCKS;cb+=1u){let bw=q4_bits[row_block_base+cb];let scale=q4_scale(row_block_base+cb);let act_base=cb*8u;value+=scale*({{ q4_block_dot("bw", "act_base") | indent(14) }});}if(value>best_value||(value==best_value&&vocab<best_token)){best_value=value;best_token=vocab;} } } }subgroup_values[lid]=best_value;subgroup_ids[lid]=best_token;}workgroupBarrier();if(lid==0u){var value=F32_MIN;var index=0u;for(var i=0u;i<select(WORKGROUP_SIZE,NUM_SUBGROUPS,nativeLayout);i=i+1u){let candidate_value=subgroup_values[i];let candidate_id=subgroup_ids[i];if(candidate_value>value||(candidate_value==value&&candidate_id<index)){value=candidate_value;index=candidate_id;} }best_values_out[block]=value;best_ids_out[block]=index;} }`],["decode-lm-head-q4-argmax-tiled.wgsl.jinja",`{% set nibbleHelpers = quantBits != 2 %}{% set packed4x8CompatUnpackU8 = quantBits != 2 %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const VOCAB_SIZE:u32={{ vocabSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const HIDDEN_SIZE_BLOCKS:u32={{ hiddenSizeBlocks }}u;
{% if quantBits == 2 or not colMajor %}const W_LM_HEAD_Q4:u32={{ lmHeadOffset }}u;
{% endif %}const W_NORM_VEC4:u32={{ finalNormOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const NUM_BLOCKS:u32={{ numBlocks }}u;const F32_MIN:f32={{ f32Min }};var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;var<workgroup>shared_acts:array<vec4<f32>,{{ hiddenSizeVec4 }}>;
{% if quantBits == 2 %}var<workgroup>shared_asum:array<f32,{{ hiddenSizeBlocks }}>;
{% endif %}{% include "./lm-head-scan-helpers.wgsl.jinja" %}{% include "ops/_shared/matmul-argmax-hidden4.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let block=wid.x;let lid=lid3.x;for(var i=lid;i<HIDDEN_SIZE_VEC4;i=i+WORKGROUP_SIZE){shared_acts[i]=hidden4(i)*vec4<f32>(norm_weight[W_NORM_VEC4+i]);}workgroupBarrier();
{% if quantBits == 2 %}for(var i=lid;i<HIDDEN_SIZE_BLOCKS;i=i+WORKGROUP_SIZE){let b8=i*8u;var sv=vec4<f32>(0.0);for(var k=0u;k<8u;k=k+1u){sv=sv+shared_acts[b8+k];}shared_asum[i]=dot(sv,vec4<f32>(1.0));}workgroupBarrier();
{% endif %}var best_token=0u;var best_value=F32_MIN;for(var vocab=block*WORKGROUP_SIZE+lid;vocab<VOCAB_SIZE;vocab=vocab+WORKGROUP_SIZE*NUM_BLOCKS){var acc=0.0;
{% if quantBits == 2 or not colMajor %}let row_block_base=W_LM_HEAD_Q4+vocab*HIDDEN_SIZE_BLOCKS;
{% endif %}{% if quantBits == 2 %}let row_q2_base=row_block_base/2u;for(var cb2=0u;cb2<HIDDEN_SIZE_BLOCKS/2u;cb2=cb2+1u){let w=q4_bits[row_q2_base+cb2];let s0=unpack2x16float(q4_scales[row_block_base+cb2*2u]);let s1=unpack2x16float(q4_scales[row_block_base+cb2*2u+1u]);let a0=cb2*16u;let b0=cb2*2u;acc=acc+s0.x*(q2wp(w.x,shared_acts[a0+0u],shared_acts[a0+1u],shared_acts[a0+2u],shared_acts[a0+3u])+q2wp(w.y,shared_acts[a0+4u],shared_acts[a0+5u],shared_acts[a0+6u],shared_acts[a0+7u]))+s0.y*shared_asum[b0];acc=acc+s1.x*(q2wp(w.z,shared_acts[a0+8u],shared_acts[a0+9u],shared_acts[a0+10u],shared_acts[a0+11u])+q2wp(w.w,shared_acts[a0+12u],shared_acts[a0+13u],shared_acts[a0+14u],shared_acts[a0+15u]))+s1.y*shared_asum[b0+1u];}
{% elif scalesPacked2 %}{% if colMajor %}{% else %}{% endif %}for(var cb=0u;cb<HIDDEN_SIZE_BLOCKS;cb=cb+2u){
{% if colMajor %}let bw0=q4_bits[cb*VOCAB_SIZE+vocab];let bw1=q4_bits[(cb+1u)*VOCAB_SIZE+vocab];let sp=unpack2x16float(q4_scales[(cb>>1u)*VOCAB_SIZE+vocab]);
{% else %}let bw0=q4_bits[row_block_base+cb];let bw1=q4_bits[row_block_base+cb+1u];let sp=unpack2x16float(q4_scales[(row_block_base+cb)>>1u]);
{% endif %}let act0=cb*8u;acc=acc+sp.x*({{ q4_block_dot("bw0", "act0") | indent(8) }});let act1=act0+8u;acc=acc+sp.y*({{ q4_block_dot("bw1", "act1") | indent(8) }});}
{% else %}{% if colMajor %}{% endif %}for(var cb=0u;cb<HIDDEN_SIZE_BLOCKS;cb=cb+1u){
{% if colMajor %}let bw=q4_bits[cb*VOCAB_SIZE+vocab];let s=f32(q4_scales[cb*VOCAB_SIZE+vocab]);
{% else %}let bw=q4_bits[row_block_base+cb];let s=f32(q4_scales[row_block_base+cb]);
{% endif %}let act_base=cb*8u;acc=acc+s*({{ q4_block_dot("bw", "act_base") | indent(8) }});}
{% endif %}if(acc>best_value){best_value=acc;best_token=vocab;} }
{% if topK > 1 %}var masked_value=best_value;
{% set topKStripe = "block" %}{% filter indent(2, true) %}{% include "./lm-head-topk-tree.wgsl.jinja" %}{% endfilter %}{% else %}{% include "ops/_shared/argmax-wg-tree.wgsl.jinja" %}{% endif %}}`],["decode-lm-head-q8-argmax-tiled.wgsl.jinja",`{% set packed4x8CompatDotI8 = true %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/quant-dot4-packed.wgsl.jinja" %}const VOCAB_SIZE:u32={{ vocabSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const NUM_BLOCKS:u32={{ numBlocks }}u;var<workgroup>best_values:array<f32,{{ workgroupSize }}>;var<workgroup>best_indices:array<u32,{{ workgroupSize }}>;@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let block=wid.x;let lid=lid3.x;let activation_scale=norm_scale[0];var best_token=0u;var best_value={{ f32Min }};for(var vocab=block*WORKGROUP_SIZE+lid;vocab<VOCAB_SIZE;vocab=vocab+WORKGROUP_SIZE*NUM_BLOCKS){let weight_base=vocab*HIDDEN_SIZE_VEC4;var acc=0i;for(var dim=0u;dim<HIDDEN_SIZE_VEC4;dim=dim+1u){acc=acc+dot4_packed(norm_q8[dim],lm_head_q8[weight_base+dim]);}let value=f32(acc)*activation_scale*lm_head_scales[vocab];if(value>best_value){best_value=value;best_token=vocab;} }
{% include "ops/_shared/argmax-wg-tree.wgsl.jinja" %}}`],["lm-head-scan-helpers.wgsl.jinja",`{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% macro q4_block_dot(bw, act) %}q4wp({{ bw }}.x,shared_acts[{{ act }}+0u],shared_acts[{{ act }}+4u])+q4wp({{ bw }}.y,shared_acts[{{ act }}+1u],shared_acts[{{ act }}+5u])+q4wp({{ bw }}.z,shared_acts[{{ act }}+2u],shared_acts[{{ act }}+6u])+
q4wp({{ bw }}.w, shared_acts[{{ act }} + 3u], shared_acts[{{ act }} + 7u]){% endmacro %}{% if quantBits == 2 %}fn q2v(word:u32,sh:u32)->vec4<f32>{return vec4<f32>(f32((word>>sh)&3u),f32((word>>(sh+2u))&3u),f32((word>>(sh+4u))&3u),f32((word>>(sh+6u))&3u));}fn q2wp(word:u32,x0:vec4<f32>,x1:vec4<f32>,x2:vec4<f32>,x3:vec4<f32>)->f32{return dot(q2v(word,0u),x0)+dot(q2v(word,8u),x1)+dot(q2v(word,16u),x2)+dot(q2v(word,24u),x3);}
{% endif %}`],["lm-head-topk-tree.wgsl.jinja","for(var k=0u;k<{{ topK }}u;k=k+1u){best_values[lid]=masked_value;best_indices[lid]=best_token;workgroupBarrier();var stride_k=WORKGROUP_SIZE/2u;loop{if(lid<stride_k){let other_value=best_values[lid+stride_k];let other_index=best_indices[lid+stride_k];if(other_value>best_values[lid]||(other_value==best_values[lid]&&other_index<best_indices[lid])){best_values[lid]=other_value;best_indices[lid]=other_index;} }workgroupBarrier();if(stride_k==1u){break;}stride_k=stride_k/2u;}if(lid==0u){best_values_out[{{ topKStripe }}*{{ topK }}u+k]=best_values[0];best_ids_out[{{ topKStripe }}*{{ topK }}u+k]=best_indices[0];}let winner_token=best_indices[0];workgroupBarrier();if(best_token==winner_token){masked_value=F32_MIN;} }"]]}],["com.xenova.LlamaDecodeQkvProj",{manifest:{domain:"com.xenova",name:"LlamaDecodeQkvProj",sinceVersion:1,inputs:{normedT:{onnx:"Normed",dtype:"N",rank:1,optional:!0},hiddenT:{onnx:"Hidden",dtype:"H",rank:1,optional:!0},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},normWeightsT:{onnx:"NormWeights",dtype:"W",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},kBitsT:{onnx:"KBits",dtype:"uint32",rank:1,optional:!0},kScalesT:{onnx:"KScales",dtype:"KS",rank:1,optional:!0},vBitsT:{onnx:"VBits",dtype:"uint32",rank:1,optional:!0},vScalesT:{onnx:"VScales",dtype:"VS",rank:1,optional:!0},qT:{onnx:"Q",dtype:"float32",rank:1,optional:!0},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1,optional:!0},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1,optional:!0},cosT:{onnx:"RopeCos",dtype:"float32",rank:1,optional:!0},sinT:{onnx:"RopeSin",dtype:"float32",rank:1,optional:!0}},outputs:{qT:{onnx:"Q",dtype:"float32",rank:1,optional:!0},qGateT:{onnx:"QGate",dtype:"float32",rank:1,optional:!0},kT:{onnx:"K",dtype:"float32",rank:1,optional:!0},vT:{onnx:"V",dtype:"float32",rank:1,optional:!0},qkvT:{onnx:"QKV",dtype:"O",rank:1,optional:!0},kCacheT:{onnx:"KCache",dtype:"float32",rank:1,optional:!0},vCacheT:{onnx:"VCache",dtype:"float32",rank:1,optional:!0},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1,optional:!0},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1,optional:!0}},args:{layer:{kind:"u32",required:!1},cacheLen:{kind:"u32",onnx:"cache_len",required:!1},pastLen:{kind:"u32",onnx:"past_len",required:!1},hiddenSize:{kind:"u32",onnx:"hidden_size",required:!1},tokens:{kind:"u32",required:!1},numKvHeads:{kind:"u32",onnx:"num_kv_heads"},numHeads:{kind:"u32",onnx:"num_heads",required:!1},headDim:{kind:"u32",onnx:"head_dim"},rotaryDim:{kind:"u32",onnx:"rotary_dim",required:!1},rmsEps:{kind:"f32",onnx:"rms_eps",required:!1},normOffset:{kind:"u32",onnx:"input_norm_offset",required:!1},preNormed:{kind:"u32",onnx:"pre_normed_input",required:!1},qOffset:{kind:"u32",onnx:"q_proj_offset",required:!1},qSize:{kind:"u32",onnx:"q_out_size",required:!1},kOffset:{kind:"u32",onnx:"k_proj_offset",required:!1},vOffset:{kind:"u32",onnx:"v_proj_offset",required:!1},format:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","lut2_32","lut2_16","q6_k","q5_k","q3_k","q2_k","q1_0"]},fuseKv:{kind:"u32",onnx:"fuse_kv_append",required:!1},kvDstStart:{kind:"u32",onnx:"kv_cache_dst_start",required:!1},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1},iqRaw:{kind:"u32",onnx:"iq_raw_variant",required:!1},gridOffset:{kind:"u32",onnx:"offset",required:!1},kGridOffset:{kind:"u32",onnx:"offset",required:!1},vGridOffset:{kind:"u32",onnx:"offset",required:!1},kFormat:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q8_0","iq","lut32","lut16","q6_k","q5_k","lut32_compact","lut16_compact","lut2_32","lut2_16"]},kLut:{kind:"u32",onnx:"dequant_lut_id",required:!1},kIqRaw:{kind:"u32",onnx:"iq_raw_variant",required:!1},vFormat:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q8_0","iq","lut32","lut16","q6_k","q5_k","lut32_compact","lut16_compact","lut2_32","lut2_16"]},vLut:{kind:"u32",onnx:"dequant_lut_id",required:!1},vIqRaw:{kind:"u32",onnx:"iq_raw_variant",required:!1}},typeConstraints:{N:["float32","float16"],H:["float32","float16"],S:["float32","float16","uint32"],KS:["float32","float16","uint32"],VS:["float32","float16","uint32"],W:["float32","float16"],O:["float32","float16"],C:["float32","float16"]},tunables:{GATED_DENSE_WORKGROUP_SIZE:{default:64},GATED_Q4_WORKGROUP_SIZE:{default:32},GATED_Q1_WORKGROUP_SIZE:{default:64},GATED_Q4_TILE_ROWS:{default:2}},derive:{tokenRows:"args.tokens if args.tokens is defined else 1",deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",subgroupCoveringWorkgroup:'min(deviceWorkgroupCap, max(32, device.adapterInfo.subgroupMinSize)) if (device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize")) else 32',fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',lutcWords:"1 if (lut2 == 1 and args.lut == 9) else (5 if lutc16 == 1 else 3)",q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWave32:'device.features.has("subgroups") and wave32Adapter',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32SubgroupsUsable:"subgroupsWave32 or pinSubgroupSize32",subgroupIdBuiltins:'device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id")',q4knc:'1 if fmt == "q4_k_native_compact" else 0',iqRawSel:"args.iqRaw if args.iqRaw else 0",kFmt:"args.kFormat if args.kFormat else fmt",kQuantBits:'8 if (kFmt == "q8_0" or kFmt == "iq") else 4',kHasMin:'1 if kFmt == "q4_k" else 0',kIq:'1 if kFmt == "iq" else 0',kLut16:'1 if kFmt == "lut16" else 0',kLut2:'1 if (kFmt == "lut2_32" or kFmt == "lut2_16") else 0',kQ6k:'1 if kFmt == "q6_k" else 0',kQ5k:'1 if kFmt == "q5_k" else 0',kScalesPerBlock:"2 if (kHasMin or kIq or kLut16) else 1",kLutc:'1 if (kFmt == "lut32_compact" or kFmt == "lut16_compact" or kLut2 == 1) else 0',kLutc16:'1 if (kFmt == "lut16_compact" or kFmt == "lut2_16") else 0',kLutcWords:"1 if (kLut2 == 1 and args.kLut == 9) else (5 if kLutc16 == 1 else 3)",kIqRawSel:"args.kIqRaw if args.kIqRaw else 0",vFmt:"args.vFormat if args.vFormat else fmt",vQuantBits:'8 if (vFmt == "q8_0" or vFmt == "iq") else 4',vHasMin:'1 if vFmt == "q4_k" else 0',vIq:'1 if vFmt == "iq" else 0',vLut16:'1 if vFmt == "lut16" else 0',vLut2:'1 if (vFmt == "lut2_32" or vFmt == "lut2_16") else 0',vQ6k:'1 if vFmt == "q6_k" else 0',vQ5k:'1 if vFmt == "q5_k" else 0',vScalesPerBlock:"2 if (vHasMin or vIq or vLut16) else 1",vLutc:'1 if (vFmt == "lut32_compact" or vFmt == "lut16_compact" or vLut2 == 1) else 0',vLutc16:'1 if (vFmt == "lut16_compact" or vFmt == "lut2_16") else 0',vLutcWords:"1 if (vLut2 == 1 and args.vLut == 9) else (5 if vLutc16 == 1 else 3)",vIqRawSel:"args.vIqRaw if args.vIqRaw else 0",qIq1Family:"iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8",kIq1Family:"kIqRawSel == 1 or kIqRawSel == 7 or kIqRawSel == 8",vIq1Family:"vIqRawSel == 1 or vIqRawSel == 7 or vIqRawSel == 8",qGridInBitsOk:"(args.gridOffset is defined and args.gridOffset % 2 == 0 and dim(shapes.bitsT, 0) >= args.gridOffset + 4096) if qIq1Family else true",kGridInBitsOk:"(args.kGridOffset is defined and args.kGridOffset % 2 == 0 and dim(shapes.kBitsT, 0) >= args.kGridOffset + 4096) if kIq1Family else true",vGridInBitsOk:"(args.vGridOffset is defined and args.vGridOffset % 2 == 0 and dim(shapes.vBitsT, 0) >= args.vGridOffset + 4096) if vIq1Family else true",splitPacksPresent:"present.kBitsT and present.kScalesT and present.vBitsT and present.vScalesT",projectionArgsPresent:"(args.hiddenSize is defined and args.qOffset is defined and args.kOffset is defined and args.vOffset is defined)",sgRuntime:'device.features.has("subgroups") and not wave32SubgroupsUsable',portableMr4Preferred:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize < device.adapterInfo.subgroupMaxSize and device.adapterInfo.subgroupMaxSize <= 32',qkvMaxBlocks:"(max(args.qOffset + (args.qSize if args.qSize else args.hiddenSize) * (args.hiddenSize / 32), max(args.kOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 32), args.vOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 32)))) if projectionArgsPresent else 0",qkvSuperblocks:"ceilDiv(qkvMaxBlocks, 8)",baseContract:'projectionArgsPresent and lut2 == 0 and iqRawSel == 0 and (not present.qGateT and (ranks.hiddenT == 1 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.qT == 1 and ranks.kT == 1 and ranks.vT == 1 and fmt != "q1_0" and tensorDtypes.bitsT == "uint32" and tensorDtypes.qT == "float32" and tensorDtypes.kT == "float32" and tensorDtypes.vT == "float32" and args.hiddenSize > 0 and args.hiddenSize % 32 == 0 and args.numKvHeads > 0 and args.headDim > 0 and (quantBits == 4 or quantBits == 8) and (not (q6k or q5k or q3k or q2k or q4knc or lutc) or (args.qOffset % 8 == 0 and args.kOffset % 8 == 0 and args.vOffset % 8 == 0)) and (tensorDtypes.scalesT == "uint32" if (q4knc or lutc) else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")) and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.qT, 0) == (args.qSize if args.qSize else args.hiddenSize) and dim(shapes.kT, 0) == args.numKvHeads * args.headDim and dim(shapes.vT, 0) == args.numKvHeads * args.headDim and dim(shapes.bitsT, 0) % 4 == 0 and dim(shapes.bitsT, 0) >= (qkvMaxBlocks * 4 if (q4knc or lutc) else (qkvSuperblocks * 52 if q6k else (qkvSuperblocks * 44 if q5k else (qkvSuperblocks * 28 if q3k else (qkvSuperblocks * 20 if q2k else qkvMaxBlocks * quantBits))))) and dim(shapes.scalesT, 0) >= (qkvSuperblocks * 5 if q4knc else (qkvSuperblocks * lutcWords if lutc else (qkvSuperblocks * 2 if (q2k or q5k) else (qkvSuperblocks if (q6k or q3k) else qkvMaxBlocks * (2 if (hasMin or iq or lut16) else 1)))))))',gatedDenseWorkgroupLimit:"min(tunables.GATED_DENSE_WORKGROUP_SIZE, deviceWorkgroupCap)",gatedDenseWorkgroupSize:"max(1, pow2ceil(gatedDenseWorkgroupLimit + 1) / 2)",gatedQ4WorkgroupLimit:"min(tunables.GATED_Q4_WORKGROUP_SIZE, deviceWorkgroupCap)",gatedQ4WorkgroupSize:"max(1, pow2ceil(gatedQ4WorkgroupLimit + 1) / 2)",gatedQ1WorkgroupLimit:"min(tunables.GATED_Q1_WORKGROUP_SIZE, deviceWorkgroupCap)",gatedQ1WorkgroupSize:"max(1, pow2ceil(gatedQ1WorkgroupLimit + 1) / 2)",gatedQSize:"args.qSize if args.qSize else 0",gatedBlocksPerRow:"(args.hiddenSize / 32) if projectionArgsPresent else 0",gatedKvSize:"(args.numKvHeads * args.headDim) if projectionArgsPresent else 0",gatedQMaxBlocks:"(args.qOffset + 2 * gatedQSize * gatedBlocksPerRow) if projectionArgsPresent else 0",gatedKMaxBlocks:"(args.kOffset + gatedKvSize * gatedBlocksPerRow) if projectionArgsPresent else 0",gatedVMaxBlocks:"(args.vOffset + gatedKvSize * gatedBlocksPerRow) if projectionArgsPresent else 0",gatedMaxBlocks:"max(gatedQMaxBlocks, max(gatedKMaxBlocks, gatedVMaxBlocks))",gatedQSupers:"ceilDiv(gatedQMaxBlocks, 8)",gatedKSupers:"ceilDiv(gatedKMaxBlocks, 8)",gatedVSupers:"ceilDiv(gatedVMaxBlocks, 8)",gatedSupers:"ceilDiv(gatedMaxBlocks, 8)",gatedBitsNeeded:"ceilDiv(gatedMaxBlocks, 8) * 12 if iqRawSel == 7 else (ceilDiv(gatedMaxBlocks, 8) * 14 if iqRawSel == 8 else (ceilDiv(gatedMaxBlocks, 8) * 27 if iqRawSel == 4 else (gatedMaxBlocks * 2 if (iqRawSel == 1 or iqRawSel == 2) else (gatedSupers * 52 if q6k else (gatedSupers * 44 if q5k else gatedMaxBlocks * (2 if lut2 else quantBits))))))",gatedScalesNeeded:"ceilDiv(gatedMaxBlocks, 8) if (iqRawSel == 4 or iqRawSel == 2 or iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8) else (gatedSupers * lutcWords if lutc else (gatedSupers if q6k else (gatedSupers * 2 if q5k else gatedMaxBlocks * (2 if (hasMin or iq or lut16) else 1))))",gatedQBitsNeeded:"ceilDiv(gatedQMaxBlocks, 8) * 12 if iqRawSel == 7 else (ceilDiv(gatedQMaxBlocks, 8) * 14 if iqRawSel == 8 else (ceilDiv(gatedQMaxBlocks, 8) * 27 if iqRawSel == 4 else (gatedQMaxBlocks * 2 if (iqRawSel == 1 or iqRawSel == 2) else (gatedQSupers * 52 if q6k else (gatedQSupers * 44 if q5k else gatedQMaxBlocks * (2 if lut2 else quantBits))))))",gatedQScalesNeeded:"ceilDiv(gatedQMaxBlocks, 8) if (iqRawSel == 4 or iqRawSel == 2 or iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8) else (gatedQSupers * lutcWords if lutc else (gatedQSupers if q6k else (gatedQSupers * 2 if q5k else gatedQMaxBlocks * (2 if (hasMin or iq or lut16) else 1))))",gatedKBitsNeeded:"ceilDiv(gatedKMaxBlocks, 8) * 12 if kIqRawSel == 7 else (ceilDiv(gatedKMaxBlocks, 8) * 14 if kIqRawSel == 8 else (ceilDiv(gatedKMaxBlocks, 8) * 27 if kIqRawSel == 4 else (gatedKMaxBlocks * 2 if (kIqRawSel == 1 or kIqRawSel == 2) else (gatedKSupers * 52 if kQ6k else (gatedKSupers * 44 if kQ5k else gatedKMaxBlocks * (2 if kLut2 else kQuantBits))))))",gatedKScalesNeeded:"ceilDiv(gatedKMaxBlocks, 8) if (kIqRawSel == 4 or kIqRawSel == 2 or kIqRawSel == 1 or kIqRawSel == 7 or kIqRawSel == 8) else (gatedKSupers * kLutcWords if kLutc else (gatedKSupers if kQ6k else (gatedKSupers * 2 if kQ5k else gatedKMaxBlocks * kScalesPerBlock)))",gatedVBitsNeeded:"ceilDiv(gatedVMaxBlocks, 8) * 12 if vIqRawSel == 7 else (ceilDiv(gatedVMaxBlocks, 8) * 14 if vIqRawSel == 8 else (ceilDiv(gatedVMaxBlocks, 8) * 27 if vIqRawSel == 4 else (gatedVMaxBlocks * 2 if (vIqRawSel == 1 or vIqRawSel == 2) else (gatedVSupers * 52 if vQ6k else (gatedVSupers * 44 if vQ5k else gatedVMaxBlocks * (2 if vLut2 else vQuantBits))))))",gatedVScalesNeeded:"ceilDiv(gatedVMaxBlocks, 8) if (vIqRawSel == 4 or vIqRawSel == 2 or vIqRawSel == 1 or vIqRawSel == 7 or vIqRawSel == 8) else (gatedVSupers * vLutcWords if vLutc else (gatedVSupers if vQ6k else (gatedVSupers * 2 if vQ5k else gatedVMaxBlocks * vScalesPerBlock)))",gatedKSuperGeometry:"(args.hiddenSize % 256 == 0) if projectionArgsPresent else false",gatedTileFits:"projectionArgsPresent and args.qSize is defined and tunables.GATED_Q4_TILE_ROWS >= 1 and args.headDim % tunables.GATED_Q4_TILE_ROWS == 0 and args.qSize % tunables.GATED_Q4_TILE_ROWS == 0 and (args.numKvHeads * args.headDim) % tunables.GATED_Q4_TILE_ROWS == 0",gatedTileRows:"tunables.GATED_Q4_TILE_ROWS if gatedTileFits else 1",gatedDenseResourcesFit:"(gatedDenseWorkgroupSize * 4 <= device.limits.maxComputeWorkgroupStorageSize and 2 * gatedQSize + 2 * args.numKvHeads * args.headDim <= device.limits.maxComputeWorkgroupsPerDimension)",gatedQ4ResourcesFit:"(gatedQ4WorkgroupSize * 4 <= device.limits.maxComputeWorkgroupStorageSize and 2 * gatedQSize + 2 * args.numKvHeads * args.headDim <= device.limits.maxComputeWorkgroupsPerDimension)",gatedQ1ResourcesFit:"(gatedQ1WorkgroupSize * 16 <= device.limits.maxComputeWorkgroupStorageSize and ceilDiv(2 * gatedQSize + 2 * args.numKvHeads * args.headDim, 4) <= device.limits.maxComputeWorkgroupsPerDimension)",gatedBaseContract:'projectionArgsPresent and (ranks.normedT == 1 and ranks.qT == 1 and ranks.qGateT == 1 and ranks.kT == 1 and ranks.vT == 1 and tensorDtypes.normedT == "float32" and tensorDtypes.qT == "float32" and tensorDtypes.qGateT == "float32" and tensorDtypes.kT == "float32" and tensorDtypes.vT == "float32" and args.qSize > 0 and args.numKvHeads > 0 and args.headDim > 0 and args.qSize % args.headDim == 0 and args.hiddenSize > 0 and dim(shapes.normedT, 0) == tokenRows * args.hiddenSize and dim(shapes.qT, 0) == tokenRows * args.qSize and dim(shapes.qGateT, 0) == tokenRows * args.qSize and dim(shapes.kT, 0) == tokenRows * (args.numKvHeads * args.headDim) and dim(shapes.vT, 0) == tokenRows * (args.numKvHeads * args.headDim) and tokenRows >= 1 and tokenRows <= 4)'},bindings:{normed:{arg:"normedT",elementType:"$normedVec4"},weights:{arg:"weightsT",elementType:"$weightVec4"},q:{arg:"qT",elementType:"f32"},k:{arg:"kT",elementType:"f32"},v:{arg:"vT",elementType:"f32"},q1_bits:{arg:"bitsT",elementType:"vec4<u32>"},q1_scales:{arg:"scalesT",elementType:"$scaleScalar"},qkv:{arg:"qkvT",elementType:"$outputScalar"},hidden:{arg:"hiddenT",elementType:"$hiddenVec4"},norm_weight:{arg:"normWeightsT",elementType:"$normWeightVec4"},q4_bits:{arg:"bitsT",elementType:"vec4<u32>"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},normed_q:{arg:"normedT",name:"normed",elementType:"vec4<f32>"},q_gate:{arg:"qGateT",elementType:"f32"},q4_bits_q:{arg:"bitsT",name:"q4_bits",elementType:"$bitsElem"},cache_keys:{arg:"cacheKeysT",elementType:"$cacheScalar"},cache_values:{arg:"cacheValuesT",elementType:"$cacheScalar"},rope_cos:{arg:"cosT",elementType:"f32"},rope_sin:{arg:"sinT",elementType:"f32"},rope_params:{arg:"paramsT",name:"params",struct:[{name:"past_len",type:"u32",value:"args.pastLen if args.pastLen else 0"}]},q_bits:{arg:"bitsT",elementType:"u32"},q_scales:{arg:"scalesT",elementType:"u32"},k_cache:{arg:"kCacheT",elementType:"f32"},v_cache:{arg:"vCacheT",elementType:"f32"},kv_params:{name:"p",struct:[{name:"kvDstStart",type:"u32",value:"args.kvDstStart if args.kvDstStart else 0"}]}},variants:[{id:"dense",priority:0,when:["projectionArgsPresent","not present.qGateT","ranks.normedT == 1","ranks.weightsT == 1","ranks.qT == 1","ranks.kT == 1","ranks.vT == 1",'(tensorDtypes.normedT == "float32" or tensorDtypes.normedT == "float16")','(tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")','tensorDtypes.qT == "float32"','tensorDtypes.kT == "float32"','tensorDtypes.vT == "float32"',"f16Ok(dtypes.N) and f16Ok(dtypes.W)","args.hiddenSize > 0","args.hiddenSize % 4 == 0","args.numKvHeads > 0","args.headDim > 0","args.qOffset % 4 == 0","args.kOffset % 4 == 0","args.vOffset % 4 == 0","dim(shapes.normedT, 0) == args.hiddenSize","dim(shapes.qT, 0) == args.hiddenSize","dim(shapes.kT, 0) == args.numKvHeads * args.headDim","dim(shapes.vT, 0) == args.numKvHeads * args.headDim","dim(shapes.weightsT, 0) >= max(args.qOffset + args.hiddenSize * args.hiddenSize, args.kOffset + args.numKvHeads * args.headDim * args.hiddenSize, args.vOffset + args.numKvHeads * args.headDim * args.hiddenSize)","tokenRows == 1"],derive:{fusedRmsNorm:"false",normedVec4:'"vec4<f16>" if tensorDtypes.normedT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",kvSize:"args.numKvHeads * args.headDim",totalOutputs:"args.hiddenSize + 2 * args.numKvHeads * args.headDim",wQVec4:"args.qOffset / 4",wKVec4:"args.kOffset / 4",wVVec4:"args.vOffset / 4",workgroupSize:64,weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',weightScalar:"dtypes.W",hiddenSizeVec4:"args.hiddenSize / 4"},passes:[{id:"main",name:"LlamaDecodeQkv",shader:"llama-decode-qkv.wgsl.jinja",bindings:["normed","weights","q","k","v"],dispatch:{x:"args.hiddenSize + 2 * args.numKvHeads * args.headDim"}}]},{id:"dense_norm",priority:0,when:["projectionArgsPresent","not present.qGateT","args.rmsEps is defined","args.normOffset is defined","ranks.hiddenT == 1","ranks.weightsT == 1","ranks.qT == 1","ranks.kT == 1","ranks.vT == 1",'tensorDtypes.hiddenT == "float32"','(tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")','tensorDtypes.qT == "float32"','tensorDtypes.kT == "float32"','tensorDtypes.vT == "float32"',"f16Ok(dtypes.W)","args.hiddenSize > 0","args.hiddenSize % 4 == 0","args.numKvHeads > 0","args.headDim > 0","args.normOffset % 4 == 0","args.qOffset % 4 == 0","args.kOffset % 4 == 0","args.vOffset % 4 == 0","dim(shapes.hiddenT, 0) == args.hiddenSize","dim(shapes.qT, 0) == args.hiddenSize","dim(shapes.kT, 0) == args.numKvHeads * args.headDim","dim(shapes.vT, 0) == args.numKvHeads * args.headDim","dim(shapes.weightsT, 0) >= args.normOffset + args.hiddenSize","dim(shapes.weightsT, 0) >= max(args.qOffset + args.hiddenSize * args.hiddenSize, args.kOffset + args.numKvHeads * args.headDim * args.hiddenSize, args.vOffset + args.numKvHeads * args.headDim * args.hiddenSize)","tokenRows == 1"],derive:{fusedRmsNorm:"true",hiddenSize:"args.hiddenSize",kvSize:"args.numKvHeads * args.headDim",totalOutputs:"args.hiddenSize + 2 * args.numKvHeads * args.headDim",rmsEps:"args.rmsEps",normOffsetVec4:"args.normOffset / 4",wQVec4:"args.qOffset / 4",wKVec4:"args.kOffset / 4",wVVec4:"args.vOffset / 4",workgroupSize:64,weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',weightScalar:"dtypes.W",hiddenSizeVec4:"args.hiddenSize / 4"},passes:[{id:"main",name:"LlamaDecodeQkvNorm",shader:"llama-decode-qkv.wgsl.jinja",bindings:[{arg:"hiddenT",name:"hidden",elementType:"vec4<f32>"},"weights","q","k","v"],dispatch:{x:"args.hiddenSize + 2 * args.numKvHeads * args.headDim"}}]},{id:"q1",priority:0,when:["projectionArgsPresent",'present.bitsT and fmt == "q1_0"',"not present.qGateT","ranks.normedT == 1","ranks.bitsT == 1","ranks.scalesT == 1","ranks.qkvT == 1",'(tensorDtypes.normedT == "float32" or tensorDtypes.normedT == "float16")','tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"f16Ok(dtypes.N) and f16Ok(dtypes.S) and f16Ok(dtypes.O)","args.hiddenSize > 0","args.hiddenSize % 128 == 0","args.numKvHeads > 0","args.headDim > 0","(args.numKvHeads * args.headDim) % 4 == 0","dim(shapes.normedT, 0) == args.hiddenSize","dim(shapes.qkvT, 0) == args.hiddenSize + 2 * args.numKvHeads * args.headDim","dim(shapes.bitsT, 0) >= max(args.qOffset + args.hiddenSize * (args.hiddenSize / 128), args.kOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 128), args.vOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 128)) * 4","dim(shapes.scalesT, 0) >= max(args.qOffset + args.hiddenSize * (args.hiddenSize / 128), args.kOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 128), args.vOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 128))","tokenRows == 1"],derive:{useSubgroups:'device.features.has("subgroups")',workgroupSize:"subgroupCoveringWorkgroup",normedVec4:'"vec4<f16>" if tensorDtypes.normedT == "float16" else "vec4<f32>"',scaleScalar:"dtypes.S",outputScalar:"dtypes.O",hiddenSize:"args.hiddenSize",hiddenSizeQ1Words:"args.hiddenSize / 32",blocksPerRow:"args.hiddenSize / 128",kvSize:"args.numKvHeads * args.headDim",totalOutputs:"args.hiddenSize + 2 * args.numKvHeads * args.headDim",qOffset:"args.qOffset",kOffset:"args.kOffset",vOffset:"args.vOffset"},passes:[{id:"main",name:"LlamaDecodeQkvQ1",shader:"decode-qkv-q1.wgsl.jinja",bindings:["normed","q1_bits","q1_scales","qkv"],dispatch:{x:"ceil((args.hiddenSize + 2 * args.numKvHeads * args.headDim) / 4)"}}]},{id:"q1_norm",priority:0,when:["projectionArgsPresent",'present.bitsT and fmt == "q1_0"',"not present.qGateT","args.rmsEps is defined","args.normOffset is defined","ranks.hiddenT == 1","ranks.bitsT == 1","ranks.scalesT == 1","ranks.qkvT == 1","ranks.normWeightsT == 1",'(tensorDtypes.hiddenT == "float32" or tensorDtypes.hiddenT == "float16")','tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")','(tensorDtypes.normWeightsT == "float32" or tensorDtypes.normWeightsT == "float16")',"f16Ok(dtypes.H) and f16Ok(dtypes.S) and f16Ok(dtypes.O) and f16Ok(dtypes.W)","args.hiddenSize > 0","args.hiddenSize % 128 == 0","args.numKvHeads > 0","args.headDim > 0","(args.numKvHeads * args.headDim) % 4 == 0","args.normOffset % 4 == 0","dim(shapes.hiddenT, 0) == args.hiddenSize","dim(shapes.qkvT, 0) == args.hiddenSize + 2 * args.numKvHeads * args.headDim","dim(shapes.normWeightsT, 0) >= args.normOffset + args.hiddenSize","dim(shapes.bitsT, 0) >= max(args.qOffset + args.hiddenSize * (args.hiddenSize / 128), args.kOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 128), args.vOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 128)) * 4","dim(shapes.scalesT, 0) >= max(args.qOffset + args.hiddenSize * (args.hiddenSize / 128), args.kOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 128), args.vOffset + args.numKvHeads * args.headDim * (args.hiddenSize / 128))","tokenRows == 1"],derive:{useSubgroups:'device.features.has("subgroups")',workgroupSize:"subgroupCoveringWorkgroup",hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',normWeightVec4:'"vec4<f16>" if tensorDtypes.normWeightsT == "float16" else "vec4<f32>"',scaleScalar:"dtypes.S",outputScalar:"dtypes.O",hiddenSize:"args.hiddenSize",hiddenSizeQ1Words:"args.hiddenSize / 32",blocksPerRow:"args.hiddenSize / 128",kvSize:"args.numKvHeads * args.headDim",totalOutputs:"args.hiddenSize + 2 * args.numKvHeads * args.headDim",rmsEps:"args.rmsEps",normOffsetVec4:"args.normOffset / 4",qOffset:"args.qOffset",kOffset:"args.kOffset",vOffset:"args.vOffset"},passes:[{id:"main",name:"LlamaDecodeQkvNormQ1",shader:"decode-qkv-norm-q1.wgsl.jinja",bindings:["hidden","q1_bits","q1_scales","qkv","norm_weight"],dispatch:{x:"ceil((args.hiddenSize + 2 * args.numKvHeads * args.headDim) / 4)"}}]},{id:"q4",priority:0,when:["args.fuseKv != 1","not args.preNormed","baseContract","ranks.normWeightsT == 1",'(tensorDtypes.normWeightsT == "float32" or tensorDtypes.normWeightsT == "float16")',"f16Ok(dtypes.H) and f16Ok(dtypes.S) and f16Ok(dtypes.W)","args.normOffset % 4 == 0","dim(shapes.normWeightsT, 0) >= args.normOffset + args.hiddenSize","tokenRows == 1","(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (128 if (0) else (subgroupCoveringWorkgroup)) + 8 <= device.limits.maxComputeWorkgroupStorageSize"],derive:{scaleScalar:"dtypes.S",hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',normWeightVec4:'"vec4<f16>" if tensorDtypes.normWeightsT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",blocksPerRow:"args.hiddenSize / 32",kvSize:"args.numKvHeads * args.headDim",rmsEps:"args.rmsEps",normOffsetVec4:"args.normOffset / 4",preNormed:"0",multiRow:"0",regRows:"1",fuseKv:"0",lutId:"args.lut if args.lut else 0",workgroupSize:"subgroupCoveringWorkgroup",useSubgroups:"wave32SubgroupsUsable",qSize:"args.qSize if args.qSize else args.hiddenSize",totalOutputs:"(args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim",wQ:"args.qOffset",wK:"args.kOffset",wV:"args.vOffset",q4knCompact:"q4knc"},passes:[{id:"main",name:"LlamaDecodeQkvNormQ4",shader:"decode-qkv-norm-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","norm_weight","q","k","v"],dispatch:{x:"(args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim"},subgroupCollectivesWidth:"portable"}]},{cases:[{id:"q4_prenormed",priority:0,guard:["args.fuseKv != 1"],tokenGuard:["tokenRows == 1"],demote:null,multiRow:"0",regRows:"1",fuseKv:"0",kvBind:[],dispatchX:"(args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (128 if (0) else (subgroupCoveringWorkgroup)) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{id:"q4_prenormed_mr2_kv",priority:4,guard:["args.fuseKv == 1","projectionArgsPresent and (args.qSize if args.qSize else args.hiddenSize) % 2 == 0","(args.numKvHeads * args.headDim) % 2 == 0","projectionArgsPresent and ((args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim) % 8 == 0"],tokenGuard:["tokenRows == 1"],demote:null,multiRow:"1",regRows:"2",fuseKv:"1",kvBind:["k_cache","v_cache","kv_params"],dispatchX:"ceil(((args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim) / 8)",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (128 if (1) else (subgroupCoveringWorkgroup)) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{id:"q4_prenormed_mr4",priority:2,guard:["args.fuseKv != 1","projectionArgsPresent and (args.qSize if args.qSize else args.hiddenSize) % 4 == 0","projectionArgsPresent and (args.numKvHeads * args.headDim) % 4 == 0","projectionArgsPresent and ((args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim) % 16 == 0"],tokenGuard:[],demote:null,multiRow:"1",regRows:"4",fuseKv:"0",kvBind:[],dispatchX:"ceil(((args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim) / 16)",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (128 if (1) else (subgroupCoveringWorkgroup)) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{id:"q4_prenormed_mr2",priority:3,guard:["args.fuseKv != 1","projectionArgsPresent and (args.qSize if args.qSize else args.hiddenSize) % 2 == 0","projectionArgsPresent and (args.numKvHeads * args.headDim) % 2 == 0","projectionArgsPresent and ((args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim) % 8 == 0"],tokenGuard:[],demote:["portableMr4Preferred"],multiRow:"1",regRows:"2",fuseKv:"0",kvBind:[],dispatchX:"ceil(((args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim) / 8)",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * (128 if (1) else (subgroupCoveringWorkgroup)) + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"{id}",priority:"{priority}",when:["{guard}","args.preNormed == 1","baseContract","f16Ok(dtypes.H) and f16Ok(dtypes.S)","{tokenGuard}","{sgFoldStorageGate0}"],demoteWhen:"{demote}",derive:{scaleScalar:"dtypes.S",hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',hiddenSize:"args.hiddenSize",blocksPerRow:"args.hiddenSize / 32",kvSize:"args.numKvHeads * args.headDim",preNormed:"1",multiRow:"{multiRow}",regRows:"{regRows}",fuseKv:"{fuseKv}",lutId:"args.lut if args.lut else 0",workgroupSize:"subgroupCoveringWorkgroup",useSubgroups:"wave32SubgroupsUsable",qSize:"args.qSize if args.qSize else args.hiddenSize",totalOutputs:"(args.qSize if args.qSize else args.hiddenSize) + 2 * args.numKvHeads * args.headDim",wQ:"args.qOffset",wK:"args.kOffset",wV:"args.vOffset",q4knCompact:"q4knc"},passes:[{id:"main",name:"LlamaDecodeQkvNormQ4",shader:"decode-qkv-norm-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","q","k","v","{kvBind}"],dispatch:{x:"{dispatchX}"},subgroupCollectivesWidth:"portable"}]}]},{cases:[{id:"dense_norm_rope_cache",priority:0,qAlign:["args.qOffset % 4 == 0"],denseWeights:["dim(shapes.weightsT, 0) >= max(args.normOffset + args.hiddenSize, max(args.qOffset + args.numHeads * args.headDim * args.hiddenSize, max(args.kOffset + args.numKvHeads * args.headDim * args.hiddenSize, args.vOffset + args.numKvHeads * args.headDim * args.hiddenSize)))"],nativeQuery:[],qPackedOffset:null,nativeFlag:null,name:"LlamaDecodeQkvRopeCacheNorm",qPack:[],requires:null},{id:"q4_query_dense_kv_norm_rope_cache",priority:10,qAlign:[],denseWeights:[],nativeQuery:['args.format == "q4_k_native_compact"',"args.hiddenSize % 256 == 0","args.qOffset % 8 == 0","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','tensorDtypes.scalesT == "uint32"',"dim(shapes.bitsT, 0) >= (args.qOffset + args.numHeads * args.headDim * args.hiddenSize / 32) * 4","dim(shapes.scalesT, 0) >= (args.qOffset + args.numHeads * args.headDim * args.hiddenSize / 32) * 5 / 8","dim(shapes.weightsT, 0) >= max(args.normOffset + args.hiddenSize, args.kOffset + args.numKvHeads * args.headDim * args.hiddenSize, args.vOffset + args.numKvHeads * args.headDim * args.hiddenSize)"],qPackedOffset:"args.qOffset",nativeFlag:!0,name:"LlamaDecodeQkvNativeQueryRopeCacheNorm",qPack:["q_bits","q_scales"],requires:{limits:{maxStorageBuffersPerShaderStage:9}}}],variants:[{id:"{id}",priority:"{priority}",when:["projectionArgsPresent","not present.qGateT","args.layer is defined","args.cacheLen is defined","args.numHeads is defined","args.rmsEps is defined","args.normOffset is defined","ranks.hiddenT == 1","ranks.weightsT == 1","ranks.qT == 1","(ranks.cacheKeysT == 1 or ranks.cacheKeysT == 5)","(ranks.cacheValuesT == 1 or ranks.cacheValuesT == 5)","(ranks.cosT == 1 or ranks.cosT == 2)","(ranks.sinT == 1 or ranks.sinT == 2)",'(tensorDtypes.hiddenT == "float32" or tensorDtypes.hiddenT == "float16")','(tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")','tensorDtypes.qT == "float32"',"tensorDtypes.cacheKeysT == tensorDtypes.cacheValuesT",'(tensorDtypes.cacheKeysT == "float32" or tensorDtypes.cacheKeysT == "float16")',"f16Ok(dtypes.H) and f16Ok(dtypes.W) and f16Ok(dtypes.C)","args.cacheLen > 0","args.hiddenSize > 0","args.hiddenSize % 4 == 0","args.normOffset % 4 == 0","args.numHeads > 0","args.numKvHeads > 0","args.headDim > 0","args.headDim % 2 == 0","{qAlign}","args.kOffset % 4 == 0","args.vOffset % 4 == 0","dim(shapes.hiddenT, 0) == args.hiddenSize","dim(shapes.qT, 0) == args.numHeads * args.headDim","numel(shapes.cacheKeysT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim","numel(shapes.cacheValuesT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim","((args.pastLen if args.pastLen else 0) >= args.cacheLen or numel(shapes.cosT) >= ((args.pastLen if args.pastLen else 0) + 1) * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2))","((args.pastLen if args.pastLen else 0) >= args.cacheLen or numel(shapes.sinT) >= ((args.pastLen if args.pastLen else 0) + 1) * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2))","{denseWeights}","(not args.rotaryDim or args.rotaryDim > 0)","(not args.rotaryDim or args.rotaryDim % 2 == 0)","(not args.rotaryDim or args.rotaryDim <= args.headDim)","tokenRows == 1","{nativeQuery}"],derive:{hiddenVec4:'"vec4<f16>" if tensorDtypes.hiddenT == "float16" else "vec4<f32>"',cacheScalar:"dtypes.C",layer:"args.layer",cacheLen:"args.cacheLen",hiddenSize:"args.hiddenSize",rmsEps:"args.rmsEps",normOffsetVec4:"args.normOffset / 4",numKvHeads:"args.numKvHeads",headDim:"args.headDim",headDimHalf:"((args.rotaryDim if args.rotaryDim else args.headDim) / 2)",qPairs:"args.numHeads * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2)",vOffset:"args.numHeads * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2) + args.numKvHeads * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2)",totalOutputs:"args.numHeads * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2) + args.numKvHeads * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2) + args.numKvHeads * args.headDim + args.numHeads * (args.headDim - (args.rotaryDim if args.rotaryDim else args.headDim)) + args.numKvHeads * (args.headDim - (args.rotaryDim if args.rotaryDim else args.headDim))",wQVec4:"args.qOffset / 4",wKVec4:"args.kOffset / 4",wVVec4:"args.vOffset / 4",rotaryDim:"(args.rotaryDim if args.rotaryDim else args.headDim)",passPerHead:"(args.headDim - (args.rotaryDim if args.rotaryDim else args.headDim))",qPassStart:"args.numHeads * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2) + args.numKvHeads * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2) + args.numKvHeads * args.headDim",kPassStart:"args.numHeads * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2) + args.numKvHeads * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2) + args.numKvHeads * args.headDim + args.numHeads * (args.headDim - (args.rotaryDim if args.rotaryDim else args.headDim))",qPackedOffset:"{qPackedOffset}",nativeQuery:"{nativeFlag}",workgroupSize:64,weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',weightScalar:"dtypes.W",hiddenSizeVec4:"args.hiddenSize / 4"},passes:[{id:"main",name:"{name}",shader:"decode-qkv-rope-cache-norm.wgsl.jinja",bindings:["hidden","weights","{qPack}","q","cache_keys","cache_values","rope_cos","rope_sin","rope_params"],dispatch:{x:"args.numHeads * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2) + args.numKvHeads * ((args.rotaryDim if args.rotaryDim else args.headDim) / 2) + args.numKvHeads * args.headDim + args.numHeads * (args.headDim - (args.rotaryDim if args.rotaryDim else args.headDim)) + args.numKvHeads * (args.headDim - (args.rotaryDim if args.rotaryDim else args.headDim))"}}],requires:"{requires}"}]},{id:"dense_gated",priority:10,when:["gatedBaseContract","not present.bitsT","ranks.weightsT == 1",'(tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")',"f16Ok(tensorDtypes.weightsT)","gatedDenseResourcesFit","args.hiddenSize % 4 == 0","args.qOffset % 4 == 0","args.kOffset % 4 == 0","args.vOffset % 4 == 0","dim(shapes.weightsT, 0) >= args.qOffset + 2 * args.qSize * args.hiddenSize","dim(shapes.weightsT, 0) >= args.kOffset + (args.numKvHeads * args.headDim) * args.hiddenSize","dim(shapes.weightsT, 0) >= args.vOffset + (args.numKvHeads * args.headDim) * args.hiddenSize"],derive:{useSubgroups:'device.features.has("subgroups")',workgroupSize:"gatedDenseWorkgroupSize",qSize:"args.qSize",kvSize:"(args.numKvHeads * args.headDim)",headDim:"args.headDim",total:"2 * args.qSize + 2 * (args.numKvHeads * args.headDim)",qOffsetVec4:"args.qOffset / 4",kOffsetVec4:"args.kOffset / 4",vOffsetVec4:"args.vOffset / 4",mRows:"tokenRows",weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',weightScalar:"dtypes.W",hiddenSizeVec4:"args.hiddenSize / 4"},passes:[{id:"main",name:"LlamaDecodeQkvGated",shader:"decode-qkv-gated.wgsl.jinja",bindings:["normed_q","weights","q","q_gate","k","v"],dispatch:{x:"2 * args.qSize + 2 * (args.numKvHeads * args.headDim)"}}]},{id:"q4_gated",priority:10,when:["gatedBaseContract","present.bitsT","not present.kBitsT","not present.vBitsT",'fmt != "q1_0"',"not (q3k or q2k or q4knc)","(gatedKSuperGeometry and args.qOffset % 8 == 0 and args.kOffset % 8 == 0 and args.vOffset % 8 == 0) if (q6k or q5k or lutc or iqRawSel == 4 or iqRawSel == 2 or qIq1Family) else true","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "uint32" if lutc else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16"))',"iqRawSel == 0 or iqRawSel == 1 or iqRawSel == 2 or iqRawSel == 4 or iqRawSel == 7 or iqRawSel == 8","qGridInBitsOk","(args.lut is defined and args.lut != 0) if lutc else true","true if lutc else f16Ok(tensorDtypes.scalesT)","gatedQ4ResourcesFit","args.hiddenSize % 32 == 0","(quantBits == 4 or quantBits == 8)","dim(shapes.bitsT, 0) >= gatedBitsNeeded","dim(shapes.scalesT, 0) >= gatedScalesNeeded"],derive:{useSubgroups:'device.features.has("subgroups")',scaleScalar:"dtypes.S",blocksPerRow:"args.hiddenSize / 32",qSize:"args.qSize",kvSize:"(args.numKvHeads * args.headDim)",headDim:"args.headDim",total:"2 * args.qSize + 2 * (args.numKvHeads * args.headDim)",lutId:"args.lut if args.lut else 0",mRows:"tokenRows",workgroupSize:"gatedQ4WorkgroupSize",wRows:"gatedTileRows",bitsElem:'"u32" if (iqRawSel == 4 or iqRawSel == 2 or qIq1Family or lut2) else "vec4<u32>"',iqGridInBits:"1 if qIq1Family else 0",iqGridBase:"args.gridOffset if args.gridOffset else 0",iqDot:"1 if (iqRawSel == 4 or iqRawSel == 2 or qIq1Family) else 0",iq1m:"iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8",iq1s:"iqRawSel == 7",iq1md:"iqRawSel == 8",iq2xxs:"iqRawSel == 2",iq2xs:"iqRawSel == 6",iq2s:"iqRawSel == 5",iq3xxs:"iqRawSel == 3",iq3s:"iqRawSel == 4",qOffset:"args.qOffset",kOffset:"args.kOffset",vOffset:"args.vOffset"},passes:[{id:"main",name:"LlamaDecodeQkvGatedQ4",shader:"decode-qkv-gated-q4.wgsl.jinja",bindings:["normed_q","q4_bits_q","q4_scales","q","q_gate","k","v"],dispatch:{x:"ceilDiv(2 * args.qSize + 2 * (args.numKvHeads * args.headDim), gatedTileRows)"}}]},{id:"q4_gated_split",priority:11,when:["gatedBaseContract","present.bitsT","splitPacksPresent","args.kFormat is defined","args.vFormat is defined",'fmt != "q1_0"',"not (q3k or q2k or q4knc)","(gatedKSuperGeometry and args.qOffset % 8 == 0) if (q6k or q5k or lutc or iqRawSel == 4 or iqRawSel == 2 or qIq1Family) else true","(gatedKSuperGeometry and args.kOffset % 8 == 0) if (kQ6k or kQ5k or kLutc or kIqRawSel == 4 or kIqRawSel == 2 or kIq1Family) else true","(gatedKSuperGeometry and args.vOffset % 8 == 0) if (vQ6k or vQ5k or vLutc or vIqRawSel == 4 or vIqRawSel == 2 or vIq1Family) else true","ranks.bitsT == 1","ranks.scalesT == 1","ranks.kBitsT == 1","ranks.kScalesT == 1","ranks.vBitsT == 1","ranks.vScalesT == 1",'tensorDtypes.bitsT == "uint32"','tensorDtypes.kBitsT == "uint32"','tensorDtypes.vBitsT == "uint32"','(tensorDtypes.scalesT == "uint32" if lutc else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16"))',"(iqRawSel == 0 or iqRawSel == 1 or iqRawSel == 2 or iqRawSel == 4 or iqRawSel == 7 or iqRawSel == 8) and (kIqRawSel == 0 or kIqRawSel == 1 or kIqRawSel == 2 or kIqRawSel == 4 or kIqRawSel == 7 or kIqRawSel == 8) and (vIqRawSel == 0 or vIqRawSel == 1 or vIqRawSel == 2 or vIqRawSel == 4 or vIqRawSel == 7 or vIqRawSel == 8)","qGridInBitsOk","kGridInBitsOk","vGridInBitsOk",'(tensorDtypes.kScalesT == "uint32" if kLutc else (tensorDtypes.kScalesT == "float32" or tensorDtypes.kScalesT == "float16"))','(tensorDtypes.vScalesT == "uint32" if vLutc else (tensorDtypes.vScalesT == "float32" or tensorDtypes.vScalesT == "float16"))',"(args.lut is defined and args.lut != 0) if lutc else true","(args.kLut is defined and args.kLut != 0) if kLutc else true","(args.vLut is defined and args.vLut != 0) if vLutc else true","(true if lutc else f16Ok(tensorDtypes.scalesT)) and (true if kLutc else f16Ok(tensorDtypes.kScalesT)) and (true if vLutc else f16Ok(tensorDtypes.vScalesT))","gatedQ4ResourcesFit","args.hiddenSize % 32 == 0","(quantBits == 4 or quantBits == 8)","(kQuantBits == 4 or kQuantBits == 8)","(vQuantBits == 4 or vQuantBits == 8)","dim(shapes.bitsT, 0) >= gatedQBitsNeeded","dim(shapes.scalesT, 0) >= gatedQScalesNeeded","dim(shapes.kBitsT, 0) >= gatedKBitsNeeded","dim(shapes.kScalesT, 0) >= gatedKScalesNeeded","dim(shapes.vBitsT, 0) >= gatedVBitsNeeded","dim(shapes.vScalesT, 0) >= gatedVScalesNeeded"],derive:{useSubgroups:'device.features.has("subgroups")',scaleScalar:"dtypes.S",blocksPerRow:"args.hiddenSize / 32",qSize:"args.qSize",kvSize:"(args.numKvHeads * args.headDim)",headDim:"args.headDim",total:"2 * args.qSize + 2 * (args.numKvHeads * args.headDim)",lutId:"args.lut if args.lut else 0",mRows:"tokenRows",workgroupSize:"gatedQ4WorkgroupSize",wRows:"gatedTileRows",bitsElem:'"u32" if (iqRawSel == 4 or iqRawSel == 2 or qIq1Family or lut2) else "vec4<u32>"',iqGridInBits:"1 if qIq1Family else 0",iqGridBase:"args.gridOffset if args.gridOffset else 0",iqDot:"1 if (iqRawSel == 4 or iqRawSel == 2 or qIq1Family) else 0",iq1m:"iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8",iq1s:"iqRawSel == 7",iq1md:"iqRawSel == 8",iq2xxs:"iqRawSel == 2",iq2xs:"iqRawSel == 6",iq2s:"iqRawSel == 5",iq3xxs:"iqRawSel == 3",iq3s:"iqRawSel == 4",qOffset:"args.qOffset",kOffset:"args.kOffset",vOffset:"args.vOffset"},passes:[{id:"q",name:"LlamaDecodeQkvGatedQ4SplitQ",shader:"decode-qkv-gated-q4.wgsl.jinja",derive:{region:'"q"',indexEnd:"2 * args.qSize"},bindings:["normed_q","q4_bits_q","q4_scales","q","q_gate"],dispatch:{x:"ceilDiv(2 * args.qSize, gatedTileRows)"}},{id:"k",name:"LlamaDecodeQkvGatedQ4SplitK",shader:"decode-qkv-gated-q4.wgsl.jinja",derive:{region:'"k"',indexBase:"2 * args.qSize",indexEnd:"2 * args.qSize + (args.numKvHeads * args.headDim)",scaleScalar:"dtypes.KS",quantBits:"kQuantBits",hasMin:"kHasMin",iq:"kIq",lutId:"args.kLut if args.kLut else 0",lut16:"kLut16",lutc:"kLutc",lutc16:"kLutc16",lut2:"kLut2",q6k:"kQ6k",q5k:"kQ5k",iq3s:"1 if kIqRawSel == 4 else 0",iq2xxs:"1 if kIqRawSel == 2 else 0",iq1m:"1 if kIq1Family else 0",iq1s:"1 if kIqRawSel == 7 else 0",iq1md:"1 if kIqRawSel == 8 else 0",iqGridInBits:"1 if kIq1Family else 0",iqGridBase:"args.kGridOffset if args.kGridOffset else 0",iqDot:"1 if (kIqRawSel == 4 or kIqRawSel == 2 or kIq1Family) else 0",kBitsElem:'"u32" if (kIqRawSel == 4 or kIqRawSel == 2 or kIq1Family or kLut2) else "vec4<u32>"'},bindings:["normed_q",{arg:"kBitsT",name:"q4_bits",elementType:"$kBitsElem"},{arg:"kScalesT",name:"q4_scales",elementType:"$scaleScalar"},"k"],dispatch:{x:"ceilDiv((args.numKvHeads * args.headDim), gatedTileRows)"}},{id:"v",name:"LlamaDecodeQkvGatedQ4SplitV",shader:"decode-qkv-gated-q4.wgsl.jinja",derive:{region:'"v"',indexBase:"2 * args.qSize + (args.numKvHeads * args.headDim)",indexEnd:"2 * args.qSize + 2 * (args.numKvHeads * args.headDim)",scaleScalar:"dtypes.VS",quantBits:"vQuantBits",hasMin:"vHasMin",iq:"vIq",lutId:"args.vLut if args.vLut else 0",lut16:"vLut16",lutc:"vLutc",lutc16:"vLutc16",lut2:"vLut2",q6k:"vQ6k",q5k:"vQ5k",iq3s:"1 if vIqRawSel == 4 else 0",iq2xxs:"1 if vIqRawSel == 2 else 0",iq1m:"1 if vIq1Family else 0",iq1s:"1 if vIqRawSel == 7 else 0",iq1md:"1 if vIqRawSel == 8 else 0",iqGridInBits:"1 if vIq1Family else 0",iqGridBase:"args.vGridOffset if args.vGridOffset else 0",iqDot:"1 if (vIqRawSel == 4 or vIqRawSel == 2 or vIq1Family) else 0",vBitsElem:'"u32" if (vIqRawSel == 4 or vIqRawSel == 2 or vIq1Family or vLut2) else "vec4<u32>"'},bindings:["normed_q",{arg:"vBitsT",name:"q4_bits",elementType:"$vBitsElem"},{arg:"vScalesT",name:"q4_scales",elementType:"$scaleScalar"},"v"],dispatch:{x:"ceilDiv((args.numKvHeads * args.headDim), gatedTileRows)"}}]},{id:"q1_gated",priority:10,when:["gatedBaseContract",'present.bitsT and fmt == "q1_0"',"ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"f16Ok(tensorDtypes.scalesT)","gatedQ1ResourcesFit","args.headDim % 4 == 0","(args.numKvHeads * args.headDim) % 4 == 0","args.hiddenSize % 128 == 0","dim(shapes.bitsT, 0) >= max(args.qOffset + 2 * args.qSize * (args.hiddenSize / 128), args.kOffset + (args.numKvHeads * args.headDim) * (args.hiddenSize / 128), args.vOffset + (args.numKvHeads * args.headDim) * (args.hiddenSize / 128)) * 4","dim(shapes.scalesT, 0) >= max(args.qOffset + 2 * args.qSize * (args.hiddenSize / 128), args.kOffset + (args.numKvHeads * args.headDim) * (args.hiddenSize / 128), args.vOffset + (args.numKvHeads * args.headDim) * (args.hiddenSize / 128))","tokenRows == 1"],derive:{useSubgroups:'device.features.has("subgroups")',scaleScalar:"dtypes.S",workgroupSize:"gatedQ1WorkgroupSize",blocksPerRow:"args.hiddenSize / 128",inWords:"args.hiddenSize / 32",qSize:"args.qSize",kvSize:"(args.numKvHeads * args.headDim)",headDim:"args.headDim",total:"2 * args.qSize + 2 * (args.numKvHeads * args.headDim)",qOffset:"args.qOffset",kOffset:"args.kOffset",vOffset:"args.vOffset"},passes:[{id:"main",name:"LlamaDecodeQkvGatedQ1",shader:"decode-qkv-gated-q1.wgsl.jinja",bindings:["normed_q","q1_bits","q1_scales","q","q_gate","k","v"],dispatch:{x:"ceil((2 * args.qSize + 2 * (args.numKvHeads * args.headDim)) / 4)"}}]}]},assets:[["decode-qkv-gated-q1.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const BPR:u32={{ blocksPerRow }}u;const IN_WORDS:u32={{ inWords }}u;const Q_SIZE:u32={{ qSize }}u;const KV_SIZE:u32={{ kvSize }}u;const HEAD_DIM:u32={{ headDim }}u;const TOTAL:u32={{ total }}u;const W_Q_Q1:u32={{ qOffset }}u;const W_K_Q1:u32={{ kOffset }}u;const W_V_Q1:u32={{ vOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const TILE_ROWS:u32=4u;var<workgroup>partials0:array<f32,{{ workgroupSize }}>;var<workgroup>partials1:array<f32,{{ workgroupSize }}>;var<workgroup>partials2:array<f32,{{ workgroupSize }}>;var<workgroup>partials3:array<f32,{{ workgroupSize }}>;
{% include "ops/_shared/q1-dot-helpers.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})
{% include "ops/_shared/decode-entry-point.wgsl.jinja" %}let index0=wid.x*TILE_ROWS;let lid=lid3.x;if(index0>=TOTAL){return;}var block_offset=W_Q_Q1;var row0=index0;if(index0<Q_SIZE){let head=index0/HEAD_DIM;let d=index0%HEAD_DIM;row0=head*(2u*HEAD_DIM)+d;}else if(index0<2u*Q_SIZE){let qg=index0-Q_SIZE;let head=qg/HEAD_DIM;let d=qg%HEAD_DIM;row0=head*(2u*HEAD_DIM)+HEAD_DIM+d;}else if(index0<2u*Q_SIZE+KV_SIZE){block_offset=W_K_Q1;row0=index0-2u*Q_SIZE;}else{block_offset=W_V_Q1;row0=index0-2u*Q_SIZE-KV_SIZE;}let tile=row0/4u;
{% include "ops/_shared/q1-tile4-dot-normed.wgsl.jinja" %}{% set q1ReduceVector = true %}{% set q1ReduceComment = "full-qkv" %}{% include "ops/_shared/q1-tile4-reduce.wgsl.jinja" %}if(lid==0u){let v0=partials0[0];let v1=partials1[0];let v2=partials2[0];let v3=partials3[0];if(index0<Q_SIZE){q[index0+0u]=v0;q[index0+1u]=v1;q[index0+2u]=v2;q[index0+3u]=v3;}else if(index0<2u*Q_SIZE){let qg=index0-Q_SIZE;q_gate[qg+0u]=v0;q_gate[qg+1u]=v1;q_gate[qg+2u]=v2;q_gate[qg+3u]=v3;}else if(index0<2u*Q_SIZE+KV_SIZE){let kIdx=index0-2u*Q_SIZE;k[kIdx+0u]=v0;k[kIdx+1u]=v1;k[kIdx+2u]=v2;k[kIdx+3u]=v3;}else{let vIdx=index0-2u*Q_SIZE-KV_SIZE;v[vIdx+0u]=v0;v[vIdx+1u]=v1;v[vIdx+2u]=v2;v[vIdx+3u]=v3;} } }`],["decode-qkv-gated-q4.wgsl.jinja",`{% set emitQuantDotHelpers = not (lut16 or lutc16) %}{% set nibbleHelpers = not (q5k or q6k) %}{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{% set packed4x8CompatUnpackI8 = (quantBits == 8 and emitQuantDotHelpers) or q6k or (iq1m | default(false)) %}{% set packed4x8CompatUnpackU8 = ((((iq2xxs | default(false)) or (iq3s | default(false))) and (iqGridBound | default(false))) or (not ((iq1m | default(false)) or (iq2xxs | default(false)) or (iq3s | default(false))) and not lutId and (((quantBits != 8) and nibbleHelpers) or q5k or q6k))) %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const BPR:u32={{ blocksPerRow }}u;const Q_SIZE:u32={{ qSize }}u;const KV_SIZE:u32={{ kvSize }}u;const HEAD_DIM:u32={{ headDim }}u;
{% if indexEnd is not defined %}const TOTAL:u32={{ total }}u;
{% endif %}const W_Q_Q4:u32={{ qOffset }}u;const W_K_Q4:u32={{ kOffset }}u;const W_V_Q4:u32={{ vOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>partials:array<f32,{{ workgroupSize }}>;
{% if (iq2xxs is defined and iq2xxs) or (iq3s is defined and iq3s) or (iq1m is defined and iq1m) %}{% set iqInline = true %}{% include "ops/_shared/iq-grid.wgsl.jinja" %}{% include "ops/_shared/iqx-raw-dequant-helpers.wgsl.jinja" %}{% elif lutId %}{% set emitQuantValueHelpers = not (lut2 | default(false)) %}{% set emitLut2DotHelper = not ((lut2 | default(false)) and (lutc16 | default(false))) %}{% include "ops/_shared/q4nl-dequant-helpers.wgsl.jinja" %}{% set emitQuantValueHelpers = true %}{% else %}{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% endif %}@compute@workgroup_size({{ workgroupSize }})
{% set entrySubgroupIds = (useSubgroups | default(false)) and (subgroupIdBuiltins | default(false)) %}{% set wgTotal = "reduce_total" if entrySubgroupIds else "partials[0]" %}{% include "ops/_shared/decode-entry-point.wgsl.jinja" %}let index=wid.x{{ " * " ~ wRows ~ "u" if wRows > 1 else "" }}{{ " + " ~ indexBase ~ "u" if indexBase is defined else "" }};let lid=lid3.x;if(index>={{ indexEnd ~ "u" if indexEnd is defined else "TOTAL" }}){return;}var region_block_base=W_Q_Q4;if(index<2u*Q_SIZE){var row=index;if(index>=Q_SIZE){row=index-Q_SIZE;}let head=row/HEAD_DIM;let d=row%HEAD_DIM;var q_proj_row=head*(2u*HEAD_DIM)+d;if(index>=Q_SIZE){q_proj_row=q_proj_row+HEAD_DIM;}region_block_base=W_Q_Q4+q_proj_row*BPR;}else if(index<2u*Q_SIZE+KV_SIZE){region_block_base=W_K_Q4+(index-2u*Q_SIZE)*BPR;}else{region_block_base=W_V_Q4+(index-2u*Q_SIZE-KV_SIZE)*BPR;}
{% set q4NormedHbaseComment = false %}{% set q4VerboseLutComment = false %}{% include "ops/_shared/q4-normed-block-dot.wgsl.jinja" %}{% for w in range(wRows) %}{% for m in range(mRows) %}{% set accN = ("acc" if w == 0 else "accW" ~ w) ~ ("" if m == 0 else "M" ~ m) %}{% set qOff = "" if m == 0 else (m ~ "u * Q_SIZE + ") %}{% set kvOff = "" if m == 0 else (m ~ "u * KV_SIZE + ") %}{% if w > 0 or m > 0 %}{% if m > 0 %}{% else %}{% endif %}workgroupBarrier();{
{% endif %}{% if w > 0 %}let index=index+{{ w }}u;
{% endif %}{% if useSubgroups %}{% set reduceAcc = accN %}{% include "ops/_shared/workgroup-reduce-subgroup-arm.wgsl.jinja" %}{% else %}partials[lid]={{ accN }};workgroupBarrier();{{ wgsl_tree_fold(["partials"], form="head", barrierFirst=true) }}
{% endif %}if(lid==0u){let value={{ wgTotal }};
{% if region is not defined %}if(index<Q_SIZE){q[{{ qOff }}index]=value;}else if(index<2u*Q_SIZE){q_gate[{{ qOff }}index-Q_SIZE]=value;}else if(index<2u*Q_SIZE+KV_SIZE){k[{{ kvOff }}index-2u*Q_SIZE]=value;}else{v[{{ kvOff }}index-2u*Q_SIZE-KV_SIZE]=value;}
{% elif region == "q" %}if(index<Q_SIZE){q[{{ qOff }}index]=value;}else{q_gate[{{ qOff }}index-Q_SIZE]=value;}
{% elif region == "k" %}k[{{ kvOff }}index-2u*Q_SIZE]=value;
{% else %}v[{{ kvOff }}index-2u*Q_SIZE-KV_SIZE]=value;
{% endif %}}
{% if w > 0 or m > 0 %}}
{% endif %}{% endfor %}{% endfor %}}`],["decode-qkv-gated.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const Q_SIZE:u32={{ qSize }}u;const KV_SIZE:u32={{ kvSize }}u;const HEAD_DIM:u32={{ headDim }}u;const TOTAL:u32={{ total }}u;const W_Q_VEC4:u32={{ qOffsetVec4 }}u;const W_K_VEC4:u32={{ kOffsetVec4 }}u;const W_V_VEC4:u32={{ vOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>partials:array<f32,{{ workgroupSize }}>;@compute@workgroup_size({{ workgroupSize }})
{% set entrySubgroupIds = (useSubgroups | default(false)) and (subgroupIdBuiltins | default(false)) %}{% set wgTotal = "reduce_total" if entrySubgroupIds else "partials[0]" %}{% include "ops/_shared/decode-entry-point.wgsl.jinja" %}let index=wid.x;let lid=lid3.x;if(index>=TOTAL){return;}var row=index;var weight_base=W_Q_VEC4+row*HIDDEN_SIZE_VEC4;if(index>=Q_SIZE&&index<2u*Q_SIZE){row=index-Q_SIZE;}if(index<2u*Q_SIZE){let head=row/HEAD_DIM;let d=row%HEAD_DIM;var q_proj_row=head*(2u*HEAD_DIM)+d;if(index>=Q_SIZE){q_proj_row=q_proj_row+HEAD_DIM;}weight_base=W_Q_VEC4+q_proj_row*HIDDEN_SIZE_VEC4;}else if(index>=2u*Q_SIZE&&index<2u*Q_SIZE+KV_SIZE){row=index-2u*Q_SIZE;weight_base=W_K_VEC4+row*HIDDEN_SIZE_VEC4;}else if(index>=2u*Q_SIZE+KV_SIZE){row=index-2u*Q_SIZE-KV_SIZE;weight_base=W_V_VEC4+row*HIDDEN_SIZE_VEC4;}var acc=0.0;
{% for m in range(1, mRows) %}var accM{{ m }}=0.0;
{% endfor %}for(var dim=lid;dim<HIDDEN_SIZE_VEC4;dim=dim+WORKGROUP_SIZE){
{% if weightScalar == "f16" %}acc=acc+f32(dot(vec4<f16>(normed[dim]),weights[weight_base+dim]));
{% for m in range(1, mRows) %}accM{{ m }}=accM{{ m }}+f32(dot(vec4<f16>(normed[{{ m }}u*HIDDEN_SIZE_VEC4+dim]),weights[weight_base+dim]));
{% endfor %}{% else %}acc=acc+dot(normed[dim],weights[weight_base+dim]);
{% for m in range(1, mRows) %}accM{{ m }}=accM{{ m }}+dot(normed[{{ m }}u*HIDDEN_SIZE_VEC4+dim],weights[weight_base+dim]);
{% endfor %}{% endif %}}
{% for m in range(mRows) %}{% set reduceAcc = "acc" if m == 0 else "accM" ~ m %}{% set qOff = "" if m == 0 else (m ~ "u * Q_SIZE + ") %}{% set kvOff = "" if m == 0 else (m ~ "u * KV_SIZE + ") %}{% if m > 0 %}workgroupBarrier();{
{% endif %}{% include "ops/_shared/workgroup-reduce-scalar.wgsl.jinja" %}if(lid==0u){if(index<Q_SIZE){q[{{ qOff }}row]={{ wgTotal }};}else if(index<2u*Q_SIZE){q_gate[{{ qOff }}row]={{ wgTotal }};}else if(index<2u*Q_SIZE+KV_SIZE){k[{{ kvOff }}row]={{ wgTotal }};}else{v[{{ kvOff }}row]={{ wgTotal }};} }
{% if m > 0 %}}
{% endif %}{% endfor %}}`],["decode-qkv-norm-q1.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const HIDDEN_SIZE_Q1_WORDS:u32={{ hiddenSizeQ1Words }}u;const BPR:u32={{ blocksPerRow }}u;const KV_SIZE:u32={{ kvSize }}u;const TOTAL_OUTPUTS:u32={{ totalOutputs }}u;const RMS_EPS:f32={{ rmsEps }};const W_NORM_VEC4:u32={{ normOffsetVec4 }}u;const W_Q_Q1:u32={{ qOffset }}u;const W_K_Q1:u32={{ kOffset }}u;const W_V_Q1:u32={{ vOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const TILE_ROWS:u32=4u;var<workgroup>partials0:array<f32,{{ workgroupSize }}>;var<workgroup>partials1:array<f32,{{ workgroupSize }}>;var<workgroup>partials2:array<f32,{{ workgroupSize }}>;var<workgroup>partials3:array<f32,{{ workgroupSize }}>;var<workgroup>partials_ss:array<f32,{{ workgroupSize }}>;
{% include "ops/_shared/q1-dot-helpers.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})
{% include "ops/_shared/decode-entry-point.wgsl.jinja" %}let index0=wid.x*TILE_ROWS;let lid=lid3.x;if(index0>=TOTAL_OUTPUTS){return;}var block_offset=W_Q_Q1;var tile=index0/TILE_ROWS;let out_base=index0;if(index0>=HIDDEN_SIZE&&index0<HIDDEN_SIZE+KV_SIZE){block_offset=W_K_Q1;tile=(index0-HIDDEN_SIZE)/TILE_ROWS;}else if(index0>=HIDDEN_SIZE+KV_SIZE){block_offset=W_V_Q1;tile=(index0-HIDDEN_SIZE-KV_SIZE)/TILE_ROWS;}var acc0=0.0;var acc1=0.0;var acc2=0.0;var acc3=0.0;var ss=0.0;for(var word_index=lid;word_index<HIDDEN_SIZE_Q1_WORDS;word_index=word_index+WORKGROUP_SIZE){
{% include "ops/_shared/q1-hidden-rmsnorm-load.wgsl.jinja" %}let dot4=q1_dot_word_t4(block_offset,tile,BPR,col_block,word_id,n0,n1,n2,n3,n4,n5,n6,n7);acc0+=dot4.x;acc1+=dot4.y;acc2+=dot4.z;acc3+=dot4.w;}
{% if useSubgroups %}let subgroupTotal0=subgroupAdd(acc0);let subgroupTotal1=subgroupAdd(acc1);let subgroupTotal2=subgroupAdd(acc2);let subgroupTotal3=subgroupAdd(acc3);let subgroupTotal4=subgroupAdd(ss);let elected=subgroupElect();partials0[lid]=select(0.0,subgroupTotal0,elected);partials1[lid]=select(0.0,subgroupTotal1,elected);partials2[lid]=select(0.0,subgroupTotal2,elected);partials3[lid]=select(0.0,subgroupTotal3,elected);partials_ss[lid]=select(0.0,subgroupTotal4,elected);workgroupBarrier();if(lid==0u){var total0=0.0;var total1=0.0;var total2=0.0;var total3=0.0;var total4=0.0;for(var i=0u;i<WORKGROUP_SIZE;i=i+1u){total0=total0+partials0[i];total1=total1+partials1[i];total2=total2+partials2[i];total3=total3+partials3[i];total4=total4+partials_ss[i];}partials0[0]=total0;partials1[0]=total1;partials2[0]=total2;partials3[0]=total3;partials_ss[0]=total4;}workgroupBarrier();
{% else %}partials0[lid]=acc0;partials1[lid]=acc1;partials2[lid]=acc2;partials3[lid]=acc3;partials_ss[lid]=ss;workgroupBarrier();{{ wgsl_tree_fold(["partials0", "partials1", "partials2", "partials3", "partials_ss"]) }}
{% endif %}if(lid==0u){let scale=inverseSqrt(partials_ss[0]/f32(HIDDEN_SIZE)+RMS_EPS);
{% if outputScalar == "f32" %}qkv[out_base+0u]=partials0[0]*scale;qkv[out_base+1u]=partials1[0]*scale;qkv[out_base+2u]=partials2[0]*scale;qkv[out_base+3u]=partials3[0]*scale;
{% else %}qkv[out_base+0u]={{ outputScalar }}(partials0[0]*scale);qkv[out_base+1u]={{ outputScalar }}(partials1[0]*scale);qkv[out_base+2u]={{ outputScalar }}(partials2[0]*scale);qkv[out_base+3u]={{ outputScalar }}(partials3[0]*scale);
{% endif %}} }`],["decode-qkv-norm-q4.wgsl.jinja",`{% set emitQuantDotHelpers = not (lut16 or lutc16) %}{% set nibbleHelpers = not (q2k or q3k or q5k or q6k) %}{% set packed4x8CompatUnpackI8 = (quantBits == 8 and emitQuantDotHelpers) or q3k or q6k %}{% set packed4x8CompatUnpackU8 = not lutId and (((quantBits != 8) and nibbleHelpers) or q2k or q3k or q5k or q6k) %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% if not preNormed %}const HIDDEN_SIZE:u32={{ hiddenSize }}u;
{% endif %}const BPR:u32={{ blocksPerRow }}u;const Q_SIZE:u32={{ qSize }}u;const KV_SIZE:u32={{ kvSize }}u;
{% if not multiRow %}const TOTAL_OUTPUTS:u32={{ totalOutputs }}u;
{% endif %}{% if not preNormed %}const RMS_EPS:f32={{ rmsEps }};const W_NORM_VEC4:u32={{ normOffsetVec4 }}u;
{% endif %}const W_Q_Q4:u32={{ wQ }}u;const W_K_Q4:u32={{ wK }}u;const W_V_Q4:u32={{ wV }}u;
{% if not multiRow %}const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% endif %}{% include "ops/_shared/subgroup-runtime-fold.wgsl.jinja" %}{% if useSubgroups or sgRuntime %}{% set sgSafeStorageSize = 128 if multiRow else workgroupSize %}{% include "ops/_shared/subgroup-runtime-proof.wgsl.jinja" %}{% endif %}{% if not useSubgroups and not sgRuntime %}{% if multiRow %}{% set partialsRowVector = true %}var<workgroup>partials4:array<vec{{ regRows }}<f32>,128>;
{% else %}var<workgroup>partials:array<f32,{{ workgroupSize }}>;
{% if not preNormed %}var<workgroup>partials_ss:array<f32,{{ workgroupSize }}>;
{% endif %}{% endif %}{% endif %}{% if lutId %}{% include "ops/_shared/q4nl-dequant-helpers.wgsl.jinja" %}{% else %}{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% endif %}{% if multiRow %}@compute@workgroup_size(128){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "" }}){let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}let cluster=lid/32u;let lane=lid%32u;let dim0=(wid.x*4u+cluster)*{{ regRows }}u;var seg_base=W_Q_Q4;var seg_out0=dim0;if(dim0>=Q_SIZE&&dim0<Q_SIZE+KV_SIZE){seg_base=W_K_Q4;seg_out0=dim0-Q_SIZE;}else if(dim0>=Q_SIZE+KV_SIZE){seg_base=W_V_Q4;seg_out0=dim0-Q_SIZE-KV_SIZE;}let base0=seg_base+seg_out0*BPR;
{% include "ops/_shared/q4-normed-projection-multirow.wgsl.jinja" %}{% include "ops/_shared/projection-multirow-reduce.wgsl.jinja" %}if(lane==0u){if(dim0<Q_SIZE){
{% for r in range(regRows) %}q[seg_out0+{{ r }}u]=total{{ r }};
{% endfor %}}else if(dim0<Q_SIZE+KV_SIZE){
{% for r in range(regRows) %}k[seg_out0+{{ r }}u]=total{{ r }};
{% if fuseKv %}k_cache[p.kvDstStart+seg_out0+{{ r }}u]=total{{ r }};
{% endif %}{% endfor %}}else{
{% for r in range(regRows) %}v[seg_out0+{{ r }}u]=total{{ r }};
{% if fuseKv %}v_cache[p.kvDstStart+seg_out0+{{ r }}u]=total{{ r }};
{% endif %}{% endfor %}} } }
{% else %}@compute@workgroup_size({{ workgroupSize }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "" }}){let index=wid.x;let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}if(index>=TOTAL_OUTPUTS){return;}var output_index=index;var row_block_base=W_Q_Q4+index*BPR;if(index>=Q_SIZE&&index<Q_SIZE+KV_SIZE){output_index=index-Q_SIZE;row_block_base=W_K_Q4+output_index*BPR;}else if(index>=Q_SIZE+KV_SIZE){output_index=index-Q_SIZE-KV_SIZE;row_block_base=W_V_Q4+output_index*BPR;}
{% include "ops/_shared/q4-normed-projection-single-row.wgsl.jinja" %}if(lid==0u){
{% if preNormed %}let out=acc_total;
{% else %}let out=acc_total*inverseSqrt(ss_total/f32(HIDDEN_SIZE)+RMS_EPS);
{% endif %}if(index<Q_SIZE){q[output_index]=out;}else if(index<Q_SIZE+KV_SIZE){k[output_index]=out;}else{v[output_index]=out;} } }
{% endif %}`],["decode-qkv-q1.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const HIDDEN_SIZE_Q1_WORDS:u32={{ hiddenSizeQ1Words }}u;const BPR:u32={{ blocksPerRow }}u;const KV_SIZE:u32={{ kvSize }}u;const TOTAL_OUTPUTS:u32={{ totalOutputs }}u;const W_Q_Q1:u32={{ qOffset }}u;const W_K_Q1:u32={{ kOffset }}u;const W_V_Q1:u32={{ vOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const TILE_ROWS:u32=4u;var<workgroup>partials0:array<f32,{{ workgroupSize }}>;var<workgroup>partials1:array<f32,{{ workgroupSize }}>;var<workgroup>partials2:array<f32,{{ workgroupSize }}>;var<workgroup>partials3:array<f32,{{ workgroupSize }}>;
{% include "ops/_shared/q1-dot-helpers.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})
{% include "ops/_shared/decode-entry-point.wgsl.jinja" %}let index0=wid.x*TILE_ROWS;let lid=lid3.x;if(index0>=TOTAL_OUTPUTS){return;}var block_offset=W_Q_Q1;var tile=index0/4u;let out_base=index0;if(index0>=HIDDEN_SIZE&&index0<HIDDEN_SIZE+KV_SIZE){block_offset=W_K_Q1;tile=(index0-HIDDEN_SIZE)/4u;}else if(index0>=HIDDEN_SIZE+KV_SIZE){block_offset=W_V_Q1;tile=(index0-HIDDEN_SIZE-KV_SIZE)/4u;}var acc0=0.0;var acc1=0.0;var acc2=0.0;var acc3=0.0;for(var word_index=lid;word_index<HIDDEN_SIZE_Q1_WORDS;word_index=word_index+WORKGROUP_SIZE){let col_block=word_index/4u;let word_id=word_index%4u;let base4=word_index*8u;
{% for i in range(8) %}let n{{ i }}={{ "vec4<f32>(" if normedVec4 != "vec4<f32>" else "" }}normed[base4{{ " + " ~ i ~ "u" if i else "" }}]{{ ")" if normedVec4 != "vec4<f32>" else "" }};
{% endfor %}let dot4=q1_dot_word_t4(block_offset,tile,BPR,col_block,word_id,n0,n1,n2,n3,n4,n5,n6,n7);acc0+=dot4.x;acc1+=dot4.y;acc2+=dot4.z;acc3+=dot4.w;}
{% set q1ReduceVector = false %}{% set q1ReduceComment = "" %}{% include "ops/_shared/q1-tile4-reduce.wgsl.jinja" %}if(lid==0u){
{% if outputScalar == "f32" %}qkv[out_base+0u]=partials0[0];qkv[out_base+1u]=partials1[0];qkv[out_base+2u]=partials2[0];qkv[out_base+3u]=partials3[0];
{% else %}qkv[out_base+0u]={{ outputScalar }}(partials0[0]);qkv[out_base+1u]={{ outputScalar }}(partials1[0]);qkv[out_base+2u]={{ outputScalar }}(partials2[0]);qkv[out_base+3u]={{ outputScalar }}(partials3[0]);
{% endif %}} }`],["decode-qkv-rope-cache-norm.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const LAYER:u32={{ layer }}u;const CACHE_LEN:u32={{ cacheLen }}u;const HIDDEN_SIZE:u32={{ hiddenSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const NUM_KV_HEADS:u32={{ numKvHeads }}u;const HEAD_DIM:u32={{ headDim }}u;const HEAD_DIM_HALF:u32={{ headDimHalf }}u;const Q_PAIRS:u32={{ qPairs }}u;const V_OFFSET:u32={{ vOffset }}u;
{% if passPerHead > 0 %}const ROTARY_DIM:u32={{ rotaryDim }}u;const PASS_PER_HEAD:u32={{ passPerHead }}u;const Q_PASS_START:u32={{ qPassStart }}u;const K_PASS_START:u32={{ kPassStart }}u;
{% endif %}const TOTAL_OUTPUTS:u32={{ totalOutputs }}u;const RMS_EPS:f32={{ rmsEps }};const W_NORM_VEC4:u32={{ normOffsetVec4 }}u;
{% if nativeQuery | default(false) %}const W_Q_Q4:u32={{ qPackedOffset }}u;
{% else %}const W_Q_VEC4:u32={{ wQVec4 }}u;
{% endif %}const W_K_VEC4:u32={{ wKVec4 }}u;const W_V_VEC4:u32={{ wVVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>partials0:array<f32,{{ workgroupSize }}>;var<workgroup>partials1:array<f32,{{ workgroupSize }}>;var<workgroup>partials_ss:array<f32,{{ workgroupSize }}>;fn store_cache(value:f32)->{{ cacheScalar }} {return {{ cacheScalar }}(value);}fn rope_index(pos:u32,d:u32)->u32{return pos*HEAD_DIM_HALF+d;}
{% include "ops/_shared/kv-cache-index.wgsl.jinja" %}{% if nativeQuery | default(false) %}fn query_weight(row:u32,col4:u32)->vec4<{{ weightScalar }}>{let block=W_Q_Q4+row*(HIDDEN_SIZE/32u)+col4/8u;let word=q_bits[block*4u+(col4&3u)];let shift=(col4&4u);let nib=(vec4<u32>(word)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(shift)))&vec4<u32>(15u);let sb=block>>3u;let j=block&7u;let dm=unpack2x16float(q_scales[sb*5u]);let sm=q_scales[sb*5u+1u+(j>>1u)]>>((j&1u)*12u);let d=dm.x*f32(sm&63u);let m=dm.y*f32((sm>>6u)&63u);return vec4<{{ weightScalar }}>(vec4<f32>(nib)*d-vec4<f32>(m));}
{% endif %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let index=wid.x;let lid=lid3.x;if(index>=TOTAL_OUTPUTS||params.past_len>=CACHE_LEN){return;}let is_q=index<Q_PAIRS;let is_k=index>=Q_PAIRS&&index<V_OFFSET;
{% if passPerHead > 0 %}let is_q_pass=index>=Q_PASS_START&&index<K_PASS_START;let is_k_pass=index>=K_PASS_START;
{% endif %}var pair_index=index;var row0=0u;var row1=0u;var weight_base0=W_V_VEC4;var weight_base1=W_V_VEC4;if(is_q){let head=pair_index/HEAD_DIM_HALF;let d=pair_index%HEAD_DIM_HALF;row0=head*HEAD_DIM+d;row1=row0+HEAD_DIM_HALF;
{% if nativeQuery | default(false) %}weight_base0=row0*HIDDEN_SIZE_VEC4;weight_base1=row1*HIDDEN_SIZE_VEC4;
{% else %}weight_base0=W_Q_VEC4+row0*HIDDEN_SIZE_VEC4;weight_base1=W_Q_VEC4+row1*HIDDEN_SIZE_VEC4;
{% endif %}}else if(is_k){pair_index=index-Q_PAIRS;let head=pair_index/HEAD_DIM_HALF;let d=pair_index%HEAD_DIM_HALF;row0=head*HEAD_DIM+d;row1=row0+HEAD_DIM_HALF;weight_base0=W_K_VEC4+row0*HIDDEN_SIZE_VEC4;weight_base1=W_K_VEC4+row1*HIDDEN_SIZE_VEC4;
{% if passPerHead > 0 %}}else if(is_q_pass||is_k_pass){let local=select(index-K_PASS_START,index-Q_PASS_START,is_q_pass);row0=(local/PASS_PER_HEAD)*HEAD_DIM+ROTARY_DIM+(local%PASS_PER_HEAD);
{% if nativeQuery | default(false) %}weight_base0=select(W_K_VEC4,0u,is_q_pass)+row0*HIDDEN_SIZE_VEC4;
{% else %}weight_base0=select(W_K_VEC4,W_Q_VEC4,is_q_pass)+row0*HIDDEN_SIZE_VEC4;
{% endif %}}else{row0=index-V_OFFSET;weight_base0=W_V_VEC4+row0*HIDDEN_SIZE_VEC4;}
{% else %}}else{row0=index-V_OFFSET;weight_base0=W_V_VEC4+row0*HIDDEN_SIZE_VEC4;}
{% endif %}var acc0=0.0;var acc1=0.0;var ss=0.0;for(var dim=lid;dim<HIDDEN_SIZE_VEC4;dim=dim+WORKGROUP_SIZE){let h=vec4<f32>(hidden[dim]);ss=ss+dot(h,h);let x=h*vec4<f32>(weights[W_NORM_VEC4+dim]);
{% if nativeQuery | default(false) %}var w0:vec4<{{ weightScalar }}>;var w1:vec4<{{ weightScalar }}>;if(is_q{{ " || is_q_pass" if passPerHead > 0 else "" }}){w0=query_weight(row0,dim);if(is_q){w1=query_weight(row1,dim);} }else{w0=weights[weight_base0+dim];if(is_k){w1=weights[weight_base1+dim];} }
{% endif %}{% if weightScalar == "f16" %}let xh=vec4<f16>(x);
{% if nativeQuery | default(false) %}acc0=acc0+f32(dot(xh,w0));
{% else %}acc0=acc0+f32(dot(xh,weights[weight_base0+dim]));
{% endif %}if(is_q||is_k){
{% if nativeQuery | default(false) %}acc1=acc1+f32(dot(xh,w1));
{% else %}acc1=acc1+f32(dot(xh,weights[weight_base1+dim]));
{% endif %}}
{% else %}{% if nativeQuery | default(false) %}acc0=acc0+dot(x,w0);
{% else %}acc0=acc0+dot(x,weights[weight_base0+dim]);
{% endif %}if(is_q||is_k){
{% if nativeQuery | default(false) %}acc1=acc1+dot(x,w1);
{% else %}acc1=acc1+dot(x,weights[weight_base1+dim]);
{% endif %}}
{% endif %}}partials0[lid]=acc0;partials1[lid]=acc1;partials_ss[lid]=ss;workgroupBarrier();{{ wgsl_tree_fold(["partials0", "partials1", "partials_ss"]) }} if(lid==0u){let scale=inverseSqrt(partials_ss[0]/f32(HIDDEN_SIZE)+RMS_EPS);let a0=partials0[0]*scale;let a1=partials1[0]*scale;if(is_q||is_k){let d=row0%HEAD_DIM;let head=row0/HEAD_DIM;let rope=rope_index(params.past_len,d);let c=rope_cos[rope];let s=rope_sin[rope];let out0=a0*c-a1*s;let out1=a1*c+a0*s;if(is_q){q[row0]=out0;q[row1]=out1;}else{cache_keys[cache_index(params.past_len,head,d)]=store_cache(out0);cache_keys[cache_index(params.past_len,head,d+HEAD_DIM_HALF)]=store_cache(out1);}
{% if passPerHead > 0 %}}else if(is_q_pass){q[row0]=a0;}else if(is_k_pass){cache_keys[cache_index(params.past_len,row0/HEAD_DIM,row0%HEAD_DIM)]=store_cache(a0);}else{let head=row0/HEAD_DIM;let d=row0%HEAD_DIM;cache_values[cache_index(params.past_len,head,d)]=store_cache(a0);}
{% else %}}else{let head=row0/HEAD_DIM;let d=row0%HEAD_DIM;cache_values[cache_index(params.past_len,head,d)]=store_cache(a0);}
{% endif %}} }`],["llama-decode-qkv.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const HIDDEN_SIZE_VEC4:u32={{ hiddenSizeVec4 }}u;const KV_SIZE:u32={{ kvSize }}u;const TOTAL_OUTPUTS:u32={{ totalOutputs }}u;
{% if fusedRmsNorm %}const RMS_EPS:f32={{ rmsEps }};const W_NORM_VEC4:u32={{ normOffsetVec4 }}u;
{% endif %}const W_Q_VEC4:u32={{ wQVec4 }}u;const W_K_VEC4:u32={{ wKVec4 }}u;const W_V_VEC4:u32={{ wVVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% if fusedRmsNorm %}{% endif %}var<workgroup>partials:array<f32,{{ workgroupSize }}>;
{% if fusedRmsNorm %}var<workgroup>partials_ss:array<f32,{{ workgroupSize }}>;
{% else %}fn load_normed(value:{{ normedVec4 }})->vec4<f32>{return vec4<f32>(value);}
{% endif %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let index=wid.x;let lid=lid3.x;if(index>=TOTAL_OUTPUTS){return;}var weight_base=W_Q_VEC4+index*HIDDEN_SIZE_VEC4;var output_index=index;if(index>=HIDDEN_SIZE&&index<HIDDEN_SIZE+KV_SIZE){output_index=index-HIDDEN_SIZE;weight_base=W_K_VEC4+output_index*HIDDEN_SIZE_VEC4;}else if(index>=HIDDEN_SIZE+KV_SIZE){output_index=index-HIDDEN_SIZE-KV_SIZE;weight_base=W_V_VEC4+output_index*HIDDEN_SIZE_VEC4;}var acc=0.0;
{% if fusedRmsNorm %}var ss=0.0;
{% endif %}for(var in_dim=lid;in_dim<HIDDEN_SIZE_VEC4;in_dim=in_dim+WORKGROUP_SIZE){
{% if fusedRmsNorm %}let h=vec4<f32>(hidden[in_dim]);ss=ss+dot(h,h);
{% endif %}let n={{ "h * vec4<f32>(weights[W_NORM_VEC4 + in_dim])" if fusedRmsNorm else "load_normed(normed[in_dim])" }};
{% if weightScalar == "f16" %}acc=acc+f32(dot(vec4<f16>(n),weights[weight_base+in_dim]));
{% else %}acc=acc+dot(n,weights[weight_base+in_dim]);
{% endif %}}partials[lid]=acc;
{% if fusedRmsNorm %}partials_ss[lid]=ss;
{% endif %}workgroupBarrier();var stride=WORKGROUP_SIZE/2u;loop{if(lid<stride){partials[lid]=partials[lid]+partials[lid+stride];
{% if fusedRmsNorm %}partials_ss[lid]=partials_ss[lid]+partials_ss[lid+stride];
{% endif %}}workgroupBarrier();if(stride==1u){break;}stride=stride/2u;}if(lid==0u){
{% if fusedRmsNorm %}let out=partials[0]*inverseSqrt(partials_ss[0]/f32(HIDDEN_SIZE)+RMS_EPS);
{% endif %}if(index<HIDDEN_SIZE){q[output_index]={{ "out" if fusedRmsNorm else "partials[0]" }};}else if(index<HIDDEN_SIZE+KV_SIZE){k[output_index]={{ "out" if fusedRmsNorm else "partials[0]" }};}else{v[output_index]={{ "out" if fusedRmsNorm else "partials[0]" }};} } }`]]}],["com.xenova.LlamaDecodeResidualProjection",{manifest:{domain:"com.xenova",name:"LlamaDecodeResidualProjection",sinceVersion:1,inputs:{hiddenT:{onnx:"Hidden",dtype:"float32",rank:1},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},attnT:{onnx:"Attn",dtype:"A",rank:1,optional:!0},intermediateT:{onnx:"Intermediate",dtype:"I",rank:1,optional:!0},upT:{onnx:"Up",dtype:"float32",rank:1,optional:!0},colBitsT:{onnx:"ColBits",dtype:"uint32",rank:1,optional:!0},colScalesT:{onnx:"ColScales",dtype:"S",rank:1,optional:!0},gridT:{onnx:"Grid",dtype:"uint32",rank:1,optional:!0}},outputs:{hiddenT:{onnx:"Hidden",dtype:"float32",rank:"ranks.hiddenT",shape:"shapes.hiddenT"}},args:{hiddenSize:{kind:"u32",onnx:"hidden_size"},inFeatures:{kind:"u32",onnx:"in_features",required:!1},oOffset:{kind:"u32",onnx:"o_proj_offset",required:!1},intermediateSize:{kind:"u32",onnx:"intermediate_size",required:!1},downOffset:{kind:"u32",onnx:"down_proj_offset",required:!1},colOffset:{kind:"u32",onnx:"column_pack_word_offset",required:!1},scaleOffset:{kind:"u32",onnx:"transposed_scale_offset",required:!1},format:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","lut2_32","lut2_16","q3_k","q2_k","q6_k","q5_k","q1_0"]},iqRaw:{kind:"u32",onnx:"iq_raw_pack_format",required:!1},actRelu2:{kind:"u32",onnx:"activation_relu_squared",required:!1},hasMin:{kind:"u32",onnx:"kquant_has_min",required:!1},colElem:{kind:"u32",onnx:"column_pack_element_major",required:!1},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1},tokens:{kind:"u32",required:!1}},typeConstraints:{W:["float32","float16"],S:["float32","float16","uint32"],A:["float32","float16"],I:["float32","float16"]},tunables:{OUTPUTS_PER_WG:{default:2},DENSE_WORKGROUP_SIZE:{default:64},Q1_WORKGROUP_SIZE:{default:64},OPROJ_Q1_WORKGROUP_SIZE:{default:32},Q1_GENERAL_WORKGROUP_SIZE:{default:64}},derive:{tokenRows:"args.tokens if args.tokens is defined else 1",deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",subgroupCoveringWorkgroup:'min(deviceWorkgroupCap, max(32, device.adapterInfo.subgroupMinSize)) if (device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize")) else 32',fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',lutcWords:"1 if (lut2 == 1 and args.lut == 9) else (5 if lutc16 == 1 else 3)",q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWave32:'device.features.has("subgroups") and wave32Adapter',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32SubgroupsUsable:"subgroupsWave32 or pinSubgroupSize32",subgroupIdBuiltins:'device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id")',q4knc:'1 if fmt == "q4_k_native_compact" else 0',iqRawSel:"args.iqRaw if args.iqRaw else 0",oprojMode:"present.attnT and not present.intermediateT and not present.upT and not present.colBitsT and not present.colScalesT",legacyMode:"args.inFeatures is undefined",generalMode:"args.inFeatures is defined",inputSize:"args.inFeatures if args.inFeatures is defined else args.hiddenSize",denseRoute:"present.weightsT and not present.bitsT and not present.scalesT",q1Route:'not present.weightsT and present.bitsT and present.scalesT and fmt == "q1_0"',q4Route:'not present.weightsT and present.bitsT and present.scalesT and fmt != "q1_0"',oprojFormatOk:'args.format is undefined or args.format == "q4_0" or args.format == "q4_k" or args.format == "q8_0" or args.format == "iq" or args.format == "lut32" or args.format == "lut16" or args.format == "q6_k" or args.format == "q5_k" or args.format == "q3_k" or args.format == "q2_k" or args.format == "q4_k_native_compact" or args.format == "lut32_compact" or args.format == "lut16_compact" or args.format == "lut2_32" or args.format == "lut2_16"',sgRuntime:'device.features.has("subgroups") and not wave32SubgroupsUsable',portableMr4Preferred:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize < device.adapterInfo.subgroupMaxSize and device.adapterInfo.subgroupMaxSize <= 32',denseWorkgroupLimit:"min(tunables.DENSE_WORKGROUP_SIZE, deviceWorkgroupCap)",denseWorkgroupSize:"max(1, pow2ceil(denseWorkgroupLimit + 1) / 2)",denseWorkgroupBytes:"denseWorkgroupSize * tunables.OUTPUTS_PER_WG * 4",denseDeviceOk:"tunables.OUTPUTS_PER_WG >= 1 and tunables.OUTPUTS_PER_WG <= denseWorkgroupSize and denseWorkgroupBytes <= device.limits.maxComputeWorkgroupStorageSize",denseGeneralResourcesFit:"denseWorkgroupSize * 4 <= device.limits.maxComputeWorkgroupStorageSize and args.hiddenSize <= device.limits.maxComputeWorkgroupsPerDimension",q1WorkgroupLimit:"min((tunables.OPROJ_Q1_WORKGROUP_SIZE if oprojMode else tunables.Q1_WORKGROUP_SIZE), deviceWorkgroupCap)",q1WorkgroupSize:"max(1, pow2ceil(q1WorkgroupLimit + 1) / 2)",q1WorkgroupBytes:"q1WorkgroupSize * 4 * 4",q1DeviceOk:"q1WorkgroupBytes <= device.limits.maxComputeWorkgroupStorageSize",q1GeneralWorkgroupLimit:"min(tunables.Q1_GENERAL_WORKGROUP_SIZE, deviceWorkgroupCap)",q1GeneralWorkgroupSize:"max(1, pow2ceil(q1GeneralWorkgroupLimit + 1) / 2)",q1GeneralResourcesFit:"q1GeneralWorkgroupSize * 16 <= device.limits.maxComputeWorkgroupStorageSize and ceilDiv(args.hiddenSize, 4) <= device.limits.maxComputeWorkgroupsPerDimension",q4WorkgroupSize:"32",q4MultiRowWorkgroupSize:"q4WorkgroupSize * 4",q4WorkgroupFits:"q4WorkgroupSize == 32 and q4WorkgroupSize <= deviceWorkgroupCap and q4WorkgroupSize * 4 <= device.limits.maxComputeWorkgroupStorageSize",q4ResourcesFit:"q4WorkgroupFits and args.hiddenSize <= device.limits.maxComputeWorkgroupsPerDimension",q4MultiRowResourcesFit:"q4WorkgroupFits and q4MultiRowWorkgroupSize <= deviceWorkgroupCap and q4MultiRowWorkgroupSize * 4 <= device.limits.maxComputeWorkgroupStorageSize",stageActivationsMeasured:'wave32SubgroupsUsable and has(device.adapterInfo, "architecture") and (device.adapterInfo.architecture == "pascal" or device.adapterInfo.architecture == "maxwell") and (fmt == "q4_0" or fmt == "q8_0" or fmt == "q3_k" or fmt == "q4_k" or fmt == "q5_k" or fmt == "q6_k" or fmt == "q4_k_native_compact" or fmt == "lut2_32") and 8192 + 544 <= device.limits.maxComputeWorkgroupStorageSize',oprojMaxBlocks:"(args.oOffset + args.hiddenSize * (inputSize / 32)) if (oprojMode and args.oOffset is defined and args.hiddenSize is defined) else 0",oprojSuperblocks:"ceilDiv(oprojMaxBlocks, 8)",downMaxBlocks:"(args.downOffset + args.hiddenSize * (args.intermediateSize / 32)) if (args.downOffset is defined and args.hiddenSize is defined and args.intermediateSize is defined) else 0",downSuperblocks:"ceilDiv(downMaxBlocks, 8)",oprojBaseContract:'oprojMode and args.oOffset is defined and ranks.hiddenT == 1 and ranks.attnT == 1 and tensorDtypes.hiddenT == "float32" and args.hiddenSize > 0 and inputSize > 0 and dim(shapes.hiddenT, 0) == tokenRows * args.hiddenSize and dim(shapes.attnT, 0) == tokenRows * inputSize and tokenRows >= 1 and tokenRows <= 4',oprojQ4Contract:'iqRawSel == 0 and oprojFormatOk and oprojBaseContract and q4Route and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.attnT == "float32" and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "uint32" if (q4knc or lutc) else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")) and f16Ok(tensorDtypes.scalesT) and inputSize % 32 == 0 and (quantBits == 4 or quantBits == 8) and (not (q6k or q5k or q3k or q2k or q4knc or lutc) or args.oOffset % 8 == 0) and dim(shapes.bitsT, 0) % (2 if lut2 else 4) == 0 and dim(shapes.bitsT, 0) >= (oprojMaxBlocks * (2 if lut2 else 4) if (q4knc or lutc) else (oprojSuperblocks * 20 if q2k else (oprojSuperblocks * 52 if q6k else (oprojSuperblocks * 44 if q5k else (oprojSuperblocks * 28 if q3k else oprojMaxBlocks * quantBits))))) and dim(shapes.scalesT, 0) >= (oprojSuperblocks * 5 if q4knc else (oprojSuperblocks * lutcWords if lutc else (oprojSuperblocks * 2 if q2k else (oprojSuperblocks if q6k else (oprojSuperblocks * 2 if q5k else (oprojSuperblocks if q3k else oprojMaxBlocks * (2 if (hasMin or iq or lut16) else 1)))))))',oprojIqBitsNeeded:"oprojSuperblocks * (20 if iqRawSel == 5 else (18 if iqRawSel == 6 else 12)) if iqRawSel >= 5 else oprojMaxBlocks * 2",oprojIqRawContract:'(iqRawSel == 1 or iqRawSel == 2 or iqRawSel == 5 or iqRawSel == 6 or iqRawSel == 7) and oprojBaseContract and present.bitsT and present.scalesT and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(tensorDtypes.scalesT) and tensorDtypes.attnT == "float32" and inputSize % 256 == 0 and args.oOffset % 8 == 0 and dim(shapes.bitsT, 0) >= oprojIqBitsNeeded and dim(shapes.scalesT, 0) >= oprojSuperblocks',downBaseContract:'args.intermediateSize is defined and not present.attnT and present.intermediateT and not present.upT and not present.colBitsT and not present.colScalesT and args.downOffset is defined and ranks.hiddenT == 1 and ranks.intermediateT == 1 and tensorDtypes.hiddenT == "float32" and (tensorDtypes.intermediateT == "float32" or tensorDtypes.intermediateT == "float16") and args.hiddenSize > 0 and args.intermediateSize > 0 and dim(shapes.hiddenT, 0) == tokenRows * args.hiddenSize and dim(shapes.intermediateT, 0) == tokenRows * args.intermediateSize and tokenRows >= 1 and tokenRows <= 4',downSparseContract:'args.intermediateSize is defined and not present.attnT and present.upT and present.colBitsT and present.colScalesT and not present.intermediateT and not present.weightsT and not present.bitsT and not present.scalesT and args.colOffset is defined and args.scaleOffset is defined and args.actRelu2 == 1 and ranks.hiddenT == 1 and ranks.upT == 1 and ranks.colBitsT == 1 and ranks.colScalesT == 1 and tensorDtypes.hiddenT == "float32" and tensorDtypes.upT == "float32" and tensorDtypes.colBitsT == "uint32" and (tensorDtypes.colScalesT == "float32" or tensorDtypes.colScalesT == "float16") and f16Ok(dtypes.S) and args.hiddenSize > 0 and args.hiddenSize % 32 == 0 and args.intermediateSize > 0 and args.intermediateSize % 32 == 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.upT, 0) == args.intermediateSize and dim(shapes.colBitsT, 0) >= args.colOffset + args.intermediateSize * (args.hiddenSize / 8) and dim(shapes.colScalesT, 0) >= args.scaleOffset + (args.intermediateSize / 32) * args.hiddenSize * (2 if args.hasMin else 1)',downQ4Contract:'iqRawSel == 0 and downBaseContract and present.bitsT and fmt != "q1_0" and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "uint32" if (q4knc or lutc) else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")) and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.intermediateT) and args.intermediateSize % 32 == 0 and (quantBits == 4 or quantBits == 8) and (not (q6k or q5k or q3k or q2k or q4knc or lutc) or args.downOffset % 8 == 0) and dim(shapes.bitsT, 0) % (2 if lut2 else 4) == 0 and dim(shapes.bitsT, 0) >= (downMaxBlocks * (2 if lut2 else 4) if (q4knc or lutc) else (downSuperblocks * 20 if q2k else (downSuperblocks * 52 if q6k else (downSuperblocks * 44 if q5k else (downSuperblocks * 28 if q3k else downMaxBlocks * quantBits))))) and dim(shapes.scalesT, 0) >= (downSuperblocks * 5 if q4knc else (downSuperblocks * lutcWords if lutc else (downSuperblocks * 2 if q2k else (downSuperblocks if q6k else (downSuperblocks * 2 if q5k else (downSuperblocks if q3k else downMaxBlocks * (2 if (hasMin or iq or lut16) else 1)))))))',downIqBitsNeeded:"downSuperblocks * (20 if iqRawSel == 5 else (18 if iqRawSel == 6 else (12 if iqRawSel == 7 else 14))) if iqRawSel >= 5 else downMaxBlocks * (3 if iqRawSel == 3 else 2)",downIqRawContract:'iqRawSel >= 1 and iqRawSel <= 8 and iqRawSel != 4 and downBaseContract and present.bitsT and present.scalesT and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and f16Ok(tensorDtypes.scalesT) and f16Ok(tensorDtypes.intermediateT) and args.intermediateSize % 256 == 0 and args.downOffset % 8 == 0 and dim(shapes.bitsT, 0) >= downIqBitsNeeded and dim(shapes.scalesT, 0) >= downSuperblocks',iqGridRows:"256 if iqRawSel == 3 else (512 if iqRawSel == 2 else (2048 if iqRawSel == 5 else (1024 if iqRawSel == 6 else 4096)))",iqGridBoundOk:'present.gridT and ranks.gridT == 1 and tensorDtypes.gridT == "uint32" and dim(shapes.gridT, 0) >= iqGridRows'},bindings:{hidden:{arg:"hiddenT",elementType:"f32"},weights:{arg:"weightsT",elementType:"$weightVec4"},activation:{arg:"attnT",elementType:"$activationVec4"},q1_bits:{arg:"bitsT",elementType:"vec4<u32>"},q1_scales:{arg:"scalesT",elementType:"$scaleScalar"},attn_out:{arg:"attnT",elementType:"vec4<f32>"},q4_bits:{arg:"bitsT",elementType:"$bitsElem"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},intermediate:{arg:"upT",elementType:"vec4<f32>"},col_bits:{arg:"colBitsT",elementType:"u32"},scales_t:{arg:"colScalesT",elementType:"$scaleScalar"},activation_main:{arg:"intermediateT",name:"activation",elementType:"$activationVec4"},intermediate_main:{arg:"intermediateT",name:"intermediate",elementType:"$intermediateVec4"},q4_bits_u32:{arg:"bitsT",name:"q4_bits",elementType:"u32"},iq_grid_data:{arg:"gridT",elementType:"vec2<u32>"}},variants:[{id:"oproj_dense",priority:0,when:["legacyMode","oprojBaseContract","denseRoute","denseDeviceOk","ranks.weightsT == 1",'tensorDtypes.attnT == "float32"','(tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")',"f16Ok(dtypes.W)","args.hiddenSize % 4 == 0","args.oOffset % 4 == 0","dim(shapes.weightsT, 0) >= args.oOffset + args.hiddenSize * args.hiddenSize","tokenRows == 1"],derive:{weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',weightScalar:"dtypes.W",activationVec4:'"vec4<f32>"',outputSize:"args.hiddenSize",inputSizeVec4:"args.hiddenSize / 4",weightOffsetVec4:"args.oOffset / 4",outputsPerWg:"tunables.OUTPUTS_PER_WG",assumeFullBlocks:"args.hiddenSize % tunables.OUTPUTS_PER_WG == 0",widenActivation:!1},passes:[{id:"main",name:"LlamaDecodeOProj",shader:"llama-decode-dense-proj.wgsl.jinja",bindings:["hidden","weights","activation"],dispatch:{x:"ceilDiv(args.hiddenSize, tunables.OUTPUTS_PER_WG)"}}]},{id:"oproj_q1",priority:0,when:["legacyMode","oprojBaseContract","q1Route","q1DeviceOk","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")','(tensorDtypes.attnT == "float32" or tensorDtypes.attnT == "float16")',"f16Ok(dtypes.S) and f16Ok(dtypes.A)","args.hiddenSize % 128 == 0","dim(shapes.bitsT, 0) >= (args.oOffset + args.hiddenSize * (args.hiddenSize / 128)) * 4","dim(shapes.scalesT, 0) >= args.oOffset + args.hiddenSize * (args.hiddenSize / 128)","tokenRows == 1"],derive:{useSubgroups:'device.features.has("subgroups")',activationVec4:'"vec4<f16>" if tensorDtypes.attnT == "float16" else "vec4<f32>"',scaleScalar:"dtypes.S",outputSize:"args.hiddenSize",inputSizeQ1Words:"args.hiddenSize / 32",blocksPerRow:"args.hiddenSize / 128",weightOffset:"args.oOffset"},passes:[{id:"main",name:"LlamaDecodeOProjQ1",shader:"llama-decode-q1-proj.wgsl.jinja",bindings:["hidden","q1_bits","q1_scales","activation"],dispatch:{x:"ceil(args.hiddenSize / 4)"}}]},{id:"oproj_dense_general",priority:0,when:["generalMode","oprojBaseContract","denseRoute",'tensorDtypes.attnT == "float32"',"ranks.weightsT == 1",'(tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")',"f16Ok(tensorDtypes.weightsT)","denseGeneralResourcesFit","inputSize % 4 == 0","args.oOffset % 4 == 0","dim(shapes.weightsT, 0) >= args.oOffset + args.hiddenSize * inputSize","tokenRows == 1"],derive:{useSubgroups:'device.features.has("subgroups")',weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',weightScalar:"dtypes.W",workgroupSize:"denseWorkgroupSize",hiddenSize:"args.hiddenSize",inFeaturesVec4:"inputSize / 4",oOffsetVec4:"args.oOffset / 4"},passes:[{id:"main",name:"LlamaDecodeOProjGeneral",shader:"decode-o-proj-general.wgsl.jinja",bindings:["hidden","weights",{arg:"attnT",name:"x",elementType:"vec4<f32>"}],dispatch:{x:"args.hiddenSize"}}]},{id:"oproj_q1_general",priority:0,when:["generalMode","oprojBaseContract","q1Route",'tensorDtypes.attnT == "float32"',"ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"f16Ok(tensorDtypes.scalesT)","q1GeneralResourcesFit","inputSize % 128 == 0","dim(shapes.bitsT, 0) >= (args.oOffset + args.hiddenSize * (inputSize / 128)) * 4","dim(shapes.scalesT, 0) >= args.oOffset + args.hiddenSize * (inputSize / 128)","args.hiddenSize % 4 == 0","tokenRows == 1"],derive:{useSubgroups:'device.features.has("subgroups")',scaleScalar:"dtypes.S",workgroupSize:"q1GeneralWorkgroupSize",hiddenSize:"args.hiddenSize",blocksPerRow:"inputSize / 128",inWords:"inputSize / 32",oOffset:"args.oOffset"},passes:[{id:"main",name:"LlamaDecodeOProjQ1General",shader:"decode-o-proj-q1-general.wgsl.jinja",bindings:["hidden","q1_bits","q1_scales","attn_out"],dispatch:{x:"ceil(args.hiddenSize / 4)"}}]},{id:"oproj_q4",priority:0,when:["oprojQ4Contract","q4ResourcesFit","tokenRows == 1","(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * subgroupCoveringWorkgroup + 8 <= device.limits.maxComputeWorkgroupStorageSize"],derive:{outputProjection:!0,useSubgroups:"wave32SubgroupsUsable",scaleScalar:"dtypes.S",hiddenSize:"args.hiddenSize",blocksPerRow:"inputSize / 32",oOffset:"args.oOffset",multiRow:"0",regRows:"1",scalesPk2:"0",workgroupSize:"subgroupCoveringWorkgroup",lutId:"args.lut if args.lut else 0",q4knCompact:"q4knc",bitsElem:'"u32" if lut2 else "vec4<u32>"',packMeta:'"q4_scales"'},passes:[{id:"main",name:"LlamaDecodeOProjQ4",shader:"decode-residual-projection-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","attn_out"],dispatch:{x:"args.hiddenSize"},subgroupCollectivesWidth:"portable"}]},{id:"down_sparse_mr2",priority:0,when:["downSparseContract","not args.colElem","tokenRows == 1","(not (wave32SubgroupsUsable) and not sgRuntime) or 520 <= device.limits.maxComputeWorkgroupStorageSize"],derive:{useSubgroups:"wave32SubgroupsUsable",scaleScalar:"dtypes.S",hiddenSize:"args.hiddenSize",blocksPerRow:"args.intermediateSize / 32",colOffset:"args.colOffset",scaleOffset:"args.scaleOffset",quantBits:"4",hasMin:"1 if args.hasMin else 0"},passes:[{id:"main",name:"LlamaDecodeDownProjSparseQ4",shader:"decode-down-proj-sparse-q4.wgsl.jinja",bindings:["hidden","intermediate","col_bits","scales_t"],dispatch:{x:"ceil(args.hiddenSize / 8)"},subgroupCollectivesWidth:"portable"}]},{id:"down_col_stationary",priority:1,when:["downSparseContract","args.colElem == 1","tokenRows == 1"],derive:{scaleScalar:"dtypes.S",hiddenSize:"args.hiddenSize",intermediateSize:"args.intermediateSize",colOffset:"args.colOffset",scaleOffset:"args.scaleOffset",hasMin:"1 if args.hasMin else 0",colSplits:8,colRowsPerWorkgroup:128,colMergeWorkgroupSize:256},intermediates:[{id:"down_partials",dtype:"float32",shape:"[colSplits * args.hiddenSize]"}],passes:[{id:"cols",name:"LlamaDecodeDownProjColStationary",shader:"decode-down-proj-col-stationary.wgsl.jinja",bindings:["intermediate","col_bits","scales_t",{scratch:"down_partials",name:"partials",elementType:"f32"}],dispatch:{x:"ceil(args.hiddenSize / colRowsPerWorkgroup)",y:"colSplits"}},{id:"merge",name:"LlamaDecodeDownProjColMerge",shader:"decode-down-proj-col-merge.wgsl.jinja",bindings:["hidden",{scratch:"down_partials",name:"partials",buffer:"read-only-storage",elementType:"f32"}],dispatch:{x:"ceil(args.hiddenSize / colMergeWorkgroupSize)"}}]},{id:"down_dense",priority:0,when:["downBaseContract","not present.bitsT","denseDeviceOk","ranks.weightsT == 1",'(tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")',"f16Ok(tensorDtypes.weightsT)","f16Ok(tensorDtypes.intermediateT)","args.intermediateSize % 4 == 0","args.downOffset % 4 == 0","dim(shapes.weightsT, 0) >= args.downOffset + args.hiddenSize * args.intermediateSize","tokenRows == 1"],derive:{weightVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',weightScalar:"dtypes.W",activationVec4:'"vec4<f16>" if tensorDtypes.intermediateT == "float16" else "vec4<f32>"',outputSize:"args.hiddenSize",inputSizeVec4:"args.intermediateSize / 4",weightOffsetVec4:"args.downOffset / 4",outputsPerWg:"tunables.OUTPUTS_PER_WG",assumeFullBlocks:"args.hiddenSize % tunables.OUTPUTS_PER_WG == 0",widenActivation:!0},passes:[{id:"main",name:"LlamaDecodeDownProj",shader:"llama-decode-dense-proj.wgsl.jinja",bindings:["hidden","weights","activation_main"],dispatch:{x:"ceilDiv(args.hiddenSize, tunables.OUTPUTS_PER_WG)"}}]},{id:"down_q4",priority:0,when:["downQ4Contract","tokenRows == 1","(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * subgroupCoveringWorkgroup + 8 <= device.limits.maxComputeWorkgroupStorageSize"],derive:{useSubgroups:"wave32SubgroupsUsable",scaleScalar:"dtypes.S",hiddenSize:"args.hiddenSize",blocksPerRow:"args.intermediateSize / 32",multiRow:"0",regRows:"1",workgroupSize:"subgroupCoveringWorkgroup",lutId:"args.lut if args.lut else 0",q4knCompact:"q4knc",bitsElem:'"u32" if lut2 else "vec4<u32>"',packMeta:'"q4_scales"',intermediateVec4:'"vec4<f16>" if tensorDtypes.intermediateT == "float16" else "vec4<f32>"',downOffset:"args.downOffset",actRelu2:"1 if args.actRelu2 else 0"},passes:[{id:"main",name:"LlamaDecodeDownProjQ4",shader:"decode-residual-projection-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","intermediate_main"],dispatch:{x:"args.hiddenSize"},subgroupCollectivesWidth:"portable"}]},{id:"down_q1",priority:0,when:["downBaseContract",'present.bitsT and fmt == "q1_0"',"q1DeviceOk","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"f16Ok(tensorDtypes.scalesT)","f16Ok(tensorDtypes.intermediateT)","args.hiddenSize % 4 == 0","args.intermediateSize % 128 == 0","dim(shapes.bitsT, 0) >= (args.downOffset + args.hiddenSize * (args.intermediateSize / 128)) * 4","dim(shapes.scalesT, 0) >= args.downOffset + args.hiddenSize * (args.intermediateSize / 128)","tokenRows == 1"],derive:{useSubgroups:'device.features.has("subgroups")',activationVec4:'"vec4<f16>" if tensorDtypes.intermediateT == "float16" else "vec4<f32>"',scaleScalar:"dtypes.S",outputSize:"args.hiddenSize",inputSizeQ1Words:"args.intermediateSize / 32",blocksPerRow:"args.intermediateSize / 128",weightOffset:"args.downOffset"},passes:[{id:"main",name:"LlamaDecodeDownProjQ1",shader:"llama-decode-q1-proj.wgsl.jinja",bindings:["hidden","q1_bits","q1_scales","activation_main"],dispatch:{x:"ceil(args.hiddenSize / 4)"}}]},{cases:[{suffix:"mr2",priority:3,preference:["portableMr4Preferred","stageActivationsMeasured and args.hiddenSize % 16 == 0 and args.hiddenSize >= 4096"],divisor:8,regRows:"2",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"mr4",priority:2,preference:null,divisor:16,regRows:"4",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"oproj_q4_{suffix}",priority:"{priority}",when:["oprojQ4Contract","args.hiddenSize % {divisor} == 0","q4MultiRowResourcesFit","ceilDiv(args.hiddenSize, {divisor}) <= device.limits.maxComputeWorkgroupsPerDimension","{sgFoldStorageGate0}"],demoteWhen:"{preference}",derive:{outputProjection:!0,useSubgroups:"wave32SubgroupsUsable",scaleScalar:'"u32" if (hasMin == 1 and tensorDtypes.scalesT == "float16") else dtypes.S',hiddenSize:"args.hiddenSize",blocksPerRow:"inputSize / 32",oOffset:"args.oOffset",multiRow:"1",regRows:"{regRows}",wgClusters:"4",scalesPk2:'1 if (hasMin == 1 and tensorDtypes.scalesT == "float16") else 0',mRows:"tokenRows",stageActivations:"stageActivationsMeasured",workgroupSize:"subgroupCoveringWorkgroup",lutId:"args.lut if args.lut else 0",q4knCompact:"q4knc",bitsElem:'"u32" if lut2 else "vec4<u32>"',packMeta:'"q4_scales"'},passes:[{id:"main",name:"LlamaDecodeOProjQ4",shader:"decode-residual-projection-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","attn_out"],dispatch:{x:"ceil(args.hiddenSize / {divisor})"},subgroupCollectivesWidth:"portable"}]}]},{cases:[{rows:2,priority:3,preference:["portableMr4Preferred","stageActivationsMeasured and args.hiddenSize % 16 == 0 and args.hiddenSize >= 4096"],divisor:8,regRows:"2",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{rows:4,priority:2,preference:null,divisor:16,regRows:"4",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"down_q4_mr{rows}",priority:"{priority}",when:["downQ4Contract","args.hiddenSize % {divisor} == 0","{sgFoldStorageGate0}"],demoteWhen:"{preference}",derive:{outputProjection:!1,useSubgroups:"wave32SubgroupsUsable",scaleScalar:"dtypes.S",hiddenSize:"args.hiddenSize",blocksPerRow:"args.intermediateSize / 32",multiRow:"1",regRows:"{regRows}",mRows:"tokenRows",stageActivations:"stageActivationsMeasured",workgroupSize:"subgroupCoveringWorkgroup",lutId:"args.lut if args.lut else 0",q4knCompact:"q4knc",bitsElem:'"u32" if lut2 else "vec4<u32>"',packMeta:'"q4_scales"',intermediateVec4:'"vec4<f16>" if tensorDtypes.intermediateT == "float16" else "vec4<f32>"',downOffset:"args.downOffset",actRelu2:"1 if args.actRelu2 else 0"},passes:[{id:"main",name:"LlamaDecodeDownProjQ4",shader:"decode-residual-projection-q4.wgsl.jinja",bindings:["hidden","q4_bits","q4_scales","intermediate_main"],dispatch:{x:"ceil(args.hiddenSize / {divisor})"},subgroupCollectivesWidth:"portable"}]}]},{cases:[{suffix:"",prio:0,demote:null,multiRow:"0",regRows:"1",guard:["tokenRows == 1"],mrFits:[],gridBound:null,gridBind:[],dispatchX:"args.hiddenSize",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * subgroupCoveringWorkgroup + 8 <= device.limits.maxComputeWorkgroupStorageSize",sgFoldStorageGate1:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * subgroupCoveringWorkgroup + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"_mr2",prio:3,demote:["portableMr4Preferred","tokenRows >= 4"],multiRow:"1",regRows:"2",guard:["args.hiddenSize % 8 == 0"],mrFits:["q4MultiRowResourcesFit","ceilDiv(args.hiddenSize, 8) <= device.limits.maxComputeWorkgroupsPerDimension"],gridBound:null,gridBind:[],dispatchX:"ceil(args.hiddenSize / 8)",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize",sgFoldStorageGate1:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"_mr4",prio:2,demote:null,multiRow:"1",regRows:"4",guard:["args.hiddenSize % 16 == 0"],mrFits:["q4MultiRowResourcesFit","ceilDiv(args.hiddenSize, 16) <= device.limits.maxComputeWorkgroupsPerDimension"],gridBound:null,gridBind:[],dispatchX:"ceil(args.hiddenSize / 16)",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize",sgFoldStorageGate1:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"_grid",prio:10,demote:null,multiRow:"0",regRows:"1",guard:["iqGridBoundOk","tokenRows == 1"],mrFits:[],gridBound:!0,gridBind:["iq_grid_data"],dispatchX:"args.hiddenSize",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * subgroupCoveringWorkgroup + 8 <= device.limits.maxComputeWorkgroupStorageSize",sgFoldStorageGate1:"(not (wave32SubgroupsUsable) and not sgRuntime) or 4 * subgroupCoveringWorkgroup + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"_mr2_grid",prio:13,demote:["portableMr4Preferred","tokenRows >= 4"],multiRow:"1",regRows:"2",guard:["args.hiddenSize % 8 == 0","iqGridBoundOk"],mrFits:["q4MultiRowResourcesFit","ceilDiv(args.hiddenSize, 8) <= device.limits.maxComputeWorkgroupsPerDimension"],gridBound:!0,gridBind:["iq_grid_data"],dispatchX:"ceil(args.hiddenSize / 8)",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize",sgFoldStorageGate1:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"_mr4_grid",prio:12,demote:null,multiRow:"1",regRows:"4",guard:["args.hiddenSize % 16 == 0","iqGridBoundOk"],mrFits:["q4MultiRowResourcesFit","ceilDiv(args.hiddenSize, 16) <= device.limits.maxComputeWorkgroupsPerDimension"],gridBound:!0,gridBind:["iq_grid_data"],dispatchX:"ceil(args.hiddenSize / 16)",sgFoldStorageGate0:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize",sgFoldStorageGate1:"(not (wave32SubgroupsUsable) and not sgRuntime) or 512 + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"down_iq_raw{suffix}",priority:"{prio}",when:["downIqRawContract","{guard}","{sgFoldStorageGate0}"],demoteWhen:"{demote}",derive:{outputProjection:!1,useSubgroups:"wave32SubgroupsUsable",scaleScalar:"dtypes.S",hiddenSize:"args.hiddenSize",blocksPerRow:"args.intermediateSize / 32",multiRow:"{multiRow}",regRows:"{regRows}",iqGridBound:"{gridBound}",mRows:"tokenRows",workgroupSize:"subgroupCoveringWorkgroup",quantBits:4,hasMin:!1,iq:!1,lutId:0,lut16:!1,q2k:!1,q6k:!1,q5k:!1,q3k:!1,q4knCompact:!1,lutc:!1,lutc16:!1,iqDot:!0,intermediateVec4:'"vec4<f16>" if tensorDtypes.intermediateT == "float16" else "vec4<f32>"',downOffset:"args.downOffset",actRelu2:"1 if args.actRelu2 else 0",iq1m:"iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8",iq1s:"iqRawSel == 7",iq1md:"iqRawSel == 8",iq2xxs:"iqRawSel == 2",iq2xs:"iqRawSel == 6",iq2s:"iqRawSel == 5",iq3xxs:"iqRawSel == 3",iq3s:"iqRawSel == 4"},passes:[{id:"main",name:"LlamaDecodeDownProjIqRaw",shader:"decode-residual-projection-q4.wgsl.jinja",bindings:["hidden","q4_bits_u32","q4_scales","intermediate_main","{gridBind}"],dispatch:{x:"{dispatchX}"},subgroupCollectivesWidth:"portable"}]},{id:"oproj_iq_raw{suffix}",priority:"{prio}",when:["oprojIqRawContract","{guard}","{mrFits}","{sgFoldStorageGate1}"],demoteWhen:"{demote}",derive:{outputProjection:!0,useSubgroups:"wave32SubgroupsUsable",scaleScalar:"dtypes.S",hiddenSize:"args.hiddenSize",blocksPerRow:"inputSize / 32",multiRow:"{multiRow}",regRows:"{regRows}",iqGridBound:"{gridBound}",mRows:"tokenRows",workgroupSize:"subgroupCoveringWorkgroup",oOffset:"args.oOffset",wgClusters:"4",quantBits:4,hasMin:!1,iq:!1,lutId:0,lut16:!1,q2k:!1,q6k:!1,q5k:!1,q3k:!1,q4knCompact:!1,lutc:!1,lutc16:!1,iqDot:!0,iq1m:"iqRawSel == 1 or iqRawSel == 7 or iqRawSel == 8",iq1s:"iqRawSel == 7",iq1md:"iqRawSel == 8",iq2xxs:"iqRawSel == 2",iq2xs:"iqRawSel == 6",iq2s:"iqRawSel == 5",iq3xxs:"iqRawSel == 3",iq3s:"iqRawSel == 4"},passes:[{id:"main",name:"LlamaDecodeOProjIqRaw",shader:"decode-residual-projection-q4.wgsl.jinja",bindings:["hidden","q4_bits_u32","q4_scales","attn_out","{gridBind}"],dispatch:{x:"{dispatchX}"},subgroupCollectivesWidth:"portable"}]}]}]},assets:[["decode-down-proj-col-merge.wgsl.jinja","{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const NSPLIT:u32={{ colSplits }}u;@compute@workgroup_size({{ colMergeWorkgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let o=gid.x;if(o>=HIDDEN_SIZE){return;}var s=0.0;for(var k=0u;k<NSPLIT;k=k+1u){s=s+partials[k*HIDDEN_SIZE+o];}hidden[o]=hidden[o]+s;}"],["decode-down-proj-col-stationary.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const KQUADS:u32={{ intermediateSize }}u/4u;const NEIGHTH:u32={{ hiddenSize }}u/8u;const COL_OFF:u32={{ colOffset }}u;const SCALE_OFF:u32={{ scaleOffset }}u;const NSPLIT:u32={{ colSplits }}u;const QCHUNK:u32=(KQUADS+NSPLIT-1u)/NSPLIT;fn act(v:vec4<f32>)->vec4<f32>{let r=max(v,vec4<f32>(0.0));return r*r;}@compute@workgroup_size({{ colRowsPerWorkgroup }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let o=wid.x*{{ colRowsPerWorkgroup }}u+lid3.x;let inRange=o<HIDDEN_SIZE;let oc=min(o,HIDDEN_SIZE-1u);let oword=oc>>3u;let oshift=(oc&7u)*4u;let qEnd=min((wid.y+1u)*QCHUNK,KQUADS);var acc=0.0;for(var q=wid.y*QCHUNK;q<qEnd;q=q+1u){let u=intermediate[q];let x=act(u);if(all(x==vec4<f32>(0.0))){continue;}let j0=q*4u;let jb=j0>>5u;
{% if hasMin %}let sp=SCALE_OFF+(jb*HIDDEN_SIZE+oc)*2u;let scale=f32(scales_t[sp]);let bias=f32(scales_t[sp+1u]);
{% else %}let scale=f32(scales_t[SCALE_OFF+jb*HIDDEN_SIZE+oc]);
{% endif %}for(var c=0u;c<4u;c=c+1u){if(x[c]!=0.0){let w=col_bits[COL_OFF+(j0+c)*NEIGHTH+oword];let nib=f32((w>>oshift)&0xFu)-8.0;
{% if hasMin %}acc=fma(fma(scale,nib,bias),x[c],acc);
{% else %}acc=fma(scale*nib,x[c],acc);
{% endif %}} } }if(inRange){partials[wid.y*HIDDEN_SIZE+o]=acc;} }`],["decode-down-proj-sparse-q4.wgsl.jinja",`{% set packed4x8CompatUnpackU8 = true %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
const HIDDEN_SIZE:u32={{ hiddenSize }}u;const BPR:u32={{ blocksPerRow }}u;const NHALF:u32={{ hiddenSize }}u/2u;const COL_OFF:u32={{ colOffset }}u;const SCALE_OFF:u32={{ scaleOffset }}u;fn spread4(g:u32)->u32{return(g&0xFu)|(((g>>4u)&0xFu)<<8u)|(((g>>8u)&0xFu)<<16u)|(((g>>12u)&0xFu)<<24u);}
{% include "ops/_shared/subgroup-runtime-fold.wgsl.jinja" %}{% if useSubgroups or sgRuntime %}{% set sgSafeStorageSize = 128 %}{% include "ops/_shared/subgroup-runtime-proof.wgsl.jinja" %}{% endif %}{% if not useSubgroups and not sgRuntime %}var<workgroup>partials4:array<f32,128>;
{% endif %}{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}fn act(v:vec4<f32>)->vec4<f32>{let r=max(v,vec4<f32>(0.0));return r*r;}@compute@workgroup_size(128){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "" }}){let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}let cluster=lid/32u;let lane=lid%32u;let dim0=(wid.x*4u+cluster)*2u;let ohalf=dim0>>1u;var acc0=0.0;var acc1=0.0;for(var b=lane;b<BPR;b=b+32u){let hbase=b*8u;
{% for i in range(8) %}let u{{ i }}=intermediate[hbase{{ " + " ~ i ~ "u" if i else "" }}];
{% endfor %}let mx=max(max(max(u0,u1),max(u2,u3)),max(max(u4,u5),max(u6,u7)));if(max(max(mx.x,mx.y),max(mx.z,mx.w))<=0.0){continue;}
{% for i in range(8) %}let x{{ i }}=act(u{{ i }});
{% endfor %}let qb=b*8u;let a0=any(u0>vec4<f32>(0.0));var c0=0u;if(a0){c0=col_bits[COL_OFF+(qb+0u)*NHALF+ohalf];}let g0r0=select(0x8888u,c0&0xFFFFu,a0);let g0r1=select(0x8888u,c0>>16u,a0);let a1=any(u1>vec4<f32>(0.0));var c1=0u;if(a1){c1=col_bits[COL_OFF+(qb+1u)*NHALF+ohalf];}let g1r0=select(0x8888u,c1&0xFFFFu,a1);let g1r1=select(0x8888u,c1>>16u,a1);let a2=any(u2>vec4<f32>(0.0));var c2=0u;if(a2){c2=col_bits[COL_OFF+(qb+2u)*NHALF+ohalf];}let g2r0=select(0x8888u,c2&0xFFFFu,a2);let g2r1=select(0x8888u,c2>>16u,a2);let a3=any(u3>vec4<f32>(0.0));var c3=0u;if(a3){c3=col_bits[COL_OFF+(qb+3u)*NHALF+ohalf];}let g3r0=select(0x8888u,c3&0xFFFFu,a3);let g3r1=select(0x8888u,c3>>16u,a3);let a4=any(u4>vec4<f32>(0.0));var c4=0u;if(a4){c4=col_bits[COL_OFF+(qb+4u)*NHALF+ohalf];}let g4r0=select(0x8888u,c4&0xFFFFu,a4);let g4r1=select(0x8888u,c4>>16u,a4);let a5=any(u5>vec4<f32>(0.0));var c5=0u;if(a5){c5=col_bits[COL_OFF+(qb+5u)*NHALF+ohalf];}let g5r0=select(0x8888u,c5&0xFFFFu,a5);let g5r1=select(0x8888u,c5>>16u,a5);let a6=any(u6>vec4<f32>(0.0));var c6=0u;if(a6){c6=col_bits[COL_OFF+(qb+6u)*NHALF+ohalf];}let g6r0=select(0x8888u,c6&0xFFFFu,a6);let g6r1=select(0x8888u,c6>>16u,a6);let a7=any(u7>vec4<f32>(0.0));var c7=0u;if(a7){c7=col_bits[COL_OFF+(qb+7u)*NHALF+ohalf];}let g7r0=select(0x8888u,c7&0xFFFFu,a7);let g7r1=select(0x8888u,c7>>16u,a7);let w0r0=spread4(g0r0)|(spread4(g4r0)<<4u);let w1r0=spread4(g1r0)|(spread4(g5r0)<<4u);let w2r0=spread4(g2r0)|(spread4(g6r0)<<4u);let w3r0=spread4(g3r0)|(spread4(g7r0)<<4u);let w0r1=spread4(g0r1)|(spread4(g4r1)<<4u);let w1r1=spread4(g1r1)|(spread4(g5r1)<<4u);let w2r1=spread4(g2r1)|(spread4(g6r1)<<4u);let w3r1=spread4(g3r1)|(spread4(g7r1)<<4u);
{% if hasMin %}let sa=dot(x0+x1+x2+x3+x4+x5+x6+x7,vec4<f32>(1.0));let sp=SCALE_OFF+(b*HIDDEN_SIZE+dim0)*2u;let scale0=f32(scales_t[sp]);let bias0=f32(scales_t[sp+1u]);let scale1=f32(scales_t[sp+2u]);let bias1=f32(scales_t[sp+3u]);acc0=acc0+scale0*(q4wp(w0r0,x0,x4)+q4wp(w1r0,x1,x5)+q4wp(w2r0,x2,x6)+q4wp(w3r0,x3,x7))+bias0*sa;acc1=acc1+scale1*(q4wp(w0r1,x0,x4)+q4wp(w1r1,x1,x5)+q4wp(w2r1,x2,x6)+q4wp(w3r1,x3,x7))+bias1*sa;
{% else %}let scale0=f32(scales_t[SCALE_OFF+b*HIDDEN_SIZE+dim0]);let scale1=f32(scales_t[SCALE_OFF+b*HIDDEN_SIZE+dim0+1u]);acc0=acc0+scale0*(q4wp(w0r0,x0,x4)+q4wp(w1r0,x1,x5)+q4wp(w2r0,x2,x6)+q4wp(w3r0,x3,x7));acc1=acc1+scale1*(q4wp(w0r1,x0,x4)+q4wp(w1r1,x1,x5)+q4wp(w2r1,x2,x6)+q4wp(w3r1,x3,x7));
{% endif %}}
{% if useSubgroups or sgRuntime %}{% if useSubgroups %}let total0=sg_safe_sum(acc0,lid,32u,sgNative);let total1=sg_safe_sum(acc1,lid,32u,sgNative);
{% else %}{{ sg_fold_cluster32([["acc0", "total0"], ["acc1", "total1"]]) }}
{% endif %}if(lane==0u){hidden[dim0]=hidden[dim0]+total0;hidden[dim0+1u]=hidden[dim0+1u]+total1;}
{% else %}{% for r in range(2) %}partials4[lid]=acc{{ r }};workgroupBarrier();var stride{{ r }}=16u;loop{if(stride{{ r }}==0u){break;}if(lane<stride{{ r }}){partials4[lid]=partials4[lid]+partials4[lid+stride{{ r }}];}workgroupBarrier();stride{{ r }}=stride{{ r }}/2u;}let total{{ r }}=partials4[cluster*32u];
{% if not loop.last %}workgroupBarrier();
{% endif %}{% endfor %}if(lane==0u){hidden[dim0]=hidden[dim0]+total0;hidden[dim0+1u]=hidden[dim0+1u]+total1;}
{% endif %}}`],["decode-o-proj-general.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const IN_FEATURES_VEC4:u32={{ inFeaturesVec4 }}u;const W_VEC4:u32={{ oOffsetVec4 }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>partials:array<f32,{{ workgroupSize }}>;@compute@workgroup_size({{ workgroupSize }})
{% set entrySubgroupIds = (useSubgroups | default(false)) and (subgroupIdBuiltins | default(false)) %}{% set wgTotal = "reduce_total" if entrySubgroupIds else "partials[0]" %}{% include "ops/_shared/decode-entry-point.wgsl.jinja" %}let out_dim=wid.x;let lid=lid3.x;if(out_dim>=HIDDEN_SIZE){return;}let weight_base=W_VEC4+out_dim*IN_FEATURES_VEC4;var acc=0.0;for(var i=lid;i<IN_FEATURES_VEC4;i=i+WORKGROUP_SIZE){
{% if weightScalar == "f16" %}acc=acc+f32(dot(vec4<f16>(x[i]),weights[weight_base+i]));
{% else %}acc=acc+dot(x[i],weights[weight_base+i]);
{% endif %}}
{% include "ops/_shared/workgroup-reduce-scalar.wgsl.jinja" %}if(lid==0u){hidden[out_dim]=hidden[out_dim]+{{ wgTotal }};} }`],["decode-o-proj-q1-general.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const BPR:u32={{ blocksPerRow }}u;const IN_WORDS:u32={{ inWords }}u;const W_O_Q1:u32={{ oOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;const TILE_ROWS:u32=4u;var<workgroup>partials0:array<f32,{{ workgroupSize }}>;var<workgroup>partials1:array<f32,{{ workgroupSize }}>;var<workgroup>partials2:array<f32,{{ workgroupSize }}>;var<workgroup>partials3:array<f32,{{ workgroupSize }}>;
{% include "ops/_shared/q1-dot-helpers.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})
{% include "ops/_shared/decode-entry-point.wgsl.jinja" %}let index0=wid.x*TILE_ROWS;let lid=lid3.x;if(index0>=HIDDEN_SIZE){return;}let tile=index0/4u;var acc0=0.0;var acc1=0.0;var acc2=0.0;var acc3=0.0;for(var word_index=lid;word_index<IN_WORDS;word_index=word_index+WORKGROUP_SIZE){let col_block=word_index/4u;let word_id=word_index%4u;let base4=word_index*8u;
{% for i in range(8) %}let n{{ i }}=attn_out[base4{{ " + " ~ i ~ "u" if i else "" }}];
{% endfor %}let dot4=q1_dot_word_t4(W_O_Q1,tile,BPR,col_block,word_id,n0,n1,n2,n3,n4,n5,n6,n7);acc0+=dot4.x;acc1+=dot4.y;acc2+=dot4.z;acc3+=dot4.w;}
{% set q1ReduceVector = true %}{% set q1ReduceComment = "output-projection" %}{% include "ops/_shared/q1-tile4-reduce.wgsl.jinja" %}if(lid==0u){hidden[index0+0u]=hidden[index0+0u]+partials0[0];hidden[index0+1u]=hidden[index0+1u]+partials1[0];hidden[index0+2u]=hidden[index0+2u]+partials2[0];hidden[index0+3u]=hidden[index0+3u]+partials3[0];} }`],["decode-q4-projection-multirow-tail.wgsl.jinja",`{% set hoistFormat = hoistFormat | default((
"iq3xxs"if(iq3xxs|default(false))else"iq2xxs"if((iq2xxs|default(false))and(regRows|default(1))>=4)else"iq2xs"if(iq2xs|default(false))else"iq2s"if(iq2s|default(false))else
  "iq1" if (iq1m | default(false)) else "") if (mRows > 1 and not (iqGridBound | default(false))) else "") %}{% set hoistBlockWeights = hoistBlockWeights | default(hoistFormat != "") %}{% set hoistTwoScale = hoistFormat == "iq1" or hoistFormat == "iq2xs" or hoistFormat == "iq2s" %}{% if hoistBlockWeights %}{% for r in range(regRows) %}let blk{{ r }}=base0+b+{{ r }}u*BPR;
{% endfor %}{% if hoistFormat == "iq1" %}{% include "./iq1-block-weights.wgsl.jinja" %}{% elif hoistFormat == "iq3xxs" %}{% include "./iq3-block-weights.wgsl.jinja" %}{% else %}{% include "./iq2-block-weights.wgsl.jinja" %}{% endif %}{% for m in range(mRows) %}{% set rowAcc = "acc" if m == 0 else "accM" ~ m ~ "_" %}{% if m > 0 %}{let hbase{{ m }}={{ m }}u*BPR*8u+hbase;
{% for i in range(8) %}let x{{ i }}={{ xOpen }}hbase{{ m }}{{ " + " ~ i ~ "u" if i else "" }}{{ xClose }};
{% endfor %}{% endif %}{% for r in range(regRows) %}{{ iq_dot_acc(r ~ "", rowAcc ~ r, "two" if hoistTwoScale else "one") | trim }}
{% endfor %}{% if m > 0 %}}
{% endif %}{% endfor %}{% else %}{% include "ops/_shared/q4-decode-row-block-accumulate.wgsl.jinja" %}{% for m in range(1, mRows) %}{let hbase{{ m }}={{ m }}u*BPR*8u+hbase;
{% for i in range(8) %}let x{{ i }}={{ xOpen }}hbase{{ m }}{{ " + " ~ i ~ "u" if i else "" }}{{ xClose }};
{% endfor %}{% set accPrefix = "accM" ~ m ~ "_" %}{% include "ops/_shared/q4-decode-row-block-accumulate.wgsl.jinja" %}{% set accPrefix = "acc" %}}
{% endfor %}{% endif %}}
{% if stageX | default(false) %}}
{% endif %}{% for m in range(mRows) %}{% set accPrefix = "acc" if m == 0 else "accM" ~ m ~ "_" %}{% set totalPrefix = "total" if m == 0 else "totalM" ~ m ~ "_" %}{% set sgTotalsSlot = "cluster * " ~ (mRows * regRows) ~ "u + " ~ (m * regRows) ~ "u + " %}{% include "ops/_shared/projection-multirow-reduce.wgsl.jinja" %}{% endfor %}{% set accPrefix = "acc" %}{% set totalPrefix = "total" %}{% if (sgPath | default("")) == "general" %}{{ sg_cluster_store(sgClusters) }} let dim0={{ sgDim0 }};
{% for m in range(mRows) %}{% for r in range(regRows) %}let {{ "total" if m == 0 else "totalM" ~ m ~ "_" }}{{ r }}=sgTotals[cluster*{{ mRows * regRows }}u+{{ m * regRows + r }}u];
{% endfor %}{% endfor %}{% endif %}if({{ "activeRank" if (sgPath | default("")) == "general" else "lane" }}==0u){
{% for m in range(mRows) %}{% set rowTotal = "total" if m == 0 else "totalM" ~ m ~ "_" %}{% set rowBase = "" if m == 0 else m ~ "u * HIDDEN_SIZE + " %}{% for r in range(regRows) %}hidden[{{ rowBase }}dim0+{{ r }}u]=hidden[{{ rowBase }}dim0+{{ r }}u]+{{ rowTotal }}{{ r }};
{% endfor %}{% endfor %}}
{% if sgOwned %}{{ sg_cluster_close(sgPath) }}
{% endif %}`],["decode-q4-projection-single-row-tail.wgsl.jinja",`{% include "ops/_shared/q4-decode-row-block-accumulate.wgsl.jinja" %}}
{% if useSubgroups %}let total=sg_safe_sum(acc0,lid,SG_SAFE_WG,sgNative);
{% elif sgRuntime %}{{ sg_fold_workgroup([["acc0", "total"]]) }}
{% else %}partials[lid]=acc0;workgroupBarrier();
{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ wgsl_tree_fold(["partials"]) }} let total=partials[0];
{% endif %}if(lid==0u){hidden[dim]=hidden[dim]+total;} }`],["decode-residual-projection-q4.wgsl.jinja",`{% set mRows = mRows if mRows is defined else 1 %}{% set hoistFormat = (
"iq3xxs"if(iq3xxs|default(false))else"iq2xxs"if((iq2xxs|default(false))and(regRows|default(1))>=4)else"iq2xs"if(iq2xs|default(false))else"iq2s"if(iq2s|default(false))else
  "iq1" if (iq1m | default(false)) else "") if (mRows > 1 and not (iqGridBound | default(false))) else "" %}{% set hoistBlockWeights = hoistFormat != "" %}{% set xOpen = "attn_out[" if outputProjection else "act(vec4<f32>(intermediate[" %}{% set xClose = "]" if outputProjection else "]))" %}{% macro x_loads(base, note=false) %}{% if note and not outputProjection %}let hbase={{ base }};
{% else %}let hbase={{ base }};
{% endif %}{% for i in range(8) %}let x{{ i }}={{ xOpen }}hbase{{ " + " ~ i ~ "u" if i else "" }}{{ xClose }};
{% endfor %}{% endmacro %}{% set iqPrimitives = not (iq1m is defined and iq1m) %}{% set emitQuantDotHelpers = not (lut16 or lutc16) %}{% set nibbleHelpers = not (q2k or q3k or q5k or q6k) %}{% set packed4x8CompatUnpackI8 = (quantBits == 8 and emitQuantDotHelpers) or q3k or q6k or (iq1m | default(false)) %}{% set packed4x8CompatUnpackU8 = (iq2s | default(false)) or (((iq2xxs | default(false)) or (iq2xs | default(false)) or (iq3xxs | default(false))) and (iqGridBound | default(false))) or (not ((iq1m | default(false)) or (iq2xxs | default(false)) or (iq2xs | default(false)) or (iq2s | default(false)) or (iq3xxs | default(false))) and not lutId and (((quantBits != 8) and nibbleHelpers) or q2k or q3k or q5k or q6k)) %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% if not multiRow or mRows > 1 %}const HIDDEN_SIZE:u32={{ hiddenSize }}u;
{% endif %}{% if outputProjection %}const BPR:u32={{ blocksPerRow }}u;const W_O_Q4:u32={{ oOffset }}u;
{% else %}const BPR:u32={{ blocksPerRow }}u;const W_DOWN_Q4:u32={{ downOffset }}u;
{% endif %}{% if not multiRow %}const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% endif %}{% set sgWorkgroupSize = (32 * wgClusters if outputProjection else 128) if multiRow else workgroupSize %}{% include "ops/_shared/subgroup-runtime-fold.wgsl.jinja" %}{% set stageX = (stageActivations | default(false)) and multiRow and mRows == 1 and (useSubgroups or sgRuntime) and not ((iq1s | default(false)) or (iq1md | default(false))) %}{% set sgOwned = multiRow and useSubgroups and not stageX and (subgroupIdBuiltins | default(false)) %}{% if sgOwned %}{% include "ops/_shared/subgroup-owned-clusters.wgsl.jinja" %}{% set sgClusters = (wgClusters if outputProjection else 4) ~ "u" %}{% set sgDim0 = "(wid.x * " ~ sgClusters ~ " + cluster) * " ~ regRows ~ "u" %}{% elif useSubgroups or sgRuntime %}{% set sgSafeStorageSize = sgWorkgroupSize %}{% include "ops/_shared/subgroup-runtime-proof.wgsl.jinja" %}{% endif %}{% if stageX %}const X_STAGE_BLOCKS:u32=64u;var<workgroup>x_sh:array<vec4<f32>,512>;fn x_slot(j:u32)->u32{return j^((j>>3u)&7u);}
{% endif %}{% if not useSubgroups and not sgRuntime %}{% if multiRow %}{% set partialsRowVector = true %}var<workgroup>partials4:array<vec{{ regRows }}<f32>,{{ sgWorkgroupSize }}>;
{% else %}var<workgroup>partials:array<f32,{{ workgroupSize }}>;
{% endif %}{% endif %}{% if (iq1m is defined and iq1m) or (iq2xxs is defined and iq2xxs) or (iq2xs is defined and iq2xs) or (iq2s is defined and iq2s) or (iq3xxs is defined and iq3xxs) %}{% set iqInline = true %}{% include "ops/_shared/iq-grid.wgsl.jinja" %}{% include "ops/_shared/iqx-raw-dequant-helpers.wgsl.jinja" %}{% elif lutId %}{% set emitQuantValueHelpers = not (lut2 | default(false)) %}{% set emitLut2DotHelper = not ((lut2 | default(false)) and (lutc16 | default(false))) %}{% include "ops/_shared/q4nl-dequant-helpers.wgsl.jinja" %}{% set emitQuantValueHelpers = true %}{% else %}{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% endif %}{% if not outputProjection %}{% if actRelu2 %}fn act(v:vec4<f32>)->vec4<f32>{let r=max(v,vec4<f32>(0.0));return r*r;}
{% else %}fn act(v:vec4<f32>)->vec4<f32>{return v;}
{% endif %}{% endif %}{% if multiRow %}{% if outputProjection %}{% else %}{% endif %}@compute@workgroup_size({{ 32 * wgClusters if outputProjection else 128 }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>{{ sgClusterBuiltins if sgOwned else (", @builtin(local_invocation_id) lid3: vec3<u32>" ~ (", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "")) }}){
{% for sgPath in (sgClusterPaths if sgOwned else [""]) %}{% set laneStride = "activeCount" if sgPath == "general" else "32u" %}{% if sgOwned %}{{ sg_cluster_open(sgPath, sgClusters, (wgClusters if outputProjection else 4) * regRows * mRows) }}
{% else %}let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}let cluster=lid/32u;let lane=lid%32u;
{% endif %}let dim0=(wid.x*{{ wgClusters if outputProjection else 4 }}u+cluster)*{{ regRows }}u;let base0={{ "W_O_Q4" if outputProjection else "W_DOWN_Q4" }}+dim0*BPR;
{% for m in range(mRows) %}{% for r in range(regRows) %}var {{ "acc" ~ r if m == 0 else "accM" ~ m ~ "_" ~ r }}=0.0;
{% endfor %}{% endfor %}{% if ((iq1s | default(false)) or (iq1md | default(false))) and mRows == 1 %}let dqPu=((BPR/2u)/{{ laneStride }})*{{ laneStride }};for(var dqP=lane;dqP<dqPu;dqP=dqP+{{ laneStride }}){let b=2u*dqP;{{ x_loads("b * 8u") | trim }} let dqK=b&7u;let dqMo=8u+(dqK>>1u);
{% if iq1md | default(false) %}let dqCo=12u+(dqK>>2u);let dqCs=8u*(dqK&3u);
{% endif %}{% for r in range(regRows) %}let dpS{{ r }}=(base0+b+{{ r }}u*BPR)>>3u;
{% if iq1md | default(false) %}let dpW{{ r }}=dpS{{ r }}*14u;let dpM{{ r }}=q4_bits[dpW{{ r }}+dqMo];let dpC{{ r }}=q4_bits[dpW{{ r }}+dqCo]>>dqCs;let dpD{{ r }}=f32(q4_scales[dpS{{ r }}]);{{ iq1_dense_dot("pa" ~ r, "acc" ~ r, true, "q4_bits[dpW" ~ r ~ " + dqK]", "(dpM" ~ r ~ " & 0xFFFFu) | ((dpC" ~ r ~ " & 0x3Fu) << 16u)", "dpD" ~ r) | trim }}
{% else %}let dpW{{ r }}=dpS{{ r }}*12u;let dpM{{ r }}=q4_bits[dpW{{ r }}+dqMo];let dpD{{ r }}=f32(q4_scales[dpS{{ r }}]);{{ iq1_dense_dot("pa" ~ r, "acc" ~ r, false, "q4_bits[dpW" ~ r ~ " + dqK]", "dpM" ~ r ~ " & 0xFFFFu", "dpD" ~ r) | trim }}
{% endif %}{% endfor %}{ {{ x_loads("b * 8u + 8u") | trim }}
{% for r in range(regRows) %}{% if iq1md | default(false) %}{{ iq1_dense_dot("pb" ~ r, "acc" ~ r, true, "q4_bits[dpW" ~ r ~ " + dqK + 1u]", "(dpM" ~ r ~ " >> 16u) | (((dpC" ~ r ~ " >> 8u) & 0x3Fu) << 16u)", "dpD" ~ r) | trim }}
{% else %}{{ iq1_dense_dot("pb" ~ r, "acc" ~ r, false, "q4_bits[dpW" ~ r ~ " + dqK + 1u]", "dpM" ~ r ~ " >> 16u", "dpD" ~ r) | trim }}
{% endif %}{% endfor %}} }for(var b=2u*dqPu+lane;b<BPR;b=b+{{ laneStride }}){ {{ x_loads("b * 8u") | trim }}
{% elif stageX %}for(var c0=0u;c0<BPR;c0=c0+X_STAGE_BLOCKS){workgroupBarrier();for(var i=lid;i<X_STAGE_BLOCKS*8u;i=i+{{ 32 * wgClusters if outputProjection else 128 }}u){
{% if blocksPerRow % 64 != 0 %}if(c0*8u+i<BPR*8u){x_sh[x_slot(i)]={{ xOpen }}c0*8u+i{{ xClose }};}
{% else %}x_sh[x_slot(i)]={{ xOpen }}c0*8u+i{{ xClose }};
{% endif %}}workgroupBarrier();for(var b=c0+lane;b<min(c0+X_STAGE_BLOCKS,BPR);b=b+32u){let hl=(b-c0)*8u;
{% for i in range(8) %}let x{{ i }}=x_sh[x_slot(hl+{{ i }}u)];
{% endfor %}{% else %}for(var b=lane;b<BPR;b=b+{{ laneStride }}){ {{ x_loads("b * 8u") | trim }}
{% endif %}{% include "./decode-q4-projection-multirow-tail.wgsl.jinja" %}{% endfor %}}
{% else %}{% if outputProjection %}{% else %}{% endif %}@compute@workgroup_size({{ workgroupSize }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>{{ ", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "" }}){let dim=wid.x;let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}if(dim>=HIDDEN_SIZE){return;}let base0={{ "W_O_Q4" if outputProjection else "W_DOWN_Q4" }}+dim*BPR;var acc0=0.0;
{% if (iq1s | default(false)) or (iq1md | default(false)) %}let dqPu=((BPR/2u)/WORKGROUP_SIZE)*WORKGROUP_SIZE;for(var dqP=lid;dqP<dqPu;dqP=dqP+WORKGROUP_SIZE){let b=2u*dqP;{{ x_loads("b * 8u", true) | trim }} let dqK=b&7u;let dqMo=8u+(dqK>>1u);let dpS=(base0+b)>>3u;
{% if iq1md | default(false) %}let dpW=dpS*14u;let dpM=q4_bits[dpW+dqMo];let dpC=q4_bits[dpW+12u+(dqK>>2u)]>>(8u*(dqK&3u));let dpD=f32(q4_scales[dpS]);{{ iq1_dense_dot("pa", "acc0", true, "q4_bits[dpW + dqK]", "(dpM & 0xFFFFu) | ((dpC & 0x3Fu) << 16u)", "dpD") | trim }}
{% else %}let dpW=dpS*12u;let dpM=q4_bits[dpW+dqMo];let dpD=f32(q4_scales[dpS]);{{ iq1_dense_dot("pa", "acc0", false, "q4_bits[dpW + dqK]", "dpM & 0xFFFFu", "dpD") | trim }}
{% endif %}{ {{ x_loads("b * 8u + 8u", true) | trim }}
{% if iq1md | default(false) %}{{ iq1_dense_dot("pb", "acc0", true, "q4_bits[dpW + dqK + 1u]", "(dpM >> 16u) | (((dpC >> 8u) & 0x3Fu) << 16u)", "dpD") | trim }}
{% else %}{{ iq1_dense_dot("pb", "acc0", false, "q4_bits[dpW + dqK + 1u]", "dpM >> 16u", "dpD") | trim }}
{% endif %}} }for(var b=2u*dqPu+lid;b<BPR;b=b+WORKGROUP_SIZE){
{% else %}for(var b=lid;b<BPR;b=b+WORKGROUP_SIZE){
{% endif %}{{ x_loads("b * 8u", true) | trim }}
{% include "./decode-q4-projection-single-row-tail.wgsl.jinja" %}{% endif %}`],["iq1-block-weights.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}{% if iq1s | default(false) %}let dqK=b&7u;let dqMo=8u+(dqK>>1u);let dqSh=16u*(dqK&1u);
{% for r in range(regRows) %}let dqS{{ r }}=blk{{ r }}>>3u;let sbw{{ r }}=dqS{{ r }}*12u;{{ iq1s_regs(r ~ "", "q4_bits[sbw" ~ r ~ " + dqK]", "(q4_bits[sbw" ~ r ~ " + dqMo] >> dqSh) & 0xFFFFu", "f32(q4_scales[dqS" ~ r ~ "])", iqSfx) | trim }}
{% endfor %}{% else %}{% if iq1md | default(false) %}let dqK=b&7u;let dqMo=8u+(dqK>>1u);let dqSh=16u*(dqK&1u);let dqCo=12u+(dqK>>2u);let dqCs=8u*(dqK&3u);
{% endif %}{% for r in range(regRows) %}{% if iq1md | default(false) %}let dqS{{ r }}=blk{{ r }}>>3u;let sbw{{ r }}=dqS{{ r }}*14u;{{ iq1_regs(r ~ "", "q4_bits[sbw" ~ r ~ " + dqK]", "((q4_bits[sbw" ~ r ~ " + dqMo] >> dqSh) & 0xFFFFu) | (((q4_bits[sbw" ~ r ~ " + dqCo] >> dqCs) & 0x3Fu) << 16u)", "f32(q4_scales[dqS" ~ r ~ "])", iqSfx) | trim }}
{% else %}let w{{ r }}=blk{{ r }}*2u;{{ iq1_regs(r ~ "", "q4_bits[w" ~ r ~ "]", "q4_bits[w" ~ r ~ " + 1u]", "f32(q4_scales[blk" ~ r ~ " >> 3u])", iqSfx) | trim }}
{% endif %}{% endfor %}{% endif %}`],["iq2-block-weights.wgsl.jinja",`{% set iqSfx = iqSfx | default("") %}{% if iq2xxs | default(false) %}{% for r in range(regRows) %}let w{{ r }}=blk{{ r }}*2u;{{ iq2xxs_regs(r ~ "", "q4_bits[w" ~ r ~ "]", "q4_bits[w" ~ r ~ " + 1u]", "f32(q4_scales[blk" ~ r ~ " >> 3u])", iqSfx) | trim }}
{% endfor %}{% elif iq2xs | default(false) %}{% for r in range(regRows) %}let sbw{{ r }}=(blk{{ r }}>>3u)*18u;let k{{ r }}=blk{{ r }}&7u;{{ iq2xs_regs(r ~ "", "q4_bits[sbw" ~ r ~ " + 2u * k" ~ r ~ "]", "q4_bits[sbw" ~ r ~ " + 2u * k" ~ r ~ " + 1u]",
      "(q4_bits[sbw" ~ r ~ " + 16u + (k" ~ r ~ " >> 2u)] >> ((k" ~ r ~ " & 3u) * 8u)) & 0xFFu", "f32(q4_scales[blk" ~ r ~ " >> 3u])", iqSfx) | trim }}
{% endfor %}{% else %}{% for r in range(regRows) %}let sbw{{ r }}=(blk{{ r }}>>3u)*20u;let k{{ r }}=blk{{ r }}&7u;{{ iq2s_regs(r ~ "", "q4_bits[sbw" ~ r ~ " + 2u * k" ~ r ~ "]", "q4_bits[sbw" ~ r ~ " + 2u * k" ~ r ~ " + 1u]",
      "(q4_bits[sbw" ~ r ~ " + 16u + (k" ~ r ~ " >> 2u)] >> ((k" ~ r ~ " & 3u) * 8u)) & 0xFFu",
      "(q4_bits[sbw" ~ r ~ " + 18u + (k" ~ r ~ " >> 2u)] >> ((k" ~ r ~ " & 3u) * 8u)) & 0xFFu", "f32(q4_scales[blk" ~ r ~ " >> 3u])", iqSfx) | trim }}
{% endfor %}{% endif %}`],["iq3-block-weights.wgsl.jinja",`{% for r in range(regRows) %}let iw{{ r }}=blk{{ r }}*3u;{{ iq3xxs_regs(r ~ "", "q4_bits[iw" ~ r ~ "]", "q4_bits[iw" ~ r ~ " + 1u]", "q4_bits[iw" ~ r ~ " + 2u]", "f32(q4_scales[blk" ~ r ~ " >> 3u])") | trim }}
{% endfor %}`],["llama-decode-dense-proj.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }}
{% if not assumeFullBlocks %}const OUTPUT_SIZE:u32={{ outputSize }}u;
{% endif %}const INPUT_SIZE_VEC4:u32={{ inputSizeVec4 }}u;const WEIGHT_OFFSET_VEC4:u32={{ weightOffsetVec4 }}u;const WG:u32={{ denseWorkgroupSize }}u;const OUTPUTS_PER_WG:u32={{ outputsPerWg }}u;var<workgroup>partials:array<f32,WG*OUTPUTS_PER_WG>;@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let output_base=wid.x*OUTPUTS_PER_WG;let lid=lid3.x;
{% if not assumeFullBlocks %}if(output_base>=OUTPUT_SIZE){return;}
{% endif %}var acc:array<f32,OUTPUTS_PER_WG>;for(var row=0u;row<OUTPUTS_PER_WG;row=row+1u){acc[row]=0.0;}for(var input_index=lid;input_index<INPUT_SIZE_VEC4;input_index=input_index+WG){let value={{ "vec4<f32>(activation[input_index])" if widenActivation else "activation[input_index]" }};
{% if weightScalar == "f16" %}let value_f16=vec4<f16>(value);
{% endif %}for(var row=0u;row<OUTPUTS_PER_WG;row=row+1u){
{% if not assumeFullBlocks %}if(output_base+row<OUTPUT_SIZE){
{% endif %}let weight_index=WEIGHT_OFFSET_VEC4+(output_base+row)*INPUT_SIZE_VEC4+input_index;acc[row]=acc[row]+{{ "f32(dot(value_f16, weights[weight_index]))" if weightScalar == "f16" else "dot(value, weights[weight_index])" }};
{% if not assumeFullBlocks %}}
{% endif %}} }for(var row=0u;row<OUTPUTS_PER_WG;row=row+1u){partials[lid*OUTPUTS_PER_WG+row]=acc[row];}workgroupBarrier();var stride=WG/2u;loop{if(lid<stride){for(var row=0u;row<OUTPUTS_PER_WG;row=row+1u){let dst=lid*OUTPUTS_PER_WG+row;let src=(lid+stride)*OUTPUTS_PER_WG+row;partials[dst]=partials[dst]+partials[src];} }workgroupBarrier();if(stride==1u){break;}stride=stride/2u;}if(lid<OUTPUTS_PER_WG){let output_index=output_base+lid;
{% if not assumeFullBlocks %}if(output_index<OUTPUT_SIZE){hidden[output_index]=hidden[output_index]+partials[lid];}
{% else %}hidden[output_index]=hidden[output_index]+partials[lid];
{% endif %}} }`],["llama-decode-q1-proj.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const OUTPUT_SIZE:u32={{ outputSize }}u;const INPUT_SIZE_Q1_WORDS:u32={{ inputSizeQ1Words }}u;const BLOCKS_PER_ROW:u32={{ blocksPerRow }}u;const WEIGHT_OFFSET:u32={{ weightOffset }}u;const WORKGROUP_SIZE:u32={{ q1WorkgroupSize }}u;const TILE_ROWS:u32=4u;var<workgroup>partials0:array<f32,WORKGROUP_SIZE>;var<workgroup>partials1:array<f32,WORKGROUP_SIZE>;var<workgroup>partials2:array<f32,WORKGROUP_SIZE>;var<workgroup>partials3:array<f32,WORKGROUP_SIZE>;
{% include "ops/_shared/q1-dot-helpers.wgsl.jinja" %}@compute@workgroup_size(WORKGROUP_SIZE,1,1)
{% include "ops/_shared/decode-entry-point.wgsl.jinja" %}let output_base=wid.x*TILE_ROWS;let lid=lid3.x;if(output_base>=OUTPUT_SIZE){return;}let tile=output_base/TILE_ROWS;var acc0=0.0;var acc1=0.0;var acc2=0.0;var acc3=0.0;for(var word_index=lid;word_index<INPUT_SIZE_Q1_WORDS;word_index=word_index+WORKGROUP_SIZE){let col_block=word_index/4u;let word_id=word_index%4u;let activation_base=word_index*8u;
{% for i in range(8) %}let v{{ i }}={{ "vec4<f32>(" if activationVec4 != "vec4<f32>" else "" }}activation[activation_base{{ " + " ~ i ~ "u" if i else "" }}]{{ ")" if activationVec4 != "vec4<f32>" else "" }};
{% endfor %}let dot4=q1_dot_word_t4(WEIGHT_OFFSET,tile,BLOCKS_PER_ROW,col_block,word_id,v0,v1,v2,v3,v4,v5,v6,v7);acc0=acc0+dot4.x;acc1=acc1+dot4.y;acc2=acc2+dot4.z;acc3=acc3+dot4.w;}
{% set q1ReduceVector = false %}{% set q1ReduceComment = "" %}{% include "ops/_shared/q1-tile4-reduce.wgsl.jinja" %}if(lid==0u){hidden[output_base]=hidden[output_base]+partials0[0];hidden[output_base+1u]=hidden[output_base+1u]+partials1[0];hidden[output_base+2u]=hidden[output_base+2u]+partials2[0];hidden[output_base+3u]=hidden[output_base+3u]+partials3[0];} }`]]}],["com.xenova.LlamaEmbed",{manifest:{domain:"com.xenova",name:"LlamaEmbed",sinceVersion:1,inputs:{inputT:{onnx:"InputToken",dtype:"uint32",rank:1},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},signsT:{onnx:"Signs",dtype:"float32",rank:1,optional:!0}},outputs:{hiddenT:{onnx:"Hidden",dtype:"Y",rank:"2 if args.seqLen is defined else 1",shape:"[args.seqLen, args.hiddenSize] if args.seqLen is defined else [args.hiddenSize]"}},args:{hiddenSize:{kind:"u32",onnx:"hidden_size"},vocabSize:{kind:"u32",onnx:"vocab_size"},embedOffset:{kind:"u32",onnx:"embed_offset",required:!1},tokenOffset:{kind:"u32",onnx:"token_offset",required:!1},pastLen:{kind:"u32",onnx:"past_len",required:!1},cacheLen:{kind:"u32",onnx:"cache_len",required:!1},seqLen:{kind:"u32",onnx:"seq_len",required:!1},format:{kind:"string",required:!1,oneOf:["q1_0","q8_rows","q4_0","q4_k","q4_k_native_compact","q6_k","q5_k","q3_k","q2_k","lut2_32"]},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1},rotationBlockSize:{kind:"u32",onnx:"rotation_block_size",required:!1}},typeConstraints:{W:["float32","float16"],S:["float32","float16","uint32"],Y:["float32","float16"]},tunables:{EMBED_OUT_VEC4S:{default:4}},derive:{decodeMode:"args.seqLen is undefined",prefillMode:"args.seqLen is defined",denseMode:"present.weightsT and not present.bitsT",q1Mode:'present.bitsT and present.scalesT and args.format == "q1_0"',q8Mode:'present.bitsT and present.scalesT and args.format == "q8_rows"',rotationMode:"present.signsT or args.rotationBlockSize is defined",decodeShape:'decodeMode and ranks.inputT == 1 and ranks.hiddenT == 1 and tensorDtypes.inputT == "uint32" and tensorDtypes.hiddenT == "float32" and args.hiddenSize > 0 and args.vocabSize > 0 and dim(shapes.hiddenT, 0) == args.hiddenSize and dim(shapes.inputT, 0) > (args.tokenOffset if args.tokenOffset else 0)',decodeContract:"decodeShape and not rotationMode",rotationContract:'decodeShape and present.signsT and args.rotationBlockSize is defined and ranks.signsT == 1 and tensorDtypes.signsT == "float32" and numel(shapes.signsT) >= args.hiddenSize and args.rotationBlockSize in [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096] and args.hiddenSize % args.rotationBlockSize == 0',prefillContract:'prefillMode and ranks.inputT == 1 and ranks.hiddenT == 2 and tensorDtypes.inputT == "uint32" and args.hiddenSize > 0 and args.seqLen > 0 and args.vocabSize > 0 and dim(shapes.hiddenT, 0) == args.seqLen and dim(shapes.hiddenT, 1) == args.hiddenSize and numel(shapes.inputT) >= args.seqLen and not rotationMode',prefillF16Ok:"f16Ok(dtypes.Y)",kq4Mode:'present.bitsT and present.scalesT and (args.format == "q4_0" or args.format == "q4_k")',kq4Blocks:"args.vocabSize * (args.hiddenSize / 32) if (args.vocabSize is defined and args.hiddenSize is defined) else 0",kq4Tables:'kq4Mode and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16") and args.hiddenSize % 32 == 0 and dim(shapes.bitsT, 0) >= kq4Blocks * 4 and dim(shapes.scalesT, 0) >= kq4Blocks * (2 if args.format == "q4_k" else 1)',lut2Tables:'present.bitsT and present.scalesT and args.format == "lut2_32" and args.lut is defined and args.lut in [8, 9] and not args.embedOffset and ranks.bitsT == 1 and ranks.scalesT == 1 and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and args.hiddenSize % 256 == 0 and dim(shapes.bitsT, 0) >= kq4Blocks * 2 and dim(shapes.scalesT, 0) >= (kq4Blocks / 8) * (3 if args.lut == 8 else 1)',kSupers:"kq4Blocks / 8",kBitsWords:'52 if args.format == "q6_k" else (44 if args.format == "q5_k" else (20 if args.format == "q2_k" else 28))',kScaleWords:'2 if (args.format == "q5_k" or args.format == "q2_k") else 1',embedOutVec4sWant:"max(1, min(tunables.EMBED_OUT_VEC4S, 8))",embedOutVec4s:"embedOutVec4sWant if args.hiddenSize % (4 * embedOutVec4sWant) == 0 else 1",hiddenSize:"args.hiddenSize",workgroupSize:64,wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32Effective:"wave32Adapter or pinSubgroupSize32"},bindings:{input_token:{arg:"inputT",elementType:"u32"},weights:{arg:"weightsT",elementType:"$weightScalar"},hidden:{arg:"hiddenT",elementType:"f32"},params:{arg:"paramsT",struct:[{name:"past_len",type:"u32",value:"args.pastLen if args.pastLen else 0",required:!0,internal:!0},{name:"cache_len",type:"u32",value:"args.cacheLen if args.cacheLen else 0",required:!0,internal:!0},{name:"token_offset",type:"u32",value:"args.tokenOffset if args.tokenOffset else 0"}]},q1_bits:{arg:"bitsT",elementType:"vec4<u32>"},q1_scales:{arg:"scalesT",elementType:"$scaleScalar"},embed_q8:{arg:"bitsT",elementType:"u32"},embed_scales:{arg:"scalesT",elementType:"$scaleScalar"},tokens:{arg:"inputT",elementType:"u32"},hidden_main:{arg:"hiddenT",name:"hidden",elementType:"$outVec4"},params_main:{name:"params",struct:[{name:"embed_offset",type:"u32",value:"args.embedOffset"},{name:"seq_len",type:"u32",value:"args.seqLen"}]},params_seq_len:{name:"params",struct:[{name:"seq_len",type:"u32",value:"args.seqLen"}]},q4_bits:{arg:"bitsT",elementType:"u32"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},signs:{arg:"signsT",elementType:"f32"},q6_bits:{arg:"bitsT",elementType:"u32"},q6_scales:{arg:"scalesT",elementType:"$scaleScalar"},q4kn_bits:{arg:"bitsT",elementType:"u32"},q4knc_metadata:{arg:"scalesT",elementType:"u32"},q5_bits:{arg:"bitsT",elementType:"u32"},q5_scales:{arg:"scalesT",elementType:"$scaleScalar"},q2_bits:{arg:"bitsT",elementType:"u32"},q2_scales:{arg:"scalesT",elementType:"$scaleScalar"},q3_bits:{arg:"bitsT",elementType:"u32"},q3_scales:{arg:"scalesT",elementType:"$scaleScalar"},weights_2:{arg:"weightsT",name:"weights",elementType:"$weightVec4"},hidden_2:{arg:"hiddenT",name:"hidden",elementType:"$outScalar"},lut2_bits:{arg:"bitsT",elementType:"u32"},lut2_meta:{arg:"scalesT",elementType:"u32"}},variants:[{id:"dense",priority:0,when:["decodeContract","denseMode","ranks.weightsT == 1",'(tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")',"f16Ok(dtypes.W)","dim(shapes.weightsT, 0) >= args.embedOffset + args.vocabSize * args.hiddenSize"],derive:{weightScalar:"dtypes.W",embedOffset:"args.embedOffset"},passes:[{id:"main",name:"LlamaEmbed",shader:"embed.wgsl.jinja",bindings:["input_token","weights","hidden","params"],dispatch:{x:"ceil(args.hiddenSize / workgroupSize)"}}]},{id:"q1",priority:0,when:["decodeContract","q1Mode","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"f16Ok(dtypes.S)","args.hiddenSize % 128 == 0","dim(shapes.bitsT, 0) >= (args.embedOffset + args.vocabSize * (args.hiddenSize / 128)) * 4","dim(shapes.scalesT, 0) >= args.embedOffset + args.vocabSize * (args.hiddenSize / 128)"],derive:{scaleScalar:"dtypes.S",embedOffset:"args.embedOffset"},passes:[{id:"main",name:"LlamaEmbedQ1",shader:"embed-q1.wgsl.jinja",bindings:["input_token","q1_bits","q1_scales","hidden","params"],dispatch:{x:"ceil(args.hiddenSize / workgroupSize)"}}]},{id:"q8",priority:0,when:["decodeContract","q8Mode","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"f16Ok(dtypes.S)","args.hiddenSize % 4 == 0","dim(shapes.bitsT, 0) >= args.vocabSize * (args.hiddenSize / 4)","dim(shapes.scalesT, 0) >= args.vocabSize"],derive:{scaleScalar:"dtypes.S",cols4:"args.hiddenSize / 4"},passes:[{id:"main",name:"LlamaEmbedQ8",shader:"embed-q8.wgsl.jinja",bindings:["input_token","embed_q8","embed_scales","hidden","params"],dispatch:{x:"ceil(args.hiddenSize / workgroupSize)"}}]},{id:"prefill_dense_vec4",priority:1,when:["prefillContract","denseMode","ranks.weightsT == 1",'(tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")',"prefillF16Ok and f16Ok(dtypes.W)","numel(shapes.weightsT) >= (args.embedOffset if args.embedOffset else 0) + args.vocabSize * args.hiddenSize","args.hiddenSize % (4 * embedOutVec4s) == 0","(args.embedOffset if args.embedOffset else 0) % 4 == 0"],derive:{outScalar:"dtypes.Y",weightVec4:'"vec4<f16>" if dtypes.W == "f16" else "vec4<f32>"',outVec4:'"vec4<f16>" if dtypes.Y == "f16" else "vec4<f32>"',outVec4s:"embedOutVec4s"},passes:[{id:"main",name:"LlamaPrefillEmbedVec4",shader:"prefill-embed.wgsl.jinja",bindings:["tokens","weights_2","hidden_main","params_main"],dispatch:{threads:"args.seqLen * (args.hiddenSize / (4 * embedOutVec4s))",workgroupSize:"workgroupSize"}}]},{id:"prefill_dense",priority:0,when:["prefillContract","denseMode","ranks.weightsT == 1",'(tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16")',"prefillF16Ok and f16Ok(dtypes.W)","numel(shapes.weightsT) >= (args.embedOffset if args.embedOffset else 0) + args.vocabSize * args.hiddenSize"],derive:{weightScalar:"dtypes.W",outScalar:"dtypes.Y"},passes:[{id:"main",name:"LlamaPrefillEmbed",shader:"prefill-embed.wgsl.jinja",bindings:["tokens","weights","hidden_2","params_main"],dispatch:{threads:"args.seqLen * args.hiddenSize",workgroupSize:"workgroupSize"}}]},{id:"prefill_q8",priority:0,when:["prefillContract","q8Mode","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"prefillF16Ok and f16Ok(dtypes.S)","args.hiddenSize % 4 == 0","dim(shapes.bitsT, 0) >= args.vocabSize * (args.hiddenSize / 4)","dim(shapes.scalesT, 0) >= args.vocabSize"],derive:{scaleScalar:"dtypes.S",outScalar:"dtypes.Y",cols4:"args.hiddenSize / 4",outVec4:'"vec4<f16>" if dtypes.Y == "f16" else "vec4<f32>"',outVec4s:"embedOutVec4s"},passes:[{id:"main",name:"LlamaPrefillEmbedQ8",shader:"prefill-embed-q8.wgsl.jinja",bindings:["tokens","embed_q8","embed_scales","hidden_main","params_seq_len"],dispatch:{threads:"args.seqLen * (args.hiddenSize / (4 * embedOutVec4s))",workgroupSize:"workgroupSize"}}]},{id:"prefill_q1",priority:0,when:["prefillContract","q1Mode","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"prefillF16Ok and f16Ok(dtypes.S)","args.hiddenSize % 128 == 0","dim(shapes.bitsT, 0) >= ((args.embedOffset if args.embedOffset else 0) + args.vocabSize * (args.hiddenSize / 128)) * 4","dim(shapes.scalesT, 0) >= (args.embedOffset if args.embedOffset else 0) + args.vocabSize * (args.hiddenSize / 128)"],derive:{scaleScalar:"dtypes.S",outScalar:"dtypes.Y",embedOffset:"args.embedOffset if args.embedOffset else 0",blocksPerRow:"args.hiddenSize / 128",outVec4:'"vec4<f16>" if dtypes.Y == "f16" else "vec4<f32>"',outVec4s:"embedOutVec4s"},passes:[{id:"main",name:"LlamaPrefillEmbedQ1",shader:"prefill-embed-q1.wgsl.jinja",bindings:["tokens","q1_bits","q1_scales","hidden_main","params_seq_len"],dispatch:{threads:"args.seqLen * (args.hiddenSize / (4 * embedOutVec4s))",workgroupSize:"workgroupSize"}}]},{id:"kq4",priority:0,when:["decodeContract","kq4Tables","f16Ok(dtypes.S)"],derive:{scaleScalar:"dtypes.S",hasMin:'1 if args.format == "q4_k" else 0',blocksPerRow:"args.hiddenSize / 32"},passes:[{id:"main",name:"LlamaEmbedKq4",shader:"embed-kq4.wgsl.jinja",bindings:["input_token","q4_bits","q4_scales","hidden","params"],dispatch:{x:"ceil(args.hiddenSize / workgroupSize)"}}]},{id:"kq4_inverse_rotation_subgroup",priority:1,when:["rotationContract","kq4Tables","f16Ok(dtypes.S)","args.rotationBlockSize >= 32",'device.wgslLanguageFeatures.has("subgroup_id")',"args.rotationBlockSize * 4 <= device.limits.maxComputeWorkgroupStorageSize"],requires:{features:["subgroups"]},derive:{scaleScalar:"dtypes.S",hasMin:'1 if args.format == "q4_k" else 0',blocksPerRow:"args.hiddenSize / 32",rotationBlock:"args.rotationBlockSize",workgroupSize:"512 if args.rotationBlockSize >= 4096 and device.limits.maxComputeInvocationsPerWorkgroup >= 512 and device.limits.maxComputeWorkgroupSizeX >= 512 else min(256, args.rotationBlockSize)",registers:"args.rotationBlockSize / workgroupSize",useSubgroups:!0,sgMinWidth:'32 if wave32Effective else (device.adapterInfo.subgroupMinSize if has(device.adapterInfo, "subgroupMinSize") else 4)',sgMaxWidth:'32 if wave32Effective else (device.adapterInfo.subgroupMaxSize if has(device.adapterInfo, "subgroupMaxSize") else 128)',lut2:!1},passes:[{id:"main",name:"LlamaEmbedKq4Rotated",shader:"embed-rotated.wgsl.jinja",subgroupCollectivesWidth:"portable",bindings:["input_token","q4_bits","q4_scales","signs","hidden","params"],dispatch:{x:"args.hiddenSize / args.rotationBlockSize"}}]},{id:"kq4_inverse_rotation",priority:0,when:["rotationContract","kq4Tables","f16Ok(dtypes.S)","args.rotationBlockSize * 4 <= device.limits.maxComputeWorkgroupStorageSize"],derive:{scaleScalar:"dtypes.S",hasMin:'1 if args.format == "q4_k" else 0',blocksPerRow:"args.hiddenSize / 32",rotationBlock:"args.rotationBlockSize",workgroupSize:"min(256, args.rotationBlockSize)",registers:"args.rotationBlockSize / workgroupSize",useSubgroups:!1,sgMinWidth:0,sgMaxWidth:0,lut2:!1},passes:[{id:"main",name:"LlamaEmbedKq4Rotated",shader:"embed-rotated.wgsl.jinja",bindings:["input_token","q4_bits","q4_scales","signs","hidden","params"],dispatch:{x:"args.hiddenSize / args.rotationBlockSize"}}]},{id:"prefill_kq4",priority:0,when:["prefillContract","kq4Tables","prefillF16Ok and f16Ok(dtypes.S)"],derive:{scaleScalar:"dtypes.S",outScalar:"dtypes.Y",hasMin:'1 if args.format == "q4_k" else 0',blocksPerRow:"args.hiddenSize / 32",outVec4:'"vec4<f16>" if dtypes.Y == "f16" else "vec4<f32>"',outVec4s:"embedOutVec4s"},passes:[{id:"main",name:"LlamaPrefillEmbedKq4",shader:"prefill-embed-kq4.wgsl.jinja",bindings:["tokens","q4_bits","q4_scales","hidden_main","params_seq_len"],dispatch:{threads:"args.seqLen * (args.hiddenSize / (4 * embedOutVec4s))",workgroupSize:"workgroupSize"}}]},{id:"lut2",priority:0,when:["decodeContract","lut2Tables"],derive:{blocksPerRow:"args.hiddenSize / 32",lut:"args.lut"},passes:[{id:"main",name:"LlamaEmbedLut2",shader:"embed-lut2.wgsl.jinja",bindings:["input_token","lut2_bits","lut2_meta","hidden","params"],dispatch:{x:"ceil(args.hiddenSize / workgroupSize)"}}]},{id:"lut2_inverse_rotation_subgroup",priority:1,when:["rotationContract","lut2Tables","args.rotationBlockSize >= 32",'device.wgslLanguageFeatures.has("subgroup_id")',"args.rotationBlockSize * 4 <= device.limits.maxComputeWorkgroupStorageSize"],requires:{features:["subgroups"]},derive:{lut:"args.lut",blocksPerRow:"args.hiddenSize / 32",rotationBlock:"args.rotationBlockSize",workgroupSize:"512 if args.rotationBlockSize >= 4096 and device.limits.maxComputeInvocationsPerWorkgroup >= 512 and device.limits.maxComputeWorkgroupSizeX >= 512 else min(256, args.rotationBlockSize)",registers:"args.rotationBlockSize / workgroupSize",useSubgroups:!0,sgMinWidth:'32 if wave32Effective else (device.adapterInfo.subgroupMinSize if has(device.adapterInfo, "subgroupMinSize") else 4)',sgMaxWidth:'32 if wave32Effective else (device.adapterInfo.subgroupMaxSize if has(device.adapterInfo, "subgroupMaxSize") else 128)',lut2:!0},passes:[{id:"main",name:"LlamaEmbedLut2Rotated",shader:"embed-rotated.wgsl.jinja",subgroupCollectivesWidth:"portable",bindings:["input_token","lut2_bits","lut2_meta","signs","hidden","params"],dispatch:{x:"args.hiddenSize / args.rotationBlockSize"}}]},{id:"lut2_inverse_rotation",priority:0,when:["rotationContract","lut2Tables","args.rotationBlockSize * 4 <= device.limits.maxComputeWorkgroupStorageSize"],derive:{lut:"args.lut",blocksPerRow:"args.hiddenSize / 32",rotationBlock:"args.rotationBlockSize",workgroupSize:"min(256, args.rotationBlockSize)",registers:"args.rotationBlockSize / workgroupSize",useSubgroups:!1,sgMinWidth:0,sgMaxWidth:0,lut2:!0},passes:[{id:"main",name:"LlamaEmbedLut2Rotated",shader:"embed-rotated.wgsl.jinja",bindings:["input_token","lut2_bits","lut2_meta","signs","hidden","params"],dispatch:{x:"args.hiddenSize / args.rotationBlockSize"}}]},{id:"prefill_lut2",priority:0,when:["prefillContract","lut2Tables","prefillF16Ok"],derive:{outScalar:"dtypes.Y",blocksPerRow:"args.hiddenSize / 32",lut:"args.lut",outVec4:'"vec4<f16>" if dtypes.Y == "f16" else "vec4<f32>"',outVec4s:"embedOutVec4s"},passes:[{id:"main",name:"LlamaPrefillEmbedLut2",shader:"prefill-embed-lut2.wgsl.jinja",bindings:["tokens","lut2_bits","lut2_meta","hidden_main","params_seq_len"],dispatch:{threads:"args.seqLen * (args.hiddenSize / (4 * embedOutVec4s))",workgroupSize:"workgroupSize"}}]},{cases:[{format:"q6k",fmt:"q6_k",kernel:"Q6K",pack:["q6_bits","q6_scales"]},{format:"q5k",fmt:"q5_k",kernel:"Q5K",pack:["q5_bits","q5_scales"]},{format:"q2k",fmt:"q2_k",kernel:"Q2K",pack:["q2_bits","q2_scales"]},{format:"q3k",fmt:"q3_k",kernel:"Q3K",pack:["q3_bits","q3_scales"]}],variants:[{id:"{format}",priority:0,when:["decodeContract",'present.bitsT and present.scalesT and args.format == "{fmt}"',"ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"args.hiddenSize % 256 == 0","dim(shapes.bitsT, 0) >= kSupers * kBitsWords","dim(shapes.scalesT, 0) >= kSupers * kScaleWords","f16Ok(dtypes.S)"],derive:{scaleScalar:"dtypes.S",blocksPerRow:"args.hiddenSize / 32",isPrefill:!1},passes:[{id:"main",name:"LlamaEmbed{kernel}",shader:"embed-{format}.wgsl.jinja",bindings:["input_token","{pack}","hidden","params"],dispatch:{x:"ceil(args.hiddenSize / workgroupSize)"}}]},{id:"prefill_{format}",priority:0,when:["prefillContract",'present.bitsT and present.scalesT and args.format == "{fmt}"',"ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"args.hiddenSize % 256 == 0","dim(shapes.bitsT, 0) >= kSupers * kBitsWords","dim(shapes.scalesT, 0) >= kSupers * kScaleWords","prefillF16Ok and f16Ok(dtypes.S)"],derive:{scaleScalar:"dtypes.S",outScalar:"dtypes.Y",blocksPerRow:"args.hiddenSize / 32",isPrefill:!0,outVec4:'"vec4<f16>" if dtypes.Y == "f16" else "vec4<f32>"',outVec4s:"embedOutVec4s"},passes:[{id:"main",name:"LlamaPrefillEmbed{kernel}",shader:"embed-{format}.wgsl.jinja",bindings:["tokens","{pack}","hidden_main","params_seq_len"],dispatch:{threads:"args.seqLen * (args.hiddenSize / (4 * embedOutVec4s))",workgroupSize:"workgroupSize"}}]}]},{id:"q4knc",priority:0,when:["decodeContract",'present.bitsT and present.scalesT and args.format == "q4_k_native_compact"',"ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','tensorDtypes.scalesT == "uint32"',"args.hiddenSize % 256 == 0","dim(shapes.bitsT, 0) >= args.vocabSize * (args.hiddenSize / 256) * 32","dim(shapes.scalesT, 0) >= args.vocabSize * (args.hiddenSize / 256) * 5"],derive:{superblocksPerRow:"args.hiddenSize / 256"},passes:[{id:"main",name:"LlamaEmbedQ4KNativeCompact",shader:"embed-q4knc.wgsl.jinja",bindings:["input_token","q4kn_bits","q4knc_metadata","hidden","params"],dispatch:{x:"ceil(args.hiddenSize / workgroupSize)"}}]},{id:"prefill_q4knc",priority:0,when:["prefillContract",'present.bitsT and present.scalesT and args.format == "q4_k_native_compact"',"ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.bitsT == "uint32"','tensorDtypes.scalesT == "uint32"',"prefillF16Ok","args.hiddenSize % 256 == 0","dim(shapes.bitsT, 0) >= args.vocabSize * (args.hiddenSize / 256) * 32","dim(shapes.scalesT, 0) >= args.vocabSize * (args.hiddenSize / 256) * 5"],derive:{outScalar:"dtypes.Y",superblocksPerRow:"args.hiddenSize / 256",outVec4:'"vec4<f16>" if dtypes.Y == "f16" else "vec4<f32>"',outVec4s:"embedOutVec4s"},passes:[{id:"main",name:"LlamaPrefillEmbedQ4KNativeCompact",shader:"prefill-embed-q4knc.wgsl.jinja",bindings:["tokens","q4kn_bits","q4knc_metadata","hidden_main","params_seq_len"],dispatch:{threads:"args.seqLen * (args.hiddenSize / (4 * embedOutVec4s))",workgroupSize:"workgroupSize"}}]}]},assets:[["embed-kq4.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const BPR:u32={{ blocksPerRow }}u;@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let dim=gid.x;if(dim>=HIDDEN_SIZE){return;}let token=input_token[params.token_offset];let blk=token*BPR+dim/32u;let eib=dim%32u;let word=q4_bits[blk*4u+(eib%16u)/4u];let nib=f32((word>>((eib%4u)*8u+select(0u,4u,eib>=16u)))&0xFu)-8.0;
{% if hasMin %}hidden[dim]=nib*f32(q4_scales[blk*2u])+f32(q4_scales[blk*2u+1u]);
{% else %}hidden[dim]=nib*f32(q4_scales[blk]);
{% endif %}}`],["embed-lut2.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const BPR:u32={{ blocksPerRow }}u;
{% include "./lut2-value.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let dim=gid.x;if(dim>=HIDDEN_SIZE){return;}hidden[dim]=lut2_value(input_token[params.token_offset],dim);}`],["embed-native-k-driver.wgsl.jinja",`{% if isPrefill %}const OUT_VEC4S:u32={{ outVec4s }}u;const ROW_VEC4S:u32=HIDDEN_SIZE/(4u*OUT_VEC4S);@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let g=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let total=params.seq_len*ROW_VEC4S;if(g>=total){return;}let token_index=g/ROW_VEC4S;let span=g%ROW_VEC4S;let dim=span*4u*OUT_VEC4S;let block=tokens[token_index]*BLOCKS_PER_ROW+dim/32u;let lo=dim&31u;
{% for v in range(outVec4s) %}hidden[g*OUT_VEC4S+{{ v }}u]={{ outVec4 }}(
{% for c in range(4) %}{{ outScalar }}({{ nativeKValueFn }}(block,lo+{{ v * 4 + c }}u)){{ "" if c == 3 else "," }}
{% endfor %});
{% endfor %}}
{% else %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let dim=gid.x;if(dim>=HIDDEN_SIZE){return;}let block=input_token[params.token_offset]*BLOCKS_PER_ROW+dim/32u;hidden[dim]={{ nativeKValueFn }}(block,dim&31u);}
{% endif %}`],["embed-q1.wgsl.jinja","{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const W_EMBED_Q1:u32={{ embedOffset }}u;fn q1_scale(block:u32)->f32{return f32(q1_scales[block]);}fn q1_weight(block_offset:u32,row:u32,cols:u32,col:u32)->f32{let blocks_per_row=cols/128u;let block=block_offset+row*blocks_per_row+col/128u;let bit_index=col%128u;let block_words=q1_bits[block];let word=block_words[bit_index/32u];let bit=(word>>(bit_index%32u))&1u;let scale=q1_scale(block);return select(-scale,scale,bit==1u);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let dim=gid.x;if(dim>=HIDDEN_SIZE){return;}let token=input_token[params.token_offset];hidden[dim]=q1_weight(W_EMBED_Q1,token,HIDDEN_SIZE,dim);}"],["embed-q2k.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const BLOCKS_PER_ROW:u32={{ blocksPerRow }}u;fn q2_value(block:u32,element_in_block:u32)->f32{let superblock=block>>3u;let j=block&7u;let base=superblock*20u;let plane=element_in_block>>4u;let e=element_in_block&15u;let code_word=q2_bits[base+j*2u+plane];let code=(code_word>>((e&3u)*8u+(e>>2u)*2u))&3u;let scale_word=q2_bits[base+16u+(j>>1u)];let scale_byte=(scale_word>>(((j&1u)*2u+plane)*8u))&255u;let d=f32(q2_scales[superblock*2u]);let dmin=f32(q2_scales[superblock*2u+1u]);return d*f32(scale_byte&15u)*f32(code)-dmin*f32(scale_byte>>4u);}
{% set nativeKValueFn = "q2_value" %}{% include "./embed-native-k-driver.wgsl.jinja" %}`],["embed-q3k.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const BLOCKS_PER_ROW:u32={{ blocksPerRow }}u;fn q3_value(block:u32,element_in_block:u32)->f32{let superblock=block>>3u;let j=block&7u;let base=superblock*28u;let plane=element_in_block>>4u;let e=element_in_block&15u;let code_word=q3_bits[base+j*2u+plane];let code2=(code_word>>((e&3u)*8u+(e>>2u)*2u))&3u;let h_word=q3_bits[base+16u+j];let hi_bit=(h_word>>((e&3u)*8u+(e>>2u)+plane*4u))&1u;let scale_word=q3_bits[base+24u+(j>>1u)];let scale_byte=(scale_word>>(((j&1u)*2u+plane)*8u))&255u;let sub_scale=select(i32(scale_byte),i32(scale_byte)-256,scale_byte>=128u);let code=i32(code2|(hi_bit<<2u))-4;return f32(code)*(f32(q3_scales[superblock])*f32(sub_scale));}
{% set nativeKValueFn = "q3_value" %}{% include "./embed-native-k-driver.wgsl.jinja" %}`],["embed-q4knc.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const SUPERBLOCKS_PER_ROW:u32={{ superblocksPerRow }}u;
{% include "./q4knc-value.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let dim=gid.x;if(dim>=HIDDEN_SIZE){return;}let token=input_token[params.token_offset];let element_in_superblock=dim&255u;let superblock=token*SUPERBLOCKS_PER_ROW+dim/256u;hidden[dim]=q4knc_value(superblock,element_in_superblock/32u,element_in_superblock&31u);}`],["embed-q5k.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const BLOCKS_PER_ROW:u32={{ blocksPerRow }}u;fn q5_value(block:u32,element_in_block:u32)->f32{let superblock=block>>3u;let j=block&7u;let base=superblock*44u;let word=q5_bits[base+j*4u+((element_in_block&15u)>>2u)];let nibble=(word>>((element_in_block&3u)*8u+select(0u,4u,element_in_block>=16u)))&15u;let hi=(q5_bits[base+32u+j]>>element_in_block)&1u;let sc=(q5_bits[base+40u+(j>>2u)]>>((j&3u)*8u))&255u;let m=(q5_bits[base+42u+(j>>2u)]>>((j&3u)*8u))&255u;let d=f32(q5_scales[superblock*2u]);let dmin=f32(q5_scales[superblock*2u+1u]);return d*f32(sc)*f32(nibble|(hi<<4u))-dmin*f32(m);}
{% set nativeKValueFn = "q5_value" %}{% include "./embed-native-k-driver.wgsl.jinja" %}`],["embed-q6k.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const BLOCKS_PER_ROW:u32={{ blocksPerRow }}u;fn q6_value(block:u32,element_in_block:u32)->f32{let superblock=block>>3u;let j=block&7u;let plane=element_in_block>>4u;let e=element_in_block&15u;let word=q6_bits[superblock*52u+j*4u+(e>>2u)];let nibble=(word>>((e&3u)*8u+plane*4u))&15u;let hi_word=q6_bits[superblock*52u+32u+j*2u+plane];let hi2=(hi_word>>((e&3u)*8u+(e>>2u)*2u))&3u;let scale_word=q6_bits[superblock*52u+48u+(j>>1u)];let scale_byte=(scale_word>>(((j&1u)*2u+plane)*8u))&255u;let sub_scale=select(i32(scale_byte),i32(scale_byte)-256,scale_byte>=128u);let code=i32(nibble|(hi2<<4u))-32;let scale=f32(q6_scales[superblock])*f32(sub_scale);return f32(code)*scale;}
{% set nativeKValueFn = "q6_value" %}{% include "./embed-native-k-driver.wgsl.jinja" %}`],["embed-q8.wgsl.jinja",'{% set packed4x8CompatUnpackI8 = true %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const COLS4:u32={{ cols4 }}u;@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let dim=gid.x;if(dim>=HIDDEN_SIZE){return;}let token=input_token[params.token_offset];let word=embed_q8[token*COLS4+dim/4u];var lanes=unpack4xI8(word);let scale=f32(embed_scales[token]);hidden[dim]=f32(lanes[dim&3u])*scale;}'],["embed-rotated.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const BPR:u32={{ blocksPerRow }}u;
{% if useSubgroups %}const WORKGROUP_SIZE:u32={{ workgroupSize }}u;
{% endif %}{% if lut2 %}{% include "./lut2-value.wgsl.jinja" %}{% else %}{% include "./kq4-nibble.wgsl.jinja" %}fn kq4_value(token:u32,dim:u32)->f32{let blk=token*BPR+dim/32u;
{% if hasMin %}return kq4_nibble(blk,dim%32u)*f32(q4_scales[blk*2u])+f32(q4_scales[blk*2u+1u]);
{% else %}return kq4_nibble(blk,dim%32u)*f32(q4_scales[blk]);
{% endif %}}
{% endif %}var<workgroup>exchange:array<f32,{{ rotationBlock }}>;
{% include "ops/_shared/hadamard-column-butterfly.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }}){{ " @subgroup_size(32)" if useSubgroups and pinSubgroupSize32 else "" }} fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_index)lid:u32{{ "," if useSubgroups else "" }}
{% if useSubgroups %}@builtin(subgroup_invocation_id)lane:u32,@builtin(subgroup_id)group:u32,@builtin(num_subgroups)numSg:u32{{ ",\\n  @builtin(subgroup_size) sgSize: u32" if not wave32Effective else "" }}
{% endif %}){
{% if useSubgroups %}let fullSubgroups=numSg*{{ "32u" if wave32Effective else "sgSize" }}==WORKGROUP_SIZE;
{% else %}let tid=lid;
{% endif %}let token=input_token[params.token_offset];let base=wg.x*{{ rotationBlock }}u;
{% for path in (["full", "general"] if useSubgroups else ["portable"]) %}{% set full = path == "full" %}{% if full %}if(fullSubgroups){let tid=group*{{ "32u" if wave32Effective else "sgSize" }}+lane;
{% elif path == "general" %}}else{let tid=lid;
{% endif %}{% for r in range(registers | int) %}var v{{ r }}={{ "lut2_value" if lut2 else "kq4_value" }}(token,base+{{ r * workgroupSize }}u+tid);
{% endfor %}{% include "ops/_shared/hadamard-block-stages.wgsl.jinja" %}{% for r in range(registers | int) %}hidden[base+{{ r * workgroupSize }}u+tid]=v{{ r }}*inverseSqrt(f32({{ rotationBlock }}u))*signs[base+{{ r * workgroupSize }}u+tid];
{% endfor %}{% endfor %}{% if useSubgroups %}}
{% endif %}}`],["embed.wgsl.jinja","{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const W_EMBED:u32={{ embedOffset }}u;fn load_weight(value:{{ weightScalar }})->f32{return f32(value);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let dim=gid.x;if(dim>=HIDDEN_SIZE){return;}let token=input_token[params.token_offset];hidden[dim]=load_weight(weights[W_EMBED+token*HIDDEN_SIZE+dim]);}"],["kq4-nibble.wgsl.jinja","fn kq4_nibble(blk:u32,eib:u32)->f32{let word=q4_bits[blk*4u+(eib%16u)/4u];return f32((word>>((eib%4u)*8u+select(0u,4u,eib>=16u)))&0xFu)-8.0;}"],["lut2-codes.wgsl.jinja",`{% include "ops/_shared/f16-bits-to-f32.wgsl.jinja" %}fn lut2_scale(blk:u32)->f32{
{% if lut == 9 %}return f16_bits_to_f32(lut2_meta[blk>>3u]>>(((blk>>2u)&1u)*16u));
{% else %}return f16_bits_to_f32(lut2_meta[(blk>>3u)*3u+((blk&7u)>>2u)]>>(((blk>>1u)&1u)*16u));
{% endif %}}fn lut2_code(blk:u32,eib:u32)->f32{return f32((lut2_bits[blk*2u+(eib>>4u)]>>((eib&15u)*2u))&3u)-1.0;}`],["lut2-value.wgsl.jinja",'{% include "./lut2-codes.wgsl.jinja" %}fn lut2_value(token:u32,dim:u32)->f32{let blk=token*BPR+dim/32u;return lut2_code(blk,dim%32u)*lut2_scale(blk);}'],["prefill-embed-kq4.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const BPR:u32={{ blocksPerRow }}u;const OUT_VEC4S:u32={{ outVec4s }}u;const ROW_VEC4S:u32=HIDDEN_SIZE/(4u*OUT_VEC4S);
{% include "./kq4-nibble.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let g=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let total=params.seq_len*ROW_VEC4S;if(g>=total){return;}let t=g/ROW_VEC4S;let span=g%ROW_VEC4S;let d=span*4u*OUT_VEC4S;let token=tokens[t];let blk=token*BPR+d/32u;let lo=d%32u;
{% if hasMin %}let scale=f32(q4_scales[blk*2u]);let bias=f32(q4_scales[blk*2u+1u]);
{% else %}let scale=f32(q4_scales[blk]);
{% endif %}{% for v in range(outVec4s) %}hidden[g*OUT_VEC4S+{{ v }}u]={{ outVec4 }}(
{% for c in range(4) %}{% if hasMin %}{{ outScalar }}(kq4_nibble(blk,lo+{{ v * 4 + c }}u)*scale+bias){{ "" if c == 3 else "," }}
{% else %}{{ outScalar }}(kq4_nibble(blk,lo+{{ v * 4 + c }}u)*scale){{ "" if c == 3 else "," }}
{% endif %}{% endfor %});
{% endfor %}}`],["prefill-embed-lut2.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const BPR:u32={{ blocksPerRow }}u;const OUT_VEC4S:u32={{ outVec4s }}u;const ROW_VEC4S:u32=HIDDEN_SIZE/(4u*OUT_VEC4S);
{% include "./lut2-codes.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let g=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let total=params.seq_len*ROW_VEC4S;if(g>=total){return;}let d=(g%ROW_VEC4S)*4u*OUT_VEC4S;let blk=tokens[g/ROW_VEC4S]*BPR+d/32u;let lo=d%32u;let scale=lut2_scale(blk);
{% for v in range(outVec4s) %}hidden[g*OUT_VEC4S+{{ v }}u]={{ outVec4 }}(
{% for c in range(4) %}{{ outScalar }}(lut2_code(blk,lo+{{ v * 4 + c }}u)*scale){{ "" if c == 3 else "," }}
{% endfor %});
{% endfor %}}`],["prefill-embed-q1.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const W_EMBED_Q1:u32={{ embedOffset }}u;const BPR:u32={{ blocksPerRow }}u;const OUT_VEC4S:u32={{ outVec4s }}u;const ROW_VEC4S:u32=HIDDEN_SIZE/(4u*OUT_VEC4S);@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let gi=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let total=params.seq_len*ROW_VEC4S;if(gi>=total){return;}let t=gi/ROW_VEC4S;let span=gi%ROW_VEC4S;let d=span*4u*OUT_VEC4S;let token=tokens[t];let block=W_EMBED_Q1+token*BPR+d/128u;let bit_index=d%128u;let word=q1_bits[block][bit_index/32u];let bits=word>>(bit_index%32u);let scale=f32(q1_scales[block]);let out_base=(t*HIDDEN_SIZE)/4u+span*OUT_VEC4S;
{% for v in range(outVec4s) %}hidden[out_base+{{ v }}u]={{ outVec4 }}(
{% for c in range(4) %}{{ outScalar }}(select(-scale,scale,((bits>>{{ v * 4 + c }}u)&1u)==1u)){{ "" if c == 3 else "," }}
{% endfor %});
{% endfor %}}`],["prefill-embed-q4knc.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const SUPERBLOCKS_PER_ROW:u32={{ superblocksPerRow }}u;const OUT_VEC4S:u32={{ outVec4s }}u;const ROW_VEC4S:u32=HIDDEN_SIZE/(4u*OUT_VEC4S);
{% include "./q4knc-value.wgsl.jinja" %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let g=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let total=params.seq_len*ROW_VEC4S;if(g>=total){return;}let token_index=g/ROW_VEC4S;let span=g%ROW_VEC4S;let dim=span*4u*OUT_VEC4S;let element_in_superblock=dim&255u;let superblock=tokens[token_index]*SUPERBLOCKS_PER_ROW+dim/256u;let block_in_superblock=element_in_superblock/32u;let lo=element_in_superblock&31u;
{% for v in range(outVec4s) %}hidden[g*OUT_VEC4S+{{ v }}u]={{ outVec4 }}(
{% for c in range(4) %}{{ outScalar }}(q4knc_value(superblock,block_in_superblock,lo+{{ v * 4 + c }}u)){{ "" if c == 3 else "," }}
{% endfor %});
{% endfor %}}`],["prefill-embed-q8.wgsl.jinja",`{% set packed4x8CompatUnpackI8 = true %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;const COLS4:u32={{ cols4 }}u;const OUT_VEC4S:u32={{ outVec4s }}u;const ROW_VEC4S:u32=HIDDEN_SIZE/(4u*OUT_VEC4S);@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let g=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let total=params.seq_len*ROW_VEC4S;if(g>=total){return;}let t=g/ROW_VEC4S;let span=g%ROW_VEC4S;let token=tokens[t];let scale=f32(embed_scales[token]);let base=token*COLS4+span*OUT_VEC4S;
{% for v in range(outVec4s) %}let lanes{{ v }}=unpack4xI8(embed_q8[base+{{ v }}u]);hidden[g*OUT_VEC4S+{{ v }}u]={{ outVec4 }}(
{% for c in range(4) %}{{ outScalar }}(f32(lanes{{ v }}[{{ c }}])*scale){{ "" if c == 3 else "," }}
{% endfor %});
{% endfor %}}`],["prefill-embed.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const HIDDEN_SIZE:u32={{ hiddenSize }}u;
{% if outVec4s %}const OUT_VEC4S:u32={{ outVec4s }}u;const ROW_VEC4S:u32=HIDDEN_SIZE/(4u*OUT_VEC4S);
{% endif %}{% if outVec4s %}{% else %}{% endif %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let g=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;
{% if outVec4s %}let total=params.seq_len*ROW_VEC4S;if(g>=total){return;}let t=g/ROW_VEC4S;let span=g%ROW_VEC4S;let src=(params.embed_offset+tokens[t]*HIDDEN_SIZE)/4u+span*OUT_VEC4S;
{% for v in range(outVec4s) %}hidden[g*OUT_VEC4S+{{ v }}u]={{ outVec4 }}(weights[src+{{ v }}u]);
{% endfor %}{% else %}let total=params.seq_len*HIDDEN_SIZE;if(g>=total){return;}let t=g/HIDDEN_SIZE;let d=g%HIDDEN_SIZE;let token=tokens[t];hidden[g]={{ outScalar }}(weights[params.embed_offset+token*HIDDEN_SIZE+d]);
{% endif %}}`],["q4knc-value.wgsl.jinja","fn q4knc_scale_min(superblock:u32,block:u32)->vec2<u32>{let carrier=q4knc_metadata[superblock*5u+1u+(block>>1u)];let pair=carrier>>((block&1u)*12u);return vec2<u32>(pair&63u,(pair>>6u)&63u);}fn q4knc_value(superblock:u32,block:u32,element_in_block:u32)->f32{let word=q4kn_bits[superblock*32u+block*4u+(element_in_block%16u)/4u];let nibble=(word>>((element_in_block%4u)*8u+select(0u,4u,element_in_block>=16u)))&0xFu;let sm=q4knc_scale_min(superblock,block);let dm=unpack2x16float(q4knc_metadata[superblock*5u]);let dl=dm.x*f32(sm.x);let ml=dm.y*f32(sm.y);let product=dl*f32(nibble);return product-ml;}"]]}],["com.xenova.LlamaPrefillDequant",{manifest:{domain:"com.xenova",name:"LlamaPrefillDequant",sinceVersion:1,inputs:{bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1},denseT:{onnx:"Dense",dtype:"D",rank:1}},outputs:{denseT:{onnx:"Dense",dtype:"D",rank:1,shape:["args.denseElems"]}},args:{denseElems:{kind:"u32",onnx:"dense_elems"},blockOffset:{kind:"u32",onnx:"block_offset"},outRows:{kind:"u32",onnx:"out_rows"},inFeatures:{kind:"u32",onnx:"in_features"},dstElemOffset:{kind:"u32",onnx:"dst_elem_offset"},format:{kind:"string",oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","lut2_32","lut2_16","q5_0","q2_k","q3_k","q6_k","q5_k","q1_0"]},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1}},typeConstraints:{S:["float32","float16","uint32"],D:["float32","float16"]},derive:{fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',lutcWords:"1 if (lut2 == 1 and args.lut == 9) else (5 if lutc16 == 1 else 3)",q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',q4knc:'1 if fmt == "q4_k_native_compact" else 0',dqBlocks:"(args.blockOffset + args.outRows * (args.inFeatures / 32)) if (args.blockOffset is defined and args.outRows is defined and args.inFeatures is defined) else 0",scaleScalar:"dtypes.S",denseScalar:"dtypes.D",workgroupSize:64},when:["ranks.bitsT == 1","ranks.scalesT == 1","ranks.denseT == 1",'tensorDtypes.bitsT == "uint32"','(tensorDtypes.denseT == "float32" or tensorDtypes.denseT == "float16")',"f16Ok(tensorDtypes.scalesT)","f16Ok(tensorDtypes.denseT)","args.outRows > 0","args.inFeatures > 0","args.denseElems >= args.dstElemOffset + args.outRows * args.inFeatures","numel(shapes.denseT) >= args.dstElemOffset + args.outRows * args.inFeatures"],bindings:{dense:{arg:"denseT",elementType:"$denseScalar"},params:{struct:[{name:"block_offset",type:"u32",value:"args.blockOffset"},{name:"out_rows",type:"u32",value:"args.outRows"},{name:"in_features",type:"u32",value:"args.inFeatures"},{name:"dst_elem_offset",type:"u32",value:"args.dstElemOffset"}]}},variants:[{id:"q4",priority:0,when:['present.bitsT and fmt != "q1_0"','(tensorDtypes.scalesT == "uint32" if (q4knc or lutc) else (tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16"))',"(quantBits == 4 or quantBits == 8)","args.inFeatures % 32 == 0 and ((q4knc == 0 and lutc == 0) or args.blockOffset % 8 == 0)","numel(shapes.scalesT) >= (ceilDiv(dqBlocks, 8) * 5 if q4knc else (ceilDiv(dqBlocks, 8) * lutcWords if lutc else (dqBlocks / 4 if q2k else (dqBlocks / 8 if q6k else (dqBlocks / 4 if q5k else (dqBlocks / 8 if q3k else dqBlocks * (2 if (hasMin or iq or lut16) else 1)))))))","numel(shapes.bitsT) >= dqBlocks * (4 if q4knc else (2.5 if q2k else (6.5 if q6k else (5.5 if q5k else (3.5 if q3k else (5 if q5 else (2 if lut2 else quantBits)))))))"],derive:{lutId:"args.lut if args.lut else 0"},passes:[{id:"main",name:"LlamaPrefillDequant",shader:"prefill-dequant.wgsl.jinja",bindings:[{arg:"bitsT",name:"q4_bits"},{arg:"scalesT",name:"q4_scales",elementType:"$scaleScalar"},"dense","params"],dispatch:{threads:"args.outRows * args.inFeatures / 4",workgroupSize:"workgroupSize"}}]},{id:"q1",priority:0,when:['present.bitsT and fmt == "q1_0"','(tensorDtypes.scalesT == "float32" or tensorDtypes.scalesT == "float16")',"args.outRows % 4 == 0","args.inFeatures % 128 == 0","numel(shapes.bitsT) >= (args.blockOffset + args.outRows * (args.inFeatures / 128)) * 4","numel(shapes.scalesT) >= args.blockOffset + args.outRows * (args.inFeatures / 128)"],passes:[{id:"main",name:"LlamaPrefillDequant.Q1",shader:"prefill-dequant-q1.wgsl.jinja",bindings:[{arg:"bitsT",name:"q1_bits",elementType:"vec4<u32>"},{arg:"scalesT",name:"q1_scales",elementType:"$scaleScalar"},"dense","params"],dispatch:{threads:"args.outRows * args.inFeatures / 4",workgroupSize:"workgroupSize"}}]}]},assets:[["prefill-dequant-q1.wgsl.jinja","{{ env.wgsl.resourceDeclarations }}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let group=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let total4=(params.out_rows*params.in_features)/4u;if(group>=total4){return;}let elem=group*4u;let r=elem/params.in_features;let c=elem%params.in_features;let bpr=params.in_features/128u;let tile=r/4u;let rit=r%4u;let col_block=c/128u;let word_id=(c%128u)/32u;let bit0=c%32u;let group_idx=tile*bpr+col_block;let bit_vec=q1_bits[params.block_offset+group_idx*4u+word_id];let sign_word=select(select(bit_vec.x,bit_vec.y,rit==1u),select(bit_vec.z,bit_vec.w,rit==3u),rit>=2u);let bits4=(vec4<u32>(sign_word)>>vec4<u32>(bit0,bit0+1u,bit0+2u,bit0+3u))&vec4<u32>(1u);let signs=select(vec4<f32>(-1.0),vec4<f32>(1.0),bits4==vec4<u32>(1u));let scale=f32(q1_scales[params.block_offset+group_idx*4u+rit]);let v=signs*scale;let o=params.dst_elem_offset+elem;dense[o]={{ denseScalar }}(v.x);dense[o+1u]={{ denseScalar }}(v.y);dense[o+2u]={{ denseScalar }}(v.z);dense[o+3u]={{ denseScalar }}(v.w);}"],["prefill-dequant.wgsl.jinja",`{% include "ops/_shared/lut2-block-scale.wgsl.jinja" %}{% set emitQuantDotHelpers = false %}{% set packed4x8CompatUnpackI8 = quantBits == 8 or q3k or q6k %}{% set packed4x8CompatUnpackU8 = not (quantBits == 8 or (lut2 | default(false)) or lutId or q5 or q2k or q3k or q6k or q5k) %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% if lutId %}{% set emitQuantValueHelpers = not (lut2 | default(false)) %}{% set emitLut2DotHelper = false %}{% include "ops/_shared/q4nl-dequant-helpers.wgsl.jinja" %}{% endif %}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let group=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let total4=(params.out_rows*params.in_features)/4u;if(group>=total4){return;}let elem=group*4u;let r=elem/params.in_features;let c=elem%params.in_features;let bpr=params.in_features/32u;let blk=params.block_offset+r*bpr+c/32u;let eib=c%32u;
{% if quantBits == 8 %}let word=q4_bits[blk*8u+eib/4u];
{% if iq %}let scale=f32(q4_scales[blk*2u+select(0u,1u,eib>=16u)]);
{% else %}let scale=f32(q4_scales[blk]);
{% endif %}let v=vec4<f32>(unpack4xI8(word))*scale;
{% elif lut2 is defined and lut2 %}let word=q4_bits[blk*2u+(eib>>4u)];
{% if lutId in [8, 9] %}let scale={{ lut2_native_scale("blk", "q4_scales", lutId) }};let db2=0u;
{% elif lutc16 %}let mb2=(blk>>3u)*5u;let ci2=(blk&7u)*2u+select(0u,1u,eib>=16u);let b2=(q4_scales[mb2+(ci2>>2u)]>>((ci2&3u)*8u))&255u;let scale=bitcast<f32>(q4_scales[mb2+4u])*lutScale(b2&15u);let db2=((b2>>(4u+((eib>>3u)&1u)))&1u)*8u;
{% else %}let mb2=(blk>>3u)*3u;let b2=(q4_scales[mb2+((blk&7u)>>2u)]>>((blk&3u)*8u))&255u;let scale=bitcast<f32>(q4_scales[mb2+2u])*lutScale(b2&15u);let db2=((b2>>4u)&1u)*8u;
{% endif %}let v=q2q(word,(eib&15u)*2u,db2)*scale;
{% elif lutId %}let word=q4_bits[blk*4u+(eib%16u)/4u];
{% if lutc is defined and lutc and lutc16 %}let mbL=(blk>>3u)*5u;let ciL=(blk&7u)*2u+select(0u,1u,eib>=16u);let scale=bitcast<f32>(q4_scales[mbL+4u])*lutScale((q4_scales[mbL+(ciL>>2u)]>>((ciL&3u)*8u))&255u);
{% elif lutc is defined and lutc %}let mbL=(blk>>3u)*3u;let scale=bitcast<f32>(q4_scales[mbL+2u])*lutScale((q4_scales[mbL+((blk&7u)>>2u)]>>((blk&3u)*8u))&255u);
{% elif lut16 %}let scale=f32(q4_scales[blk*2u+select(0u,1u,eib>=16u)]);
{% else %}let scale=f32(q4_scales[blk]);
{% endif %}var nib:vec4<f32>;if(eib<16u){nib=q4_lo(word);}else{nib=q4_hi(word);}let v=nib*scale;
{% elif q5 %}let g5=(blk>>2u)*20u;let l5=blk&3u;let word=q4_bits[g5+l5*4u+(eib%16u)/4u];let qh=q4_bits[g5+16u+l5];let scale=f32(q4_scales[blk]);var q5n:vec4<u32>;if(eib<16u){q5n=(vec4<u32>(word)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(15u);}else{q5n=(vec4<u32>(word)>>vec4<u32>(4u,12u,20u,28u))&vec4<u32>(15u);}q5n=q5n|(((vec4<u32>(qh)>>(vec4<u32>(0u,1u,2u,3u)+vec4<u32>(eib)))&vec4<u32>(1u))<<vec4<u32>(4u));let v=(vec4<f32>(q5n)-16.0)*scale;
{% elif q2k %}let sb=blk>>3u;let j2=blk&7u;let g2=sb*20u;let cw=q4_bits[g2+2u*j2+select(0u,1u,eib>=16u)];let scb=(q4_bits[g2+16u+(j2>>1u)]>>((j2&1u)*16u+select(0u,8u,eib>=16u)))&0xFFu;let d2=f32(q4_scales[sb*2u]);let dm2=f32(q4_scales[sb*2u+1u]);let w2=(eib%16u)/4u;let q2=(vec4<u32>(cw)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(2u*w2)))&vec4<u32>(3u);let v=vec4<f32>(q2)*(d2*f32(scb&15u))-vec4<f32>(dm2*f32(scb>>4u));
{% elif q3k %}let sb=blk>>3u;let j3=blk&7u;let g3=sb*28u;let plane3=select(0u,1u,eib>=16u);let cw3=q4_bits[g3+2u*j3+plane3];let hw3=q4_bits[g3+16u+j3];let s83=unpack4xI8(q4_bits[g3+24u+(j3>>1u)])[(j3&1u)*2u+plane3];let w3=(eib%16u)/4u;let q3c=(vec4<u32>(cw3)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(2u*w3)))&vec4<u32>(3u);let h3=(vec4<u32>(hw3)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(4u*plane3+w3)))&vec4<u32>(1u);let v=(vec4<f32>(q3c|(h3<<vec4<u32>(2u)))-4.0)*(f32(q4_scales[sb])*f32(s83));
{% elif q6k %}let sb=blk>>3u;let j6=blk&7u;let g6=sb*52u;let w6=(eib%16u)/4u;let word=q4_bits[g6+4u*j6+w6];let hw=q4_bits[g6+32u+2u*j6+select(0u,1u,eib>=16u)];let sc8=unpack4xI8(q4_bits[g6+48u+(j6>>1u)])[(j6&1u)*2u+select(0u,1u,eib>=16u)];let scale=f32(q4_scales[sb])*f32(sc8);var q6n:vec4<u32>;if(eib<16u){q6n=(vec4<u32>(word)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(15u);}else{q6n=(vec4<u32>(word)>>vec4<u32>(4u,12u,20u,28u))&vec4<u32>(15u);}let hi2=(vec4<u32>(hw)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(2u*w6)))&vec4<u32>(3u);let v=(vec4<f32>(q6n|(hi2<<vec4<u32>(4u)))-32.0)*scale;
{% elif q5k %}let sb=blk>>3u;let j5=blk&7u;let g5k=sb*44u;let w5=(eib%16u)/4u;let word=q4_bits[g5k+4u*j5+w5];let qh=q4_bits[g5k+32u+j5];let sc5=f32((q4_bits[g5k+40u+(j5>>2u)]>>((j5&3u)*8u))&0xFFu);let mn5=f32((q4_bits[g5k+42u+(j5>>2u)]>>((j5&3u)*8u))&0xFFu);let scEff=f32(q4_scales[sb*2u])*sc5;let mEff=f32(q4_scales[sb*2u+1u])*mn5;var q5n:vec4<u32>;if(eib<16u){q5n=(vec4<u32>(word)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(15u);}else{q5n=(vec4<u32>(word)>>vec4<u32>(4u,12u,20u,28u))&vec4<u32>(15u);}q5n=q5n|(((vec4<u32>(qh)>>(vec4<u32>(0u,1u,2u,3u)+vec4<u32>(eib)))&vec4<u32>(1u))<<vec4<u32>(4u));let v=vec4<f32>(q5n)*scEff-vec4<f32>(mEff);
{% elif q4knc %}let word=q4_bits[blk*4u+(eib%16u)/4u];let sbC=blk>>3u;let jC=blk&7u;let smBaseC=sbC*5u;let dmC=unpack2x16float(q4_scales[smBaseC]);let smPairC=q4_scales[smBaseC+1u+(jC>>1u)]>>((jC&1u)*12u);let dlC=dmC.x*f32(smPairC&63u);let mlC=dmC.y*f32((smPairC>>6u)&63u);var nibC:vec4<f32>;if(eib<16u){nibC=vec4<f32>(unpack4xU8(word&0x0F0F0F0Fu));}else{nibC=vec4<f32>(unpack4xU8((word>>4u)&0x0F0F0F0Fu));}let v=nibC*dlC-vec4<f32>(mlC);
{% else %}let word=q4_bits[blk*4u+(eib%16u)/4u];
{% if hasMin %}let scale=f32(q4_scales[blk*2u]);let bias=f32(q4_scales[blk*2u+1u]);
{% else %}let scale=f32(q4_scales[blk]);
{% endif %}var nib:vec4<f32>;if(eib<16u){nib=vec4<f32>(unpack4xU8(word&0x0F0F0F0Fu))-8.0;}else{nib=vec4<f32>(unpack4xU8((word>>4u)&0x0F0F0F0Fu))-8.0;}
{% if hasMin %}let v=nib*scale+bias;
{% else %}let v=nib*scale;
{% endif %}{% endif %}let o=params.dst_elem_offset+elem;dense[o]={{ denseScalar }}(v.x);dense[o+1u]={{ denseScalar }}(v.y);dense[o+2u]={{ denseScalar }}(v.z);dense[o+3u]={{ denseScalar }}(v.w);}`]]}],["com.xenova.LlamaPrefillMatmul",{manifest:{domain:"com.xenova",name:"LlamaPrefillMatmul",sinceVersion:1,inputs:{aT:{onnx:"A",dtype:"A",rank:2},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},bitsT:{onnx:"Bits",dtype:"uint32",rank:1,optional:!0},scalesT:{onnx:"Scales",dtype:"S",rank:1,optional:!0},realLenT:{onnx:"RealLen",dtype:"uint32",rank:1,optional:!0},yT:{onnx:"Y",dtype:"Y",rank:2},gridT:{onnx:"Grid",dtype:"uint32",rank:1,optional:!0},bT:{dtype:"float32",rank:1,optional:!0},rT:{dtype:"float32",rank:2,optional:!0}},outputs:{yT:{onnx:"Y",dtype:"Y",rank:2,shape:["args.M","args.outStride if args.outStride else args.outFeatures"]},partialsT:{onnx:"Partials",dtype:"float32",rank:1,optional:!0}},args:{M:{kind:"u32"},inFeatures:{kind:"u32",onnx:"in_features"},outFeatures:{kind:"u32",onnx:"out_features"},weightOffset:{kind:"u32",onnx:"weight_offset",required:!1},blockOffset:{kind:"u32",onnx:"block_offset",required:!1},outStride:{kind:"u32",onnx:"out_stride",required:!1},dstColStart:{kind:"u32",onnx:"dst_col_start",required:!1},format:{kind:"string",required:!1,oneOf:["q4_0","q4_k","q4_k_native_compact","q8_0","iq","lut32","lut16","lut32_compact","lut16_compact","q5_0","q2_k","q3_k","q6_k","q5_k","q1_0","lut2_32","lut2_16","bf16","f16"]},iqRaw:{kind:"u32",onnx:"iq_raw_pack_format",required:!1},lut:{kind:"u32",onnx:"dequant_lut_id",required:!1},exact:{kind:"u32",onnx:"exact_f32_tiles",required:!1},activationQuant:{kind:"string",required:!1,oneOf:["int8"]},kSplits:{kind:"u32",onnx:"k_splits",required:!1},denseTailStart:{kind:"u32",required:!1,onnx:"dense_tail_start"},activation:{kind:"string",required:!1,oneOf:["none","tanh","erf"]}},typeConstraints:{A:["float32","float16"],W:["float32","float16"],S:["float32","float16","uint32"],Y:["float32","float16"]},tunables:{PORTABLE_LUT4_MVEC:{default:4},SMALL_M_SPLIT_K:{default:1},SMALL_M_N128_SPLIT_K:{default:1},SMALL_M_SPLIT_K_VALUE:{default:4},SMALL_M_DEEP_SPLIT_K:{default:16},SMALL_M_N128_EXPAND_SPLIT_K:{default:4},SMALL_M_N128_NARROW_SPLIT_K:{default:16},DENSE_GEMV_MROWS:{default:1},DENSE_GEMV_CLUSTERS:{default:4},DENSE_GEMV_REG_ROWS:{default:2},LUT2_PREFILL_STATIC_SCHEDULE:{default:1},LUT2_PREFILL_K_SPLITS:{default:8},LUT2_PREFILL_POLICY_ROWS:{default:0},LUT2_PREFILL_REGISTER_ROWS:{default:1},LUT2_PREFILL_WIDE_COLUMNS:{default:1},LUT2_PREFILL_BATCHED_RESULTS:{default:1}},derive:{fmt:'args.format if args.format else "q4_0"',quantBits:'8 if (fmt == "q8_0" or fmt == "q8_0_pair" or fmt == "iq") else 4',q8Pair:'1 if fmt == "q8_0_pair" else 0',hasMin:'1 if fmt == "q4_k" else 0',iq:'1 if fmt == "iq" else 0',lut16:'1 if fmt == "lut16" else 0',lut2:'1 if (fmt == "lut2_32" or fmt == "lut2_16") else 0',lutc:'1 if (fmt == "lut32_compact" or fmt == "lut16_compact" or lut2 == 1) else 0',lutc16:'1 if (fmt == "lut16_compact" or fmt == "lut2_16") else 0',lutcWords:"1 if (lut2 == 1 and args.lut == 9) else (5 if lutc16 == 1 else 3)",q5:'1 if fmt == "q5_0" else 0',q2k:'1 if fmt == "q2_k" else 0',q3k:'1 if fmt == "q3_k" else 0',q6k:'1 if fmt == "q6_k" else 0',q5k:'1 if fmt == "q5_k" else 0',wave32Adapter:'has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 32 and device.adapterInfo.subgroupMaxSize == 32',subgroupsWave32:'device.features.has("subgroups") and wave32Adapter',canPinSubgroupSize32:'device.features.has("subgroups") and device.features.has("subgroup-size-control") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= 32 and device.adapterInfo.subgroupMaxSize >= 32',pinSubgroupSize32:"canPinSubgroupSize32 and not wave32Adapter",wave32Effective:"wave32Adapter or pinSubgroupSize32",wave32SubgroupsUsable:"subgroupsWave32 or pinSubgroupSize32",variableSubgroup16To32:'device.features.has("subgroups") and has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize == 16 and device.adapterInfo.subgroupMaxSize == 32',subgroupIdBuiltins:'device.features.has("subgroups") and device.wgslLanguageFeatures.has("subgroup_id")',decodeGemvSmallM:"args.M >= 1 and args.M <= 4",wideRowTile:"args.M >= 64 and ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 128",fusedWideRows:"args.M >= 128 and ceil(args.M / 128) * ceil(args.outFeatures / 32) >= 128",fusedNStarved:"ceil(args.outFeatures / 32) <= 96",fusedNarrowTile:"args.M <= 8 or (args.M <= 16 and fusedNStarved)",fusedMPerThread:"4 if fusedWideRows else 1",fusedBm:"128 if fusedWideRows else (8 if fusedNarrowTile else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))",fusedBn:"32 if fusedWideRows else (16 if fusedNarrowTile else 32)",fusedNTile:"4 if fusedWideRows else (16 if fusedNarrowTile else (4 if (args.M > 16 and args.outFeatures >= 8192) else 8))",fusedAccs:"8 if fusedWideRows else (1 if fusedNarrowTile else (8 if (args.M > 16 and args.outFeatures >= 8192) else 4))",q1Present:'present.bitsT and present.scalesT and fmt == "q1_0"',q4Present:'present.bitsT and present.scalesT and fmt != "q1_0" and fmt != "q4_k_native_compact" and fmt != "q5_k" and lutc == 0 and (args.iqRaw if args.iqRaw else 0) == 0',iq3xxsRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 3",iq3sRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 4",iq3xxsBlocks:"(args.blockOffset if iq3xxsRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq3sSupers:"ceilDiv(iq3xxsBlocks, 8)",iq3xxsContract:'iq3xxsRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq3xxsBlocks * 3 and numel(shapes.scalesT) >= iq3xxsBlocks / 8',iq3sContract:'iq3sRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq3sSupers * 27 and numel(shapes.scalesT) >= iq3sSupers',q4knCompactPresent:'present.bitsT and present.scalesT and fmt == "q4_k_native_compact" and not args.denseTailStart',q5kPresent:'present.bitsT and present.scalesT and fmt == "q5_k"',q1Blocks:"(args.blockOffset if q1Present else 0) + args.outFeatures * (args.inFeatures / 128)",q1BaseContract:'q1Present and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and (tensorDtypes.yT in ["float32", "float16"]) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 128 == 0 and args.outFeatures > 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= q1Blocks * 4 and numel(shapes.scalesT) >= q1Blocks',q4Blocks:"(args.blockOffset if q4Present else 0) + args.outFeatures * (args.inFeatures / 32)",splitCount:"args.kSplits if args.kSplits else 1",q4BaseContract:'q4Present and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and (quantBits == 4 or quantBits == 8) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 32 == 0 and args.outFeatures > 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.scalesT) >= (q4Blocks / 4 if q2k else (q4Blocks / 8 if (q6k or q3k) else q4Blocks * (2 if (hasMin or iq or lut16) else 1))) and numel(shapes.bitsT) >= q4Blocks * (2.5 if q2k else (3.5 if q3k else (6.5 if q6k else (5 if q5 else quantBits))))',decodeGemvContract:'q4BaseContract and decodeGemvSmallM and splitCount == 1 and tensorDtypes.aT == "float32" and tensorDtypes.yT == "float32" and not q5 and numel(shapes.bitsT) % 4 == 0 and (not (q2k or q3k or q6k) or (args.blockOffset % 8 == 0 and args.inFeatures % 256 == 0))',q4knCompactBlocks:"(args.blockOffset if q4knCompactPresent else 0) + args.outFeatures * (args.inFeatures / 32)",q4knCompactBaseContract:'q4knCompactPresent and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) % 4 == 0 and numel(shapes.bitsT) >= q4knCompactBlocks * 4 and numel(shapes.scalesT) >= (q4knCompactBlocks / 8) * 5',q4knCompactDecodeGemvContract:'q4knCompactBaseContract and tensorDtypes.aT == "float32" and tensorDtypes.yT == "float32" and decodeGemvSmallM and splitCount == 1 and not present.realLenT',lutcPresent:"present.bitsT and present.scalesT and lutc == 1 and lut2 == 0 and (args.iqRaw if args.iqRaw else 0) == 0",lutcBlocks:"(args.blockOffset if lutcPresent else 0) + args.outFeatures * (args.inFeatures / 32)",lutcBaseContract:'lutcPresent and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and (tensorDtypes.aT in ["float32", "float16"]) and f16Ok(tensorDtypes.aT) and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and (tensorDtypes.yT in ["float32", "float16"]) and f16Ok(tensorDtypes.yT) and args.lut is defined and args.lut != 0 and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) % 4 == 0 and numel(shapes.bitsT) >= lutcBlocks * 4 and numel(shapes.scalesT) >= (lutcBlocks / 8) * lutcWords',lutcDecodeGemvContract:"lutcBaseContract and decodeGemvSmallM and splitCount == 1",lut2Present:"present.bitsT and present.scalesT and lut2 == 1 and (args.iqRaw if args.iqRaw else 0) == 0",lut2Blocks:"(args.blockOffset if lut2Present else 0) + args.outFeatures * (args.inFeatures / 32)",lut2Int8:'args.activationQuant is defined and args.activationQuant == "int8"',lut2ShapeContract:'lut2Present and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and (tensorDtypes.aT in ["float32", "float16"]) and f16Ok(tensorDtypes.aT) and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "uint32" and (tensorDtypes.yT in ["float32", "float16"]) and f16Ok(tensorDtypes.yT) and args.lut is defined and args.lut != 0 and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) % 2 == 0 and numel(shapes.bitsT) >= lut2Blocks * 2 and numel(shapes.scalesT) >= (lut2Blocks / 8) * lutcWords',lut2BaseContract:"lut2ShapeContract and not lut2Int8",lut2DoubleKTilePreferred:'lut2Present and args.lut in [8, 9] and variableSubgroup16To32 and ((tensorDtypes.aT == "float16" and tensorDtypes.yT == "float16" and args.M >= 64) or (tensorDtypes.aT == "float32" and fusedWideRows)) and device.limits.maxComputeWorkgroupStorageSize >= (fusedBm + fusedBn) * 2 * 8 * 8',lut2DecodeGemvContract:"lut2BaseContract and decodeGemvSmallM and splitCount == 1",q5kBlocks:"(args.blockOffset if q5kPresent else 0) + args.outFeatures * (args.inFeatures / 32)",q5kBaseContract:'q5kPresent and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and tensorDtypes.scalesT == "float32" and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.scalesT) >= q5kBlocks / 4 and numel(shapes.bitsT) >= q5kBlocks * 5.5',q5kDecodeGemvContract:'q5kBaseContract and (tensorDtypes.aT in ["float32", "float16"]) and f16Ok(tensorDtypes.aT) and (tensorDtypes.yT in ["float32", "float16"]) and f16Ok(tensorDtypes.yT) and decodeGemvSmallM and splitCount == 1 and numel(shapes.bitsT) % 4 == 0',lut2PolicyRows:"max(args.M, tunables.LUT2_PREFILL_POLICY_ROWS) if (lut2Present and args.lut in [8, 9]) else args.M",sgmatTallSubtile:"1 if (args.M > 16 and ceil(args.outFeatures / 64) * ceil(lut2PolicyRows / 32) >= 128) else 0",iq2sRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 5",iq2xsRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 6",iq2sBlocks:"(args.blockOffset if iq2sRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq2xsBlocks:"(args.blockOffset if iq2xsRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq2sSupers:"ceilDiv(iq2sBlocks, 8)",iq2xsSupers:"ceilDiv(iq2xsBlocks, 8)",iq2sContract:'iq2sRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq2sSupers * 20 and numel(shapes.scalesT) >= iq2sSupers',iq2xsContract:'iq2xsRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq2xsSupers * 18 and numel(shapes.scalesT) >= iq2xsSupers',iq1Raw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 1",iq1Blocks:"(args.blockOffset if iq1Raw else 0) + args.outFeatures * (args.inFeatures / 32)",iq1Contract:'iq1Raw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq1Blocks * 2 and numel(shapes.scalesT) >= iq1Blocks / 8',iq2xxsRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 2",iq2xxsBlocks:"(args.blockOffset if iq2xxsRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq2xxsContract:'iq2xxsRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= iq2xxsBlocks * 2 and numel(shapes.scalesT) >= iq2xxsBlocks / 8',denseOutStride:"args.outStride if args.outStride else args.outFeatures",denseGemvClusters:"tunables.DENSE_GEMV_CLUSTERS",denseGemvRegRows:"tunables.DENSE_GEMV_REG_ROWS",denseGemvCols:"denseGemvClusters * denseGemvRegRows",denseGemvMRowsContract:'present.weightsT and ranks.aT == 2 and ranks.weightsT == 1 and ranks.yT == 2 and (tensorDtypes.aT in ["float16", "float32"]) and (tensorDtypes.weightsT in ["float16", "float32"]) and f16Ok(tensorDtypes.weightsT) and f16Ok(tensorDtypes.aT) and f16Ok(tensorDtypes.yT) and (tensorDtypes.yT in ["float16", "float32"]) and args.M >= 1 and args.M <= 16 and args.inFeatures > 0 and args.inFeatures % 128 == 0 and args.outFeatures > 0 and denseGemvCols > 0 and args.outFeatures % denseGemvCols == 0 and (args.weightOffset if args.weightOffset else 0) % 4 == 0 and (args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * denseOutStride and numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures',denseWholeBlockDest:"(args.dstColStart if args.dstColStart else 0) == 0 and denseOutStride == args.outFeatures",denseSgmatSmallMContract:'present.weightsT and ranks.aT == 2 and ranks.weightsT == 1 and ranks.yT == 2 and tensorDtypes.aT == "float16" and tensorDtypes.weightsT == "float16" and tensorDtypes.yT == "float16" and args.M >= 1 and args.M <= 16 and args.inFeatures > 0 and args.inFeatures % 32 == 0 and args.outFeatures > 0 and args.outFeatures % 64 == 0 and (args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * denseOutStride and numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures',iq1sRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 7",iq1sBlocks:"(args.blockOffset if iq1sRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq1sContract:'iq1sRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= (iq1sBlocks / 8) * 12 and numel(shapes.scalesT) >= iq1sBlocks / 8',iq1mdRaw:"present.bitsT and present.scalesT and (args.iqRaw if args.iqRaw else 0) == 8",iq1mdBlocks:"(args.blockOffset if iq1mdRaw else 0) + args.outFeatures * (args.inFeatures / 32)",iq1mdContract:'iq1mdRaw and ranks.aT == 2 and ranks.bitsT == 1 and ranks.scalesT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and (tensorDtypes.scalesT in ["float32", "float16"]) and f16Ok(tensorDtypes.scalesT) and args.M > 0 and args.inFeatures > 0 and args.inFeatures % 256 == 0 and args.outFeatures > 0 and args.blockOffset % 8 == 0 and args.dstColStart + args.outFeatures <= args.outStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outStride and numel(shapes.bitsT) >= (iq1mdBlocks / 8) * 14 and numel(shapes.scalesT) >= iq1mdBlocks / 8',iqGridMin:"4096 if (iq1Raw or iq1sRaw or iq1mdRaw) else (2048 if iq2sRaw else (1024 if iq2xsRaw else (512 if (iq3sRaw or iq2xxsRaw) else 256)))",inFeatures:"args.inFeatures",outFeatures:"args.outFeatures",aVec4:'"vec4<f16>" if tensorDtypes.aT == "float16" else "vec4<f32>"',wScalar:'"f16"',clusters:1,regRows:1,mRows:"args.M",useSubgroups:"wave32SubgroupsUsable",sgRuntime:'device.features.has("subgroups") and not wave32SubgroupsUsable',nTile:"fusedNTile",nPerThread:1,aScalar:"dtypes.A",splitKReduceWorkgroupSize:256,lutId:0,scaleScalar:"dtypes.S",f16Tiles:"1",kBlocks:"1",bn:"fusedBn",bm:"fusedBm",accs:"fusedAccs",tallSubtile:"sgmatTallSubtile",blocksPerRow:"args.inFeatures / 32",q1BitsElement:'"u32"',lutM:"16 if (args.M >= 128 or (args.M >= 16 and args.inFeatures <= 5120 and args.outFeatures > 1024)) else (1 if args.outFeatures <= 1024 else (4 if args.M >= 4 else args.M))",lutN:"256",assumeBiasNegHalfScale:!0,scaleOnly:!0,lutScalar:'"f16"',mSize:"args.M",scalesPk2:0,denseVectorContract:'present.weightsT and ranks.aT == 2 and ranks.weightsT == 1 and ranks.yT == 2 and tensorDtypes.aT == "float32" and tensorDtypes.weightsT == "float32" and tensorDtypes.yT == "float32" and args.M > 16 and args.inFeatures > 0 and args.inFeatures % 4 == 0 and args.outFeatures > 0 and args.outFeatures < 1024 and (args.weightOffset if args.weightOffset else 0) % 4 == 0 and (args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * denseOutStride and numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures',denseGemvParallelContract:'present.weightsT and ranks.aT == 2 and ranks.weightsT == 1 and ranks.yT == 2 and (tensorDtypes.aT in ["float16", "float32"]) and (tensorDtypes.weightsT in ["float16", "float32"]) and f16Ok(tensorDtypes.weightsT) and f16Ok(tensorDtypes.aT) and f16Ok(tensorDtypes.yT) and (tensorDtypes.yT in ["float16", "float32"]) and args.M == 1 and args.inFeatures >= 1024 and args.inFeatures % 4 == 0 and args.outFeatures > 0 and args.outFeatures <= 128 and (args.weightOffset if args.weightOffset else 0) % 4 == 0 and (args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * denseOutStride and numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures',lut2PrefillSplits:"min(max(1, tunables.LUT2_PREFILL_K_SPLITS), args.inFeatures / 32)",lut2PrefillTileCount:"ceil(args.outFeatures / (128 if args.M <= 16 else 64)) * ceil(lut2PolicyRows / (16 if args.M <= 16 else 32))",halfWordContract:'present.bitsT and (args.format == "bf16" or args.format == "f16") and not present.scalesT and not present.weightsT and not present.realLenT and not present.gridT and not args.denseTailStart and not args.blockOffset and not args.weightOffset and denseWholeBlockDest and ranks.aT == 2 and ranks.bitsT == 1 and ranks.yT == 2 and tensorDtypes.bitsT == "uint32" and tensorDtypes.aT == "float32" and tensorDtypes.yT == "float32" and args.M > 0 and args.inFeatures > 0 and args.outFeatures > 0 and numel(shapes.aT) >= args.M * args.inFeatures and numel(shapes.yT) >= args.M * args.outFeatures and numel(shapes.bitsT) * 2 >= args.outFeatures * args.inFeatures',halfWordSgmatContract:"halfWordContract and wave32Effective and args.M >= 64 and args.inFeatures % 16 == 0 and args.outFeatures % 8 == 0 and ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension and floor(args.M / 64) <= device.limits.maxComputeWorkgroupsPerDimension and 128 <= device.limits.maxComputeInvocationsPerWorkgroup",q8ExactSgmatContract:'q4BaseContract and fmt == "q8_0" and args.exact == 1 and denseWholeBlockDest and not present.weightsT and not present.realLenT and not present.partialsT and not present.gridT and not args.denseTailStart and tensorDtypes.aT == "float32" and tensorDtypes.yT == "float32" and wave32Effective and args.M >= 64 and args.outFeatures % 8 == 0 and ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension and floor(args.M / 64) <= device.limits.maxComputeWorkgroupsPerDimension and 128 <= device.limits.maxComputeInvocationsPerWorkgroup and ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 28',hasEpilogue:'present.bT or present.rT or (args.activation is defined and args.activation != "none")',epilogueOk:'(not present.bT or (tensorDtypes.bT == "float32" and ranks.bT == 1 and dim(shapes.bT, 0) == args.outFeatures)) and (not present.rT or (tensorDtypes.rT == "float32" and ranks.rT == 2 and sameShape(shapes.rT, shapes.yT)))',epilogueSupported:'tensorDtypes.aT == "float32" and tensorDtypes.yT == "float32" and dim(shapes.yT, 0) == args.M and dim(shapes.yT, 1) == args.outFeatures and not args.dstColStart and (not args.outStride or args.outStride == args.outFeatures) and not present.realLenT and not present.partialsT and not args.denseTailStart and splitCount == 1 and (halfWordContract or (q4Present and args.format == "q8_0" and args.exact == 1))'},bindings:{y:{arg:"yT",elementType:"f16"},params:{struct:[{name:"weight_offset",type:"u32",value:"args.weightOffset"},{name:"m",type:"u32",value:"args.M"}]},a_a_t:{arg:"aT",name:"a",elementType:"$aVec4"},weights:{arg:"weightsT",elementType:"$wScalar"},y_y_t:{arg:"yT",name:"y",elementType:"$yScalar"},partials:{arg:"partialsT",elementType:"f32"},partials_f32:{arg:"partialsT",name:"partials",buffer:"read-only-storage",elementType:"f32"},params_m:{name:"params",struct:[{name:"m",type:"u32",value:"args.M"}]},a:{arg:"aT",elementType:"$aScalar"},q4_bits:{arg:"bitsT",elementType:"u32"},q4_scales:{arg:"scalesT",elementType:"$scaleScalar"},params_main:{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset"},{name:"m",type:"u32",value:"args.M"}]},a_main:{arg:"aT",name:"a",elementType:"vec4<f32>"},y_f32:{arg:"yT",name:"y",elementType:"f32"},q4_metadata:{arg:"scalesT",elementType:"u32"},q4_bits_main:{arg:"bitsT",name:"q4_bits",elementType:"vec4<u32>"},q1_bits:{arg:"bitsT",elementType:"$q1BitsElement"},q1_scales:{arg:"scalesT",elementType:"$scaleScalar"},q1_bits_u32:{arg:"bitsT",name:"q1_bits",elementType:"u32"},real_len:{arg:"realLenT",elementType:"u32"},partials_partials:{name:"partials",buffer:"storage",elementType:"f32"},partials_reduce:{name:"partials",buffer:"read-only-storage",elementType:"f32"},params__uniform:{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset"}]},iq_grid_data:{arg:"gridT",elementType:"vec2<u32>"},bf16_x:{arg:"aT",name:"x",elementType:"f32"},bf16_w:{arg:"bitsT",name:"w",elementType:"vec4<u32>"},bf16_y:{arg:"yT",name:"y",elementType:"f32"},bf16_params_m:{name:"params",struct:[{name:"M",type:"u32",value:"args.M"}]},q8_sgmat_scales:{arg:"scalesT",name:"scales",elementType:"$scaleScalar"},q8_sgmat_params:{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset if args.blockOffset else 0"}]},q8_sgmat_params_m:{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset if args.blockOffset else 0"},{name:"M",type:"u32",value:"args.M"}]},bf16_weights:{arg:"bitsT",name:"weights",elementType:"u32"},bf16_params:{name:"params",struct:[{name:"weight_offset",type:"u32",value:"0"},{name:"m",type:"u32",value:"args.M"}]},epi_y:{arg:"yT",name:"y",elementType:"f32"},epi_params:{name:"epiParams",struct:[{name:"M",type:"u32",value:"args.M"},{name:"N",type:"u32",value:"args.outFeatures"}]},epi_bias:{arg:"bT",name:"bias",elementType:"f32"},epi_residual:{arg:"rT",name:"residual",elementType:"f32"}},variants:[{id:"dense_gemv_parallel",priority:105,when:["not present.realLenT","denseGemvParallelContract","useSubgroups or sgRuntime or args.inFeatures >= 4096","256 <= device.limits.maxComputeInvocationsPerWorkgroup","args.outFeatures <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","4 * (256) + (8 if (useSubgroups or sgRuntime) else 0) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],derive:{dstColStart:"args.dstColStart if args.dstColStart else 0",wVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',lanes:256},passes:[{id:"main",name:"LlamaPrefillMatmulDenseGemvParallel",shader:"prefill-matmul-dense-gemv-parallel.wgsl.jinja",bindings:["a_a_t",{arg:"weightsT",name:"weights",elementType:"$wVec4"},"y_y_t",{name:"params",struct:[{name:"weight_offset",type:"u32",value:"args.weightOffset"}]}],dispatch:{x:"args.outFeatures"},subgroupCollectivesWidth:"portable"}]},{id:"dense_vector",priority:104,when:["not present.realLenT","denseVectorContract","ceil(args.M * args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not hasEpilogue"],derive:{outStride:"denseOutStride",dstColStart:"args.dstColStart if args.dstColStart else 0",wVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',workgroupSize:64},passes:[{id:"main",name:"LlamaPrefillMatmulDenseVector",shader:"prefill-matmul-dense-vector.wgsl.jinja",bindings:["a_a_t",{arg:"weightsT",name:"weights",elementType:"$wVec4"},"y_y_t",{name:"params",struct:[{name:"weight_offset",type:"u32",value:"args.weightOffset"}]}],dispatch:{x:"ceil(args.M * args.outFeatures / workgroupSize)"}}]},{id:"subgroup_matrix",priority:100,when:["present.weightsT","denseWholeBlockDest","ranks.aT == 2","ranks.weightsT == 1","ranks.yT == 2",'tensorDtypes.aT == "float16"','tensorDtypes.weightsT == "float16"','tensorDtypes.yT == "float16"',"args.M > 0","args.inFeatures > 0","args.outFeatures > 0","args.inFeatures % 32 == 0","args.outFeatures % 64 == 0","numel(shapes.aT) >= args.M * args.inFeatures","numel(shapes.yT) >= args.M * args.outFeatures","wave32Effective","not args.denseTailStart","(14348) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},passes:[{id:"main",name:"LlamaPrefillMatmulSubgroup",shader:"prefill-matmul-subgroup.wgsl.jinja",bindings:[{arg:"aT",name:"a",elementType:"f16"},{arg:"weightsT",name:"weights",elementType:"f16"},"y","params"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil(args.M / 32)"},subgroupCollectivesWidth:"portable"}]},{id:"subgroup_matrix_small_m",priority:102,when:["denseSgmatSmallMContract","wave32Effective","ceil(args.outFeatures / 128) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(2060) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{outStride:"denseOutStride",dstColStart:"args.dstColStart if args.dstColStart else 0",aVec4:'"vec4<f16>"',yScalar:'"f16"'},passes:[{id:"main",name:"LlamaPrefillMatmulDenseSmallMSg",shader:"prefill-matmul-dense-sg-small-m.wgsl.jinja",bindings:["a_a_t","weights","y_y_t","params"],dispatch:{x:"ceil(args.outFeatures / 128)",y:"ceil(args.M / 16)"},subgroupCollectivesWidth:"portable"}]},{id:"dense_gemv_mrows",priority:104,when:["tunables.DENSE_GEMV_MROWS == 1","not present.realLenT","denseGemvMRowsContract","args.outFeatures / denseGemvCols <= device.limits.maxComputeWorkgroupsPerDimension","32 * denseGemvClusters <= device.limits.maxComputeInvocationsPerWorkgroup","not args.denseTailStart","(not useSubgroups and not sgRuntime) or 128 * (denseGemvClusters) + 8 <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],derive:{outStride:"denseOutStride",dstColStart:"args.dstColStart if args.dstColStart else 0",wVec4:'"vec4<f16>" if tensorDtypes.weightsT == "float16" else "vec4<f32>"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',clusters:"denseGemvClusters",regRows:"denseGemvRegRows"},passes:[{id:"main",name:"LlamaPrefillMatmulDenseGemvMRows",shader:"prefill-matmul-dense-gemv-mrows.wgsl.jinja",bindings:["a_a_t",{arg:"weightsT",name:"weights",elementType:"$wVec4"},"y_y_t",{name:"params",struct:[{name:"weight_offset",type:"u32",value:"args.weightOffset"}]}],dispatch:{x:"args.outFeatures / denseGemvCols"},subgroupCollectivesWidth:"portable"}]},{id:"subgroup_matrix_small_m_splitk",priority:103,when:["splitCount > 1","present.partialsT","ranks.partialsT == 1",'tensorDtypes.partialsT == "float32"',"numel(shapes.partialsT) >= args.kSplits * args.M * args.outFeatures","args.inFeatures / 32 >= args.kSplits","args.kSplits <= device.limits.maxComputeWorkgroupsPerDimension","denseSgmatSmallMContract","wave32Effective","ceil(args.outFeatures / 128) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(2060) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{outStride:"denseOutStride",dstColStart:"args.dstColStart if args.dstColStart else 0",aVec4:'"vec4<f16>"',yScalar:'"f16"',sgSplitK:1,splitBlocks:"ceil(args.inFeatures / 32 / args.kSplits)",kSplits:"args.kSplits"},passes:[{id:"main",name:"LlamaPrefillMatmulDenseSmallMSg",shader:"prefill-matmul-dense-sg-small-m.wgsl.jinja",bindings:["a_a_t","weights","partials","params"],dispatch:{x:"ceil(args.outFeatures / 128)",y:"ceil(args.M / 16)",z:"args.kSplits"},subgroupCollectivesWidth:"portable"},{id:"reduce",name:"LlamaPrefillMatmulQ4Reduce",shader:"prefill-matmul-q4-reduce.wgsl.jinja",bindings:["partials_f32","y_y_t","params_m"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"splitKReduceWorkgroupSize"}}]},{id:"tiled",priority:50,when:["present.weightsT","denseWholeBlockDest","ranks.aT == 2","ranks.weightsT == 1","ranks.yT == 2",'tensorDtypes.aT == "float16"','tensorDtypes.weightsT == "float16"','tensorDtypes.yT == "float16"',"args.M > 1","args.inFeatures > 0","args.outFeatures > 0","args.inFeatures % 64 == 0","args.weightOffset % 4 == 0","numel(shapes.aT) >= args.M * args.inFeatures","numel(shapes.yT) >= args.M * args.outFeatures","not args.denseTailStart","not hasEpilogue"],requires:{features:["shader-f16"]},derive:{hasBias:!1,tileDtype:'"f16"',hasWeightOffset:!0,outputDtype:'"float16"',kV4:"args.inFeatures / 4",mTile:"64 if wideRowTile else 32",nTile:"8 if wideRowTile else 4",nPerThread:"8 if wideRowTile else 16",mPerThread:"4 if wideRowTile else 1",outTile:"64",kTile:"64",kTileV4:"16",workgroupSize:"128",aTileElements:"1024 if wideRowTile else 512",bTileElements:"1024"},passes:[{id:"main",name:"LlamaPrefillMatmulTiled",shader:"ops/_shared/dense-tiled-matmul.wgsl.jinja",bindings:[{arg:"aT",name:"a",elementType:"vec4<f16>"},{arg:"weightsT",name:"w",elementType:"vec4<f16>"},"y",{name:"params",struct:[{name:"M",type:"u32"},{name:"weightOffsetV4",type:"u32",value:"args.weightOffset / 4"}]}],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / (64 if wideRowTile else 32)), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"tiled_reg",priority:25,when:["present.weightsT","denseWholeBlockDest","ranks.aT == 2","ranks.weightsT == 1","ranks.yT == 2",'tensorDtypes.aT == "float32"','tensorDtypes.weightsT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 128","args.inFeatures > 0","args.outFeatures >= 1024","numel(shapes.aT) >= args.M * args.inFeatures","numel(shapes.yT) >= args.M * args.outFeatures","ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 64) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not hasEpilogue"],derive:{wScalar:"dtypes.W",yScalar:"dtypes.Y"},passes:[{id:"main",name:"LlamaPrefillMatmulTiledReg",shader:"prefill-matmul-tiled-reg.wgsl.jinja",bindings:["a","weights","y_y_t","params"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{cases:[{fmt:"bf16",Name:"Bf16",bf16:!0,f16:null},{fmt:"f16",Name:"F16",bf16:null,f16:!0}],variants:[{id:"{fmt}_subgroup_matrix",priority:100,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 64 == 0","device.limits.maxComputeWorkgroupStorageSize >= 16396","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0"},passes:[{id:"main",name:"LlamaPrefillMatmul{Name}Sgmat",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"}]},{id:"{fmt}_subgroup_matrix_m_tail",priority:100,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 64 != 0","device.limits.maxComputeWorkgroupStorageSize >= 20492","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0"},passes:[{id:"bulk",name:"LlamaPrefillMatmul{Name}SgmatBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 64) * 64 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmul{Name}SgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 64u) * 2u"',directStore:!1},profile:{flops:"2 * (args.M % 64) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 64) / 32)"},subgroupCollectivesWidth:"portable"}]},{id:"{fmt}_subgroup_matrix_wide",priority:101,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 128 == 0","floor(args.M / 128) * ceil(args.outFeatures / 64) >= 256","device.limits.maxComputeWorkgroupStorageSize >= 16396","device.limits.maxComputeInvocationsPerWorkgroup >= 256","device.limits.maxComputeWorkgroupSizeX >= 256","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",tileM:128,tileN:64,tileK:32,matrixBlocks32:!0,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:8,tileGroupCols:"ceil(args.outFeatures / 64)",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0"},passes:[{id:"main",name:"LlamaPrefillMatmul{Name}SgmatWide",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 128)"},subgroupCollectivesWidth:"portable"}]},{id:"{fmt}_subgroup_matrix_wide_m_tail",priority:101,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 128 != 0","floor(args.M / 128) * ceil(args.outFeatures / 64) >= 256","device.limits.maxComputeWorkgroupStorageSize >= 20492","device.limits.maxComputeInvocationsPerWorkgroup >= 256","device.limits.maxComputeWorkgroupSizeX >= 256","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0"},passes:[{id:"bulk",name:"LlamaPrefillMatmul{Name}SgmatWideBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:128,tileN:64,tileK:32,matrixBlocks32:!0,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:8,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 128) * 128 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 128)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmul{Name}SgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 128u) * 4u"',directStore:!1},profile:{flops:"2 * (args.M % 128) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 128) / 32)"},subgroupCollectivesWidth:"portable"}]},{id:"{fmt}_tiled_reg",priority:10,when:["halfWordContract",'args.format == "{fmt}"',"ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension","16 <= device.limits.maxComputeWorkgroupSizeX","16 <= device.limits.maxComputeWorkgroupSizeY","256 <= device.limits.maxComputeInvocationsPerWorkgroup","not hasEpilogue"],derive:{yScalar:'"f32"',bf16Weights:"{bf16}",f16Weights:"{f16}"},passes:[{id:"main",name:"LlamaPrefillMatmul{Name}TiledReg",shader:"prefill-matmul-tiled-reg.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["a","bf16_weights","y_y_t","bf16_params"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{id:"q8_subgroup_matrix_exact",priority:100,when:["q8ExactSgmatContract","args.M % 64 == 0","device.limits.maxComputeWorkgroupStorageSize >= 16396","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',q8BStaging:!0,tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:!1},passes:[{id:"main",name:"LlamaPrefillMatmulQ8ExactSgmat",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"}]},{id:"q8_subgroup_matrix_exact_m_tail",priority:100,when:["q8ExactSgmatContract","args.M % 64 != 0","device.limits.maxComputeWorkgroupStorageSize >= 20492","not hasEpilogue"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',q8BStaging:!0,packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:!1},passes:[{id:"bulk",name:"LlamaPrefillMatmulQ8ExactSgmatBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 64) * 64 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmulQ8ExactSgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 64u) * 2u"',directStore:!1},profile:{flops:"2 * (args.M % 64) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 64) / 32)"},subgroupCollectivesWidth:"portable"}]},{id:"scalar",priority:0,when:["present.weightsT","denseWholeBlockDest","ranks.aT == 2","ranks.weightsT == 1","ranks.yT == 2",'(tensorDtypes.weightsT in ["float32", "float16"])','(tensorDtypes.yT in ["float32", "float16"])',"f16Ok(tensorDtypes.aT) and f16Ok(tensorDtypes.weightsT) and f16Ok(tensorDtypes.yT)","args.M > 0","args.inFeatures > 0","args.outFeatures > 0","numel(shapes.aT) >= args.M * args.inFeatures","numel(shapes.yT) >= args.M * args.outFeatures","not args.denseTailStart","not hasEpilogue"],derive:{wScalar:"dtypes.W",yScalar:"dtypes.Y",workgroupSize:64},passes:[{id:"main",name:"LlamaPrefillMatmul",shader:"prefill-matmul.wgsl.jinja",bindings:["a","weights","y_y_t",{name:"params",struct:[{name:"weight_offset",type:"u32",value:"args.weightOffset"},{name:"m",type:"u32",value:"args.M"},{name:"in_features",type:"u32",value:"args.inFeatures"},{name:"out_features",type:"u32",value:"args.outFeatures"}]}],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"workgroupSize"}}]},{cases:[{id:"tiled_f16",main_name:"LlamaPrefillMatmulQ4",rowsWhen:["not present.realLenT"],dynamicRows:null,main_bindings:["a_a_t","q4_bits","q4_scales","y_y_t","params_main"]},{id:"tiled_f16_dynamic_rows",main_name:"LlamaPrefillMatmulQ4Masked",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,main_bindings:["a_a_t","q4_bits","q4_scales","real_len","y_y_t","params_main"]}],variants:[{id:"{id}",priority:50,when:["not args.exact","{rowsWhen}","q4BaseContract",'(tensorDtypes.aT in ["float32", "float16"])','(tensorDtypes.yT in ["float32", "float16"])',"ceil(args.outFeatures / fusedBn) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / fusedBm) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not hasEpilogue"],requires:{features:["shader-f16"]},derive:{lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",mPerThread:"fusedMPerThread",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',dynamicRows:"{dynamicRows}"},passes:[{id:"main",name:"{main_name}",shader:"prefill-matmul-q4.wgsl.jinja",bindings:"{main_bindings}",dispatch:{x:"min(ceil(args.outFeatures / fusedBn), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / fusedBm), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{id:"tiled_exact",priority:0,when:["q4BaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / (16 if args.M <= 8 else 32)) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","not hasEpilogue"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1 if args.M <= 8 else 2",bn:"16 if args.M <= 8 else 32",bm:"8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16)",nTile:"16 if args.M <= 8 else (4 if (args.M > 16 and args.outFeatures >= 8192) else 8)",accs:"1 if args.M <= 8 else (8 if (args.M > 16 and args.outFeatures >= 8192) else 4)"},passes:[{id:"main",name:"LlamaPrefillMatmulQ4",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / (16 if args.M <= 8 else 32)), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}}]},{id:"q8_tiled_exact_large_m",priority:1,when:["q4BaseContract",'fmt == "q8_0"','tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"args.M >= 64","ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 24","16384 <= device.limits.maxComputeWorkgroupStorageSize","not args.denseTailStart","not present.realLenT","not hasEpilogue"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1",bn:"64",bm:"64",nTile:"8",accs:"8",mPerThread:"4"},passes:[{id:"main",name:"LlamaPrefillMatmulQ8LargeM",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}}]},{cases:[{format:"f16",priority:60,preferenceGuards:["fusedNStarved","not args.exact"],dtypeGuards:['(tensorDtypes.aT in ["float32", "float16"])','(tensorDtypes.yT in ["float32", "float16"])'],requires:{features:["shader-f16"]},f16Tiles:null,yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',activationBinding:"a_a_t"},{format:"exact",priority:10,preferenceGuards:[],dtypeGuards:['tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"'],requires:null,f16Tiles:"0",yScalar:'"f32"',activationBinding:"a_main"}],variants:[{id:"tiled_{format}_splitk",priority:"{priority}",when:["splitCount > 1","present.partialsT","ranks.partialsT == 1",'tensorDtypes.partialsT == "float32"',"numel(shapes.partialsT) >= args.kSplits * args.M * args.outFeatures","args.inFeatures / 32 >= args.kSplits","args.kSplits <= device.limits.maxComputeWorkgroupsPerDimension","{preferenceGuards}","q4BaseContract","{dtypeGuards}","ceil(args.outFeatures / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","not hasEpilogue"],requires:"{requires}",derive:{f16Tiles:"{f16Tiles}",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"2",bn:"32",bm:"32",nTile:"4",accs:"8",kSplits:"args.kSplits",splitBlocks:"ceil(args.inFeatures / 32 / args.kSplits)",yScalar:"{yScalar}"},passes:[{id:"main",name:"LlamaPrefillMatmulQ4",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["{activationBinding}","q4_bits","q4_scales","params_main","partials"],dispatch:{x:"min(ceil(args.outFeatures / 32), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 32), device.limits.maxComputeWorkgroupsPerDimension)",z:"args.kSplits"}},{id:"reduce",name:"LlamaPrefillMatmulQ4Reduce",shader:"prefill-matmul-q4-reduce.wgsl.jinja",bindings:["partials_f32","y_y_t","params_m"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"splitKReduceWorkgroupSize"}}]}]},{axes:{kindAxis:[{$key:"q4kn",kind:"q4kn_compact",contract:"q4knCompactBaseContract",name:"Q4KNativeCompact",meta:"q4_metadata",flags:{q5k:!1,q4knCompact:!0}},{$key:"q5k",kind:"q5k",contract:"q5kBaseContract",name:"Q5K",meta:"q4_scales",flags:{q5k:!0,scaleScalar:'"f32"'}},{$key:"lutc",kind:"lut_compact",contract:"lutcBaseContract",name:"LutCompact",meta:"q4_metadata",flags:{lutId:"args.lut",q5k:!1,lutc:!0}},{$key:"lut2",kind:"lut2",contract:"lut2BaseContract",name:"Lut2",meta:"q4_metadata",flags:{lutId:"args.lut",q5k:!1,lutc:!0,lut2:!0}}],rows:[{$key:"static",rowsId:"",masked:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{$key:"dynamic",rowsId:"_dynamic_rows",masked:"Masked",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,realLen:["real_len"]}]},variants:[{id:"{kind}_tiled_f16{rowsId}",priority:50,when:["not args.exact","{rowsWhen}","{contract}",'(tensorDtypes.aT in ["float32", "float16"])','(tensorDtypes.yT in ["float32", "float16"])',"ceil(args.outFeatures / fusedBn) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / fusedBm) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not hasEpilogue"],requires:{features:["shader-f16"]},derive:{f16Tiles:1,outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"2 if lut2DoubleKTilePreferred else 1",mPerThread:"fusedMPerThread",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1,dynamicRows:"{dynamicRows}"},passes:[{id:"main",name:"LlamaPrefillMatmul{name}{masked}",shader:"prefill-matmul-q4.wgsl.jinja",derive:"{flags}",bindings:["a_a_t","q4_bits","{meta}","{realLen}","y_y_t","params_main"],dispatch:{x:"min(ceil(args.outFeatures / fusedBn), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / fusedBm), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{cases:[{rowsId:"",masked:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{rowsId:"_dynamic_rows",masked:"Masked",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,realLen:["real_len"]}],variants:[{id:"lut2_int8{rowsId}",priority:50,when:["{rowsWhen}","lut2ShapeContract","lut2Int8",'args.format == "lut2_32"',"args.lut in [8, 9]","kBlocks * 32 * (bm + bn) <= device.limits.maxComputeWorkgroupStorageSize"],derive:{outStride:"args.outStride",dstColStart:"args.dstColStart",lutId:"args.lut",aGroup:"128 if args.lut == 9 else 64",kBlocks:"min(4, aGroup / 32)",bm:"64",bn:"64",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',dynamicRows:"{dynamicRows}"},intermediates:[{id:"AQ",dtype:"uint32",shape:"[args.M * args.inFeatures / 4]"},{id:"AScale",dtype:"float32",shape:"[args.M * args.inFeatures / aGroup]"}],passes:[{id:"quant",name:"LlamaPrefillMatmulLut2QuantA{masked}",shader:"prefill-lut2-quant-a.wgsl.jinja",bindings:["a_a_t","{realLen}",{scratch:"AQ",name:"aq",elementType:"vec4<u32>"},{scratch:"AScale",name:"a_scale",elementType:"f32"},"params_m"],dispatch:{x:"min(ceil(args.M * args.inFeatures / 1024), device.limits.maxComputeWorkgroupsPerDimension)"}},{id:"main",name:"LlamaPrefillMatmulLut2Int8{masked}",shader:"prefill-matmul-lut2-dp4a.wgsl.jinja",bindings:[{scratch:"AQ",name:"aq",buffer:"read-only-storage",elementType:"vec4<u32>"},{scratch:"AScale",name:"a_scale",buffer:"read-only-storage",elementType:"f32"},"q4_bits","q4_metadata","{realLen}","y_y_t","params_main"],dispatch:{x:"min(ceil(args.outFeatures / bn), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / bm), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{id:"q4kn_compact_tiled_f16_splitk",priority:60,when:["splitCount > 1","present.partialsT","ranks.partialsT == 1",'tensorDtypes.partialsT == "float32"',"numel(shapes.partialsT) >= args.kSplits * args.M * args.outFeatures","args.inFeatures / 32 >= args.kSplits","args.kSplits <= device.limits.maxComputeWorkgroupsPerDimension","fusedNStarved","not args.exact","q4knCompactBaseContract",'(tensorDtypes.aT in ["float32", "float16"])','(tensorDtypes.yT in ["float32", "float16"])',"not present.realLenT","ceil(args.outFeatures / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 32) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not hasEpilogue"],requires:{features:["shader-f16"]},derive:{f16Tiles:1,q5k:!1,q4knCompact:!0,outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:2,bn:32,bm:32,nTile:4,accs:8,mPerThread:1,yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',splitBlocks:"ceil(args.inFeatures / 32 / args.kSplits / 2) * 2",kSplits:"args.kSplits",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1},passes:[{id:"main",name:"LlamaPrefillMatmulQ4KNativeCompactSplitK",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_a_t","q4_bits","q4_metadata","partials","params_main"],dispatch:{x:"min(ceil(args.outFeatures / 32), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 32), device.limits.maxComputeWorkgroupsPerDimension)",z:"args.kSplits"}},{id:"reduce",name:"LlamaPrefillMatmulQ4KNativeCompactSplitKReduce",shader:"prefill-matmul-q4-reduce.wgsl.jinja",bindings:["partials_f32","y_y_t","params_m"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"splitKReduceWorkgroupSize"}}]},{variants:[{id:"{kind}_tiled_exact{rowsId}",priority:0,when:["{contract}",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / (16 if args.M <= 8 else 32)) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","{rowsWhen}","not hasEpilogue"],derive:{f16Tiles:0,outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1 if args.M <= 8 else 2",bn:"16 if args.M <= 8 else 32",bm:"8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16)",nTile:"16 if args.M <= 8 else (4 if (args.M > 16 and args.outFeatures >= 8192) else 8)",accs:"1 if args.M <= 8 else (8 if (args.M > 16 and args.outFeatures >= 8192) else 4)",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1,dynamicRows:"{dynamicRows}"},passes:[{id:"main",name:"LlamaPrefillMatmul{name}",shader:"prefill-matmul-q4.wgsl.jinja",derive:"{flags}",bindings:["a_main","q4_bits","{meta}","{realLen}","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / (16 if args.M <= 8 else 32)), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))), device.limits.maxComputeWorkgroupsPerDimension)"}}]}],cases:[{kind:"q4kn_compact",contract:"q4knCompactBaseContract",name:"Q4KNativeCompact",meta:"q4_metadata",flags:{q5k:!1,q4knCompact:!0},rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{kind:"q5k",contract:"q5kBaseContract",name:"Q5K",meta:"q4_scales",flags:{q5k:!0,scaleScalar:'"f32"'},rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{kind:"lut_compact",contract:"lutcBaseContract",name:"LutCompact",meta:"q4_metadata",flags:{lutId:"args.lut",q5k:!1,lutc:!0},rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{kind:"lut2",contract:"lut2BaseContract",name:"Lut2",meta:"q4_metadata",flags:{lutId:"args.lut",q5k:!1,lutc:!0,lut2:!0},rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{kind:"lut2",contract:"lut2BaseContract",name:"Lut2",meta:"q4_metadata",flags:{lutId:"args.lut",q5k:!1,lutc:!0,lut2:!0},rowsId:"_dynamic_rows",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,realLen:["real_len"]}]},{variants:[{id:"{kind}_tiled_exact_large_m{rowsId}",priority:1,when:["{contract}",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"args.M >= 64","ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 24","16384 <= device.limits.maxComputeWorkgroupStorageSize","not args.denseTailStart","{rowsWhen}","not hasEpilogue"],derive:{f16Tiles:0,outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1",bn:"64",bm:"64",nTile:"8",accs:"8",mPerThread:"4",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1,dynamicRows:"{dynamicRows}"},passes:[{id:"main",name:"LlamaPrefillMatmul{name}LargeM",shader:"prefill-matmul-q4.wgsl.jinja",derive:"{flags}",bindings:["a_main","q4_bits","{meta}","{realLen}","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]}],cases:[{kind:"lut2",contract:"lut2BaseContract",name:"Lut2",meta:"q4_metadata",flags:{lutId:"args.lut",q5k:!1,lutc:!0,lut2:!0},rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{kind:"lut2",contract:"lut2BaseContract",name:"Lut2",meta:"q4_metadata",flags:{lutId:"args.lut",q5k:!1,lutc:!0,lut2:!0},rowsId:"_dynamic_rows",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,realLen:["real_len"]}]},{id:"narrow",priority:50,when:["q1BaseContract","f16Ok(dtypes.A) and f16Ok(dtypes.S) and f16Ok(dtypes.Y)","args.M <= device.limits.maxComputeWorkgroupsPerDimension","args.outFeatures % 4 == 0","args.outFeatures % 64 != 0","not args.denseTailStart","not hasEpilogue"],derive:{useSubgroups:'device.features.has("subgroups")',inWords:"args.inFeatures / 32",blocksPerRow:"args.inFeatures / 128",outTiles:"args.outFeatures / 4",inVec4:"args.inFeatures / 4",outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',q1BitsElement:'"vec4<u32>"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ1Narrow",shader:"prefill-matmul-q1-narrow.wgsl.jinja",bindings:["a_a_t","q1_bits","q1_scales","y_y_t","params_main"],dispatch:{x:"args.M"}}]},{cases:[{suffix:"_dynamic_m",nameSuffix:"DynamicM",priority:116,prefixWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"',"args.M >= 64"],dynamicM:!0,bitsBinding:"q1_bits_u32",lengthBindings:["real_len"]},{suffix:"",nameSuffix:"",priority:115,prefixWhen:["not present.realLenT",'args.M >= 64 or not (device.features.has("subgroups") and device.features.has("chromium-experimental-subgroup-matrix") and wave32Effective) or (args.M == 16 and (2 * args.inFeatures <= args.outFeatures or (3 * args.inFeatures <= 2 * args.outFeatures and args.outFeatures <= 2 * args.inFeatures) or (args.outFeatures < args.inFeatures and args.inFeatures < 4 * args.outFeatures)))'],dynamicM:null,bitsBinding:"q1_bits",lengthBindings:[]}],variants:[{id:"portable_lut4_f16{suffix}",priority:"{priority}",when:["{prefixWhen}","q1BaseContract","args.outFeatures % 64 == 0","not args.denseTailStart","not hasEpilogue"],requires:{features:["shader-f16"]},derive:{outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",tileM:"lutM",tileN:"lutN",workgroupSize:"lutN",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ1PortableLut4F16{nameSuffix}",shader:"ops/_shared/binary-lut4-matmul.wgsl.jinja",derive:{layout:'"q1_row4"',mVectorWidth:"4 if tunables.PORTABLE_LUT4_MVEC == 4 and lutM % 4 == 0 and args.outFeatures >= args.inFeatures else 1",dynamicM:"{dynamicM}"},bindings:["a_a_t","{bitsBinding}","q1_scales","{lengthBindings}","y_y_t","params_main"],dispatch:{x:"ceilDiv(args.outFeatures, lutN)",y:"min(ceilDiv(args.M, lutM), DISPATCH_FOLD_WIDTH)",z:"ceilDiv(ceilDiv(args.M, lutM), DISPATCH_FOLD_WIDTH)"}}]}]},{id:"portable_lut4_f32",priority:70,when:["q1BaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.scalesT == "float32"','tensorDtypes.yT == "float32"',"args.outFeatures % 64 == 0","not args.denseTailStart","not hasEpilogue"],derive:{lutM:"16 if (args.M >= 16 and device.limits.maxComputeWorkgroupStorageSize >= 32768) else (8 if args.M >= 8 else args.M)",scaleScalar:'"f32"',outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",tileM:"lutM",tileN:"lutN",workgroupSize:"lutN",lutScalar:'"f32"',yScalar:'"f32"',aVec4:'"vec4<f32>"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ1PortableLut4F32",shader:"ops/_shared/binary-lut4-matmul.wgsl.jinja",derive:{layout:'"q1_row4"',mVectorWidth:1},bindings:["a_a_t","q1_bits","q1_scales","y_y_t","params_main"],dispatch:{x:"ceilDiv(args.outFeatures, lutN)",y:"min(ceilDiv(args.M, lutM), DISPATCH_FOLD_WIDTH)",z:"ceilDiv(ceilDiv(args.M, lutM), DISPATCH_FOLD_WIDTH)"}}]},{id:"portable_tiled",priority:75,when:["not present.realLenT","q1BaseContract","args.outFeatures % 64 == 0","not args.denseTailStart","not hasEpilogue"],requires:{features:["shader-f16"]},derive:{portableBM:"16 if args.M <= 16 else 32",outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",blockOffset:"args.blockOffset",bm:"portableBM",nTile:"128 / portableBM",accs:"portableBM / 4",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ1PortableTiled",shader:"prefill-matmul-q1-tiled.wgsl.jinja",derive:{portable:!0},bindings:["a_a_t","q1_bits","q1_scales","y_y_t","params_m"],dispatch:{x:"min(ceilDiv(args.outFeatures, 32), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceilDiv(args.M, portableBM), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"subgroup_matrix_small_m_splitk_wg64_n64",priority:113,when:["not present.realLenT","tunables.SMALL_M_SPLIT_K == 1","wave32Effective","q1BaseContract","args.M <= 16","args.inFeatures % 512 == 0","args.outFeatures % 64 == 0","ceilDiv(args.outFeatures, 64) <= device.limits.maxComputeWorkgroupsPerDimension","ceilDiv(args.M * args.outFeatures, 256) <= device.limits.maxComputeWorkgroupsPerDimension","ceilDiv(args.M, 16) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(9228) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{effectiveSmallMSplitK:"tunables.SMALL_M_DEEP_SPLIT_K if args.inFeatures >= 8192 and 2 * args.outFeatures <= args.inFeatures else tunables.SMALL_M_SPLIT_K_VALUE",outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',splitK:"effectiveSmallMSplitK"},intermediates:[{id:"partials",dtype:"float32",shape:"[effectiveSmallMSplitK * args.M * args.outFeatures]"}],passes:[{id:"partials",name:"LlamaPrefillMatmulQ1SmallMSplitKPartials",shader:"prefill-matmul-q1-sg-small-m-wg64-n64.wgsl.jinja",bindings:["a_a_t","q1_bits_u32","q1_scales","partials_partials","params_main"],dispatch:{x:"ceilDiv(args.outFeatures, 64)",y:"ceilDiv(args.M, 16)",z:"effectiveSmallMSplitK"},subgroupCollectivesWidth:"portable"},{id:"reduce",name:"LlamaPrefillMatmulQ1SmallMSplitKReduce",shader:"prefill-matmul-q1-small-m-splitk-reduce.wgsl.jinja",bindings:["partials_reduce","y_y_t"],dispatch:{x:"ceilDiv(args.M * args.outFeatures, splitKReduceWorkgroupSize)"}}]},{id:"subgroup_matrix_small_m_n128_splitk_wg128",priority:114,when:["not present.realLenT","tunables.SMALL_M_N128_SPLIT_K == 1 and (args.outFeatures >= args.inFeatures or 4 * args.outFeatures <= args.inFeatures)","wave32Effective","q1BaseContract","args.M <= 16","args.inFeatures % (512 if args.outFeatures >= args.inFeatures else 1024) == 0","args.outFeatures % 128 == 0","ceilDiv(args.outFeatures, 128) <= device.limits.maxComputeWorkgroupsPerDimension","ceilDiv(args.M * args.outFeatures, 256) <= device.limits.maxComputeWorkgroupsPerDimension","ceilDiv(args.M, 16) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(13324) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{effectiveN128SplitK:"tunables.SMALL_M_N128_EXPAND_SPLIT_K if args.outFeatures >= args.inFeatures else (tunables.SMALL_M_N128_NARROW_SPLIT_K if args.inFeatures >= 5120 else 8)",outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',splitK:"effectiveN128SplitK"},intermediates:[{id:"partials",dtype:"float32",shape:"[effectiveN128SplitK * args.M * args.outFeatures]"}],passes:[{id:"partials",name:"LlamaPrefillMatmulQ1SmallMN128SplitKPartials",shader:"prefill-matmul-q1-sg-m16-n128-wg128.wgsl.jinja",bindings:["a_a_t","q1_bits_u32","q1_scales","partials_partials","params_main"],dispatch:{x:"ceilDiv(args.outFeatures, 128)",y:"ceilDiv(args.M, 16)",z:"effectiveN128SplitK"},subgroupCollectivesWidth:"portable"},{id:"reduce",name:"LlamaPrefillMatmulQ1SmallMN128SplitKReduce",shader:"prefill-matmul-q1-small-m-splitk-reduce.wgsl.jinja",bindings:["partials_reduce","y_y_t"],dispatch:{x:"ceilDiv(args.M * args.outFeatures, splitKReduceWorkgroupSize)"}}]},{id:"q1_subgroup_matrix",priority:100,when:["not present.realLenT","wave32Effective","q1BaseContract","args.M > 8","args.outFeatures % 64 == 0","ceilDiv(args.outFeatures, 64) <= device.limits.maxComputeWorkgroupsPerDimension","ceilDiv(args.M, 32) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(7180) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ1",shader:"prefill-matmul-q1-sg.wgsl.jinja",bindings:["a_a_t","q1_bits","q1_scales","y_y_t","params_main"],dispatch:{x:"ceilDiv(args.outFeatures, 64)",y:"ceilDiv(args.M, 32)"},subgroupCollectivesWidth:"portable"}]},{axes:{tier:[{$key:"mr2",tierId:"_mr2",priority:110,tierWhen:["args.outFeatures % 8 == 0","128 <= device.limits.maxComputeInvocationsPerWorkgroup","512 <= device.limits.maxComputeWorkgroupStorageSize","ceilDiv(args.outFeatures, 8) <= device.limits.maxComputeWorkgroupsPerDimension"],clusters:4,regRows:2,groups:"ceilDiv(args.outFeatures, 8)",sgFoldClusters:4},{$key:"plain",tierId:"",priority:105,tierWhen:["args.outFeatures <= device.limits.maxComputeWorkgroupsPerDimension"],clusters:null,regRows:null,groups:"args.outFeatures",sgFoldClusters:"clusters"}],kindAxis:[{$key:"q4kn",kind:"q4kn_compact",contract:"q4knCompactDecodeGemvContract",name:"Q4KNativeCompact",bits:"q4_bits_main",flags:{q5k:!1,q4knCompact:!0}},{$key:"lutc",kind:"lut_compact",contract:"lutcDecodeGemvContract",name:"LutCompact",bits:"q4_bits_main",flags:{lutId:"args.lut",q5k:!1,lutc:!0}},{$key:"lut2",kind:"lut2",contract:"lut2DecodeGemvContract",name:"Lut2",bits:"q4_bits",flags:{lutId:"args.lut",q5k:!1,lutc:!0,lut2:!0}}]},variants:[{id:"{kind}_decode_gemv{tierId}",priority:"{priority}",when:["{contract}","{tierWhen}","not args.denseTailStart","(not useSubgroups and not sgRuntime) or 128 * {sgFoldClusters} + 8 <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],derive:{dstColStart:"args.dstColStart",clusters:"{clusters}",regRows:"{regRows}",aF16:'tensorDtypes.aT == "float16"',yF16:'tensorDtypes.yT == "float16"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',outStride:"args.outStride",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1},passes:[{id:"main",name:"LlamaPrefillMatmul{name}Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",derive:"{flags}",bindings:["a_a_t","{bits}","q4_metadata","y_y_t","params__uniform"],dispatch:{x:"{groups}"},subgroupCollectivesWidth:"portable"}]}]},{id:"q5k_decode_gemv",priority:105,when:["q5kDecodeGemvContract","args.outFeatures <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(not useSubgroups and not sgRuntime) or 128 * clusters + 8 <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],derive:{scaleScalar:'"f32"',quantBits:4,hasMin:!1,iq:!1,lut16:!1,q2k:!1,q3k:!1,q6k:!1,q5k:!0,dstColStart:"args.dstColStart",aF16:'tensorDtypes.aT == "float16"',yF16:'tensorDtypes.yT == "float16"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',outStride:"args.outStride"},passes:[{id:"main",name:"LlamaPrefillMatmulQ5KGemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_a_t","q4_bits_main","q4_scales","y_y_t","params__uniform"],dispatch:{x:"args.outFeatures"},subgroupCollectivesWidth:"portable"}]},{id:"q4_decode_gemv",priority:105,when:["decodeGemvContract","args.outFeatures <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(not useSubgroups and not sgRuntime) or 128 * (1) + 8 <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:"1",regRows:"1",aF16:'tensorDtypes.aT == "float16"',yF16:'tensorDtypes.yT == "float16"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',outStride:"args.outStride"},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_a_t","q4_bits_main","q4_scales","y_y_t","params__uniform"],dispatch:{x:"args.outFeatures"},subgroupCollectivesWidth:"portable"}]},{cases:[{id:"q4_subgroup_matrix_splitk",when:["q4BaseContract",'(tensorDtypes.yT in ["float32", "float16"])',"not q5","(not (q2k or q6k) or (args.inFeatures % 256 == 0 and args.blockOffset % 8 == 0))"],packedOff:null,quantBits:null,name:"Q4Sg",meta:"q4_scales",flags:{lutId:"args.lut if args.lut else 0"}},{id:"q4kn_compact_subgroup_matrix_splitk",when:["q4knCompactBaseContract",'(tensorDtypes.aT in ["float32", "float16"])','(tensorDtypes.yT in ["float32", "float16"])',"not present.realLenT"],packedOff:!1,quantBits:4,name:"Q4KNativeCompactSg",meta:"q4_metadata",flags:{q5k:!1,q4knCompact:!0}},{id:"lut_compact_subgroup_matrix_splitk",when:["lutcBaseContract"],packedOff:!1,quantBits:4,name:"LutCompactSg",meta:"q4_metadata",flags:{lutId:"args.lut",q5k:!1,lutc:!0}},{id:"q5k_subgroup_matrix_splitk",when:["q5kBaseContract",'(tensorDtypes.aT in ["float32", "float16"])','(tensorDtypes.yT in ["float32", "float16"])'],packedOff:!1,quantBits:4,name:"Q5KSg",meta:"q4_scales",flags:{q5k:!0,scaleScalar:'"f32"'}}],variants:[{id:"{id}",priority:101,when:["splitCount > 1","present.partialsT","ranks.partialsT == 1",'tensorDtypes.partialsT == "float32"',"numel(shapes.partialsT) >= args.kSplits * args.M * args.outFeatures","args.inFeatures / 32 >= args.kSplits","args.kSplits <= device.limits.maxComputeWorkgroupsPerDimension","args.M > 8","args.M <= 16","args.outFeatures % 64 == 0","ceil(args.outFeatures / 128) <= device.limits.maxComputeWorkgroupsPerDimension","not args.exact","{when}","wave32Effective","not args.denseTailStart","(10252 if args.M <= 16 else (6668 if sgmatTallSubtile else 7180)) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',smallM:1,sgSplitK:1,splitBlocks:"ceil(args.inFeatures / 32 / args.kSplits / 8) * 8",kSplits:"args.kSplits",quantBits:"{quantBits}",hasMin:"{packedOff}",iq:"{packedOff}",lut16:"{packedOff}",q5:"{packedOff}",q2k:"{packedOff}",q3k:"{packedOff}",q6k:"{packedOff}"},passes:[{id:"main",name:"LlamaPrefillMatmul{name}",shader:"prefill-matmul-q4-sg.wgsl.jinja",derive:"{flags}",bindings:["a_a_t","q4_bits_main","{meta}","partials","params_main"],dispatch:{x:"ceil(args.outFeatures / 128)",y:"ceil(args.M / 16)",z:"args.kSplits"},subgroupCollectivesWidth:"portable"},{id:"reduce",name:"LlamaPrefillMatmulQ4Reduce",shader:"prefill-matmul-q4-reduce.wgsl.jinja",bindings:["partials_f32","y_y_t","params_m"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"splitKReduceWorkgroupSize"}}]}]},{id:"lut2_tiled_exact_register_rows",priority:1,when:["lut2BaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / (16 if args.M <= 8 else 32)) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","args.lut in [8, 9]","args.M > 16","args.M <= 64","16384 <= device.limits.maxComputeWorkgroupStorageSize","tunables.LUT2_PREFILL_REGISTER_ROWS == 1","not hasEpilogue"],derive:{f16Tiles:0,lutId:"args.lut",q5k:!1,outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:1,bn:"64 if ceil(args.outFeatures / 64) >= 256 else 32",bm:"64 if ceil(args.outFeatures / 32) >= 128 else 32",nTile:8,accs:"8 if ceil(args.outFeatures / 64) >= 256 else 4",lutc:!0,quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1,lut2:!0,mPerThread:"4 if ceil(args.outFeatures / 32) >= 128 else 2"},passes:[{id:"main",name:"LlamaPrefillMatmulLut2RegisterRows",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_metadata","y_f32","params_main"],dispatch:{x:"ceil(args.outFeatures / (64 if ceil(args.outFeatures / 64) >= 256 else 32))",y:"ceil(args.M / (64 if ceil(args.outFeatures / 32) >= 128 else 32))"}}]},{axes:{iqFormat:[{$key:"iq3xxs",format:"iq3xxs",kernel:"Iq3Xxs",flags:{iq3xxs:!0}},{$key:"iq3s",format:"iq3s",kernel:"Iq3S",flags:{iq3s:!0}},{$key:"iq2s",format:"iq2s",kernel:"Iq2S",flags:{iq2s:!0}},{$key:"iq2xxs",format:"iq2xxs",kernel:"Iq2Xxs",flags:{iq2xxs:!0}},{$key:"iq1",format:"iq1",kernel:"Iq1",flags:{iq1m:!0}},{$key:"iq2xs",format:"iq2xs",kernel:"Iq2Xs",flags:{iq2xs:!0}},{$key:"iq1s",format:"iq1s",kernel:"Iq1S",flags:{iq1m:!0,iq1s:!0}},{$key:"iq1md",format:"iq1md",kernel:"Iq1Md",flags:{iq1m:!0,iq1md:!0}}],grid:[{$key:"plain_static",gridId:"",f16Priority:50,exactPriority:0,gridWhen:[],iqGridBound:null,gridName:"",gridBinding:[],rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{$key:"plain_dynamic",gridId:"",f16Priority:50,exactPriority:0,gridWhen:[],iqGridBound:null,gridName:"",gridBinding:[],rowsId:"_dynamic_rows",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,realLen:["real_len"]},{$key:"grid_static",gridId:"_grid",f16Priority:55,exactPriority:5,gridWhen:["present.gridT","ranks.gridT == 1",'tensorDtypes.gridT == "uint32"',"dim(shapes.gridT, 0) >= iqGridMin"],iqGridBound:!0,gridName:"Grid",gridBinding:["iq_grid_data"],rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{$key:"grid_dynamic",gridId:"_grid",f16Priority:55,exactPriority:5,gridWhen:["present.gridT","ranks.gridT == 1",'tensorDtypes.gridT == "uint32"',"dim(shapes.gridT, 0) >= iqGridMin"],iqGridBound:!0,gridName:"Grid",gridBinding:["iq_grid_data"],rowsId:"_dynamic_rows",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,realLen:["real_len"]}]},variants:[{id:"{format}_tiled_f16{gridId}{rowsId}",priority:"{f16Priority}",when:["not args.exact","{rowsWhen}","{format}Contract",'(tensorDtypes.aT in ["float32", "float16"])','(tensorDtypes.yT in ["float32", "float16"])',"ceil(args.outFeatures / fusedBn) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / fusedBm) <= device.limits.maxComputeWorkgroupsPerDimension","{gridWhen}","not args.denseTailStart","not hasEpilogue"],requires:{features:["shader-f16"]},derive:{outStride:"args.outStride",dstColStart:"args.dstColStart",mPerThread:"fusedMPerThread",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',iqGridBound:"{iqGridBound}",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1,dynamicRows:"{dynamicRows}"},passes:[{id:"main",name:"LlamaPrefillMatmul{kernel}{gridName}",shader:"prefill-matmul-q4.wgsl.jinja",derive:"{flags}",bindings:["a_a_t","q4_bits","q4_scales","{realLen}","y_y_t","params_main","{gridBinding}"],dispatch:{x:"min(ceil(args.outFeatures / fusedBn), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / fusedBm), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"{format}_tiled_exact{gridId}{rowsId}",priority:"{exactPriority}",when:["{format}Contract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / (16 if args.M <= 8 else 32)) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))) <= device.limits.maxComputeWorkgroupsPerDimension","{gridWhen}","not args.denseTailStart","{rowsWhen}","not hasEpilogue"],derive:{outStride:"args.outStride",dstColStart:"args.dstColStart",f16Tiles:"0",bn:"16 if args.M <= 8 else 32",bm:"8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16)",nTile:"16 if args.M <= 8 else (4 if (args.M > 16 and args.outFeatures >= 8192) else 8)",accs:"1 if args.M <= 8 else (8 if (args.M > 16 and args.outFeatures >= 8192) else 4)",iqGridBound:"{iqGridBound}",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1,dynamicRows:"{dynamicRows}"},passes:[{id:"main",name:"LlamaPrefillMatmul{kernel}{gridName}",shader:"prefill-matmul-q4.wgsl.jinja",derive:"{flags}",bindings:["a_main","q4_bits","q4_scales","{realLen}","y_f32","params_main","{gridBinding}"],dispatch:{x:"min(ceil(args.outFeatures / (16 if args.M <= 8 else 32)), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{cases:[{suffix:"mr2",clusters:4,rows:2,divisor:8,priority:110,sgFoldStorageGate0:"(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"mr4",clusters:4,rows:4,divisor:16,priority:109,sgFoldStorageGate0:"(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"q4_decode_gemv_{suffix}",priority:"{priority}",when:["decodeGemvContract","args.outFeatures % {divisor} == 0","32 * {clusters} <= device.limits.maxComputeInvocationsPerWorkgroup","4 * 32 * {clusters} <= device.limits.maxComputeWorkgroupStorageSize","ceilDiv(args.outFeatures, {divisor}) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","{sgFoldStorageGate0}","not hasEpilogue"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:"{clusters}",regRows:"{rows}",outStride:"args.outStride"},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_main","q4_bits_main","q4_scales","y_f32","params__uniform"],dispatch:{x:"ceil(args.outFeatures / {divisor})"},subgroupCollectivesWidth:"portable"}]}]},{cases:[{suffix:"mr2",clusters:4,rows:2,divisor:8,priority:110,sgFoldStorageGate0:"(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"q5k_decode_gemv_{suffix}",priority:"{priority}",when:["q5kDecodeGemvContract","args.outFeatures % {divisor} == 0","32 * {clusters} <= device.limits.maxComputeInvocationsPerWorkgroup","4 * 32 * {clusters} <= device.limits.maxComputeWorkgroupStorageSize","ceilDiv(args.outFeatures, {divisor}) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","{sgFoldStorageGate0}","not hasEpilogue"],derive:{scaleScalar:'"f32"',quantBits:4,hasMin:!1,iq:!1,lut16:!1,q2k:!1,q3k:!1,q6k:!1,q5k:!0,dstColStart:"args.dstColStart",clusters:"{clusters}",regRows:"{rows}",outStride:"args.outStride"},passes:[{id:"main",name:"LlamaPrefillMatmulQ5KGemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_main","q4_bits_main","q4_scales","y_f32","params__uniform"],dispatch:{x:"ceil(args.outFeatures / {divisor})"},subgroupCollectivesWidth:"portable"}]}]},{axes:{rows:[{$key:"static",rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{$key:"dynamic",rowsId:"_dynamic_rows",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,realLen:["real_len"]}],kindAxis:[{$key:"q4",kind:"q4",contract:["q4BaseContract",'(tensorDtypes.yT == "float32" or tensorDtypes.yT == "float16")',"not q5","(not (q2k or q6k) or (args.inFeatures % 256 == 0 and args.blockOffset % 8 == 0))"],packedOff:null,quantBits:null,name:"Q4",meta:"q4_scales",flags:{lutId:"args.lut if args.lut else 0"}},{$key:"q5k",kind:"q5k",contract:["q5kBaseContract",'(tensorDtypes.aT == "float32" or tensorDtypes.aT == "float16")','(tensorDtypes.yT == "float32" or tensorDtypes.yT == "float16")'],packedOff:!1,quantBits:4,name:"Q5K",meta:"q4_scales",flags:{q5k:!0,scaleScalar:'"f32"'}},{$key:"q4kn",kind:"q4kn_compact",contract:["q4knCompactBaseContract",'(tensorDtypes.aT == "float32" or tensorDtypes.aT == "float16")','(tensorDtypes.yT == "float32" or tensorDtypes.yT == "float16")'],packedOff:!1,quantBits:4,name:"Q4KNativeCompact",meta:"q4_metadata",flags:{q5k:!1,q4knCompact:!0}}]},variants:[{id:"{kind}_subgroup_matrix{rowsId}",priority:100,when:["not args.exact","{contract}","args.M > 8","ceil(args.outFeatures / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension","args.outFeatures % 64 == 0","wave32Effective","{rowsWhen}","not args.denseTailStart","(10252 if args.M <= 16 else (6668 if sgmatTallSubtile else 7180)) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',smallM:"1 if (args.M <= 16) else 0",dynamicRows:"{dynamicRows}",quantBits:"{quantBits}",hasMin:"{packedOff}",iq:"{packedOff}",lut16:"{packedOff}",q5:"{packedOff}",q2k:"{packedOff}",q3k:"{packedOff}",q6k:"{packedOff}"},passes:[{id:"main",name:"LlamaPrefillMatmul{name}Sg",shader:"prefill-matmul-q4-sg.wgsl.jinja",derive:"{flags}",bindings:["a_a_t","q4_bits_main","{meta}","{realLen}","y_y_t","params_main"],profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},dispatch:{x:"ceil(args.outFeatures / (128 if (args.M <= 16) else 64))",y:"ceil(args.M / (16 if (args.M <= 16) else 32))"},subgroupCollectivesWidth:"portable"}]}]},{axes:{rows:[{$key:"static",rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{$key:"dynamic",rowsId:"_dynamic_rows",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,realLen:["real_len"]}],kindAxis:[{$key:"lutc",kind:"lut_compact",contract:"lutcBaseContract",name:"LutCompact",bits:"q4_bits_main",flags:{lutId:"args.lut",q5k:!1,lutc:!0}},{$key:"lut2",kind:"lut2",contract:"lut2BaseContract",name:"Lut2",bits:"q4_bits",flags:{lutId:"args.lut",staticSubgroupSchedule:"tunables.LUT2_PREFILL_STATIC_SCHEDULE == 1 and args.lut in [8, 9] and args.M > 16",q5k:!1,lutc:!0,lut2:!0}}]},variants:[{id:"{kind}_subgroup_matrix{rowsId}",priority:100,when:["not args.exact","{contract}","args.M > 8","ceil(args.outFeatures / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension","args.outFeatures % 64 == 0","wave32Effective","{rowsWhen}","not args.denseTailStart","(10252 if args.M <= 16 else (6668 if sgmatTallSubtile else 7180)) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',smallM:"1 if (args.M <= 16) else 0",dynamicRows:"{dynamicRows}",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1},passes:[{id:"main",name:"LlamaPrefillMatmul{name}Sg",shader:"prefill-matmul-q4-sg.wgsl.jinja",derive:"{flags}",bindings:["a_a_t","{bits}","q4_metadata","{realLen}","y_y_t","params_main"],dispatch:{x:"ceil(args.outFeatures / (128 if (args.M <= 16) else 64))",y:"ceil(args.M / (16 if (args.M <= 16) else 32))"},subgroupCollectivesWidth:"portable"}]}]},{variants:[{id:"lut2_subgroup_matrix_wide_columns{drainId}{rowsId}",priority:"{drainPriority}",when:["not args.exact","lut2BaseContract","args.M > 16","ceil(args.outFeatures / 128) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 32) <= device.limits.maxComputeWorkgroupsPerDimension","args.outFeatures % 64 == 0","wave32Effective","{rowsWhen}","not args.denseTailStart","args.lut in [8, 9]","not args.kSplits",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"tunables.LUT2_PREFILL_WIDE_COLUMNS == 1","{drainWhen}","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}],limits:"{drainLimits}"},derive:{outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",yScalar:'"f32"',smallM:0,dynamicRows:"{dynamicRows}",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1,tallSubtile:1,wideNativeTile:!0,resultDrainBanks:"{drainBanks}"},passes:[{id:"main",name:"LlamaPrefillMatmulLut2WideColumnsSg",shader:"prefill-matmul-q4-sg.wgsl.jinja",derive:{lutId:"args.lut",staticSubgroupSchedule:"tunables.LUT2_PREFILL_STATIC_SCHEDULE == 1",q5k:!1,lutc:!0,lut2:!0},bindings:["a_a_t","q4_bits","q4_metadata","{realLen}","y_y_t","params_main"],dispatch:{x:"ceil(args.outFeatures / 128)",y:"ceil(args.M / 32)"},subgroupCollectivesWidth:"portable",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}}],demoteWhen:["args.M <= 128","ceil(args.outFeatures / 128) * ceil(args.M / 32) < 128"]}],axes:{rows:[{$key:"static",rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{$key:"dynamic",rowsId:"_dynamic_rows",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,realLen:["real_len"]}],drain:[{$key:"single",drainId:"",drainPriority:101,drainBanks:1,drainLimits:{maxComputeInvocationsPerWorkgroup:128,maxComputeWorkgroupSizeX:128,maxComputeWorkgroupStorageSize:11280},drainWhen:[]},{$key:"batched",drainId:"_batched_results",drainPriority:102,drainBanks:4,drainLimits:{maxComputeInvocationsPerWorkgroup:128,maxComputeWorkgroupSizeX:128,maxComputeWorkgroupStorageSize:14352},drainWhen:["tunables.LUT2_PREFILL_BATCHED_RESULTS == 1"]}]}},{cases:[{rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{rowsId:"_dynamic_rows",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,realLen:["real_len"]}],variants:[{id:"lut2_subgroup_matrix_splitk{rowsId}",priority:102,when:["not args.exact","lut2BaseContract","args.M > 8","ceil(args.outFeatures / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension","args.outFeatures % 64 == 0","wave32Effective","{rowsWhen}","not args.denseTailStart","args.lut in [8, 9]",'tensorDtypes.yT == "float32"',"args.M <= 128","lut2PrefillTileCount <= 256","lut2PrefillSplits > 1","lut2PrefillSplits == floor(lut2PrefillSplits)","lut2PrefillSplits <= device.limits.maxComputeWorkgroupsPerDimension","lut2PrefillSplits * args.M * args.outFeatures * 4 <= device.limits.maxStorageBufferBindingSize","(10252 if args.M <= 16 else (6668 if sgmatTallSubtile else 7180)) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{lutId:"args.lut",staticSubgroupSchedule:"tunables.LUT2_PREFILL_STATIC_SCHEDULE == 1 and args.lut in [8, 9] and args.M > 16",q5k:!1,outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',smallM:"1 if (args.M <= 16) else 0",lutc:!0,lut2:!0,dynamicRows:"{dynamicRows}",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1,sgSplitK:1,splitBlocks:"ceil(args.inFeatures / 32 / lut2PrefillSplits)",kSplits:"lut2PrefillSplits"},passes:[{id:"main",name:"LlamaPrefillMatmulLut2Sg",shader:"prefill-matmul-q4-sg.wgsl.jinja",bindings:["a_a_t","q4_bits","q4_metadata",{scratch:"lut2Partials",name:"partials",elementType:"f32"},"params_main","{realLen}"],dispatch:{x:"ceil(args.outFeatures / (128 if (args.M <= 16) else 64))",y:"ceil(args.M / (16 if (args.M <= 16) else 32))",z:"lut2PrefillSplits"},subgroupCollectivesWidth:"portable"},{id:"reduce",name:"LlamaPrefillMatmulLut2Reduce",shader:"prefill-matmul-q4-reduce.wgsl.jinja",bindings:[{scratch:"lut2Partials",name:"partials",elementType:"f32",buffer:"read-only-storage"},"y_y_t","params_m","{realLen}"],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"splitKReduceWorkgroupSize"}}],intermediates:[{id:"lut2Partials",dtype:"float32",shape:"[lut2PrefillSplits * args.M * args.outFeatures]"}]}]},{axes:{rows:[{$key:"static",rowsId:"",rowsWhen:["not present.realLenT"],dynamicRows:null,realLen:[]},{$key:"dynamic",rowsId:"_dynamic_rows",rowsWhen:["present.realLenT",'ranks.realLenT == 1 and numel(shapes.realLenT) >= 1 and tensorDtypes.realLenT == "uint32"'],dynamicRows:!0,realLen:["real_len"]}],iqFormat:[{$key:"iq3xxs",format:"iq3xxs",kernel:"Iq3Xxs",flags:{iq3xxs:!0}},{$key:"iq3s",format:"iq3s",kernel:"Iq3S",flags:{iq3s:!0}},{$key:"iq2s",format:"iq2s",kernel:"Iq2S",flags:{iq2s:!0,iq2xs:!1}},{$key:"iq2xs",format:"iq2xs",kernel:"Iq2Xs",flags:{iq2s:!1,iq2xs:!0}},{$key:"iq2xxs",format:"iq2xxs",kernel:"Iq2Xxs",flags:{iq2xxs:!0}},{$key:"iq1",format:"iq1",kernel:"Iq1",flags:{iq1m:!0}},{$key:"iq1s",format:"iq1s",kernel:"Iq1S",flags:{iq1m:!0,iq1s:!0}},{$key:"iq1md",format:"iq1md",kernel:"Iq1Md",flags:{iq1m:!0,iq1md:!0}}]},variants:[{id:"{format}_subgroup_matrix{rowsId}",priority:100,when:["not args.exact","{format}Contract",'(tensorDtypes.aT in ["float32", "float16"])',"f16Ok(tensorDtypes.aT)",'(tensorDtypes.yT in ["float32", "float16"])',"f16Ok(tensorDtypes.yT)","args.M > 8","args.outFeatures % 64 == 0","ceil(args.outFeatures / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension","wave32Effective","{rowsWhen}","not args.denseTailStart","(10252 if args.M <= 16 else (6668 if sgmatTallSubtile else 7180)) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{q5k:!1,outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',smallM:"1 if (args.M <= 16) else 0",dynamicRows:"{dynamicRows}",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1},passes:[{id:"main",name:"LlamaPrefillMatmul{kernel}Sg",shader:"prefill-matmul-q4-sg.wgsl.jinja",derive:"{flags}",bindings:["a_a_t","q4_bits","q4_scales","{realLen}","y_y_t","params_main"],dispatch:{x:"ceil(args.outFeatures / (128 if (args.M <= 16) else 64))",y:"ceil(args.M / (16 if (args.M <= 16) else 32))"},subgroupCollectivesWidth:"portable"}]},{id:"{format}_subgroup_matrix_grid{rowsId}",priority:105,when:["not args.exact","{format}Contract",'(tensorDtypes.aT in ["float32", "float16"])',"f16Ok(tensorDtypes.aT)",'(tensorDtypes.yT in ["float32", "float16"])',"f16Ok(tensorDtypes.yT)","args.M > 8","args.outFeatures % 64 == 0","ceil(args.outFeatures / 32) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 16) <= device.limits.maxComputeWorkgroupsPerDimension","wave32Effective","present.gridT","ranks.gridT == 1",'tensorDtypes.gridT == "uint32"',"dim(shapes.gridT, 0) >= iqGridMin","{rowsWhen}","not args.denseTailStart","(10252 if args.M <= 16 else (6668 if sgmatTallSubtile else 7180)) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{q5k:!1,outStride:"args.outStride",dstColStart:"args.dstColStart if args.dstColStart else 0",yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',smallM:"1 if (args.M <= 16) else 0",iqGridBound:!0,dynamicRows:"{dynamicRows}",quantBits:4,hasMin:!1,iq:!1,lut16:!1,q5:!1,q2k:!1,q3k:!1,q6k:!1},passes:[{id:"main",name:"LlamaPrefillMatmul{kernel}SgGrid",shader:"prefill-matmul-q4-sg.wgsl.jinja",derive:"{flags}",bindings:["a_a_t","q4_bits","q4_scales","{realLen}","y_y_t","params_main","iq_grid_data"],dispatch:{x:"ceil(args.outFeatures / (128 if (args.M <= 16) else 64))",y:"ceil(args.M / (16 if (args.M <= 16) else 32))"},subgroupCollectivesWidth:"portable"}]}]},{id:"q4kn_dense_tail",priority:200,when:["present.weightsT","present.bitsT","present.scalesT",'args.format == "q4_k_native_compact"',"args.M >= 32","(args.denseTailStart if args.denseTailStart else 0) > 0","(args.denseTailStart if args.denseTailStart else 0) < args.outFeatures","(args.denseTailStart if args.denseTailStart else 0) % 64 == 0","args.outFeatures % 64 == 0","args.inFeatures % 256 == 0","wave32Effective","not present.realLenT","not present.partialsT","ranks.aT == 2","ranks.yT == 2","ranks.weightsT == 1","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.aT == "float16"','tensorDtypes.yT == "float16"','tensorDtypes.weightsT == "float16"','tensorDtypes.bitsT == "uint32"','tensorDtypes.scalesT == "uint32"',"args.inFeatures > 0","(args.blockOffset if args.blockOffset else 0) % 8 == 0","numel(shapes.aT) >= args.M * args.inFeatures","numel(shapes.yT) >= args.M * denseOutStride","(args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride","numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + (args.outFeatures - (args.denseTailStart if args.denseTailStart else 0)) * args.inFeatures","numel(shapes.bitsT) >= ((args.blockOffset if args.blockOffset else 0) + (args.denseTailStart if args.denseTailStart else 0) * args.inFeatures / 32) * 4","numel(shapes.scalesT) >= ((args.blockOffset if args.blockOffset else 0) + (args.denseTailStart if args.denseTailStart else 0) * args.inFeatures / 32) * 5 / 8","ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / 32) <= device.limits.maxComputeWorkgroupsPerDimension","(14348) <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{outStride:"denseOutStride",dstColStart:"args.dstColStart if args.dstColStart else 0",denseTailStart:"args.denseTailStart"},passes:[{id:"main",name:"LlamaPrefillMatmulQ4KNativeDenseTailSg",shader:"prefill-matmul-subgroup.wgsl.jinja",bindings:[{arg:"aT",name:"a",elementType:"f16"},{arg:"bitsT",name:"q4_bits",elementType:"u32"},"q4_metadata","y",{name:"weights",arg:"weightsT",elementType:"f16"},{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset"},{name:"m",type:"u32",value:"args.M"},{name:"weight_offset",type:"u32",value:"args.weightOffset"}]}],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil(args.M / 32)"},subgroupCollectivesWidth:"portable"}]},{id:"q4kn_dense_tail_tiled",priority:90,when:["present.weightsT","present.bitsT","present.scalesT",'args.format == "q4_k_native_compact"',"args.M >= 1","(args.denseTailStart if args.denseTailStart else 0) > 0","(args.denseTailStart if args.denseTailStart else 0) < args.outFeatures","(args.denseTailStart if args.denseTailStart else 0) % 64 == 0","args.outFeatures % 64 == 0","args.inFeatures % 256 == 0","not present.realLenT","not present.partialsT","ranks.aT == 2","ranks.yT == 2","ranks.weightsT == 1","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.aT == "float16"','tensorDtypes.yT == "float16"','tensorDtypes.weightsT == "float16"','tensorDtypes.bitsT == "uint32"','tensorDtypes.scalesT == "uint32"',"args.inFeatures > 0","(args.blockOffset if args.blockOffset else 0) % 8 == 0","numel(shapes.aT) >= args.M * args.inFeatures","numel(shapes.yT) >= args.M * denseOutStride","(args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride","numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + (args.outFeatures - (args.denseTailStart if args.denseTailStart else 0)) * args.inFeatures","numel(shapes.bitsT) >= ((args.blockOffset if args.blockOffset else 0) + (args.denseTailStart if args.denseTailStart else 0) * args.inFeatures / 32) * 4","numel(shapes.scalesT) >= ((args.blockOffset if args.blockOffset else 0) + (args.denseTailStart if args.denseTailStart else 0) * args.inFeatures / 32) * 5 / 8","not hasEpilogue"],requires:{features:["shader-f16"]},derive:{outStride:"denseOutStride",dstColStart:"args.dstColStart if args.dstColStart else 0",denseTailStart:"args.denseTailStart",q4knCompact:!0,f16Tiles:1,kBlocks:1,bm:16,bn:32,nTile:8,accs:4,mPerThread:1,yScalar:'"f16"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4KNativeDenseTailTiled",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_a_t","q4_bits","q4_metadata","y_y_t",{name:"weights",arg:"weightsT",elementType:"f16"},{name:"params",struct:[{name:"block_offset",type:"u32",value:"args.blockOffset"},{name:"m",type:"u32",value:"args.M"},{name:"weight_offset",type:"u32",value:"args.weightOffset"}]}],dispatch:{x:"min(ceil(args.outFeatures / 32), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 16), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"q4kn_dense_tail_gemv_mrows",priority:201,when:["present.weightsT","present.bitsT","present.scalesT",'args.format == "q4_k_native_compact"',"(args.denseTailStart if args.denseTailStart else 0) > 0","(args.denseTailStart if args.denseTailStart else 0) < args.outFeatures","(args.denseTailStart if args.denseTailStart else 0) % 64 == 0","args.outFeatures % 64 == 0","args.inFeatures % 256 == 0","not present.realLenT","ranks.aT == 2","ranks.yT == 2","ranks.weightsT == 1","ranks.bitsT == 1","ranks.scalesT == 1",'tensorDtypes.aT == "float16"','tensorDtypes.yT == "float16"','tensorDtypes.weightsT == "float16"','tensorDtypes.bitsT == "uint32"','tensorDtypes.scalesT == "uint32"',"args.inFeatures > 0","(args.blockOffset if args.blockOffset else 0) % 8 == 0","numel(shapes.aT) >= args.M * args.inFeatures","numel(shapes.yT) >= args.M * denseOutStride","(args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride","numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + (args.outFeatures - (args.denseTailStart if args.denseTailStart else 0)) * args.inFeatures","numel(shapes.bitsT) >= ((args.blockOffset if args.blockOffset else 0) + (args.denseTailStart if args.denseTailStart else 0) * args.inFeatures / 32) * 4","numel(shapes.scalesT) >= ((args.blockOffset if args.blockOffset else 0) + (args.denseTailStart if args.denseTailStart else 0) * args.inFeatures / 32) * 5 / 8","args.M >= 1","args.M <= 16","(args.weightOffset if args.weightOffset else 0) % 4 == 0","denseGemvCols > 0","args.outFeatures % denseGemvCols == 0","args.outFeatures / denseGemvCols <= device.limits.maxComputeWorkgroupsPerDimension","32 * denseGemvClusters <= device.limits.maxComputeInvocationsPerWorkgroup","(not useSubgroups and not sgRuntime) or 128 * (denseGemvClusters) + 8 <= device.limits.maxComputeWorkgroupStorageSize","not hasEpilogue"],requires:{features:["shader-f16"]},derive:{outStride:"denseOutStride",dstColStart:"args.dstColStart if args.dstColStart else 0",aVec4:'"vec4<f16>"',wVec4:'"vec4<f16>"',yScalar:'"f16"',clusters:"denseGemvClusters",regRows:"denseGemvRegRows",denseTailStart:"args.denseTailStart"},passes:[{id:"main",name:"LlamaPrefillMatmulQ4KNativeDenseTailMRows",shader:"prefill-matmul-dense-gemv-mrows.wgsl.jinja",bindings:["a_a_t",{arg:"weightsT",name:"weights",elementType:"$wVec4"},{arg:"bitsT",name:"q4_bits",elementType:"u32"},"q4_metadata","y_y_t",{name:"params",struct:[{name:"weight_offset",type:"u32",value:"args.weightOffset"},{name:"block_offset",type:"u32",value:"args.blockOffset"}]}],dispatch:{x:"args.outFeatures / denseGemvCols"},subgroupCollectivesWidth:"portable"}]},{cases:[{width:32,bias:"full",mode:"none",biasGuard:"true",biasBinding:[]},{width:64,bias:"full",mode:"none",biasGuard:"true",biasBinding:[]}],variants:[{id:"subgroup_matrix_16x16x16_w{width}_{bias}",priority:-1e3,demoteWhen:["true"],when:["not args.exact","present.weightsT","not present.bitsT and not present.scalesT and not present.gridT and not present.partialsT","not args.denseTailStart","not present.realLenT","ranks.aT == 2 and ranks.weightsT == 1 and ranks.yT == 2",'tensorDtypes.aT == "float16" and tensorDtypes.weightsT == "float16"','tensorDtypes.yT in ["float16", "float32"]',"numel(shapes.aT) >= args.M * args.inFeatures","numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures","numel(shapes.yT) >= args.M * denseOutStride","(args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride","{biasGuard}","(args.M) > 0 and (args.outFeatures) > 0 and (args.inFeatures) > 0","ceilDiv((args.M), 32) <= device.limits.maxComputeWorkgroupsPerDimension and ceilDiv((args.outFeatures), 32) <= device.limits.maxComputeWorkgroupsPerDimension","(1) <= device.limits.maxComputeWorkgroupsPerDimension","128 <= device.limits.maxComputeInvocationsPerWorkgroup","128 <= device.limits.maxComputeWorkgroupSizeX","8240 <= device.limits.maxComputeWorkgroupStorageSize",'has(device.adapterInfo, "subgroupMatrixConfigs") and subgroupMatrixConfigAvailable("f16", "f32", 16, 16, 16)','(has(device.adapterInfo, "subgroupMinSize") and has(device.adapterInfo, "subgroupMaxSize") and device.adapterInfo.subgroupMinSize <= {width} and device.adapterInfo.subgroupMaxSize >= {width}) or (not (has(device.adapterInfo, "subgroupMinSize") or has(device.adapterInfo, "subgroupMaxSize")) and {width} == 32)',"not hasEpilogue"],requires:{features:["shader-f16","subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f16",resultComponentType:"f32",M:16,N:16,K:16}]},derive:{M:"args.M",N:"args.outFeatures",K:"args.inFeatures",matrixM:16,matrixN:16,matrixK:16,matrixStageK:32,matrixSubgroupWidth:"{width}",operandScalar:'"f16"',accumulatorScalar:'"f32"',outputScalar:"dtypes.Y",matrixAOffset:0,matrixBOffset:"args.weightOffset if args.weightOffset else 0",matrixARowStride:"args.inFeatures",matrixAKStride:1,matrixBKStride:1,matrixBColStride:"args.inFeatures",matrixYStride:"denseOutStride",matrixYColumnOffset:"args.dstColStart if args.dstColStart else 0",matrixAlpha:1,matrixBiasMode:'"{mode}"',matrixEpilogue:'"none"',matrixBatched:!1},passes:[{id:"main",name:"SubgroupMatrix16x16x16.W{width}.{bias}",shader:"ops/_shared/matrix-shaped-projection-gemm.wgsl.jinja",bindings:[{arg:"aT",name:"a",elementType:"f16"},{arg:"weightsT",name:"b",elementType:"f16"},"{biasBinding}",{arg:"yT",name:"y",elementType:"$outputScalar"}],dispatch:{x:"ceilDiv(N, 32)",y:"ceilDiv(M, 32)",z:"1"},subgroupCollectivesWidth:"portable"}]}]},{id:"scalar_dense_region",priority:0,when:["present.weightsT","ranks.aT == 2","ranks.weightsT == 1","ranks.yT == 2",'(tensorDtypes.weightsT in ["float32", "float16"])','(tensorDtypes.yT in ["float32", "float16"])',"f16Ok(tensorDtypes.aT) and f16Ok(tensorDtypes.weightsT) and f16Ok(tensorDtypes.yT)","args.M > 0","args.inFeatures > 0","args.outFeatures > 0","numel(shapes.aT) >= args.M * args.inFeatures","not args.denseTailStart","not denseWholeBlockDest","not present.bitsT and not present.scalesT and not present.gridT and not present.partialsT and not present.realLenT","(args.dstColStart if args.dstColStart else 0) + args.outFeatures <= denseOutStride","numel(shapes.yT) >= args.M * denseOutStride","numel(shapes.weightsT) >= (args.weightOffset if args.weightOffset else 0) + args.outFeatures * args.inFeatures","64 <= device.limits.maxComputeInvocationsPerWorkgroup and 64 <= device.limits.maxComputeWorkgroupSizeX","not hasEpilogue"],derive:{wScalar:"dtypes.W",yScalar:"dtypes.Y",workgroupSize:64},passes:[{id:"main",name:"LlamaPrefillMatmul",shader:"prefill-matmul-dense-region.wgsl.jinja",bindings:["a","weights","y_y_t",{name:"params",struct:[{name:"weight_offset",type:"u32",value:"args.weightOffset"},{name:"m",type:"u32",value:"args.M"},{name:"in_features",type:"u32",value:"args.inFeatures"},{name:"out_features",type:"u32",value:"args.outFeatures"},{name:"out_stride",type:"u32",value:"denseOutStride"},{name:"dst_col",type:"u32",value:"args.dstColStart if args.dstColStart else 0"}]}],dispatch:{threads:"args.M * args.outFeatures",workgroupSize:"workgroupSize"}}]},{cases:[{fmt:"bf16",Name:"Bf16",bf16:!0,f16:null},{fmt:"f16",Name:"F16",bf16:null,f16:!0}],variants:[{id:"{fmt}_subgroup_matrix__epi_0_0",priority:110,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 64 == 0","device.limits.maxComputeWorkgroupStorageSize >= 16396","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","not present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!1,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0",hasResidual:!1,gemmEpilogue:'"fastgelu" if args.activation == "tanh" else ("erfgelu" if args.activation == "erf" else "none")',reuseMatrixBForEpilogue:!0},passes:[{id:"main",name:"LlamaPrefillMatmul{Name}Sgmat",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"}]},{id:"{fmt}_subgroup_matrix_m_tail__epi_0_0",priority:100,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 64 != 0","device.limits.maxComputeWorkgroupStorageSize >= 20492","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","not present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0",epiHasBias:!1,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"bulk",name:"LlamaPrefillMatmul{Name}SgmatBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 64) * 64 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmul{Name}SgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 64u) * 2u"',directStore:!1},profile:{flops:"2 * (args.M % 64) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 64) / 32)"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"{fmt}_tiled_reg__epi_0_0",priority:10,when:["halfWordContract",'args.format == "{fmt}"',"ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension","16 <= device.limits.maxComputeWorkgroupSizeX","16 <= device.limits.maxComputeWorkgroupSizeY","256 <= device.limits.maxComputeInvocationsPerWorkgroup","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","not present.rT"],derive:{yScalar:'"f32"',bf16Weights:"{bf16}",f16Weights:"{f16}",epiHasBias:!1,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmul{Name}TiledReg",shader:"prefill-matmul-tiled-reg.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["a","bf16_weights","y_y_t","bf16_params"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{id:"q8_subgroup_matrix_exact__epi_0_0",priority:110,when:["q8ExactSgmatContract","args.M % 64 == 0","device.limits.maxComputeWorkgroupStorageSize >= 16396","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","not present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',q8BStaging:!0,tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!1,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:!1,hasResidual:!1,gemmEpilogue:'"fastgelu" if args.activation == "tanh" else ("erfgelu" if args.activation == "erf" else "none")',reuseMatrixBForEpilogue:!0},passes:[{id:"main",name:"LlamaPrefillMatmulQ8ExactSgmat",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"}]},{id:"q8_subgroup_matrix_exact_m_tail__epi_0_0",priority:100,when:["q8ExactSgmatContract","args.M % 64 != 0","device.limits.maxComputeWorkgroupStorageSize >= 20492","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","not present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',q8BStaging:!0,packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:!1,epiHasBias:!1,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"bulk",name:"LlamaPrefillMatmulQ8ExactSgmatBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 64) * 64 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmulQ8ExactSgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 64u) * 2u"',directStore:!1},profile:{flops:"2 * (args.M % 64) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 64) / 32)"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"tiled_exact__epi_0_0",priority:0,when:["q4BaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / (16 if args.M <= 8 else 32)) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","not present.rT"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1 if args.M <= 8 else 2",bn:"16 if args.M <= 8 else 32",bm:"8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16)",nTile:"16 if args.M <= 8 else (4 if (args.M > 16 and args.outFeatures >= 8192) else 8)",accs:"1 if args.M <= 8 else (8 if (args.M > 16 and args.outFeatures >= 8192) else 4)",epiHasBias:!1,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / (16 if args.M <= 8 else 32)), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"q8_tiled_exact_large_m__epi_0_0",priority:1,when:["q4BaseContract",'fmt == "q8_0"','tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"args.M >= 64","ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 24","16384 <= device.limits.maxComputeWorkgroupStorageSize","not args.denseTailStart","not present.realLenT","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","not present.rT"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1",bn:"64",bm:"64",nTile:"8",accs:"8",mPerThread:"4",epiHasBias:!1,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ8LargeM",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"q4_decode_gemv__epi_0_0",priority:105,when:["decodeGemvContract","args.outFeatures <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(not useSubgroups and not sgRuntime) or 128 * (1) + 8 <= device.limits.maxComputeWorkgroupStorageSize","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","not present.rT"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:"1",regRows:"1",aF16:'tensorDtypes.aT == "float16"',yF16:'tensorDtypes.yT == "float16"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',outStride:"args.outStride",epiHasBias:!1,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_a_t","q4_bits_main","q4_scales","y_y_t","params__uniform"],dispatch:{x:"args.outFeatures"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{cases:[{suffix:"mr2",clusters:4,rows:2,divisor:8,priority:110,sgFoldStorageGate0:"(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"mr4",clusters:4,rows:4,divisor:16,priority:109,sgFoldStorageGate0:"(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"q4_decode_gemv_{suffix}__epi_0_0",priority:"{priority}",when:["decodeGemvContract","args.outFeatures % {divisor} == 0","32 * {clusters} <= device.limits.maxComputeInvocationsPerWorkgroup","4 * 32 * {clusters} <= device.limits.maxComputeWorkgroupStorageSize","ceilDiv(args.outFeatures, {divisor}) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","{sgFoldStorageGate0}","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","not present.rT"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:"{clusters}",regRows:"{rows}",outStride:"args.outStride",epiHasBias:!1,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_main","q4_bits_main","q4_scales","y_f32","params__uniform"],dispatch:{x:"ceil(args.outFeatures / {divisor})"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{cases:[{fmt:"bf16",Name:"Bf16",bf16:!0,f16:null},{fmt:"f16",Name:"F16",bf16:null,f16:!0}],variants:[{id:"{fmt}_subgroup_matrix__epi_0_1",priority:110,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 64 == 0","device.limits.maxComputeWorkgroupStorageSize >= 16396","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!1,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0",hasResidual:!0,gemmEpilogue:'"fastgelu" if args.activation == "tanh" else ("erfgelu" if args.activation == "erf" else "none")',reuseMatrixBForEpilogue:!0},passes:[{id:"main",name:"LlamaPrefillMatmul{Name}Sgmat",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y","epi_residual","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"}]},{id:"{fmt}_subgroup_matrix_m_tail__epi_0_1",priority:100,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 64 != 0","device.limits.maxComputeWorkgroupStorageSize >= 20492","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0",epiHasBias:!1,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"bulk",name:"LlamaPrefillMatmul{Name}SgmatBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 64) * 64 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmul{Name}SgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 64u) * 2u"',directStore:!1},profile:{flops:"2 * (args.M % 64) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 64) / 32)"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"{fmt}_tiled_reg__epi_0_1",priority:10,when:["halfWordContract",'args.format == "{fmt}"',"ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension","16 <= device.limits.maxComputeWorkgroupSizeX","16 <= device.limits.maxComputeWorkgroupSizeY","256 <= device.limits.maxComputeInvocationsPerWorkgroup","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","present.rT"],derive:{yScalar:'"f32"',bf16Weights:"{bf16}",f16Weights:"{f16}",epiHasBias:!1,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmul{Name}TiledReg",shader:"prefill-matmul-tiled-reg.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["a","bf16_weights","y_y_t","bf16_params"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{id:"q8_subgroup_matrix_exact__epi_0_1",priority:110,when:["q8ExactSgmatContract","args.M % 64 == 0","device.limits.maxComputeWorkgroupStorageSize >= 16396","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',q8BStaging:!0,tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!1,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:!1,hasResidual:!0,gemmEpilogue:'"fastgelu" if args.activation == "tanh" else ("erfgelu" if args.activation == "erf" else "none")',reuseMatrixBForEpilogue:!0},passes:[{id:"main",name:"LlamaPrefillMatmulQ8ExactSgmat",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params_m","epi_residual"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"}]},{id:"q8_subgroup_matrix_exact_m_tail__epi_0_1",priority:100,when:["q8ExactSgmatContract","args.M % 64 != 0","device.limits.maxComputeWorkgroupStorageSize >= 20492","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',q8BStaging:!0,packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:!1,epiHasBias:!1,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"bulk",name:"LlamaPrefillMatmulQ8ExactSgmatBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 64) * 64 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmulQ8ExactSgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 64u) * 2u"',directStore:!1},profile:{flops:"2 * (args.M % 64) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 64) / 32)"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"tiled_exact__epi_0_1",priority:0,when:["q4BaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / (16 if args.M <= 8 else 32)) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","present.rT"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1 if args.M <= 8 else 2",bn:"16 if args.M <= 8 else 32",bm:"8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16)",nTile:"16 if args.M <= 8 else (4 if (args.M > 16 and args.outFeatures >= 8192) else 8)",accs:"1 if args.M <= 8 else (8 if (args.M > 16 and args.outFeatures >= 8192) else 4)",epiHasBias:!1,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / (16 if args.M <= 8 else 32)), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"q8_tiled_exact_large_m__epi_0_1",priority:1,when:["q4BaseContract",'fmt == "q8_0"','tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"args.M >= 64","ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 24","16384 <= device.limits.maxComputeWorkgroupStorageSize","not args.denseTailStart","not present.realLenT","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","present.rT"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1",bn:"64",bm:"64",nTile:"8",accs:"8",mPerThread:"4",epiHasBias:!1,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ8LargeM",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"q4_decode_gemv__epi_0_1",priority:105,when:["decodeGemvContract","args.outFeatures <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(not useSubgroups and not sgRuntime) or 128 * (1) + 8 <= device.limits.maxComputeWorkgroupStorageSize","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","present.rT"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:"1",regRows:"1",aF16:'tensorDtypes.aT == "float16"',yF16:'tensorDtypes.yT == "float16"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',outStride:"args.outStride",epiHasBias:!1,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_a_t","q4_bits_main","q4_scales","y_y_t","params__uniform"],dispatch:{x:"args.outFeatures"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{cases:[{suffix:"mr2",clusters:4,rows:2,divisor:8,priority:110,sgFoldStorageGate0:"(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"mr4",clusters:4,rows:4,divisor:16,priority:109,sgFoldStorageGate0:"(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"q4_decode_gemv_{suffix}__epi_0_1",priority:"{priority}",when:["decodeGemvContract","args.outFeatures % {divisor} == 0","32 * {clusters} <= device.limits.maxComputeInvocationsPerWorkgroup","4 * 32 * {clusters} <= device.limits.maxComputeWorkgroupStorageSize","ceilDiv(args.outFeatures, {divisor}) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","{sgFoldStorageGate0}","hasEpilogue","epilogueOk","epilogueSupported","not present.bT","present.rT"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:"{clusters}",regRows:"{rows}",outStride:"args.outStride",epiHasBias:!1,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_main","q4_bits_main","q4_scales","y_f32","params__uniform"],dispatch:{x:"ceil(args.outFeatures / {divisor})"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{cases:[{fmt:"bf16",Name:"Bf16",bf16:!0,f16:null},{fmt:"f16",Name:"F16",bf16:null,f16:!0}],variants:[{id:"{fmt}_subgroup_matrix__epi_1_0",priority:110,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 64 == 0","device.limits.maxComputeWorkgroupStorageSize >= 16396","hasEpilogue","epilogueOk","epilogueSupported","present.bT","not present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!0,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!1,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0",hasResidual:!1,gemmEpilogue:'"fastgelu" if args.activation == "tanh" else ("erfgelu" if args.activation == "erf" else "none")',reuseMatrixBForEpilogue:!0},passes:[{id:"main",name:"LlamaPrefillMatmul{Name}Sgmat",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y","epi_bias","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"}]},{id:"{fmt}_subgroup_matrix_m_tail__epi_1_0",priority:100,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 64 != 0","device.limits.maxComputeWorkgroupStorageSize >= 20492","hasEpilogue","epilogueOk","epilogueSupported","present.bT","not present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0",epiHasBias:!0,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"bulk",name:"LlamaPrefillMatmul{Name}SgmatBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 64) * 64 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmul{Name}SgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 64u) * 2u"',directStore:!1},profile:{flops:"2 * (args.M % 64) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 64) / 32)"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"{fmt}_tiled_reg__epi_1_0",priority:10,when:["halfWordContract",'args.format == "{fmt}"',"ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension","16 <= device.limits.maxComputeWorkgroupSizeX","16 <= device.limits.maxComputeWorkgroupSizeY","256 <= device.limits.maxComputeInvocationsPerWorkgroup","hasEpilogue","epilogueOk","epilogueSupported","present.bT","not present.rT"],derive:{yScalar:'"f32"',bf16Weights:"{bf16}",f16Weights:"{f16}",epiHasBias:!0,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmul{Name}TiledReg",shader:"prefill-matmul-tiled-reg.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["a","bf16_weights","y_y_t","bf16_params"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{id:"q8_subgroup_matrix_exact__epi_1_0",priority:110,when:["q8ExactSgmatContract","args.M % 64 == 0","device.limits.maxComputeWorkgroupStorageSize >= 16396","hasEpilogue","epilogueOk","epilogueSupported","present.bT","not present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!0,mmaScalar:'"f32"',yScalar:'"f32"',q8BStaging:!0,tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!1,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:!1,hasResidual:!1,gemmEpilogue:'"fastgelu" if args.activation == "tanh" else ("erfgelu" if args.activation == "erf" else "none")',reuseMatrixBForEpilogue:!0},passes:[{id:"main",name:"LlamaPrefillMatmulQ8ExactSgmat",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params_m","epi_bias"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"}]},{id:"q8_subgroup_matrix_exact_m_tail__epi_1_0",priority:100,when:["q8ExactSgmatContract","args.M % 64 != 0","device.limits.maxComputeWorkgroupStorageSize >= 20492","hasEpilogue","epilogueOk","epilogueSupported","present.bT","not present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',q8BStaging:!0,packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:!1,epiHasBias:!0,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"bulk",name:"LlamaPrefillMatmulQ8ExactSgmatBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 64) * 64 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmulQ8ExactSgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 64u) * 2u"',directStore:!1},profile:{flops:"2 * (args.M % 64) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 64) / 32)"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"tiled_exact__epi_1_0",priority:0,when:["q4BaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / (16 if args.M <= 8 else 32)) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","hasEpilogue","epilogueOk","epilogueSupported","present.bT","not present.rT"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1 if args.M <= 8 else 2",bn:"16 if args.M <= 8 else 32",bm:"8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16)",nTile:"16 if args.M <= 8 else (4 if (args.M > 16 and args.outFeatures >= 8192) else 8)",accs:"1 if args.M <= 8 else (8 if (args.M > 16 and args.outFeatures >= 8192) else 4)",epiHasBias:!0,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / (16 if args.M <= 8 else 32)), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"q8_tiled_exact_large_m__epi_1_0",priority:1,when:["q4BaseContract",'fmt == "q8_0"','tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"args.M >= 64","ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 24","16384 <= device.limits.maxComputeWorkgroupStorageSize","not args.denseTailStart","not present.realLenT","hasEpilogue","epilogueOk","epilogueSupported","present.bT","not present.rT"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1",bn:"64",bm:"64",nTile:"8",accs:"8",mPerThread:"4",epiHasBias:!0,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ8LargeM",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"q4_decode_gemv__epi_1_0",priority:105,when:["decodeGemvContract","args.outFeatures <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(not useSubgroups and not sgRuntime) or 128 * (1) + 8 <= device.limits.maxComputeWorkgroupStorageSize","hasEpilogue","epilogueOk","epilogueSupported","present.bT","not present.rT"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:"1",regRows:"1",aF16:'tensorDtypes.aT == "float16"',yF16:'tensorDtypes.yT == "float16"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',outStride:"args.outStride",epiHasBias:!0,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_a_t","q4_bits_main","q4_scales","y_y_t","params__uniform"],dispatch:{x:"args.outFeatures"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{cases:[{suffix:"mr2",clusters:4,rows:2,divisor:8,priority:110,sgFoldStorageGate0:"(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"mr4",clusters:4,rows:4,divisor:16,priority:109,sgFoldStorageGate0:"(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"q4_decode_gemv_{suffix}__epi_1_0",priority:"{priority}",when:["decodeGemvContract","args.outFeatures % {divisor} == 0","32 * {clusters} <= device.limits.maxComputeInvocationsPerWorkgroup","4 * 32 * {clusters} <= device.limits.maxComputeWorkgroupStorageSize","ceilDiv(args.outFeatures, {divisor}) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","{sgFoldStorageGate0}","hasEpilogue","epilogueOk","epilogueSupported","present.bT","not present.rT"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:"{clusters}",regRows:"{rows}",outStride:"args.outStride",epiHasBias:!0,epiHasResidual:!1,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_main","q4_bits_main","q4_scales","y_f32","params__uniform"],dispatch:{x:"ceil(args.outFeatures / {divisor})"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{cases:[{fmt:"bf16",Name:"Bf16",bf16:!0,f16:null},{fmt:"f16",Name:"F16",bf16:null,f16:!0}],variants:[{id:"{fmt}_subgroup_matrix__epi_1_1",priority:110,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 64 == 0","device.limits.maxComputeWorkgroupStorageSize >= 16396","hasEpilogue","epilogueOk","epilogueSupported","present.bT","present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!0,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!1,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0",hasResidual:!0,gemmEpilogue:'"fastgelu" if args.activation == "tanh" else ("erfgelu" if args.activation == "erf" else "none")',reuseMatrixBForEpilogue:!0},passes:[{id:"main",name:"LlamaPrefillMatmul{Name}Sgmat",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y","epi_bias","epi_residual","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"}]},{id:"{fmt}_subgroup_matrix_m_tail__epi_1_1",priority:100,when:["halfWordSgmatContract",'args.format == "{fmt}"',"args.M % 64 != 0","device.limits.maxComputeWorkgroupStorageSize >= 20492","hasEpilogue","epilogueOk","epilogueSupported","present.bT","present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',bf16BStaging:"{bf16}",f16BStaging:"{f16}",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:"args.inFeatures % 32 != 0",epiHasBias:!0,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"bulk",name:"LlamaPrefillMatmul{Name}SgmatBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 64) * 64 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmul{Name}SgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 64u) * 2u"',directStore:!1},profile:{flops:"2 * (args.M % 64) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","bf16_y","bf16_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 64) / 32)"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"{fmt}_tiled_reg__epi_1_1",priority:10,when:["halfWordContract",'args.format == "{fmt}"',"ceil(args.outFeatures / 64) <= device.limits.maxComputeWorkgroupsPerDimension","16 <= device.limits.maxComputeWorkgroupSizeX","16 <= device.limits.maxComputeWorkgroupSizeY","256 <= device.limits.maxComputeInvocationsPerWorkgroup","hasEpilogue","epilogueOk","epilogueSupported","present.bT","present.rT"],derive:{yScalar:'"f32"',bf16Weights:"{bf16}",f16Weights:"{f16}",epiHasBias:!0,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmul{Name}TiledReg",shader:"prefill-matmul-tiled-reg.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["a","bf16_weights","y_y_t","bf16_params"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]},{id:"q8_subgroup_matrix_exact__epi_1_1",priority:110,when:["q8ExactSgmatContract","args.M % 64 == 0","device.limits.maxComputeWorkgroupStorageSize >= 16396","hasEpilogue","epilogueOk","epilogueSupported","present.bT","present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!0,mmaScalar:'"f32"',yScalar:'"f32"',q8BStaging:!0,tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!1,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)",packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:!1,hasResidual:!0,gemmEpilogue:'"fastgelu" if args.activation == "tanh" else ("erfgelu" if args.activation == "erf" else "none")',reuseMatrixBForEpilogue:!0},passes:[{id:"main",name:"LlamaPrefillMatmulQ8ExactSgmat",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params_m","epi_bias","epi_residual"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"}]},{id:"q8_subgroup_matrix_exact_m_tail__epi_1_1",priority:100,when:["q8ExactSgmatContract","args.M % 64 != 0","device.limits.maxComputeWorkgroupStorageSize >= 20492","hasEpilogue","epilogueOk","epilogueSupported","present.bT","present.rT"],requires:{features:["subgroups","chromium-experimental-subgroup-matrix"],subgroupMatrixConfigs:[{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8}]},derive:{hasBias:!1,mmaScalar:'"f32"',yScalar:'"f32"',q8BStaging:!0,packedMatrixNTail:"args.outFeatures % 64 != 0",packedMatrixKTail:!1,epiHasBias:!0,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"bulk",name:"LlamaPrefillMatmulQ8ExactSgmatBulk",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:64,tileN:64,tileK:32,directAStorage:!0,doubleBufferedB:!0,directStore:!0,tileGroupRows:16,tileGroupCols:"ceil(args.outFeatures / 64)"},profile:{flops:"2 * floor(args.M / 64) * 64 * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"floor(args.M / 64)"},subgroupCollectivesWidth:"portable"},{id:"tail",name:"LlamaPrefillMatmulQ8ExactSgmatTail",shader:"ops/_shared/gemm-subgroup-matrix.wgsl.jinja",derive:{tileM:32,tileN:64,tileK:32,rowTileBaseRuntime:'"(params.M / 64u) * 2u"',directStore:!1},profile:{flops:"2 * (args.M % 64) * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"},bindings:["bf16_x","bf16_w","q8_sgmat_scales","bf16_y","q8_sgmat_params_m"],dispatch:{x:"ceil(args.outFeatures / 64)",y:"ceil((args.M % 64) / 32)"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"tiled_exact__epi_1_1",priority:0,when:["q4BaseContract",'tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"ceil(args.outFeatures / (16 if args.M <= 8 else 32)) <= device.limits.maxComputeWorkgroupsPerDimension","ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","not present.realLenT","hasEpilogue","epilogueOk","epilogueSupported","present.bT","present.rT"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1 if args.M <= 8 else 2",bn:"16 if args.M <= 8 else 32",bm:"8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16)",nTile:"16 if args.M <= 8 else (4 if (args.M > 16 and args.outFeatures >= 8192) else 8)",accs:"1 if args.M <= 8 else (8 if (args.M > 16 and args.outFeatures >= 8192) else 4)",epiHasBias:!0,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / (16 if args.M <= 8 else 32)), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / (8 if args.M <= 8 else (32 if (args.M > 16 and args.outFeatures >= 8192) else 16))), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"q8_tiled_exact_large_m__epi_1_1",priority:1,when:["q4BaseContract",'fmt == "q8_0"','tensorDtypes.aT == "float32"','tensorDtypes.yT == "float32"',"args.M >= 64","ceil(args.M / 64) * ceil(args.outFeatures / 64) >= 24","16384 <= device.limits.maxComputeWorkgroupStorageSize","not args.denseTailStart","not present.realLenT","hasEpilogue","epilogueOk","epilogueSupported","present.bT","present.rT"],derive:{f16Tiles:"0",lutId:"args.lut if args.lut else 0",outStride:"args.outStride",dstColStart:"args.dstColStart",kBlocks:"1",bn:"64",bm:"64",nTile:"8",accs:"8",mPerThread:"4",epiHasBias:!0,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ8LargeM",shader:"prefill-matmul-q4.wgsl.jinja",bindings:["a_main","q4_bits","q4_scales","y_f32","params_main"],dispatch:{x:"min(ceil(args.outFeatures / 64), device.limits.maxComputeWorkgroupsPerDimension)",y:"min(ceil(args.M / 64), device.limits.maxComputeWorkgroupsPerDimension)"},profile:{flops:"2 * args.M * args.inFeatures * args.outFeatures",gemmK:"args.inFeatures"}},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{id:"q4_decode_gemv__epi_1_1",priority:105,when:["decodeGemvContract","args.outFeatures <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","(not useSubgroups and not sgRuntime) or 128 * (1) + 8 <= device.limits.maxComputeWorkgroupStorageSize","hasEpilogue","epilogueOk","epilogueSupported","present.bT","present.rT"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:"1",regRows:"1",aF16:'tensorDtypes.aT == "float16"',yF16:'tensorDtypes.yT == "float16"',yScalar:'"f16" if tensorDtypes.yT == "float16" else "f32"',outStride:"args.outStride",epiHasBias:!0,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_a_t","q4_bits_main","q4_scales","y_y_t","params__uniform"],dispatch:{x:"args.outFeatures"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]},{cases:[{suffix:"mr2",clusters:4,rows:2,divisor:8,priority:110,sgFoldStorageGate0:"(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize"},{suffix:"mr4",clusters:4,rows:4,divisor:16,priority:109,sgFoldStorageGate0:"(not useSubgroups and not sgRuntime) or 128 * (4) + 8 <= device.limits.maxComputeWorkgroupStorageSize"}],variants:[{id:"q4_decode_gemv_{suffix}__epi_1_1",priority:"{priority}",when:["decodeGemvContract","args.outFeatures % {divisor} == 0","32 * {clusters} <= device.limits.maxComputeInvocationsPerWorkgroup","4 * 32 * {clusters} <= device.limits.maxComputeWorkgroupStorageSize","ceilDiv(args.outFeatures, {divisor}) <= device.limits.maxComputeWorkgroupsPerDimension","not args.denseTailStart","{sgFoldStorageGate0}","hasEpilogue","epilogueOk","epilogueSupported","present.bT","present.rT"],derive:{lutId:"args.lut if args.lut else 0",scalesPk2:"0",dstColStart:"args.dstColStart",clusters:"{clusters}",regRows:"{rows}",outStride:"args.outStride",epiHasBias:!0,epiHasResidual:!0,epiActivation:'args.activation if args.activation is defined else "none"'},passes:[{id:"main",name:"LlamaPrefillMatmulQ4Gemv",shader:"prefill-matmul-q4-decode-gemv.wgsl.jinja",bindings:["a_main","q4_bits_main","q4_scales","y_f32","params__uniform"],dispatch:{x:"ceil(args.outFeatures / {divisor})"},subgroupCollectivesWidth:"portable"},{id:"epilogue",name:"LlamaPrefillMatmulEpilogue",shader:"prefill-matmul-epilogue.wgsl.jinja",bindings:["epi_y","epi_params","epi_bias","epi_residual"],dispatch:{x:"min(ceilDiv(args.M * args.outFeatures, 64), device.limits.maxComputeWorkgroupsPerDimension)"}}]}]}]},assets:[["prefill-lut2-quant-a.wgsl.jinja",`{% set packed4x8CompatPackI8 = true %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const K:u32={{ inFeatures }}u;const CHUNKS:u32=K/16u;const GROUP_CHUNKS:u32={{ aGroup }}u/16u;const WG:u32=64u;var<workgroup>chunkMax:array<f32,WG>;@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>){let tid=lid.x;
{% if dynamicRows | default(false) %}let m_rows=min(params.m,real_len[0]);
{% else %}let m_rows=params.m;
{% endif %}let total=m_rows*CHUNKS;let wgCount=(total+WG-1u)/WG;let groupLead=tid-tid%GROUP_CHUNKS;for(var w:u32=wg.x;w<wgCount;w=w+nwg.x){let c=w*WG+tid;let live=c<total;var v0=vec4<f32>(0.0);var v1=vec4<f32>(0.0);var v2=vec4<f32>(0.0);var v3=vec4<f32>(0.0);if(live){v0=vec4<f32>(a[c*4u]);v1=vec4<f32>(a[c*4u+1u]);v2=vec4<f32>(a[c*4u+2u]);v3=vec4<f32>(a[c*4u+3u]);}let mx=max(max(abs(v0),abs(v1)),max(abs(v2),abs(v3)));chunkMax[tid]=max(max(mx.x,mx.y),max(mx.z,mx.w));workgroupBarrier();var amax=0.0;for(var i:u32=0u;i<GROUP_CHUNKS;i=i+1u){amax=max(amax,chunkMax[groupLead+i]);}workgroupBarrier();if(live){let inv=select(0.0,127.0/amax,amax>0.0);let q0=vec4<i32>(round(v0*inv));let q1=vec4<i32>(round(v1*inv));let q2=vec4<i32>(round(v2*inv));let q3=vec4<i32>(round(v3*inv));aq[c]=vec4<u32>(pack4xI8(vec4<i32>(q0.x,q1.x,q2.x,q3.x)),pack4xI8(vec4<i32>(q0.y,q1.y,q2.y,q3.y)),pack4xI8(vec4<i32>(q0.z,q1.z,q2.z,q3.z)),pack4xI8(vec4<i32>(q0.w,q1.w,q2.w,q3.w)));if(tid==groupLead){a_scale[c/GROUP_CHUNKS]=amax/127.0;} } } }`],["prefill-matmul-dense-gemv-mrows.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const K_V4:u32={{ inFeatures }}u/4u;const DST_COL:u32={{ dstColStart }}u;const OUT_STRIDE:u32={{ outStride }}u;const CLUSTERS:u32={{ clusters }}u;const REG_ROWS:u32={{ regRows }}u;
{% include "ops/_shared/subgroup-runtime-fold.wgsl.jinja" %}{% set sgOwned = useSubgroups and (subgroupIdBuiltins | default(false)) %}{% if sgOwned %}{% include "ops/_shared/subgroup-owned-clusters.wgsl.jinja" %}{% set sgDim0 = "(wid.x * CLUSTERS + cluster) * REG_ROWS" %}{% elif useSubgroups or sgRuntime %}{% set sgSafeStorageSize = 32 * clusters %}{% include "ops/_shared/subgroup-runtime-proof.wgsl.jinja" %}{% endif %}{% if not useSubgroups and not sgRuntime %}var<workgroup>partials4:array<f32,{{ 32 * clusters }}>;
{% endif %}@compute@workgroup_size({{ 32 * clusters }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>{{ sgClusterBuiltins if sgOwned else (", @builtin(local_invocation_id) lid3: vec3<u32>" ~ (", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "")) }}){
{% for sgPath in (sgClusterPaths if sgOwned else [""]) %}{% if sgOwned %}{{ sg_cluster_open(sgPath, "CLUSTERS", clusters * regRows * mRows) }}
{% else %}let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}let cluster=lid/32u;let lane=lid%32u;
{% endif %}let dim0=(wid.x*CLUSTERS+cluster)*REG_ROWS;let wbase=params.weight_offset/4u+dim0*K_V4;
{% for m in range(mRows) %}{% for r in range(regRows) %}var acc{{ m }}_{{ r }}=0.0;
{% endfor %}{% endfor %}for(var v=lane;v<K_V4;v=v+{{ "activeCount" if sgPath == "general" else "32u" }}){
{% for r in range(regRows) %}{% if denseTailStart | default(0) %}var w{{ r }}:vec4<f32>;if(dim0+{{ r }}u<{{ denseTailStart }}u){let block=params.block_offset+(dim0+{{ r }}u)*(K_V4/8u)+v/8u;let word=q4_bits[block*4u+(v&3u)];let q=(vec4<u32>(word)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(v&4u)))&vec4<u32>(15u);let j=block&7u;let dm=unpack2x16float(q4_metadata[(block>>3u)*5u]);let sm=q4_metadata[(block>>3u)*5u+1u+(j>>1u)]>>((j&1u)*12u);let ds=dm.x*f32(sm&63u);let ms=dm.y*f32((sm>>6u)&63u);w{{ r }}=vec4<f32>(vec4<f16>(vec4<f32>(q)*ds-vec4<f32>(ms)));}else{w{{ r }}=vec4<f32>(weights[wbase+{{ r }}u*K_V4+v-{{ denseTailStart }}u*K_V4]);}
{% else %}let w{{ r }}=vec4<f32>(weights[wbase+{{ r }}u*K_V4+v]);
{% endif %}{% endfor %}{% for m in range(mRows) %}{let x=vec4<f32>(a[{{ m }}u*K_V4+v]);
{% for r in range(regRows) %}acc{{ m }}_{{ r }}=acc{{ m }}_{{ r }}+dot(w{{ r }},x);
{% endfor %}}
{% endfor %}}
{% if not useSubgroups and not sgRuntime %}{% endif %}{% for m in range(mRows) %}{% set accPrefix = "acc" ~ m ~ "_" %}{% set totalPrefix = "total" ~ m ~ "_" %}{% set sgTotalsSlot = "cluster * " ~ (mRows * regRows) ~ "u + " ~ (m * regRows) ~ "u + " %}{% include "ops/_shared/projection-multirow-reduce.wgsl.jinja" %}{% endfor %}{% set accPrefix = "acc" %}{% set totalPrefix = "total" %}{% if sgPath == "general" %}{{ sg_cluster_store("CLUSTERS") }} let dim0={{ sgDim0 }};
{% for m in range(mRows) %}{% for r in range(regRows) %}let total{{ m }}_{{ r }}=sgTotals[cluster*{{ mRows * regRows }}u+{{ m * regRows + r }}u];
{% endfor %}{% endfor %}{% endif %}if({{ "activeRank" if sgPath == "general" else "lane" }}==0u){
{% for m in range(mRows) %}{% for r in range(regRows) %}y[{{ m }}u*OUT_STRIDE+DST_COL+dim0+{{ r }}u]={{ yScalar }}(total{{ m }}_{{ r }});
{% endfor %}{% endfor %}}
{% if sgOwned %}{{ sg_cluster_close(sgPath) }}
{% endif %}{% endfor %}}`],["prefill-matmul-dense-gemv-parallel.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }}
{% include "ops/_shared/parallel-row-dot-storage.wgsl.jinja" %}{% set rowDotBase = "params.weight_offset / 4u + wid.x * " ~ ((inFeatures / 4) | int) ~ "u" %}{% set rowDotLen4 = (inFeatures / 4) | int %}{% set rowDotTerm = "dot(vec4<f32>(weights[wb+v]), vec4<f32>(a[v]))" %}{% macro row_dot_store(total) %}y[{{ dstColStart }}u + wid.x] = {{ yScalar }}({{ total }});{% endmacro %}@compute@workgroup_size({{ lanes }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_index)lid:u32{{ rowDotEntryBuiltins }}){
{% include "ops/_shared/parallel-row-dot.wgsl.jinja" %}}`],["prefill-matmul-dense-region.wgsl.jinja",'{% include "ops/_shared/flat-index-2d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(workgroupSize, "g", "") }} let total=params.m*params.out_features;if(g>=total){return;}let row=g/params.out_features;let col=g%params.out_features;let a_base=row*params.in_features;let w_base=params.weight_offset+col*params.in_features;var acc=0.0;for(var k=0u;k<params.in_features;k=k+1u){acc=acc+f32(a[a_base+k])*f32(weights[w_base+k]);}y[row*params.out_stride+params.dst_col+col]={{ yScalar }}(acc);}'],["prefill-matmul-dense-sg-small-m.wgsl.jinja",`{% set smallM = 1 %}{% include "ops/_shared/subgroup-matrix-enable.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const K:u32={{ inFeatures }}u;const K_V4:u32=K/4u;const OUT_F:u32={{ outFeatures }}u;const K_BLOCKS:u32=K/32u;
{% if sgSplitK is defined and sgSplitK %}const SPLIT_BLOCKS:u32={{ splitBlocks }}u;
{% else %}const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;
{% endif %}const TILE_ROWS:u32=16u;const TILE_COLS:u32=128u;const TILE_K:u32=32u;const SUB_COLS:u32=32u;const FRAG_DIM:u32=8u;const FRAG_ELEMS:u32=FRAG_DIM*FRAG_DIM;const WG_THREADS:u32=128u;const SG_MIN_WIDTH:u32=32u;const SUBTILES:u32=WG_THREADS/SG_MIN_WIDTH;var<workgroup>tile_A:array<f16,16*32>;var<workgroup>scratch:array<array<f32,FRAG_ELEMS>,SUBTILES>;
{% set accScalar = "f32" %}{% include "ops/_shared/subgroup-matrix-topology.wgsl.jinja" %}{% include "ops/_shared/subgroup-matrix-software-8x8.wgsl.jinja" %}{% macro matrixKernelBody(matrixSoftware) %}let m0=wid.y*TILE_ROWS;let n0=wid.x*TILE_COLS;let subtile_id=matrixRole;let base_B=subtile_id*SUB_COLS;
{% include "ops/_shared/subgroup-matrix-8x8-accumulators.wgsl.jinja" %}let am=local_idx/8u;let aq=local_idx%8u;let b_row0=min(n0+base_B,OUT_F-SUB_COLS);let b_base=params.weight_offset+b_row0*K;
{% if sgSplitK is defined and sgSplitK %}let kbEnd=min(wid.z*SPLIT_BLOCKS+SPLIT_BLOCKS,K_BLOCKS);for(var kb=wid.z*SPLIT_BLOCKS;kb<kbEnd;kb=kb+1u){
{% else %}for(var kb=0u;kb<K_BLOCKS;kb=kb+1u){
{% endif %}{% include "./prefill-quant-subgroup-stage-a.wgsl.jinja" %}workgroupBarrier();for(var step=0u;step<TILE_K;step=step+8u){let matA0={{ matrixLoad(matrixSoftware, "left", "f16", "row_major", "&tile_A", "step", "TILE_K") }};let matA1={{ matrixLoad(matrixSoftware, "left", "f16", "row_major", "&tile_A", "8u * TILE_K + step", "TILE_K") }};let boff=b_base+kb*TILE_K+step;
{% for n in range(4) %}let matB{{ n }}={{ matrixLoad(matrixSoftware, "right", "f16", "col_major", "&weights", ("boff" ~ (" + " ~ (n * 8) ~ "u * K" if n else "")), "K") }};
{% endfor %}{% for m in range(2) %}{% for n in range(4) %}matC{{ m }}{{ n }}={{ matrixMma(matrixSoftware, "f32", "matA" ~ (m), "matB" ~ (n), "matC" ~ (m) ~ (n)) }};
{% endfor %}{% endfor %}}workgroupBarrier();}let sgLane=matrixLane;
{% for ri in range(2) %}{% for ci in range(4) %}workgroupBarrier();{{ matrixStore(matrixSoftware, "row_major", "&scratch[subtile_id]", "0u", "matC" ~ (ri) ~ (ci), "FRAG_DIM") }} workgroupBarrier();for(var t=sgLane;t<FRAG_ELEMS;t=t+32u){let r={{ ri }}u*FRAG_DIM+t/FRAG_DIM;let c=base_B+{{ ci }}u*FRAG_DIM+t%FRAG_DIM;if(m0+r<params.m&&n0+c<OUT_F){
{% if sgSplitK is defined and sgSplitK %}partials[(wid.z*params.m+(m0+r))*OUT_F+n0+c]=scratch[subtile_id][t];
{% else %}y[(m0+r)*OUT_STRIDE+DST_COL+n0+c]={{ yScalar }}(scratch[subtile_id][t]);
{% endif %}} }
{% endfor %}{% endfor %}{% endmacro %}{% include "ops/_shared/subgroup-matrix-topology-entry.wgsl.jinja" %}{{ matrix_topology_entry("WG_THREADS", "WG_THREADS / 32u", "wid", "matrixLane") }}`],["prefill-matmul-dense-vector.wgsl.jinja","{{ env.wgsl.resourceDeclarations }}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){if(gid.x>={{ mSize * outFeatures }}u){return;}let row=gid.x/{{ outFeatures }}u;let col=gid.x% {{ outFeatures }}u;let ab=row*{{ (inFeatures / 4) | int }}u;let wb=params.weight_offset/4u+col*{{ (inFeatures / 4) | int }}u;var acc=0.0;for(var k=0u;k<{{ (inFeatures / 4) | int }}u;k++){let av=vec4<f32>(a[ab+k]);let wv=vec4<f32>(weights[wb+k]);acc=acc+av.x*wv.x;acc=acc+av.y*wv.y;acc=acc+av.z*wv.z;acc=acc+av.w*wv.w;}y[row*{{ outStride }}u+{{ dstColStart }}u+col]={{ yScalar }}(acc);}"],["prefill-matmul-epilogue.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }}
{% if epiActivation == "tanh" %}{% include "ops/_shared/gelu-tanh.wgsl.jinja" %}{% elif epiActivation == "erf" %}{% include "ops/_shared/gelu-erf.wgsl.jinja" %}{% endif %}@compute@workgroup_size(64)fn main(@builtin(global_invocation_id)gid:vec3<u32>,@builtin(num_workgroups)groups:vec3<u32>){let count=epiParams.M*epiParams.N;for(var e=gid.x;e<count;e+=groups.x*64u){var value=y[e];
{% if epiHasBias %}    value += bias[e % epiParams.N];
{% endif %}{% if epiHasResidual %}    value += residual[e];
{% endif %}{% if epiActivation == "tanh" %}    value = gelu_tanh(value);
{% elif epiActivation == "erf" %}    value = 0.5 * value * (1.0 + erf_approx(value * 0.7071067811865476));
{% endif %}    y[e] = value;
} }`],["prefill-matmul-lut2-dp4a.wgsl.jinja",`{% include "ops/_shared/lut2-block-scale.wgsl.jinja" %}{% set packed4x8CompatDotI8 = true %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const K:u32={{ inFeatures }}u;const N:u32={{ outFeatures }}u;const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;const BPR:u32=K/32u;const K_WORDS:u32=K/4u;const A_GROUPS:u32=K/{{ aGroup }}u;const BLOCKS_PER_GROUP:u32={{ aGroup }}u/32u;const BM:u32={{ bm }}u;const BN:u32={{ bn }}u;const K_BLOCKS:u32={{ kBlocks }}u;const RW:u32=K_BLOCKS*8u;const COL_THREADS:u32=BN/8u;const WG:u32=128u;var<workgroup>aTile:array<vec4<u32>,{{ kBlocks * 8 * bm // 4 }}>;var<workgroup>bTile:array<vec4<u32>,{{ kBlocks * 8 * bn // 4 }}>;fn widen(word:u32,p:u32)->u32{return(((word>>(2u*p))&0x03030303u)+0x7f7f7f7fu)^0x80808080u;}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>){let tid=lid.x;
{% if dynamicRows | default(false) %}let m_rows=min(params.m,real_len[0]);
{% else %}let m_rows=params.m;
{% endif %}let r4=tid/COL_THREADS;let c8=tid-r4*COL_THREADS;let n_tiles=(N+BN-1u)/BN;let m_tiles=(m_rows+BM-1u)/BM;for(var wgx:u32=wg.x;wgx<n_tiles;wgx=wgx+nwg.x){let nBase=wgx*BN;for(var wgy:u32=wg.y;wgy<m_tiles;wgy=wgy+nwg.y){let mBase=wgy*BM;
{% for i in range(32) %}var acc{{ i }}:f32=0.0;var isum{{ i }}:i32=0;
{% endfor %}for(var kt:u32=0u;kt<BPR;kt=kt+K_BLOCKS){for(var i:u32=tid;i<BM/4u*(RW/4u);i=i+WG){let sr4=i/(RW/4u);let skq=i-sr4*(RW/4u);var rows:array<vec4<u32>,4>;for(var j:u32=0u;j<4u;j=j+1u){let row=mBase+sr4*4u+j;rows[j]=vec4<u32>(0u);if(row<m_rows){rows[j]=aq[(row*K_WORDS+kt*8u)/4u+skq];} }for(var c:u32=0u;c<4u;c=c+1u){aTile[(skq*4u+c)*(BM/4u)+sr4]=vec4<u32>(rows[0][c],rows[1][c],rows[2][c],rows[3][c]);} }for(var i:u32=tid;i<BN/4u*(K_BLOCKS*2u);i=i+WG){let bc4=i/(K_BLOCKS*2u);let bh=i-bc4*(K_BLOCKS*2u);var src:array<u32,4>;var live:array<bool,4>;for(var j:u32=0u;j<4u;j=j+1u){let n=nBase+bc4*4u+j;live[j]=n<N;src[j]=0u;if(live[j]){src[j]=q4_bits[(params.block_offset+n*BPR+kt)*2u+bh];} }for(var p:u32=0u;p<4u;p=p+1u){bTile[(bh*4u+p)*(BN/4u)+bc4]=vec4<u32>(select(0u,widen(src[0],p),live[0]),select(0u,widen(src[1],p),live[1]),select(0u,widen(src[2],p),live[2]),select(0u,widen(src[3],p),live[3]));} }workgroupBarrier();for(var kw:u32=0u;kw<RW;kw=kw+1u){let av=aTile[kw*(BM/4u)+r4];let b0=bTile[kw*(BN/4u)+c8*2u];let b1=bTile[kw*(BN/4u)+c8*2u+1u];
{% set comps = ["x", "y", "z", "w"] %}{% for mm in range(4) %}{% for nn in range(8) %}isum{{ mm * 8 + nn }}=isum{{ mm * 8 + nn }}+dot4I8Packed(av.{{ comps[mm] }},b{{ nn // 4 }}.{{ comps[nn % 4] }});
{% endfor %}{% endfor %}}workgroupBarrier();if((kt+K_BLOCKS)%BLOCKS_PER_GROUP==0u){let g=kt/BLOCKS_PER_GROUP;
{% for mm in range(4) %}let ar{{ mm }}=mBase+r4*4u+{{ mm }}u;let sa{{ mm }}=select(0.0,a_scale[min(ar{{ mm }},params.m-1u)*A_GROUPS+g],ar{{ mm }}<m_rows);
{% endfor %}{% for nn in range(8) %}let bn{{ nn }}=nBase+c8*8u+{{ nn }}u;let blk{{ nn }}=params.block_offset+min(bn{{ nn }},N-1u)*BPR+kt;let sb{{ nn }}=select(0.0,{{ lut2_native_scale("blk" ~ nn, "q4_metadata", lutId) }},bn{{ nn }}<N);
{% endfor %}{% for mm in range(4) %}{% for nn in range(8) %}acc{{ mm * 8 + nn }}=fma(f32(isum{{ mm * 8 + nn }}),sa{{ mm }}*sb{{ nn }},acc{{ mm * 8 + nn }});isum{{ mm * 8 + nn }}=0;
{% endfor %}{% endfor %}} }
{% for mm in range(4) %}let m{{ mm }}=mBase+r4*4u+{{ mm }}u;if(m{{ mm }}<m_rows){
{% for nn in range(8) %}if(nBase+c8*8u+{{ nn }}u<N){
          y[m{{ mm }} * OUT_STRIDE + DST_COL + nBase + c8 * 8u + {{ nn }}u] = {% if yScalar == "f16" %}f16(acc{{ mm * 8 + nn }}){% else %}acc{{ mm * 8 + nn }}{% endif %};
}
{% endfor %}}
{% endfor %}} } }`],["prefill-matmul-q1-narrow.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const IN_WORDS:u32={{ inWords }}u;const BPR:u32={{ blocksPerRow }}u;const OUT_TILES:u32={{ outTiles }}u;const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;const IN_VEC4:u32={{ inVec4 }}u;const WORKGROUP_SIZE:u32=64u;var<workgroup>partials0:array<f32,64>;var<workgroup>partials1:array<f32,64>;var<workgroup>partials2:array<f32,64>;var<workgroup>partials3:array<f32,64>;
{% include "ops/_shared/q1-dot-helpers.wgsl.jinja" %}@compute@workgroup_size(64)
{% include "ops/_shared/decode-entry-point.wgsl.jinja" %}let t=wid.x;let lid=lid3.x;if(t>=params.m){return;}let aBase=t*IN_VEC4;for(var tile=0u;tile<OUT_TILES;tile=tile+1u){var acc0=0.0;var acc1=0.0;var acc2=0.0;var acc3=0.0;for(var word_index=lid;word_index<IN_WORDS;word_index=word_index+WORKGROUP_SIZE){let col_block=word_index/4u;let word_id=word_index%4u;let base4=aBase+word_index*8u;let n0=vec4<f32>(a[base4]);let n1=vec4<f32>(a[base4+1u]);let n2=vec4<f32>(a[base4+2u]);let n3=vec4<f32>(a[base4+3u]);let n4=vec4<f32>(a[base4+4u]);let n5=vec4<f32>(a[base4+5u]);let n6=vec4<f32>(a[base4+6u]);let n7=vec4<f32>(a[base4+7u]);let dot4=q1_dot_word_t4(params.block_offset,tile,BPR,col_block,word_id,n0,n1,n2,n3,n4,n5,n6,n7);acc0+=dot4.x;acc1+=dot4.y;acc2+=dot4.z;acc3+=dot4.w;}
{% set q1ReduceVector = true %}{% set q1ReduceComment = "" %}{% filter indent(2, true) %}{% include "ops/_shared/q1-tile4-reduce.wgsl.jinja" %}{% endfilter %}if(lid==0u){let yBase=t*OUT_STRIDE+DST_COL+tile*4u;y[yBase+0u]={{ yScalar }}(partials0[0]);y[yBase+1u]={{ yScalar }}(partials1[0]);y[yBase+2u]={{ yScalar }}(partials2[0]);y[yBase+3u]={{ yScalar }}(partials3[0]);}workgroupBarrier();} }`],["prefill-matmul-q1-sg-m16-n128-wg128.wgsl.jinja",`{% include "ops/_shared/subgroup-matrix-enable.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const K:u32={{ inFeatures }}u;const K_V4:u32=K/4u;const BPR_W:u32=K/32u;const BPR128:u32=K/128u;const SPLIT_K:u32={{ splitK }}u;const WORDS_PER_SPLIT:u32=BPR_W/SPLIT_K;const TILE_ROWS:u32=16u;const TILE_COLS:u32=128u;const TILE_K:u32=32u;var<workgroup>tile_A:array<f16,16*32>;var<workgroup>tile_B:array<f16,128*32>;var<workgroup>scratch:array<array<f32,64>,16>;
{% set accScalar = "f32" %}{% include "ops/_shared/subgroup-matrix-topology.wgsl.jinja" %}{% include "ops/_shared/subgroup-matrix-software-8x8.wgsl.jinja" %}{% macro matrixKernelBody(matrixSoftware) %}let n0=wid.x*TILE_COLS;let m0=wid.y*TILE_ROWS;let subgroup_id=matrixRole;let subgroup_col=subgroup_id*32u;let split_id=wid.z;let first_word=split_id*WORDS_PER_SPLIT;
{% include "ops/_shared/subgroup-matrix-8x8-accumulators.wgsl.jinja" %}let a_row=local_idx/8u;let a_quad=local_idx%8u;let row=n0+local_idx;let tile=row/4u;let rit=row%4u;let group_base=params.block_offset+tile*BPR128*4u;for(var slice_word=0u;slice_word<WORDS_PER_SPLIT;slice_word=slice_word+1u){let kb=first_word+slice_word;var av=vec4<f32>(0.0);if(m0+a_row<params.m){av=vec4<f32>(a[(m0+a_row)*K_V4+kb*8u+a_quad]);}let abase=a_row*TILE_K+a_quad*4u;tile_A[abase]=f16(av.x);tile_A[abase+1u]=f16(av.y);tile_A[abase+2u]=f16(av.z);tile_A[abase+3u]=f16(av.w);
{% include "./prefill-q1-subgroup-stage-b-mma.wgsl.jinja" %}}if(subgroup_id<2u){let bank=subgroup_id*8u;
{% set bank0 = "bank" %}{% filter indent(4, true) %}{% include "./subgroup-matrix-store-8-banks.wgsl.jinja" %}{% endfilter %}}workgroupBarrier();for(var i=local_idx;i<TILE_ROWS*64u;i=i+128u){let r=i/64u;let c=i%64u;if(m0+r<params.m){let b=(c/32u)*8u+(r/8u)*4u+((c%32u)/8u);let e=(r%8u)*8u+(c%8u);partials[(split_id*params.m+m0+r)*{{ outFeatures }}u+n0+c]=scratch[b][e];} }workgroupBarrier();if(subgroup_id>=2u){let bank=(subgroup_id-2u)*8u;
{% set bank0 = "bank" %}{% filter indent(4, true) %}{% include "./subgroup-matrix-store-8-banks.wgsl.jinja" %}{% endfilter %}}workgroupBarrier();for(var i=local_idx;i<TILE_ROWS*64u;i=i+128u){let r=i/64u;let c=i%64u;let out_c=64u+c;if(m0+r<params.m){let b=(c/32u)*8u+(r/8u)*4u+((c%32u)/8u);let e=(r%8u)*8u+(c%8u);partials[(split_id*params.m+m0+r)*{{ outFeatures }}u+n0+out_c]=scratch[b][e];} }
{% endmacro %}@compute@workgroup_size(128,1,1){{ " @subgroup_size(32)" if pinSubgroupSize32 else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_index)local_idx:u32,@builtin(subgroup_size)matrixPhysicalWidth:u32){let matrixRole=claimMatrixSubgroupRole(matrixPhysicalWidth,32u);let matrixAcceptedHere=matrixTopologyAccepted(local_idx,128u/32u);matrixSoftwareLane=local_idx%32u;if(matrixAcceptedHere){ {{ matrixKernelBody(false) }} }else{let matrixRole=local_idx/32u;{{ matrixKernelBody(true) }} } }`],["prefill-matmul-q1-sg-small-m-wg64-n64.wgsl.jinja",`{% include "ops/_shared/subgroup-matrix-enable.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const K:u32={{ inFeatures }}u;const K_V4:u32=K/4u;const BPR_W:u32=K/32u;const BPR128:u32=K/128u;const SPLIT_K:u32={{ splitK }}u;const WORDS_PER_SPLIT:u32=BPR_W/SPLIT_K;const TILE_ROWS:u32=16u;const TILE_COLS:u32=64u;const TILE_K:u32=32u;var<workgroup>tile_A:array<f16,16*32>;var<workgroup>tile_B:array<f16,64*32>;var<workgroup>scratch:array<array<f32,64>,16>;
{% set accScalar = "f32" %}{% include "ops/_shared/subgroup-matrix-topology.wgsl.jinja" %}{% include "ops/_shared/subgroup-matrix-software-8x8.wgsl.jinja" %}{% macro matrixKernelBody(matrixSoftware) %}let n0=wid.x*TILE_COLS;let m0=wid.y*TILE_ROWS;let subgroup_id=matrixRole;let subgroup_col=subgroup_id*32u;let split_id=wid.z;let first_word=split_id*WORDS_PER_SPLIT;
{% include "ops/_shared/subgroup-matrix-8x8-accumulators.wgsl.jinja" %}let a_row=local_idx/4u;let a_quad=(local_idx%4u)*2u;let row=n0+local_idx;let tile=row/4u;let rit=row%4u;let group_base=params.block_offset+tile*BPR128*4u;for(var slice_word=0u;slice_word<WORDS_PER_SPLIT;slice_word=slice_word+1u){let kb=first_word+slice_word;var av0=vec4<f32>(0.0);var av1=vec4<f32>(0.0);if(m0+a_row<params.m){av0=vec4<f32>(a[(m0+a_row)*K_V4+kb*8u+a_quad]);av1=vec4<f32>(a[(m0+a_row)*K_V4+kb*8u+a_quad+1u]);}let abase=a_row*TILE_K+a_quad*4u;tile_A[abase]=f16(av0.x);tile_A[abase+1u]=f16(av0.y);tile_A[abase+2u]=f16(av0.z);tile_A[abase+3u]=f16(av0.w);tile_A[abase+4u]=f16(av1.x);tile_A[abase+5u]=f16(av1.y);tile_A[abase+6u]=f16(av1.z);tile_A[abase+7u]=f16(av1.w);
{% include "./prefill-q1-subgroup-stage-b-mma.wgsl.jinja" %}}let bank=subgroup_id*8u;
{% set bank0 = "bank" %}{% filter indent(2, true) %}{% include "./subgroup-matrix-store-8-banks.wgsl.jinja" %}{% endfilter %}workgroupBarrier();for(var i=local_idx;i<TILE_ROWS*TILE_COLS;i=i+64u){let r=i/TILE_COLS;let c=i%TILE_COLS;if(m0+r<params.m){let b=(c/32u)*8u+(r/8u)*4u+((c%32u)/8u);let e=(r%8u)*8u+(c%8u);partials[(split_id*params.m+m0+r)*{{ outFeatures }}u+n0+c]=scratch[b][e];} }
{% endmacro %}@compute@workgroup_size(64,1,1){{ " @subgroup_size(32)" if pinSubgroupSize32 else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_index)local_idx:u32,@builtin(subgroup_size)matrixPhysicalWidth:u32){let matrixRole=claimMatrixSubgroupRole(matrixPhysicalWidth,32u);let matrixAcceptedHere=matrixTopologyAccepted(local_idx,64u/32u);matrixSoftwareLane=local_idx%32u;if(matrixAcceptedHere){ {{ matrixKernelBody(false) }} }else{let matrixRole=local_idx/32u;{{ matrixKernelBody(true) }} } }`],["prefill-matmul-q1-sg.wgsl.jinja",`{% include "ops/_shared/subgroup-matrix-enable.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const K:u32={{ inFeatures }}u;const K_V4:u32=K/4u;const BPR_W:u32=K/32u;const BPR128:u32=K/128u;const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;const TILE_ROWS:u32=32u;const TILE_COLS:u32=64u;const TILE_K:u32=32u;const SUB_ROWS:u32=16u;const SUB_COLS:u32=32u;const FRAG_DIM:u32=8u;const FRAG_ELEMS:u32=FRAG_DIM*FRAG_DIM;const WG_THREADS:u32=128u;const SG_MIN_WIDTH:u32=32u;const SUBTILES:u32=WG_THREADS/SG_MIN_WIDTH;var<workgroup>tile_A:array<f16,32*32>;var<workgroup>tile_B:array<f16,64*32>;var<workgroup>scratch:array<array<f32,FRAG_ELEMS>,SUBTILES>;
{% set accScalar = "f32" %}{% include "ops/_shared/subgroup-matrix-topology.wgsl.jinja" %}{% include "ops/_shared/subgroup-matrix-software-8x8.wgsl.jinja" %}{% macro matrixKernelBody(matrixSoftware) %}let m0=wid.y*TILE_ROWS;let n0=wid.x*TILE_COLS;let subtile_id=matrixRole;let subtile_idx=subtile_id/2u;let subtile_idy=subtile_id%2u;let base_A=subtile_idy*SUB_ROWS;let base_B=subtile_idx*SUB_COLS;
{% include "ops/_shared/subgroup-matrix-8x8-accumulators.wgsl.jinja" %}let am=local_idx/4u;let aq=(local_idx%4u)*2u;let l=local_idx&63u;let row=n0+l;let tile=row/4u;let rit=row%4u;let group_base=params.block_offset+tile*BPR128*4u;for(var kb=0u;kb<BPR_W;kb=kb+1u){
{% include "./prefill-quant-subgroup-stage-a.wgsl.jinja" %}if(local_idx<TILE_COLS){let col_block=kb/4u;let word_id=kb%4u;let idx=group_base+col_block*4u+word_id;let sign_word=q1_bits[idx*4u+rit];let scale=f16(f32(q1_scales[group_base+col_block*4u+rit]));let neg=-scale;let bbase=l*TILE_K;for(var j=0u;j<32u;j=j+1u){tile_B[bbase+j]=select(neg,scale,((sign_word>>j)&1u)==1u);} }workgroupBarrier();
{% include "./prefill-subgroup-matmul-epilogue.wgsl.jinja" %}{% endmacro %}{% include "ops/_shared/subgroup-matrix-topology-entry.wgsl.jinja" %}{{ matrix_topology_entry("WG_THREADS", "WG_THREADS / 32u", "wid", "matrixLane") }}`],["prefill-matmul-q1-small-m-splitk-reduce.wgsl.jinja","{{ env.wgsl.resourceDeclarations }} const M:u32={{ mSize }}u;const N:u32={{ outFeatures }}u;const SPLIT_K:u32={{ splitK }}u;const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;@compute@workgroup_size({{ splitKReduceWorkgroupSize }},1,1)fn main(@builtin(global_invocation_id)gid:vec3<u32>){let output_index=gid.x;if(output_index>=M*N){return;}var sum=0.0;for(var split=0u;split<SPLIT_K;split=split+1u){sum+=partials[split*M*N+output_index];}let row=output_index/N;let col=output_index%N;y[row*OUT_STRIDE+DST_COL+col]={{ yScalar }}(sum);}"],["prefill-matmul-q1-tiled.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const K:u32={{ inFeatures }}u;const N:u32={{ outFeatures }}u;const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;const BLOCK_OFFSET:u32={{ blockOffset }}u;const BPR:u32=K/128u;const K_V4:u32=K/4u;const BM:u32={{ bm }}u;const BN:u32=32u;const BK_V4:u32=32u;const N_TILE:u32={{ nTile }}u;const WG:u32=128u;var<workgroup>a_tile:array<vec4<f16>,{{ bm * 32 }}>;var<workgroup>b_tile:array<vec4<f16>,1024>;
{% include "ops/_shared/q1-signs.wgsl.jinja" %}fn q1_scale(index:u32)->f32{
{% if scaleScalar == "f32" %}return q1_scales[index];
{% else %}return f32(q1_scales[index]);
{% endif %}}@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>){let tid=lid3.x;let m_local=tid/N_TILE;let n_local=tid-m_local*N_TILE;let n_tiles=(N+BN-1u)/BN;let m_tiles=(params.m+BM-1u)/BM;for(var tile_n=wid.x;tile_n<n_tiles;tile_n=tile_n+nwg.x){let n_base=tile_n*BN;for(var tile_m=wid.y;tile_m<m_tiles;tile_m=tile_m+nwg.y){let m_base=tile_m*BM;let m=m_base+m_local;
{% for nn in range(accs) %}var acc{{ nn }}=0.0;
{% endfor %}for(var kb=0u;kb<BPR;kb=kb+1u){let kv_base=kb*BK_V4;for(var i=tid;i<BM*BK_V4;i=i+WG){let tm=i/BK_V4;let kv=i-tm*BK_V4;let gm=m_base+tm;if(gm<params.m){a_tile[i]=vec4<f16>(a[gm*K_V4+kv_base+kv]);}else{a_tile[i]=vec4<f16>(0.0h);} }for(var i=tid;i<BN*BK_V4;i=i+WG){let tn=i/BK_V4;let kv=i-tn*BK_V4;let row=n_base+tn;var decoded=vec4<f32>(0.0);if(row<N){let row_tile=row/4u;let row_lane=row&3u;let word_id=kv/8u;let shift=(kv&7u)*4u;let group=row_tile*BPR+kb;let sign_word=q1_bits[(BLOCK_OFFSET+group*4u+word_id)*4u+row_lane];let scale_index=BLOCK_OFFSET+group*4u+row_lane;decoded=q1_signs4_unit(sign_word,shift)*q1_scale(scale_index);}b_tile[i]=vec4<f16>(decoded);}workgroupBarrier();if(m<params.m){for(var kv=0u;kv<BK_V4;kv=kv+1u){let av=vec4<f32>(a_tile[m_local*BK_V4+kv]);
{% for nn in range(accs) %}acc{{ nn }}+=dot(av,vec4<f32>(b_tile[(n_local+{{ nn * nTile }}u)*BK_V4+kv]));
{% endfor %}} }workgroupBarrier();}if(m<params.m){let y_base=m*OUT_STRIDE+DST_COL+n_base+n_local;
{% for nn in range(accs) %}if(n_base+n_local+{{ nn * nTile }}u<N){y[y_base+{{ nn * nTile }}u]={{ yScalar }}(acc{{ nn }});}
{% endfor %}} } } }`],["prefill-matmul-q4-decode-gemv.wgsl.jinja",`{% set emitQuantDotHelpers = not ((lut2 | default(false)) or (lut16 | default(false)) or ((lutc | default(false)) and (lutc16 | default(false)))) %}{% set emitQuantValueHelpers = not (lut2 | default(false)) %}{% set emitLut2DotHelper = not (lutc16 | default(false)) %}{% set nibbleHelpers = not (q5 or q2k or q3k or q5k or q6k) %}{% set aF16 = aF16 if aF16 is defined else false %}{% set yF16 = yF16 if yF16 is defined else false %}{% set mRows = mRows %}{% set packed4x8CompatUnpackI8 = (quantBits == 8 and emitQuantDotHelpers) or q3k or q6k %}{% set packed4x8CompatUnpackU8 = not lutId and quantBits != 8 %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const BPR:u32={{ blocksPerRow }}u;const DST_COL:u32={{ dstColStart }}u;const CLUSTERS:u32={{ clusters }}u;const REG_ROWS:u32={{ regRows }}u;
{% if mRows > 1 %}const OUT_STRIDE:u32={{ outStride }}u;const A_ROW_V4:u32=BPR*8u;
{% endif %}{% include "ops/_shared/subgroup-runtime-fold.wgsl.jinja" %}{% set sgOwned = useSubgroups and (subgroupIdBuiltins | default(false)) %}{% if sgOwned %}{% include "ops/_shared/subgroup-owned-clusters.wgsl.jinja" %}{% set sgDim0 = "(wid.x * CLUSTERS + cluster) * REG_ROWS" %}{% elif useSubgroups or sgRuntime %}{% set sgSafeStorageSize = 32 * clusters %}{% include "ops/_shared/subgroup-runtime-proof.wgsl.jinja" %}{% endif %}{% if not useSubgroups and not sgRuntime %}var<workgroup>partials4:array<f32,{{ 32 * clusters }}>;
{% endif %}{% if lutId %}{% include "ops/_shared/q4nl-dequant-helpers.wgsl.jinja" %}{% else %}{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% endif %}@compute@workgroup_size({{ 32 * clusters }}){{ " @subgroup_size(32)" if (useSubgroups and pinSubgroupSize32) else "" }} fn main(@builtin(workgroup_id)wid:vec3<u32>{{ sgClusterBuiltins if sgOwned else (", @builtin(local_invocation_id) lid3: vec3<u32>" ~ (", @builtin(subgroup_invocation_id) sg_lid: u32, @builtin(subgroup_size) sg_size: u32" if (useSubgroups or sgRuntime) else "")) }}){
{% for sgPath in (sgClusterPaths if sgOwned else [""]) %}{% if sgOwned %}{{ sg_cluster_open(sgPath, "CLUSTERS", clusters * regRows * mRows) }}
{% else %}let lid=lid3.x;
{% if useSubgroups or sgRuntime %}let sgNative=sg_native_layout(lid,sg_lid,sg_size);
{% endif %}let cluster=lid/32u;let lane=lid%32u;
{% endif %}let dim0=(wid.x*CLUSTERS+cluster)*REG_ROWS;let base0=params.block_offset+dim0*BPR;
{% for r in range(regRows) %}var acc{{ r }}=0.0;
{% endfor %}{% for m in range(1, mRows) %}{% for r in range(regRows) %}var accM{{ m }}_{{ r }}=0.0;
{% endfor %}{% endfor %}for(var b=lane;b<BPR;b=b+{{ "activeCount" if sgPath == "general" else "32u" }}){let hbase=b*8u;let x0={{ "vec4<f32>(a[hbase])" if aF16 else "a[hbase]" }};let x1={{ "vec4<f32>(a[hbase + 1u])" if aF16 else "a[hbase + 1u]" }};let x2={{ "vec4<f32>(a[hbase + 2u])" if aF16 else "a[hbase + 2u]" }};let x3={{ "vec4<f32>(a[hbase + 3u])" if aF16 else "a[hbase + 3u]" }};let x4={{ "vec4<f32>(a[hbase + 4u])" if aF16 else "a[hbase + 4u]" }};let x5={{ "vec4<f32>(a[hbase + 5u])" if aF16 else "a[hbase + 5u]" }};let x6={{ "vec4<f32>(a[hbase + 6u])" if aF16 else "a[hbase + 6u]" }};let x7={{ "vec4<f32>(a[hbase + 7u])" if aF16 else "a[hbase + 7u]" }};
{% include "ops/_shared/q4-decode-row-block-accumulate.wgsl.jinja" %}{% for m in range(1, mRows) %}{let hbase{{ m }}={{ m }}u*A_ROW_V4+hbase;let x0={{ "vec4<f32>(a[hbase" ~ m ~ "])" if aF16 else "a[hbase" ~ m ~ "]" }};let x1={{ "vec4<f32>(a[hbase" ~ m ~ " + 1u])" if aF16 else "a[hbase" ~ m ~ " + 1u]" }};let x2={{ "vec4<f32>(a[hbase" ~ m ~ " + 2u])" if aF16 else "a[hbase" ~ m ~ " + 2u]" }};let x3={{ "vec4<f32>(a[hbase" ~ m ~ " + 3u])" if aF16 else "a[hbase" ~ m ~ " + 3u]" }};let x4={{ "vec4<f32>(a[hbase" ~ m ~ " + 4u])" if aF16 else "a[hbase" ~ m ~ " + 4u]" }};let x5={{ "vec4<f32>(a[hbase" ~ m ~ " + 5u])" if aF16 else "a[hbase" ~ m ~ " + 5u]" }};let x6={{ "vec4<f32>(a[hbase" ~ m ~ " + 6u])" if aF16 else "a[hbase" ~ m ~ " + 6u]" }};let x7={{ "vec4<f32>(a[hbase" ~ m ~ " + 7u])" if aF16 else "a[hbase" ~ m ~ " + 7u]" }};
{% set accPrefix = "accM" ~ m ~ "_" %}{% include "ops/_shared/q4-decode-row-block-accumulate.wgsl.jinja" %}{% set accPrefix = "acc" %}}
{% endfor %}}
{% if not useSubgroups and not sgRuntime and mRows > 1 %}{% endif %}{% for m in range(mRows) %}{% set accPrefix = "acc" if m == 0 else "accM" ~ m ~ "_" %}{% set totalPrefix = "total" if m == 0 else "totalM" ~ m ~ "_" %}{% set sgTotalsSlot = "cluster * " ~ (mRows * regRows) ~ "u + " ~ (m * regRows) ~ "u + " %}{% include "ops/_shared/projection-multirow-reduce.wgsl.jinja" %}{% endfor %}{% set accPrefix = "acc" %}{% set totalPrefix = "total" %}{% if sgPath == "general" %}{{ sg_cluster_store("CLUSTERS") }} let dim0={{ sgDim0 }};
{% for m in range(mRows) %}{% for r in range(regRows) %}let {{ "total" if m == 0 else "totalM" ~ m ~ "_" }}{{ r }}=sgTotals[cluster*{{ mRows * regRows }}u+{{ m * regRows + r }}u];
{% endfor %}{% endfor %}{% endif %}if({{ "activeRank" if sgPath == "general" else "lane" }}==0u){
{% for m in range(mRows) %}{% set rowTotal = "total" if m == 0 else "totalM" ~ m ~ "_" %}{% set rowBase = "" if m == 0 else m ~ "u * OUT_STRIDE + " %}{% for r in range(regRows) %}{% if yF16 %}y[{{ rowBase }}DST_COL+dim0+{{ r }}u]=f16({{ rowTotal }}{{ r }});
{% else %}y[{{ rowBase }}DST_COL+dim0+{{ r }}u]={{ rowTotal }}{{ r }};
{% endif %}{% endfor %}{% endfor %}}
{% if sgOwned %}{{ sg_cluster_close(sgPath) }}
{% endif %}{% endfor %}}`],["prefill-matmul-q4-reduce.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const N:u32={{ outFeatures }}u;const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;const K_SPLITS:u32={{ kSplits }}u;@compute@workgroup_size({{ splitKReduceWorkgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(splitKReduceWorkgroupSize, "flat", "params.m * N") }} let m=flat/N;let n=flat%N;
{% if dynamicRows is defined and dynamicRows %}if(m>=real_len[0]){return;}
{% endif %}var acc=0.0;for(var z=0u;z<K_SPLITS;z=z+1u){acc=acc+partials[(z*params.m+m)*N+n];}y[m*OUT_STRIDE+DST_COL+n]={{ yScalar }}(acc);}`],["prefill-matmul-q4-sg.wgsl.jinja",`{% include "ops/_shared/lut2-block-scale.wgsl.jinja" %}{% set emitQuantDotHelpers = false %}{% set emitQuantValueHelpers = not (lut2 | default(false)) %}{% set emitLut2DotHelper = false %}{% set nibbleHelpers = not (q4knCompact or q5 or q2k or q3k or q5k or q6k) %}{% include "ops/_shared/subgroup-matrix-enable.wgsl.jinja" %}{% set packed4x8CompatUnpackI8 = quantBits == 8 or q3k or q6k or (iq1m | default(false)) %}{% set packed4x8CompatUnpackU8 = (iq2xxs | default(false)) or (iq2xs | default(false)) or (iq2s | default(false)) or ((((iq3xxs | default(false)) or (iq3s | default(false))) and (iqGridBound | default(false)))) or (not ((iq1m | default(false)) or (iq2xxs | default(false)) or (iq2xs | default(false)) or (iq2s | default(false)) or (iq3xxs | default(false)) or (iq3s | default(false))) and not lutId and quantBits != 8) %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const K:u32={{ inFeatures }}u;const K_V4:u32=K/4u;const BPR:u32=K/32u;
{% if sgSplitK is defined and sgSplitK %}const N_OUT:u32={{ outFeatures }}u;const SPLIT_BLOCKS:u32={{ splitBlocks }}u;
{% else %}const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;
{% endif %}{% set wideNativeTile = wideNativeTile | default(false) %}{% if wideNativeTile %}const TILE_ROWS:u32=32u;const TILE_COLS:u32=128u;
{% elif smallM %}const TILE_ROWS:u32=16u;const TILE_COLS:u32=128u;
{% else %}const TILE_ROWS:u32=32u;const TILE_COLS:u32=64u;
{% endif %}const TILE_K:u32=32u;const SUB_ROWS:u32={{ 32 if tallSubtile else 16 }}u;const SUB_COLS:u32=32u;
{% if tallSubtile %}{% set stageAVecs = 2 if wideNativeTile else 4 %}{% set sgmatATiles = 4 %}{% endif %}const FRAG_DIM:u32=8u;const FRAG_ELEMS:u32=FRAG_DIM*FRAG_DIM;const WG_THREADS:u32={{ 64 if tallSubtile and not wideNativeTile else 128 }}u;const SG_MIN_WIDTH:u32=32u;const SUBTILES:u32=WG_THREADS/SG_MIN_WIDTH;var<workgroup>tile_A:array<f16,{{ 16 * 32 if smallM else 32 * 32 }}>;var<workgroup>tile_B:array<f16,{{ 128 * 32 if smallM or wideNativeTile else 64 * 32 }}>;
{% if wideNativeTile and (resultDrainBanks | default(1)) == 4 -%}var<workgroup>scratch:array<array<f32,FRAG_ELEMS*4u>,SUBTILES>;
{% else -%}var<workgroup>scratch:array<array<f32,FRAG_ELEMS>,SUBTILES>;
{% endif -%}{% if (iq1m is defined and iq1m) or (iq2xxs is defined and iq2xxs) or (iq2xs is defined and iq2xs) or (iq3xxs is defined and iq3xxs) or (iq3s is defined and iq3s) or (iq2s is defined and iq2s) %}{% include "ops/_shared/iq-grid.wgsl.jinja" %}{% include "ops/_shared/iqx-raw-dequant-helpers.wgsl.jinja" %}{% elif lutId %}{% if not (wideNativeTile and lutId in [8, 9]) %}{% include "ops/_shared/q4nl-dequant-helpers.wgsl.jinja" %}{% endif %}{% else %}{% include "ops/_shared/q4-dequant-helpers.wgsl.jinja" %}{% endif %}{% set accScalar = "f32" %}{% include "ops/_shared/subgroup-matrix-topology.wgsl.jinja" %}{% include "ops/_shared/subgroup-matrix-software-8x8.wgsl.jinja" %}{% if wideNativeTile and lutId in [8, 9] %}fn native_q2_stage(word:u32,sh:u32,values:vec4<f16>)->vec4<f16>{let codes=(vec4<u32>(word>>sh)>>vec4<u32>(0u,2u,4u,6u))&vec4<u32>(3u);let odd=(codes&vec4<u32>(1u))!=vec4<u32>(0u);let low=select(vec4<f16>(values.x),vec4<f16>(values.y),odd);let high=select(vec4<f16>(values.z),vec4<f16>(values.w),odd);return select(low,high,(codes&vec4<u32>(2u))!=vec4<u32>(0u));}
{% endif %}{% macro matrixKernelBody(matrixSoftware) %}let m0=wid.y*TILE_ROWS;let n0=wid.x*TILE_COLS;
{% if dynamicRows is defined and dynamicRows %}if(m0>=real_len[0]){return;}let row_limit=min(params.m,real_len[0]);
{% endif %}{% if staticSubgroupSchedule | default(false) %}let subtile_id=matrixRole;
{% else %}let subtile_id=matrixRole;
{% endif %}{% if smallM or tallSubtile %}let subtile_idx=subtile_id;let subtile_idy=0u;
{% else %}{% if staticSubgroupSchedule | default(false) %}let subtile_idx=subtile_id>>1u;let subtile_idy=subtile_id&1u;
{% else %}let subtile_idx=subtile_id/2u;let subtile_idy=subtile_id%2u;
{% endif %}{% endif %}let base_A=subtile_idy*SUB_ROWS;let base_B=subtile_idx*SUB_COLS;
{% include "ops/_shared/subgroup-matrix-8x8-accumulators.wgsl.jinja" %}{% if tallSubtile %}  var matC20: {% set matrixTypeScalar361 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar361) }};
  var matC21: {% set matrixTypeScalar362 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar362) }};
  var matC22: {% set matrixTypeScalar363 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar363) }};
  var matC23: {% set matrixTypeScalar364 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar364) }};
  var matC30: {% set matrixTypeScalar365 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar365) }};
  var matC31: {% set matrixTypeScalar366 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar366) }};
  var matC32: {% set matrixTypeScalar367 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar367) }};
  var matC33: {% set matrixTypeScalar368 %}f32{% endset %}{{ matrixType(matrixSoftware, "result", matrixTypeScalar368) }};
{% endif %}{% if wideNativeTile %}let am=local_idx/4u;let aq=(local_idx%4u)*2u;let l=min(local_idx,{{ outFeatures }}u-1u-n0);
{% elif smallM %}let am=local_idx/8u;let aq=local_idx%8u;let l=min(local_idx,{{ outFeatures }}u-1u-n0);
{% elif tallSubtile %}let am=local_idx/2u;let aq=(local_idx%2u)*4u;let l=local_idx;
{% else %}let am=local_idx/4u;let aq=(local_idx%4u)*2u;let l=local_idx&63u;
{% endif %}let row_base=params.block_offset+(n0+l)*BPR;
{% if q4knCompact is defined and q4knCompact %}var dSuper4nc=0.0;var dmSuper4nc=0.0;
{% endif %}{% if sgSplitK is defined and sgSplitK %}let kbEnd=min(wid.z*SPLIT_BLOCKS+SPLIT_BLOCKS,BPR);for(var kb=wid.z*SPLIT_BLOCKS;kb<kbEnd;kb=kb+1u){
{% else %}for(var kb=0u;kb<BPR;kb=kb+1u){
{% endif %}{% include "./prefill-quant-subgroup-stage-a.wgsl.jinja" %}if(local_idx<TILE_COLS){let blk=row_base+kb;
{% if q4knCompact is defined and q4knCompact %}let sb4nc=blk>>3u;let j4nc=blk&7u;let smBase4nc=sb4nc*5u;let smCarrier4nc=q4_metadata[smBase4nc+1u+(j4nc>>1u)];let smPair4nc=smCarrier4nc>>((j4nc&1u)*12u);let sc4nc=smPair4nc&63u;let mn4nc=(smPair4nc>>6u)&63u;if(j4nc==0u){let dm4nc=unpack2x16float(q4_metadata[smBase4nc]);dSuper4nc=dm4nc.x;dmSuper4nc=dm4nc.y;}let dw4nc=q4_bits[blk];var w0=vec4<f32>(unpack4xU8(dw4nc.x&0x0F0F0F0Fu));var w1=vec4<f32>(unpack4xU8(dw4nc.y&0x0F0F0F0Fu));var w2=vec4<f32>(unpack4xU8(dw4nc.z&0x0F0F0F0Fu));var w3=vec4<f32>(unpack4xU8(dw4nc.w&0x0F0F0F0Fu));var w4=vec4<f32>(unpack4xU8((dw4nc.x>>4u)&0x0F0F0F0Fu));var w5=vec4<f32>(unpack4xU8((dw4nc.y>>4u)&0x0F0F0F0Fu));var w6=vec4<f32>(unpack4xU8((dw4nc.z>>4u)&0x0F0F0F0Fu));var w7=vec4<f32>(unpack4xU8((dw4nc.w>>4u)&0x0F0F0F0Fu));
{% elif q5k is defined and q5k %}let sb5=blk>>3u;let j5=blk&7u;let g5=sb5*11u;let dw5=q4_bits[g5+j5];let qh5=q4_bits[g5+8u+(j5>>2u)][j5&3u];let sm5=q4_bits[g5+10u];let sc5=f32((sm5[j5>>2u]>>((j5&3u)*8u))&0xFFu);let mn5=f32((sm5[2u+(j5>>2u)]>>((j5&3u)*8u))&0xFFu);var w0=q5k_lo(dw5.x,qh5&15u);var w1=q5k_lo(dw5.y,(qh5>>4u)&15u);var w2=q5k_lo(dw5.z,(qh5>>8u)&15u);var w3=q5k_lo(dw5.w,(qh5>>12u)&15u);var w4=q5k_hi(dw5.x,(qh5>>16u)&15u);var w5=q5k_hi(dw5.y,(qh5>>20u)&15u);var w6=q5k_hi(dw5.z,(qh5>>24u)&15u);var w7=q5k_hi(dw5.w,(qh5>>28u)&15u);
{% elif q2k %}let sb2=blk>>3u;let j2=blk&7u;let g2=sb2*5u;let cv2=q4_bits[g2+(j2>>1u)];let cLo2=cv2[(j2&1u)*2u];let cHi2=cv2[(j2&1u)*2u+1u];let sm2=q4_bits[g2+4u][j2>>1u];let bLo2=(sm2>>((j2&1u)*16u))&0xFFu;let bHi2=(sm2>>((j2&1u)*16u+8u))&0xFFu;var w0=q2w(cLo2,0u);var w1=q2w(cLo2,1u);var w2=q2w(cLo2,2u);var w3=q2w(cLo2,3u);var w4=q2w(cHi2,0u);var w5=q2w(cHi2,1u);var w6=q2w(cHi2,2u);var w7=q2w(cHi2,3u);
{% elif q3k %}let sb3=blk>>3u;let j3=blk&7u;let g3=sb3*7u;let cv3=q4_bits[g3+(j3>>1u)];let cLo3=cv3[(j3&1u)*2u];let cHi3=cv3[(j3&1u)*2u+1u];let hw3=q4_bits[g3+4u+(j3>>2u)][j3&3u];let s83=unpack4xI8(q4_bits[g3+6u][j3>>1u]);var w0=q3w_lo(cLo3,hw3,0u);var w1=q3w_lo(cLo3,hw3,1u);var w2=q3w_lo(cLo3,hw3,2u);var w3=q3w_lo(cLo3,hw3,3u);var w4=q3w_hi(cHi3,hw3,0u);var w5=q3w_hi(cHi3,hw3,1u);var w6=q3w_hi(cHi3,hw3,2u);var w7=q3w_hi(cHi3,hw3,3u);
{% elif q6k %}let sb6=blk>>3u;let j6=blk&7u;let g6=sb6*13u;let dw6=q4_bits[g6+j6];let hv6=q4_bits[g6+8u+(j6>>1u)];let hLo6=hv6[(j6&1u)*2u];let hHi6=hv6[(j6&1u)*2u+1u];let s86=unpack4xI8(q4_bits[g6+12u][j6>>1u]);var w0=q6_lo(dw6.x,hLo6,0u);var w1=q6_lo(dw6.y,hLo6,1u);var w2=q6_lo(dw6.z,hLo6,2u);var w3=q6_lo(dw6.w,hLo6,3u);var w4=q6_hi(dw6.x,hHi6,0u);var w5=q6_hi(dw6.y,hHi6,1u);var w6=q6_hi(dw6.z,hHi6,2u);var w7=q6_hi(dw6.w,hHi6,3u);
{% elif iq3xxs is defined and iq3xxs %}let iw0=q4_bits[blk*3u];let iw1=q4_bits[blk*3u+1u];let isw=q4_bits[blk*3u+2u];let idb=f32(q4_scales[blk>>3u])*(0.5+f32(isw>>28u))*0.5;let isa=ksg(isw&127u);var w0=idb*sgx(iq3mag(iw0&0xFFu),isa,SGX_LO);var w1=idb*sgx(iq3mag((iw0>>8u)&0xFFu),isa,SGX_HI);let isb=ksg((isw>>7u)&127u);var w2=idb*sgx(iq3mag((iw0>>16u)&0xFFu),isb,SGX_LO);var w3=idb*sgx(iq3mag(iw0>>24u),isb,SGX_HI);let isc=ksg((isw>>14u)&127u);var w4=idb*sgx(iq3mag(iw1&0xFFu),isc,SGX_LO);var w5=idb*sgx(iq3mag((iw1>>8u)&0xFFu),isc,SGX_HI);let isd=ksg((isw>>21u)&127u);var w6=idb*sgx(iq3mag((iw1>>16u)&0xFFu),isd,SGX_LO);var w7=idb*sgx(iq3mag(iw1>>24u),isd,SGX_HI);
{% elif iq3s is defined and iq3s %}let sbw=(blk>>3u)*27u;let sk=blk&7u;let sw0=q4_bits[sbw+3u*sk];let sw1=q4_bits[sbw+3u*sk+1u];let ssg=q4_bits[sbw+3u*sk+2u];let sqh=(q4_bits[sbw+24u+(sk>>2u)]>>((sk&3u)*8u))&0xFFu;let sdb=f32(q4_scales[blk>>3u])*(1.0+2.0*f32((q4_bits[sbw+26u]>>(sk*4u))&0xFu));var w0=sdb*sgx(iq3mag((sw0&0xFFu)|((sqh<<8u)&0x100u)),ssg,SGX_LO);var w1=sdb*sgx(iq3mag(((sw0>>8u)&0xFFu)|((sqh<<7u)&0x100u)),ssg>>4u,SGX_LO);var w2=sdb*sgx(iq3mag(((sw0>>16u)&0xFFu)|((sqh<<6u)&0x100u)),ssg>>8u,SGX_LO);var w3=sdb*sgx(iq3mag((sw0>>24u)|((sqh<<5u)&0x100u)),ssg>>12u,SGX_LO);var w4=sdb*sgx(iq3mag((sw1&0xFFu)|((sqh<<4u)&0x100u)),ssg>>16u,SGX_LO);var w5=sdb*sgx(iq3mag(((sw1>>8u)&0xFFu)|((sqh<<3u)&0x100u)),ssg>>20u,SGX_LO);var w6=sdb*sgx(iq3mag(((sw1>>16u)&0xFFu)|((sqh<<2u)&0x100u)),ssg>>24u,SGX_LO);var w7=sdb*sgx(iq3mag((sw1>>24u)|((sqh<<1u)&0x100u)),ssg>>28u,SGX_LO);
{% elif iq2xs is defined and iq2xs %}let xsbw=(blk>>3u)*18u;let xk=blk&7u;let xw0=q4_bits[xsbw+2u*xk];let xw1=q4_bits[xsbw+2u*xk+1u];let xsc=(q4_bits[xsbw+16u+(xk>>2u)]>>((xk&3u)*8u))&0xFFu;let xdlo=f32(q4_scales[blk>>3u])*(0.5+f32(xsc&0xFu))*0.25;let xdhi=f32(q4_scales[blk>>3u])*(0.5+f32(xsc>>4u))*0.25;let xu0=xw0&0xFFFFu;let xu1=xw0>>16u;let xu2=xw1&0xFFFFu;let xu3=xw1>>16u;let xg0=iq_grid(xu0&0x1FFu);let xg1=iq_grid(xu1&0x1FFu);let xg2=iq_grid(xu2&0x1FFu);let xg3=iq_grid(xu3&0x1FFu);let xs0=ksg((xu0>>9u)&127u);let xs1=ksg((xu1>>9u)&127u);let xs2=ksg((xu2>>9u)&127u);let xs3=ksg((xu3>>9u)&127u);var w0=xdlo*sgx(vec4<f32>(unpack4xU8(xg0.x)),xs0,SGX_LO);var w1=xdlo*sgx(vec4<f32>(unpack4xU8(xg0.y)),xs0,SGX_HI);var w2=xdlo*sgx(vec4<f32>(unpack4xU8(xg1.x)),xs1,SGX_LO);var w3=xdlo*sgx(vec4<f32>(unpack4xU8(xg1.y)),xs1,SGX_HI);var w4=xdhi*sgx(vec4<f32>(unpack4xU8(xg2.x)),xs2,SGX_LO);var w5=xdhi*sgx(vec4<f32>(unpack4xU8(xg2.y)),xs2,SGX_HI);var w6=xdhi*sgx(vec4<f32>(unpack4xU8(xg3.x)),xs3,SGX_LO);var w7=xdhi*sgx(vec4<f32>(unpack4xU8(xg3.y)),xs3,SGX_HI);
{% elif iq2s is defined and iq2s %}let sbw=(blk>>3u)*20u;let sk=blk&7u;let sqw=q4_bits[sbw+2u*sk];let ssg=q4_bits[sbw+2u*sk+1u];let sqh=(q4_bits[sbw+16u+(sk>>2u)]>>((sk&3u)*8u))&0xFFu;let ssc=(q4_bits[sbw+18u+(sk>>2u)]>>((sk&3u)*8u))&0xFFu;let sdlo=f32(q4_scales[blk>>3u])*(0.5+f32(ssc&0xFu))*0.25;let sdhi=f32(q4_scales[blk>>3u])*(0.5+f32(ssc>>4u))*0.25;let sg0=iq_grid((sqw&0xFFu)|((sqh<<8u)&0x300u));let sg1=iq_grid(((sqw>>8u)&0xFFu)|((sqh<<6u)&0x300u));let sg2=iq_grid(((sqw>>16u)&0xFFu)|((sqh<<4u)&0x300u));let sg3=iq_grid((sqw>>24u)|((sqh<<2u)&0x300u));var w0=sdlo*sgx(vec4<f32>(unpack4xU8(sg0.x)),ssg,SGX_LO);var w1=sdlo*sgx(vec4<f32>(unpack4xU8(sg0.y)),ssg,SGX_HI);var w2=sdlo*sgx(vec4<f32>(unpack4xU8(sg1.x)),ssg>>8u,SGX_LO);var w3=sdlo*sgx(vec4<f32>(unpack4xU8(sg1.y)),ssg>>8u,SGX_HI);var w4=sdhi*sgx(vec4<f32>(unpack4xU8(sg2.x)),ssg>>16u,SGX_LO);var w5=sdhi*sgx(vec4<f32>(unpack4xU8(sg2.y)),ssg>>16u,SGX_HI);var w6=sdhi*sgx(vec4<f32>(unpack4xU8(sg3.x)),ssg>>24u,SGX_LO);var w7=sdhi*sgx(vec4<f32>(unpack4xU8(sg3.y)),ssg>>24u,SGX_HI);
{% elif iq1m is defined and iq1m %}{% if iq1s is defined and iq1s %}let sbw1=(blk>>3u)*12u;let k1=blk&7u;let oqw=q4_bits[sbw1+k1];let mt1=(q4_bits[sbw1+8u+(k1>>1u)]>>(16u*(k1&1u)))&0xFFFFu;let sg1s=((mt1>>12u)&1u)*0x88u;let omw=((mt1&7u)|(((mt1>>3u)&7u)<<4u)|sg1s)|((((mt1>>6u)&7u)|(((mt1>>9u)&7u)<<4u)|sg1s)<<8u)|((mt1>>13u)<<16u)|((mt1>>13u)<<19u);
{% elif iq1md is defined and iq1md %}let sbw1=(blk>>3u)*14u;let k1=blk&7u;let oqw=q4_bits[sbw1+k1];let omw=((q4_bits[sbw1+8u+(k1>>1u)]>>(16u*(k1&1u)))&0xFFFFu)|(((q4_bits[sbw1+12u+(k1>>2u)]>>(8u*(k1&3u)))&0x3Fu)<<16u);
{% else %}let oqw=q4_bits[blk*2u];let omw=q4_bits[blk*2u+1u];
{% endif %}let od=f32(q4_scales[blk>>3u]);let odlo=od*f32(2u*((omw>>16u)&7u)+1u);let odhi=od*f32(2u*((omw>>19u)&7u)+1u);let oh0=omw&0xFFu;let oh1=(omw>>8u)&0xFFu;let og0=iq_grid((oqw&0xFFu)|((oh0&7u)<<8u));let og1=iq_grid(((oqw>>8u)&0xFFu)|(((oh0>>4u)&7u)<<8u));let og2=iq_grid(((oqw>>16u)&0xFFu)|((oh1&7u)<<8u));let og3=iq_grid((oqw>>24u)|(((oh1>>4u)&7u)<<8u));let ot0=select(0.125,-0.125,(oh0&0x08u)!=0u);let ot1=select(0.125,-0.125,(oh0&0x80u)!=0u);let ot2=select(0.125,-0.125,(oh1&0x08u)!=0u);let ot3=select(0.125,-0.125,(oh1&0x80u)!=0u);var w0=odlo*(vec4<f32>(unpack4xI8(og0.x))+ot0);var w1=odlo*(vec4<f32>(unpack4xI8(og0.y))+ot0);var w2=odlo*(vec4<f32>(unpack4xI8(og1.x))+ot1);var w3=odlo*(vec4<f32>(unpack4xI8(og1.y))+ot1);var w4=odhi*(vec4<f32>(unpack4xI8(og2.x))+ot2);var w5=odhi*(vec4<f32>(unpack4xI8(og2.y))+ot2);var w6=odhi*(vec4<f32>(unpack4xI8(og3.x))+ot3);var w7=odhi*(vec4<f32>(unpack4xI8(og3.y))+ot3);
{% elif iq2xxs is defined and iq2xxs %}let xqw=q4_bits[blk*2u];let xsw=q4_bits[blk*2u+1u];let xdb=f32(q4_scales[blk>>3u])*(0.5+f32(xsw>>28u))*0.25;let xg0=iq_grid(xqw&0xFFu);let xg1=iq_grid((xqw>>8u)&0xFFu);let xg2=iq_grid((xqw>>16u)&0xFFu);let xg3=iq_grid(xqw>>24u);let xsa=ksg(xsw&127u);let xsb=ksg((xsw>>7u)&127u);let xsc=ksg((xsw>>14u)&127u);let xsd=ksg((xsw>>21u)&127u);var w0=xdb*sgx(vec4<f32>(unpack4xU8(xg0.x)),xsa,SGX_LO);var w1=xdb*sgx(vec4<f32>(unpack4xU8(xg0.y)),xsa,SGX_HI);var w2=xdb*sgx(vec4<f32>(unpack4xU8(xg1.x)),xsb,SGX_LO);var w3=xdb*sgx(vec4<f32>(unpack4xU8(xg1.y)),xsb,SGX_HI);var w4=xdb*sgx(vec4<f32>(unpack4xU8(xg2.x)),xsc,SGX_LO);var w5=xdb*sgx(vec4<f32>(unpack4xU8(xg2.y)),xsc,SGX_HI);var w6=xdb*sgx(vec4<f32>(unpack4xU8(xg3.x)),xsd,SGX_LO);var w7=xdb*sgx(vec4<f32>(unpack4xU8(xg3.y)),xsd,SGX_HI);
{% elif quantBits == 8 %}let dw0=q4_bits[blk*2u];let dw1=q4_bits[blk*2u+1u];var w0=vec4<f32>(unpack4xI8(dw0.x));var w1=vec4<f32>(unpack4xI8(dw0.y));var w2=vec4<f32>(unpack4xI8(dw0.z));var w3=vec4<f32>(unpack4xI8(dw0.w));var w4=vec4<f32>(unpack4xI8(dw1.x));var w5=vec4<f32>(unpack4xI8(dw1.y));var w6=vec4<f32>(unpack4xI8(dw1.z));var w7=vec4<f32>(unpack4xI8(dw1.w));
{% elif lut2 is defined and lut2 %}{% if lutId in [8, 9] %}{% if not wideNativeTile %}let dA2=0u;let dB2=0u;let dC2=0u;let dD2=0u;
{% endif %}{% else %}let m2=(blk>>3u)*{{ 5 if lutc16 else 3 }}u;
{% if lutc16 %}let c2=(blk&7u)*2u;let bL2=(q4_metadata[m2+(c2>>2u)]>>((c2&3u)*8u))&255u;let bH2=(q4_metadata[m2+((c2+1u)>>2u)]>>(((c2+1u)&3u)*8u))&255u;
{% else %}let bL2=(q4_metadata[m2+((blk&7u)>>2u)]>>((blk&3u)*8u))&255u;let bH2=bL2;
{% endif %}let dA2=((bL2>>4u)&1u)*8u;let dB2=((bL2>>{{ 5 if lutc16 else 4 }}u)&1u)*8u;let dC2=((bH2>>4u)&1u)*8u;let dD2=((bH2>>{{ 5 if lutc16 else 4 }}u)&1u)*8u;
{% endif %}let a2=q4_bits[blk*2u];let z2=q4_bits[blk*2u+1u];
{% if wideNativeTile and lutId in [8, 9] %}let native_s={{ lut2_native_scale("blk", "q4_metadata", lutId) }};let native_values=vec4<f16>(f16(native_s*-1.0),f16(native_s*0.0),f16(native_s*1.0),f16(native_s*2.0));let w0=native_q2_stage(a2,0u,native_values);let w1=native_q2_stage(a2,8u,native_values);let w2=native_q2_stage(a2,16u,native_values);let w3=native_q2_stage(a2,24u,native_values);let w4=native_q2_stage(z2,0u,native_values);let w5=native_q2_stage(z2,8u,native_values);let w6=native_q2_stage(z2,16u,native_values);let w7=native_q2_stage(z2,24u,native_values);
{% else %}var w0=q2q(a2,0u,dA2);var w1=q2q(a2,8u,dA2);var w2=q2q(a2,16u,dB2);var w3=q2q(a2,24u,dB2);var w4=q2q(z2,0u,dC2);var w5=q2q(z2,8u,dC2);var w6=q2q(z2,16u,dD2);var w7=q2q(z2,24u,dD2);
{% endif %}{% else %}let dw=q4_bits[blk];var w0=q4_lo(dw.x);var w1=q4_lo(dw.y);var w2=q4_lo(dw.z);var w3=q4_lo(dw.w);var w4=q4_hi(dw.x);var w5=q4_hi(dw.y);var w6=q4_hi(dw.z);var w7=q4_hi(dw.w);
{% endif %}{% if q4knCompact is defined and q4knCompact %}let dl4nc=dSuper4nc*f32(sc4nc);let ml4nc=dmSuper4nc*f32(mn4nc);let p0=dl4nc*w0;let p1=dl4nc*w1;let p2=dl4nc*w2;let p3=dl4nc*w3;let p4=dl4nc*w4;let p5=dl4nc*w5;let p6=dl4nc*w6;let p7=dl4nc*w7;w0=p0-ml4nc;w1=p1-ml4nc;w2=p2-ml4nc;w3=p3-ml4nc;w4=p4-ml4nc;w5=p5-ml4nc;w6=p6-ml4nc;w7=p7-ml4nc;
{% elif q5k is defined and q5k %}let scEff5=f32(q4_scales[sb5*2u])*sc5;let mEff5=f32(q4_scales[sb5*2u+1u])*mn5;w0=scEff5*w0-mEff5;w1=scEff5*w1-mEff5;w2=scEff5*w2-mEff5;w3=scEff5*w3-mEff5;w4=scEff5*w4-mEff5;w5=scEff5*w5-mEff5;w6=scEff5*w6-mEff5;w7=scEff5*w7-mEff5;
{% elif q2k %}let d2=f32(q4_scales[sb2*2u]);let dm2=f32(q4_scales[sb2*2u+1u]);let sLo2=d2*f32(bLo2&15u);let mLo2=dm2*f32(bLo2>>4u);let sHi2=d2*f32(bHi2&15u);let mHi2=dm2*f32(bHi2>>4u);w0=sLo2*w0-mLo2;w1=sLo2*w1-mLo2;w2=sLo2*w2-mLo2;w3=sLo2*w3-mLo2;w4=sHi2*w4-mHi2;w5=sHi2*w5-mHi2;w6=sHi2*w6-mHi2;w7=sHi2*w7-mHi2;
{% elif q3k %}let d3=f32(q4_scales[sb3]);let sLo3=d3*f32(s83[(j3&1u)*2u]);let sHi3=d3*f32(s83[(j3&1u)*2u+1u]);w0=sLo3*w0;w1=sLo3*w1;w2=sLo3*w2;w3=sLo3*w3;w4=sHi3*w4;w5=sHi3*w5;w6=sHi3*w6;w7=sHi3*w7;
{% elif q6k %}let d6=f32(q4_scales[sb6]);let sLo6=d6*f32(s86[(j6&1u)*2u]);let sHi6=d6*f32(s86[(j6&1u)*2u+1u]);w0=sLo6*w0;w1=sLo6*w1;w2=sLo6*w2;w3=sLo6*w3;w4=sHi6*w4;w5=sHi6*w5;w6=sHi6*w6;w7=sHi6*w7;
{% elif hasMin %}let ds=f32(q4_scales[blk*2u]);let dbias=f32(q4_scales[blk*2u+1u]);w0=ds*w0+dbias;w1=ds*w1+dbias;w2=ds*w2+dbias;w3=ds*w3+dbias;w4=ds*w4+dbias;w5=ds*w5+dbias;w6=ds*w6+dbias;w7=ds*w7+dbias;
{% elif lut2 is defined and lut2 %}{% if wideNativeTile and lutId in [8, 9] %}{% else %}{% if lutId in [8, 9] %}let s2_lo={{ lut2_native_scale("blk", "q4_metadata", lutId) }};let s2_hi=s2_lo;
{% else %}let d2s=bitcast<f32>(q4_metadata[m2+{{ 4 if lutc16 else 2 }}u]);let s2_lo=d2s*lutScale(bL2&15u);let s2_hi=d2s*lutScale(bH2&15u);
{% endif %}w0=s2_lo*w0;w1=s2_lo*w1;w2=s2_lo*w2;w3=s2_lo*w3;w4=s2_hi*w4;w5=s2_hi*w5;w6=s2_hi*w6;w7=s2_hi*w7;
{% endif %}{% elif lutc is defined and lutc and lutc16 %}let mbL=(blk>>3u)*5u;let dL=bitcast<f32>(q4_metadata[mbL+4u]);let cwL=q4_metadata[mbL+((blk&7u)>>1u)];let csL=(blk&1u)*16u;let ds_lo=dL*lutScale((cwL>>csL)&255u);let ds_hi=dL*lutScale((cwL>>(csL+8u))&255u);w0=ds_lo*w0;w1=ds_lo*w1;w2=ds_lo*w2;w3=ds_lo*w3;w4=ds_hi*w4;w5=ds_hi*w5;w6=ds_hi*w6;w7=ds_hi*w7;
{% elif lutc is defined and lutc %}let mbL=(blk>>3u)*3u;let ds=bitcast<f32>(q4_metadata[mbL+2u])*lutScale((q4_metadata[mbL+((blk&7u)>>2u)]>>((blk&3u)*8u))&255u);w0=ds*w0;w1=ds*w1;w2=ds*w2;w3=ds*w3;w4=ds*w4;w5=ds*w5;w6=ds*w6;w7=ds*w7;
{% elif (iq1m is defined and iq1m) or (iq2xxs is defined and iq2xxs) or (iq2xs is defined and iq2xs) or (iq3xxs is defined and iq3xxs) or (iq3s is defined and iq3s) or (iq2s is defined and iq2s) %}{% elif iq or lut16 %}let ds_lo=f32(q4_scales[blk*2u]);let ds_hi=f32(q4_scales[blk*2u+1u]);w0=ds_lo*w0;w1=ds_lo*w1;w2=ds_lo*w2;w3=ds_lo*w3;w4=ds_hi*w4;w5=ds_hi*w5;w6=ds_hi*w6;w7=ds_hi*w7;
{% else %}let ds=f32(q4_scales[blk]);w0=ds*w0;w1=ds*w1;w2=ds*w2;w3=ds*w3;w4=ds*w4;w5=ds*w5;w6=ds*w6;w7=ds*w7;
{% endif %}let bbase={{ "local_idx" if wideNativeTile else "l" }}*TILE_K;tile_B[bbase]=f16(w0.x);tile_B[bbase+1u]=f16(w0.y);tile_B[bbase+2u]=f16(w0.z);tile_B[bbase+3u]=f16(w0.w);tile_B[bbase+4u]=f16(w1.x);tile_B[bbase+5u]=f16(w1.y);tile_B[bbase+6u]=f16(w1.z);tile_B[bbase+7u]=f16(w1.w);tile_B[bbase+8u]=f16(w2.x);tile_B[bbase+9u]=f16(w2.y);tile_B[bbase+10u]=f16(w2.z);tile_B[bbase+11u]=f16(w2.w);tile_B[bbase+12u]=f16(w3.x);tile_B[bbase+13u]=f16(w3.y);tile_B[bbase+14u]=f16(w3.z);tile_B[bbase+15u]=f16(w3.w);tile_B[bbase+16u]=f16(w4.x);tile_B[bbase+17u]=f16(w4.y);tile_B[bbase+18u]=f16(w4.z);tile_B[bbase+19u]=f16(w4.w);tile_B[bbase+20u]=f16(w5.x);tile_B[bbase+21u]=f16(w5.y);tile_B[bbase+22u]=f16(w5.z);tile_B[bbase+23u]=f16(w5.w);tile_B[bbase+24u]=f16(w6.x);tile_B[bbase+25u]=f16(w6.y);tile_B[bbase+26u]=f16(w6.z);tile_B[bbase+27u]=f16(w6.w);tile_B[bbase+28u]=f16(w7.x);tile_B[bbase+29u]=f16(w7.y);tile_B[bbase+30u]=f16(w7.z);tile_B[bbase+31u]=f16(w7.w);}workgroupBarrier();
{% include "./prefill-subgroup-matmul-epilogue.wgsl.jinja" %}{% endmacro %}{% include "ops/_shared/subgroup-matrix-topology-entry.wgsl.jinja" %}{{ matrix_topology_entry("WG_THREADS", "WG_THREADS / 32u", "wid", "matrixLane") }}`],["prefill-matmul-q4.wgsl.jinja",`{% include "ops/_shared/lut2-block-scale.wgsl.jinja" %}{% set emitQuantDotHelpers = false %}{% set emitQuantValueHelpers = not (lut2 | default(false)) %}{% set emitLut2DotHelper = false %}{% set mPerThread = mPerThread if mPerThread is defined else 1 %}{% set rowVar = "m_row" if mPerThread > 1 else "m_local" %}{% set mBound = "m_rows" if (dynamicRows | default(false)) else "params.m" %}{% set packed4x8CompatUnpackI8 = quantBits == 8 or q3k or q6k or (iq1m | default(false)) %}{% set packed4x8CompatUnpackU8 = (iq2xxs | default(false)) or (iq2xs | default(false)) or (iq2s | default(false)) or ((((iq3xxs | default(false)) or (iq3s | default(false))) and (iqGridBound | default(false)))) or (q4knCompact | default(false)) or not (quantBits == 8 or (lut2 | default(false)) or lutId or q5 or q2k or q3k or q6k or q5k or (q4knCompact | default(false)) or (iq1m | default(false)) or (iq2xxs | default(false)) or (iq2xs | default(false)) or (iq2s | default(false)) or (iq3xxs | default(false)) or (iq3s | default(false))) %}{% include "ops/_shared/packed-4x8-compat.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}
{% if (iq1m is defined and iq1m) or (iq2xxs is defined and iq2xxs) or (iq2xs is defined and iq2xs) or (iq3xxs is defined and iq3xxs) or (iq3s is defined and iq3s) or (iq2s is defined and iq2s) %}{% include "ops/_shared/iq-grid.wgsl.jinja" %}{% include "ops/_shared/iqx-raw-dequant-helpers.wgsl.jinja" %}{% elif lutId %}{% include "ops/_shared/q4nl-dequant-helpers.wgsl.jinja" %}{% endif %}const K:u32={{ inFeatures }}u;const N:u32={{ outFeatures }}u;
{% if not kSplits %}const OUT_STRIDE:u32={{ outStride }}u;const DST_COL:u32={{ dstColStart }}u;
{% endif %}const BPR:u32=K/32u;const K_V4:u32=K/4u;const BM:u32={{ bm }}u;const BN:u32={{ bn }}u;const K_BLOCKS:u32={{ kBlocks }}u;const BK_V4:u32=K_BLOCKS*8u;const N_TILE:u32={{ nTile }}u;const WG:u32=128u;
{% if mPerThread > 1 %}const M_PER_THREAD:u32={{ mPerThread }}u;
{% endif %}{% if kSplits %}const SPLIT_BLOCKS:u32={{ splitBlocks }}u;
{% endif %}{% if f16Tiles %}alias TileVec=vec4<f16>;
{% else %}alias TileVec=vec4<f32>;
{% endif %}var<workgroup>aTile:array<TileVec,{{ bm * kBlocks * 8 }}>;var<workgroup>bTile:array<TileVec,{{ bn * kBlocks * 8 }}>;@compute@workgroup_size(WG,1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>){let tid=lid.x;
{% if dynamicRows | default(false) %}let m_rows=min(params.m,real_len[0]);
{% endif %}let m_local=tid/N_TILE;let n_local=tid-m_local*N_TILE;
{% if mPerThread > 1 %}let m_row=m_local*M_PER_THREAD;
{% endif %}let n_tiles=(N+BN-1u)/BN;let m_tiles=({{ mBound }}+BM-1u)/BM;for(var wgx:u32=wg.x;wgx<n_tiles;wgx=wgx+nwg.x){let nBase=wgx*BN;for(var wgy:u32=wg.y;wgy<m_tiles;wgy=wgy+nwg.y){let mBase=wgy*BM;let m=mBase+{{ rowVar }};
{% for nn in range(mPerThread * accs) %}var acc{{ nn }}:f32=0.0;
{% endfor %}{% if kSplits %}let ktEnd=min((wg.z+1u)*SPLIT_BLOCKS,BPR);for(var kt:u32=wg.z*SPLIT_BLOCKS;kt<ktEnd;kt=kt+K_BLOCKS){
{% else %}let ktEnd=BPR;for(var kt:u32=0u;kt<ktEnd;kt=kt+K_BLOCKS){
{% endif %}let kv0=kt*8u;for(var i:u32=tid;i<{{ bm * kBlocks * 8 }}u;i=i+WG){let tm=i/BK_V4;let kv=i-tm*BK_V4;let gm=mBase+tm;
{% if kBlocks == 1 %}if(gm<{{ mBound }}){
{% else %}if(gm<{{ mBound }}&&kt+kv/8u<ktEnd){
{% endif %}aTile[i]=TileVec(a[gm*K_V4+kv0+kv]);}else{aTile[i]=TileVec(0.0);} }for(var i:u32=tid;i<{{ bn * kBlocks * 8 }}u;i=i+WG){let tn=i/BK_V4;let kv=i-tn*BK_V4;
{% if kBlocks == 1 %}let eib=kv*4u;
{% else %}let blockInTile=kv/8u;let eib=(kv%8u)*4u;
{% endif %}let r=nBase+tn;var v=vec4<f32>(0.0);
{% if denseTailStart | default(0) %}if(r>={{ denseTailStart }}u&&r<N){let denseIndex=params.weight_offset+(r-{{ denseTailStart }}u)*K+kv0*4u+kv*4u;v=vec4<f32>(vec4<f16>(weights[denseIndex],weights[denseIndex+1u],weights[denseIndex+2u],weights[denseIndex+3u]));}else{
{% endif %}{% if kBlocks == 1 %}if(r<N){let blk=params.block_offset+r*BPR+kt;
{% else %}if(r<N&&kt+blockInTile<ktEnd){let blk=params.block_offset+r*BPR+kt+blockInTile;
{% endif %}{% if quantBits == 8 %}let word=q4_bits[blk*8u+eib/4u];
{% if iq %}let scale=f32(q4_scales[blk*2u+select(0u,1u,eib>=16u)]);
{% else %}let scale=f32(q4_scales[blk]);
{% endif %}v=vec4<f32>(unpack4xI8(word))*scale;
{% elif lut2 is defined and lut2 %}{% if lutId in [8, 9] %}let scale={{ lut2_native_scale("blk", "q4_metadata", lutId) }};let db2=0u;
{% elif lutc16 %}let mb2=(blk>>3u)*5u;let ci2=(blk&7u)*2u+select(0u,1u,eib>=16u);let b2=(q4_metadata[mb2+(ci2>>2u)]>>((ci2&3u)*8u))&255u;let scale=bitcast<f32>(q4_metadata[mb2+4u])*lutScale(b2&15u);let db2=((b2>>(4u+((eib>>3u)&1u)))&1u)*8u;
{% else %}let mb2=(blk>>3u)*3u;let b2=(q4_metadata[mb2+((blk&7u)>>2u)]>>((blk&3u)*8u))&255u;let scale=bitcast<f32>(q4_metadata[mb2+2u])*lutScale(b2&15u);let db2=((b2>>4u)&1u)*8u;
{% endif %}v=q2q(q4_bits[blk*2u+(eib>>4u)],(eib&15u)*2u,db2)*scale;
{% elif lutId %}let word=q4_bits[blk*4u+(eib%16u)/4u];
{% if lutc is defined and lutc and lutc16 %}let mbL=(blk>>3u)*5u;let ciL=(blk&7u)*2u+select(0u,1u,eib>=16u);let scale=bitcast<f32>(q4_metadata[mbL+4u])*lutScale((q4_metadata[mbL+(ciL>>2u)]>>((ciL&3u)*8u))&255u);
{% elif lutc is defined and lutc %}let mbL=(blk>>3u)*3u;let scale=bitcast<f32>(q4_metadata[mbL+2u])*lutScale((q4_metadata[mbL+((blk&7u)>>2u)]>>((blk&3u)*8u))&255u);
{% elif lut16 %}let scale=f32(q4_scales[blk*2u+select(0u,1u,eib>=16u)]);
{% else %}let scale=f32(q4_scales[blk]);
{% endif %}if(eib<16u){v=q4_lo(word)*scale;}else{v=q4_hi(word)*scale;}
{% elif q5 %}let g5=(blk>>2u)*20u;let l5=blk&3u;let word=q4_bits[g5+l5*4u+(eib%16u)/4u];let qh=q4_bits[g5+16u+l5];let scale=f32(q4_scales[blk]);var q5n:vec4<u32>;if(eib<16u){q5n=(vec4<u32>(word)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(15u);}else{q5n=(vec4<u32>(word)>>vec4<u32>(4u,12u,20u,28u))&vec4<u32>(15u);}q5n=q5n|(((vec4<u32>(qh)>>(vec4<u32>(0u,1u,2u,3u)+vec4<u32>(eib)))&vec4<u32>(1u))<<vec4<u32>(4u));v=(vec4<f32>(q5n)-16.0)*scale;
{% elif q4knCompact is defined and q4knCompact %}let sb4nc=blk>>3u;let j4nc=blk&7u;let smBase4nc=sb4nc*5u;let dm4nc=unpack2x16float(q4_metadata[smBase4nc]);let smCarrier4nc=q4_metadata[smBase4nc+1u+(j4nc>>1u)];let smPair4nc=smCarrier4nc>>((j4nc&1u)*12u);let sc4nc=smPair4nc&63u;let mn4nc=(smPair4nc>>6u)&63u;let word=q4_bits[blk*4u+(eib%16u)/4u];var nib4nc:vec4<u32>;if(eib<16u){nib4nc=unpack4xU8(word&0x0F0F0F0Fu);}else{nib4nc=unpack4xU8((word>>4u)&0x0F0F0F0Fu);}let dl4nc=dm4nc.x*f32(sc4nc);let ml4nc=dm4nc.y*f32(mn4nc);let product4nc=dl4nc*vec4<f32>(nib4nc);v=product4nc-vec4<f32>(ml4nc);
{% elif iq3xxs is defined and iq3xxs %}let g3=eib>>2u;let isw3=q4_bits[blk*3u+2u];let iwd3=q4_bits[blk*3u+(g3>>2u)];let gi3=(iwd3>>((g3&3u)*8u))&0xFFu;let is3=ksg((isw3>>((g3>>1u)*7u))&127u);let db3=f32(q4_scales[blk>>3u])*(0.5+f32(isw3>>28u))*0.5;v=db3*sgx(iq3mag(gi3),is3,select(SGX_LO,SGX_HI,(g3&1u)==1u));
{% elif iq3s is defined and iq3s %}let sbw3s=(blk>>3u)*27u;let k3s=blk&7u;let g3s=eib>>2u;let gi3s=(q4_bits[sbw3s+3u*k3s+(g3s>>2u)]>>((g3s&3u)*8u))&0xFFu;let qh3s=(q4_bits[sbw3s+24u+(k3s>>2u)]>>((k3s&3u)*8u))&0xFFu;let db3s=f32(q4_scales[blk>>3u])*(1.0+2.0*f32((q4_bits[sbw3s+26u]>>(k3s*4u))&0xFu));v=db3s*sgx(iq3mag(gi3s|(((qh3s>>g3s)&1u)<<8u)),q4_bits[sbw3s+3u*k3s+2u]>>(g3s*4u),SGX_LO);
{% elif iq2xs is defined and iq2xs %}let sbw2x=(blk>>3u)*18u;let k2x=blk&7u;let c2x=eib>>3u;let hi2x=((eib>>2u)&1u)==1u;let u2x=(q4_bits[sbw2x+2u*k2x+(c2x>>1u)]>>((c2x&1u)*16u))&0xFFFFu;let sc2x=(q4_bits[sbw2x+16u+(k2x>>2u)]>>((k2x&3u)*8u))&0xFFu;let g2x=iq_grid(u2x&0x1FFu);let db2x=f32(q4_scales[blk>>3u])*(0.5+f32(select(sc2x&0xFu,sc2x>>4u,eib>=16u)))*0.25;v=db2x*sgx(vec4<f32>(unpack4xU8(select(g2x.x,g2x.y,hi2x))),ksg((u2x>>9u)&127u),select(SGX_LO,SGX_HI,hi2x));
{% elif iq2s is defined and iq2s %}let sbw2s=(blk>>3u)*20u;let k2s=blk&7u;let c2s=eib>>3u;let hi2s=((eib>>2u)&1u)==1u;let qh2s=(q4_bits[sbw2s+16u+(k2s>>2u)]>>((k2s&3u)*8u))&0xFFu;let sc2s=(q4_bits[sbw2s+18u+(k2s>>2u)]>>((k2s&3u)*8u))&0xFFu;let g2s=iq_grid(((q4_bits[sbw2s+2u*k2s]>>(c2s*8u))&0xFFu)|(((qh2s>>(2u*c2s))&3u)<<8u));let db2s=f32(q4_scales[blk>>3u])*(0.5+f32(select(sc2s&0xFu,sc2s>>4u,eib>=16u)))*0.25;v=db2s*sgx(vec4<f32>(unpack4xU8(select(g2s.x,g2s.y,hi2s))),q4_bits[sbw2s+2u*k2s+1u]>>(c2s*8u),select(SGX_LO,SGX_HI,hi2s));
{% elif iq1m is defined and iq1m %}let c1=eib>>3u;let hi1=((eib>>2u)&1u)==1u;
{% if iq1s is defined and iq1s %}let sbw1=(blk>>3u)*12u;let k1=blk&7u;let qw1=q4_bits[sbw1+k1];let mt1=(q4_bits[sbw1+8u+(k1>>1u)]>>(16u*(k1&1u)))&0xFFFFu;let sg1s=((mt1>>12u)&1u)*0x88u;let mw1=((mt1&7u)|(((mt1>>3u)&7u)<<4u)|sg1s)|((((mt1>>6u)&7u)|(((mt1>>9u)&7u)<<4u)|sg1s)<<8u)|((mt1>>13u)<<16u)|((mt1>>13u)<<19u);
{% elif iq1md is defined and iq1md %}let sbw1=(blk>>3u)*14u;let k1=blk&7u;let qw1=q4_bits[sbw1+k1];let mw1=((q4_bits[sbw1+8u+(k1>>1u)]>>(16u*(k1&1u)))&0xFFFFu)|(((q4_bits[sbw1+12u+(k1>>2u)]>>(8u*(k1&3u)))&0x3Fu)<<16u);
{% else %}let qw1=q4_bits[blk*2u];let mw1=q4_bits[blk*2u+1u];
{% endif %}let qh1=(mw1>>((c1>>1u)*8u))&0xFFu;let sh1=(c1&1u)*4u;let g1=iq_grid(((qw1>>(c1*8u))&0xFFu)|(((qh1>>sh1)&7u)<<8u));let dt1=select(0.125,-0.125,((qh1>>(sh1+3u))&1u)!=0u);let dl1=f32(q4_scales[blk>>3u])*f32(2u*((mw1>>select(16u,19u,eib>=16u))&7u)+1u);v=dl1*(vec4<f32>(unpack4xI8(select(g1.x,g1.y,hi1)))+dt1);
{% elif iq2xxs is defined and iq2xxs %}let c2x=eib>>3u;let hi2x=((eib>>2u)&1u)==1u;let qw2x=q4_bits[blk*2u];let sw2x=q4_bits[blk*2u+1u];let g2x=iq_grid((qw2x>>(c2x*8u))&0xFFu);let is2x=ksg((sw2x>>(c2x*7u))&127u);let db2x=f32(q4_scales[blk>>3u])*(0.5+f32(sw2x>>28u))*0.25;v=db2x*sgx(vec4<f32>(unpack4xU8(select(g2x.x,g2x.y,hi2x))),is2x,select(SGX_LO,SGX_HI,hi2x));
{% elif q5k is defined and q5k %}let sb5=blk>>3u;let j5=blk&7u;let g5k=sb5*44u;let word=q4_bits[g5k+4u*j5+(eib%16u)/4u];let qh=q4_bits[g5k+32u+j5];let sc5=f32((q4_bits[g5k+40u+(j5>>2u)]>>((j5&3u)*8u))&0xFFu);let mn5=f32((q4_bits[g5k+42u+(j5>>2u)]>>((j5&3u)*8u))&0xFFu);let scEff5=f32(q4_scales[sb5*2u])*sc5;let mEff5=f32(q4_scales[sb5*2u+1u])*mn5;var q5n:vec4<u32>;if(eib<16u){q5n=(vec4<u32>(word)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(15u);}else{q5n=(vec4<u32>(word)>>vec4<u32>(4u,12u,20u,28u))&vec4<u32>(15u);}q5n=q5n|(((vec4<u32>(qh)>>(vec4<u32>(0u,1u,2u,3u)+vec4<u32>(eib)))&vec4<u32>(1u))<<vec4<u32>(4u));v=vec4<f32>(q5n)*scEff5-vec4<f32>(mEff5);
{% elif q2k %}let sb=blk>>3u;let j2=blk&7u;let g2=sb*20u;let cw=q4_bits[g2+2u*j2+select(0u,1u,eib>=16u)];let scb=(q4_bits[g2+16u+(j2>>1u)]>>((j2&1u)*16u+select(0u,8u,eib>=16u)))&0xFFu;let d2=f32(q4_scales[sb*2u]);let dm2=f32(q4_scales[sb*2u+1u]);let w2=(eib%16u)/4u;let q2=(vec4<u32>(cw)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(2u*w2)))&vec4<u32>(3u);v=vec4<f32>(q2)*(d2*f32(scb&15u))-vec4<f32>(dm2*f32(scb>>4u));
{% elif q3k %}let sb=blk>>3u;let j3=blk&7u;let g3=sb*28u;let cw3=q4_bits[g3+2u*j3+select(0u,1u,eib>=16u)];let hw3=q4_bits[g3+16u+j3];let sc83=unpack4xI8(q4_bits[g3+24u+(j3>>1u)])[(j3&1u)*2u+select(0u,1u,eib>=16u)];let scale3=f32(q4_scales[sb])*f32(sc83);let w3=(eib%16u)/4u;let q3n=(vec4<u32>(cw3)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(2u*w3)))&vec4<u32>(3u);let h3=(vec4<u32>(hw3)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(select(0u,4u,eib>=16u)+w3)))&vec4<u32>(1u);v=(vec4<f32>(q3n|(h3<<vec4<u32>(2u)))-4.0)*scale3;
{% elif q6k %}let sb=blk>>3u;let j6=blk&7u;let g6=sb*52u;let w6=(eib%16u)/4u;let word=q4_bits[g6+4u*j6+w6];let hw=q4_bits[g6+32u+2u*j6+select(0u,1u,eib>=16u)];let sc8=unpack4xI8(q4_bits[g6+48u+(j6>>1u)])[(j6&1u)*2u+select(0u,1u,eib>=16u)];let scale=f32(q4_scales[sb])*f32(sc8);var q6n:vec4<u32>;if(eib<16u){q6n=(vec4<u32>(word)>>vec4<u32>(0u,8u,16u,24u))&vec4<u32>(15u);}else{q6n=(vec4<u32>(word)>>vec4<u32>(4u,12u,20u,28u))&vec4<u32>(15u);}let hi2=(vec4<u32>(hw)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(2u*w6)))&vec4<u32>(3u);v=(vec4<f32>(q6n|(hi2<<vec4<u32>(4u)))-32.0)*scale;
{% else %}let word=q4_bits[blk*4u+(eib%16u)/4u];
{% if hasMin %}let scale=f32(q4_scales[blk*2u]);let bias=f32(q4_scales[blk*2u+1u]);
{% else %}let scale=f32(q4_scales[blk]);
{% endif %}var nib:vec4<f32>;if(eib<16u){nib=vec4<f32>(unpack4xU8(word&0x0F0F0F0Fu))-8.0;}else{nib=vec4<f32>(unpack4xU8((word>>4u)&0x0F0F0F0Fu))-8.0;}
{% if hasMin %}v=nib*scale+bias;
{% else %}v=nib*scale;
{% endif %}{% endif %}}
{% if denseTailStart | default(0) %}}
{% endif %}bTile[i]=TileVec(v);}workgroupBarrier();
{% if mPerThread > 1 %}if(m<{{ mBound }}){for(var kv:u32=0u;kv<BK_V4;kv=kv+1u){
{% for mm in range(mPerThread) %}let a_vec{{ mm }}=vec4<f32>(aTile[({{ rowVar }}+{{ mm }}u)*BK_V4+kv]);
{% endfor %}{% for nn in range(accs) %}let b_vec{{ nn }}=vec4<f32>(bTile[(n_local+{{ nn * nTile }}u)*BK_V4+kv]);
{% endfor %}{% for mm in range(mPerThread) %}{% for nn in range(accs) %}acc{{ mm * accs + nn }}=fma(a_vec{{ mm }}.w,b_vec{{ nn }}.w,fma(a_vec{{ mm }}.z,b_vec{{ nn }}.z,fma(a_vec{{ mm }}.y,b_vec{{ nn }}.y,fma(a_vec{{ mm }}.x,b_vec{{ nn }}.x,acc{{ mm * accs + nn }}))));
{% endfor %}{% endfor %}} }
{% else %}if(m<{{ mBound }}){for(var kv:u32=0u;kv<BK_V4;kv=kv+1u){let a_vec=vec4<f32>(aTile[m_local*BK_V4+kv]);
{% for nn in range(accs) %}let b_vec{{ nn }}=vec4<f32>(bTile[(n_local+{{ nn * nTile }}u)*BK_V4+kv]);acc{{ nn }}=fma(a_vec.w,b_vec{{ nn }}.w,fma(a_vec.z,b_vec{{ nn }}.z,fma(a_vec.y,b_vec{{ nn }}.y,fma(a_vec.x,b_vec{{ nn }}.x,acc{{ nn }}))));
{% endfor %}} }
{% endif %}workgroupBarrier();}
{% for mm in range(mPerThread) %}{% set mRow = "m" if mPerThread == 1 else "(m + " ~ mm ~ "u)" %}{% set aBase = mm * accs %}if({{ mRow }}<{{ mBound }}){
{% if kSplits %}{% for nn in range(accs) %}if(nBase+n_local+{{ nn * nTile }}u<N){partials[(wg.z*params.m+{{ mRow }})*N+nBase+n_local+{{ nn * nTile }}u]=acc{{ aBase + nn }};}
{% endfor %}{% else %}{% for nn in range(accs) %}if(nBase+n_local+{{ nn * nTile }}u<N){
          y[{{ mRow }} * OUT_STRIDE + DST_COL + nBase + n_local + {{ nn * nTile }}u] = {% if yScalar == "f16" %}f16(acc{{ aBase + nn }}){% else %}acc{{ aBase + nn }}{% endif %};
}
{% endfor %}{% endif %}}
{% endfor %}} } }`],["prefill-matmul-subgroup.wgsl.jinja",`{% include "ops/_shared/subgroup-matrix-enable.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const IN_F:u32={{ inFeatures }}u;const OUT_F:u32={{ (outStride | default(outFeatures)) if (denseTailStart | default(0)) else outFeatures }}u;const TILE_COLS:u32=64u;const TILE_ROWS:u32=32u;const TILE_K:u32=32u;const SUB_COLS:u32=32u;const SUB_ROWS:u32=16u;var<workgroup>tile_A:array<f16,32*32>;var<workgroup>tile_B:array<f16,64*32>;var<workgroup>scratch:array<array<f32,64>,32>;fn loadSHMA(tile_base:u32,k_idx:u32,row:u32,c_idx:u32){let a_global:u32=tile_base+row;let col:u32=c_idx*8u;for(var col_offset:u32=0u;col_offset<8u;col_offset++){let k:u32=k_idx+col+col_offset;if(a_global<params.m){tile_A[row*TILE_K+col+col_offset]=a[a_global*IN_F+k];}else{tile_A[row*TILE_K+col+col_offset]=0.0h;} } }fn loadSHMB(tile_base:u32,k_idx:u32,row:u32,c_idx:u32){let w_global:u32=tile_base+row;let col:u32=c_idx*16u;
{% if denseTailStart | default(0) %}if(w_global<{{ denseTailStart }}u){let blk=params.block_offset+w_global*(IN_F/32u)+k_idx/32u;let dm=unpack2x16float(q4_metadata[(blk>>3u)*5u]);let j=blk&7u;let sm=q4_metadata[(blk>>3u)*5u+1u+(j>>1u)]>>((j&1u)*12u);let ds=dm.x*f32(sm&63u);let ms=dm.y*f32((sm>>6u)&63u);for(var v=0u;v<4u;v++){let word=q4_bits[blk*4u+v];let q=(vec4<u32>(word)>>(vec4<u32>(0u,8u,16u,24u)+vec4<u32>(c_idx*4u)))&vec4<u32>(15u);let w=vec4<f16>(vec4<f32>(q)*ds-vec4<f32>(ms));let dst=row*TILE_K+col+v*4u;tile_B[dst]=w.x;tile_B[dst+1u]=w.y;tile_B[dst+2u]=w.z;tile_B[dst+3u]=w.w;} }else{for(var i:u32=0u;i<16u;i++){let k:u32=k_idx+col+i;tile_B[row*TILE_K+col+i]=weights[params.weight_offset+(w_global-{{ denseTailStart }}u)*IN_F+k];}
{% else %}for(var i:u32=0u;i<16u;i++){let k:u32=k_idx+col+i;tile_B[row*TILE_K+col+i]=weights[params.weight_offset+w_global*IN_F+k];
{% endif %}} }fn storeOutput(offset:u32,row:u32,col:u32,src_slot:u32,row_limit:i32){if(row_limit>0&&row<u32(row_limit)){let col2:u32=col+1u;y[offset+row*OUT_F+col]=f16(scratch[src_slot][row*8u+col]);y[offset+row*OUT_F+col2]=f16(scratch[src_slot][row*8u+col2]);} }
{% set accScalar = "f32" %}{% include "ops/_shared/subgroup-matrix-topology.wgsl.jinja" %}{% include "ops/_shared/subgroup-matrix-software-8x8.wgsl.jinja" %}{% macro matrixKernelBody(matrixSoftware) %}let a_global_base:u32=workgroup_id.y*TILE_ROWS;let w_global_base:u32=workgroup_id.x*TILE_COLS;let subtile_id:u32=matrixRole;let subtile_idx:u32=subtile_id/2u;let subtile_idy:u32=subtile_id%2u;let base_A:u32=subtile_idy*SUB_ROWS;let base_B:u32=subtile_idx*SUB_COLS;
{% include "ops/_shared/subgroup-matrix-8x8-accumulators.wgsl.jinja" %}{% set operandScalar = "f16" %}{% include "ops/_shared/subgroup-matrix-32x64-staged-kloop.wgsl.jinja" %}let row:u32=sg_id/4u;let col:u32=(sg_id%4u)*2u;
  var matrix_c_offset: u32 = (a_global_base + base_A) * OUT_F + w_global_base + base_B{% if denseTailStart | default(0) %} + {{ dstColStart }}u{% endif %};
var row_limit:i32=i32(params.m)-i32(a_global_base+base_A);let bank:u32=subtile_id*8u;
{% set bank0 = "bank + 0u" %}{% filter indent(2, true) %}{% include "./subgroup-matrix-store-8-banks.wgsl.jinja" %}{% endfilter %}workgroupBarrier();storeOutput(matrix_c_offset,row,col,bank+0u,row_limit);storeOutput(matrix_c_offset+8u,row,col,bank+1u,row_limit);storeOutput(matrix_c_offset+16u,row,col,bank+2u,row_limit);storeOutput(matrix_c_offset+24u,row,col,bank+3u,row_limit);matrix_c_offset=matrix_c_offset+8u*OUT_F;row_limit=i32(params.m)-i32(a_global_base+base_A+8u);storeOutput(matrix_c_offset,row,col,bank+4u,row_limit);storeOutput(matrix_c_offset+8u,row,col,bank+5u,row_limit);storeOutput(matrix_c_offset+16u,row,col,bank+6u,row_limit);storeOutput(matrix_c_offset+24u,row,col,bank+7u,row_limit);
{% endmacro %}{% include "ops/_shared/subgroup-matrix-topology-entry.wgsl.jinja" %}{{ matrix_topology_entry("128", "128u / 32u", "workgroup_id", "sg_id") }}`],["prefill-matmul-tiled-reg.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const K:u32={{ inFeatures }}u;const N:u32={{ outFeatures }}u;const BK:u32=16u;const BM:u32=64u;const BN:u32=64u;const TM:u32=4u;const TN:u32=4u;const K_VECS:u32=BK/4u;var<workgroup>tileA:array<array<vec4<f32>,K_VECS>,BM>;var<workgroup>tileB:array<array<vec4<f32>,K_VECS>,BN>;
{% if bf16Weights is defined and bf16Weights %}fn bf16Weight(index:u32)->f32{let word=weights[index>>1u];return bitcast<f32>(select(word<<16u,word&0xffff0000u,(index&1u)==1u));}
{% elif f16Weights is defined and f16Weights %}{% include "ops/_shared/f16-bits-to-f32.wgsl.jinja" %}fn f16Weight(index:u32)->f32{let word=weights[index>>1u];return f16_bits_to_f32(select(word,word>>16u,(index&1u)==1u));}
{% endif %}@compute@workgroup_size(16,16,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>,@builtin(num_workgroups)nwg:vec3<u32>){let li=lid.y*16u+lid.x;let woff=params.weight_offset;let n_tiles=(N+BN-1u)/BN;let m_tiles=(params.m+BM-1u)/BM;for(var nt:u32=wg.x;nt<n_tiles;nt=nt+nwg.x){let nBase=nt*BN;for(var mt:u32=wg.y;mt<m_tiles;mt=mt+nwg.y){let mBase=mt*BM;var acc:array<f32,TM*TN>;for(var i:u32=0u;i<TM*TN;i=i+1u){acc[i]=0.0;}let numTiles=(K+BK-1u)/BK;for(var kt:u32=0u;kt<numTiles;kt=kt+1u){let kBase=kt*BK;for(var idx:u32=li;idx<BM*K_VECS;idx=idx+256u){let ar=idx/K_VECS;let ac4=idx%K_VECS;let am=mBase+ar;let ak=kBase+ac4*4u;var aWord=vec4<f32>(0.0);if(am<params.m){
{% for component in range(4) %}if(ak+{{ component }}u<K){aWord[{{ component }}u]=f32(a[am*K+ak+{{ component }}u]);}
{% endfor %}}tileA[ar][ac4]=aWord;}for(var idx:u32=li;idx<BN*K_VECS;idx=idx+256u){let bc=idx/K_VECS;let br4=idx%K_VECS;let bn=nBase+bc;let bk=kBase+br4*4u;var bWord=vec4<f32>(0.0);if(bn<N){
{% for component in range(4) %}{% if bf16Weights is defined and bf16Weights %}if(bk+{{ component }}u<K){bWord[{{ component }}u]=bf16Weight(woff+bn*K+bk+{{ component }}u);}
{% elif f16Weights is defined and f16Weights %}if(bk+{{ component }}u<K){bWord[{{ component }}u]=f16Weight(woff+bn*K+bk+{{ component }}u);}
{% else %}if(bk+{{ component }}u<K){bWord[{{ component }}u]=f32(weights[woff+bn*K+bk+{{ component }}u]);}
{% endif %}{% endfor %}}tileB[bc][br4]=bWord;}workgroupBarrier();
{% set regT = "f32" %}{% set regCast = 0 %}{% filter indent(8, true) %}{% include "ops/_shared/matmul-reg-microtile-fma.wgsl.jinja" %}{% endfilter %}workgroupBarrier();}let m0=mBase+lid.y*TM;let n0=nBase+lid.x*TN;for(var ti:u32=0u;ti<TM;ti=ti+1u){let m=m0+ti;if(m>=params.m){continue;}for(var tj:u32=0u;tj<TN;tj=tj+1u){let n=n0+tj;if(n<N){y[m*N+n]={{ yScalar }}(acc[ti*TN+tj]);} } } } } }`],["prefill-matmul.wgsl.jinja",'{% include "ops/_shared/flat-index-2d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(workgroupSize, "g", "") }} let total=params.m*params.out_features;if(g>=total){return;}let row=g/params.out_features;let col=g%params.out_features;let a_base=row*params.in_features;let w_base=params.weight_offset+col*params.in_features;var acc=0.0;for(var k=0u;k<params.in_features;k=k+1u){acc=acc+f32(a[a_base+k])*f32(weights[w_base+k]);}y[g]={{ yScalar }}(acc);}'],["prefill-q1-subgroup-stage-b-mma.wgsl.jinja",'let col_block=kb/4u;let word_id=kb%4u;let idx=group_base+col_block*4u+word_id;let sign_word=q1_bits[idx*4u+rit];let scale=f16(f32(q1_scales[group_base+col_block*4u+rit]));let neg=-scale;let bbase=local_idx*TILE_K;for(var j=0u;j<TILE_K;j=j+1u){tile_B[bbase+j]=select(neg,scale,((sign_word>>j)&1u)==1u);}workgroupBarrier();for(var step=0u;step<TILE_K;step=step+8u){let matA0={{ matrixLoad(matrixSoftware, "left", "f16", "row_major", "&tile_A", "step", "TILE_K") }};let matA1={{ matrixLoad(matrixSoftware, "left", "f16", "row_major", "&tile_A", "8u * TILE_K + step", "TILE_K") }};let boff=subgroup_col*TILE_K+step;let matB0={{ matrixLoad(matrixSoftware, "right", "f16", "col_major", "&tile_B", "boff", "TILE_K") }};let matB1={{ matrixLoad(matrixSoftware, "right", "f16", "col_major", "&tile_B", "boff + 8u * TILE_K", "TILE_K") }};let matB2={{ matrixLoad(matrixSoftware, "right", "f16", "col_major", "&tile_B", "boff + 16u * TILE_K", "TILE_K") }};let matB3={{ matrixLoad(matrixSoftware, "right", "f16", "col_major", "&tile_B", "boff + 24u * TILE_K", "TILE_K") }};matC00={{ matrixMma(matrixSoftware, "f32", "matA0", "matB0", "matC00") }};matC01={{ matrixMma(matrixSoftware, "f32", "matA0", "matB1", "matC01") }};matC02={{ matrixMma(matrixSoftware, "f32", "matA0", "matB2", "matC02") }};matC03={{ matrixMma(matrixSoftware, "f32", "matA0", "matB3", "matC03") }};matC10={{ matrixMma(matrixSoftware, "f32", "matA1", "matB0", "matC10") }};matC11={{ matrixMma(matrixSoftware, "f32", "matA1", "matB1", "matC11") }};matC12={{ matrixMma(matrixSoftware, "f32", "matA1", "matB2", "matC12") }};matC13={{ matrixMma(matrixSoftware, "f32", "matA1", "matB3", "matC13") }};}workgroupBarrier();'],["prefill-quant-subgroup-stage-a.wgsl.jinja",`{% set aVecs = 4 if (stageAVecs is defined and stageAVecs == 4) else (1 if (smallM is defined and smallM) else 2) %}{% if aVecs == 4 %}{% elif aVecs == 1 %}{% else %}{% endif %}{% for v in range(aVecs) %}var av{{ v }}=vec4<f32>(0.0);
{% endfor %}if(m0+am<params.m){
{% for v in range(aVecs) %}av{{ v }}=vec4<f32>(a[(m0+am)*K_V4+kb*8u+aq{{ (" + " ~ v ~ "u") if v else "" }}]);
{% endfor %}}let abase=am*TILE_K+aq*4u;
{% for v in range(aVecs) %}{% for lane in ["x", "y", "z", "w"] %}{% set offset = 4 * v + loop.index0 %}tile_A[abase{{ (" + " ~ offset ~ "u") if offset else "" }}]=f16(av{{ v }}.{{ lane }});
{% endfor %}{% endfor %}`],["prefill-subgroup-matmul-epilogue.wgsl.jinja",`{% set stepATiles = sgmatATiles if sgmatATiles is defined else 2 %}{% set stepBTiles = 4 %}{% set rowBound = "row_limit" if (dynamicRows is defined and dynamicRows) else "params.m" %}{% set accCount = "eight" if stepATiles * stepBTiles == 8 else (stepATiles * stepBTiles)|string %}{% set staticSchedule = staticSubgroupSchedule | default(false) %}{% for stepIndex in range(4 if staticSchedule else 1) %}{% if staticSchedule %}{const step={{ stepIndex * 8 }}u;
{% else %}for(var step=0u;step<TILE_K;step=step+8u){
{% endif %}{% for m in range(stepATiles) %}let matA{{ m }}={{ matrixLoad(matrixSoftware, "left", "f16", "row_major", "&tile_A", "(base_A + " ~ (m * 8) ~ "u) * TILE_K + step", "TILE_K") }};
{% endfor %}let boff=base_B*TILE_K+step;
{% for n in range(stepBTiles) %}let matB{{ n }}={{ matrixLoad(matrixSoftware, "right", "f16", "col_major", "&tile_B", ("boff" ~ (" + " ~ (n * 8) ~ "u * TILE_K" if n else "")), "TILE_K") }};
{% endfor %}{% for m in range(stepATiles) %}{% for n in range(stepBTiles) %}matC{{ m }}{{ n }}={{ matrixMma(matrixSoftware, "f32", "matA" ~ (m), "matB" ~ (n), "matC" ~ (m) ~ (n)) }};
{% endfor %}{% endfor %}}
{% endfor %}workgroupBarrier();}
{% if (wideNativeTile | default(false)) and (resultDrainBanks | default(1)) == 4 -%}let sgLane=matrixLane;
{% for ri in range(stepATiles) %}workgroupBarrier();
{% for ci in range(stepBTiles) %}{{ matrixStore(matrixSoftware, "row_major", "&scratch[subtile_id]", (ci * 64) ~ "u", "matC" ~ (ri) ~ (ci), "FRAG_DIM") }}
{% endfor %}workgroupBarrier();
{% for ci in range(stepBTiles) %}for(var t=sgLane;t<FRAG_ELEMS;t=t+32u){let r=base_A+{{ ri }}u*FRAG_DIM+t/FRAG_DIM;let c=base_B+{{ ci }}u*FRAG_DIM+t%FRAG_DIM;if(m0+r<{{ rowBound }}&&n0+c<{{ outFeatures }}u){y[(m0+r)*OUT_STRIDE+DST_COL+n0+c]={{ yScalar }}(scratch[subtile_id][{{ ci * 64 }}u+t]);} }
{% endfor %}{% endfor %}{% else -%}let sgLane=matrixLane;
{% for ri in range(stepATiles) %}{% for ci in range(stepBTiles) %}workgroupBarrier();{{ matrixStore(matrixSoftware, "row_major", "&scratch[subtile_id]", "0u", "matC" ~ (ri) ~ (ci), "FRAG_DIM") }} workgroupBarrier();for(var t=sgLane;t<FRAG_ELEMS;t=t+32u){let r=base_A+{{ ri }}u*FRAG_DIM+t/FRAG_DIM;let c=base_B+{{ ci }}u*FRAG_DIM+t%FRAG_DIM;
    if (m0 + r < {{ rowBound }}{% if (smallM | default(false)) or (wideNativeTile | default(false)) %} && n0 + c < {{ outFeatures }}u{% endif %}) {
{% if sgSplitK is defined and sgSplitK %}partials[(wid.z*params.m+(m0+r))*N_OUT+n0+c]=scratch[subtile_id][t];
{% else %}y[(m0+r)*OUT_STRIDE+DST_COL+n0+c]={{ yScalar }}(scratch[subtile_id][t]);
{% endif %}} }
{% endfor %}{% endfor %}{% endif -%}`],["subgroup-matrix-store-8-banks.wgsl.jinja",`{% for acc in ["matC00", "matC01", "matC02", "matC03", "matC10", "matC11", "matC12", "matC13"] %}{{ matrixStore(matrixSoftware, "row_major", "&scratch[" ~ (bank0 if loop.index0 == 0 else "bank + " ~ loop.index0 ~ "u") ~ "]", "0u", acc, "8u") }}
{% endfor %}`]]}],["com.xenova.LlamaPrefillRopeCacheKv",{manifest:{domain:"com.xenova",name:"LlamaPrefillRopeCacheKv",sinceVersion:1,inputs:{projectionsT:{onnx:"Projections",dtype:"A",rank:1,optional:!0},qgT:{onnx:"Qg",dtype:"A",rank:2,optional:!0},kT:{onnx:"K",dtype:"A",rank:2,optional:!0},vT:{onnx:"V",dtype:"A",rank:2,optional:!0},qkvT:{onnx:"Qkv",dtype:"A",rank:2},weightsT:{onnx:"Weights",dtype:"W",rank:1,optional:!0},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1},cosT:{onnx:"RopeCos",dtype:"float32"},sinT:{onnx:"RopeSin",dtype:"float32"},ropePositionsT:{onnx:"RopePositions",dtype:"uint32",rank:2,optional:!0}},outputs:{qkvT:{onnx:"Qkv",dtype:"A",rank:2,shape:"shapes.qkvT"},gateT:{onnx:"Gate",dtype:"A",rank:2,optional:!0,shape:["args.seqLen","args.numHeads * args.headDim"]},cacheKeysT:{onnx:"CacheKeys",dtype:"C",rank:1,shape:"shapes.cacheKeysT"},cacheValuesT:{onnx:"CacheValues",dtype:"C",rank:1,shape:"shapes.cacheValuesT"}},args:{layer:{kind:"u32"},cacheLen:{kind:"u32",onnx:"cache_len"},pastLen:{kind:"u32",onnx:"past_len",required:!1},seqLen:{kind:"u32",onnx:"seq_len"},numHeads:{kind:"u32",onnx:"num_heads"},numKvHeads:{kind:"u32",onnx:"num_kv_heads"},headDim:{kind:"u32",onnx:"head_dim"},rotaryDim:{kind:"u32",onnx:"rotary_dim",required:!1},qNormOffset:{kind:"u32",onnx:"q_norm_offset",required:!1},kNormOffset:{kind:"u32",onnx:"k_norm_offset",required:!1},rmsEps:{kind:"f32",onnx:"rms_eps",required:!1},qgProjectionOffset:{kind:"u32",onnx:"qg_projection_offset",required:!1},kProjectionOffset:{kind:"u32",onnx:"k_projection_offset",required:!1},vProjectionOffset:{kind:"u32",onnx:"v_projection_offset",required:!1},mropeSectionT:{kind:"u32",onnx:"mrope_section_t",required:!1},mropeSectionH:{kind:"u32",onnx:"mrope_section_h",required:!1},mropeSectionW:{kind:"u32",onnx:"mrope_section_w",required:!1}},typeConstraints:{A:["float32","float16"],C:["float32","float16"],W:["float32","float16"]},derive:{deviceWorkgroupCap:"min(device.limits.maxComputeInvocationsPerWorkgroup, device.limits.maxComputeWorkgroupSizeX)",qHidden:"args.numHeads * args.headDim",kvHidden:"args.numKvHeads * args.headDim",qkvStride:"qHidden + 2 * kvHidden",totalHeads:"args.numHeads + args.numKvHeads",effectiveRotaryDim:"args.rotaryDim if args.rotaryDim else args.headDim",workgroupSize:"64",pastLen:"args.pastLen if args.pastLen else 0",legacyLayout:"not present.weightsT and not present.projectionsT and not present.qgT and not present.kT and not present.vT and not present.gateT and not present.ropePositionsT",packedNormLayout:"present.weightsT and not present.projectionsT and not present.qgT and not present.kT and not present.vT and not present.gateT",projectedNormLayout:"present.weightsT and present.projectionsT and present.gateT and not present.qgT and not present.kT and not present.vT",separateNormLayout:"present.weightsT and not present.projectionsT and present.qgT and present.kT and present.vT and present.gateT",commonNormArgs:"args.qNormOffset is defined and args.kNormOffset is defined and args.rmsEps is defined",commonShape:"ranks.qkvT == 2 and ranks.cacheKeysT == 1 and ranks.cacheValuesT == 1 and tensorDtypes.cacheKeysT == tensorDtypes.cacheValuesT and args.cacheLen > 0 and args.seqLen > 0 and args.numHeads > 0 and args.numKvHeads > 0 and args.headDim > 0 and effectiveRotaryDim > 0 and effectiveRotaryDim % 2 == 0 and effectiveRotaryDim <= args.headDim and totalHeads <= device.limits.maxComputeWorkgroupsPerDimension and workgroupSize <= deviceWorkgroupCap and 4 * workgroupSize <= device.limits.maxComputeWorkgroupStorageSize and dim(shapes.qkvT, 1) == qkvStride and numel(shapes.qkvT) >= args.seqLen * qkvStride and numel(shapes.cosT) >= (pastLen + args.seqLen) * (effectiveRotaryDim / 2) and numel(shapes.sinT) >= (pastLen + args.seqLen) * (effectiveRotaryDim / 2) and numel(shapes.cacheKeysT) >= (args.layer + 1) * args.cacheLen * kvHidden and numel(shapes.cacheValuesT) >= (args.layer + 1) * args.cacheLen * kvHidden",commonNormDtypes:'(tensorDtypes.qkvT == "float32" or tensorDtypes.qkvT == "float16") and (tensorDtypes.cacheKeysT == "float32" or tensorDtypes.cacheKeysT == "float16") and (tensorDtypes.weightsT == "float32" or tensorDtypes.weightsT == "float16") and f16Ok(tensorDtypes.qkvT) and f16Ok(tensorDtypes.cacheKeysT) and f16Ok(tensorDtypes.weightsT)',weightsShape:"ranks.weightsT == 1 and numel(shapes.weightsT) >= args.qNormOffset + args.headDim and numel(shapes.weightsT) >= args.kNormOffset + args.headDim",gateShape:"ranks.gateT == 2 and tensorDtypes.gateT == tensorDtypes.qkvT and dim(shapes.gateT, 0) == args.seqLen and dim(shapes.gateT, 1) == qHidden",activationScalar:"dtypes.A",cacheScalar:"dtypes.C",layer:"args.layer",cacheLen:"args.cacheLen",numKvHeads:"args.numKvHeads",headDim:"args.headDim",weightScalar:"dtypes.W",seqLen:"args.seqLen",numHeads:"args.numHeads",rmsEps:"args.rmsEps",qNormOffset:"args.qNormOffset",kNormOffset:"args.kNormOffset",writeRotatedK:"not args.rotaryDim",mrope:"present.ropePositionsT",mropeContract:'present.ropePositionsT and ranks.ropePositionsT == 2 and tensorDtypes.ropePositionsT == "uint32" and dim(shapes.ropePositionsT, 0) == 3 and dim(shapes.ropePositionsT, 1) == args.seqLen and args.mropeSectionT is defined and args.mropeSectionH is defined and args.mropeSectionW is defined and args.mropeSectionT + args.mropeSectionH + args.mropeSectionW == effectiveRotaryDim / 2',mropeSectionH:"args.mropeSectionH if args.mropeSectionH is defined else 0",mropeSectionW:"args.mropeSectionW if args.mropeSectionW is defined else 0"},bindings:{qkv:{arg:"qkvT",elementType:"$activationScalar"},cache_keys:{arg:"cacheKeysT",elementType:"$cacheScalar"},cache_values:{arg:"cacheValuesT",elementType:"$cacheScalar"},rope_cos:{arg:"cosT",elementType:"f32"},rope_sin:{arg:"sinT",elementType:"f32"},rope_positions:{arg:"ropePositionsT",elementType:"u32"},params:{struct:[{name:"past_len",type:"u32",value:"args.pastLen if args.pastLen else 0"},{name:"cache_len",type:"u32",value:"args.cacheLen",required:!0,internal:!0},{name:"seq_len",type:"u32",value:"args.seqLen"}]},weights:{arg:"weightsT",elementType:"$weightScalar"},gate:{arg:"gateT",elementType:"$activationScalar"},projections:{arg:"projectionsT",name:"projections",elementType:"$activationScalar"},qg:{arg:"qgT",name:"qg",elementType:"$activationScalar"},k_in:{arg:"kT",name:"k_in",elementType:"$activationScalar"},v_in:{arg:"vT",name:"v_in",elementType:"$activationScalar"}},variants:[{id:"scalar",priority:0,when:["legacyLayout","not args.rotaryDim","ranks.qkvT == 2","(ranks.cacheKeysT == 1 or ranks.cacheKeysT == 5)","(ranks.cacheValuesT == 1 or ranks.cacheValuesT == 5)","tensorDtypes.cacheKeysT == tensorDtypes.cacheValuesT",'(tensorDtypes.qkvT == "float32" or tensorDtypes.qkvT == "float16")','(tensorDtypes.cacheKeysT == "float32" or tensorDtypes.cacheKeysT == "float16")',"f16Ok(tensorDtypes.qkvT)","f16Ok(tensorDtypes.cacheKeysT)","args.cacheLen > 0","args.seqLen > 0","args.numHeads > 0","args.numKvHeads > 0","args.headDim > 0","args.headDim % 2 == 0","dim(shapes.qkvT, 1) == (args.numHeads + 2 * args.numKvHeads) * args.headDim","numel(shapes.qkvT) >= args.seqLen * (args.numHeads + 2 * args.numKvHeads) * args.headDim","numel(shapes.cosT) >= ((args.pastLen if args.pastLen else 0) + args.seqLen) * (args.headDim / 2)","numel(shapes.sinT) >= ((args.pastLen if args.pastLen else 0) + args.seqLen) * (args.headDim / 2)","numel(shapes.cacheKeysT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim","numel(shapes.cacheValuesT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim"],derive:{headDimHalf:"args.headDim / 2",qPairs:"args.numHeads * (args.headDim / 2)",kvPairs:"args.numKvHeads * (args.headDim / 2)",workPerToken:"args.numHeads * (args.headDim / 2) + args.numKvHeads * (args.headDim / 2) + args.numKvHeads * args.headDim"},passes:[{id:"main",name:"LlamaPrefillRopeCacheKv",shader:"prefill-rope-cache-kv.wgsl.jinja",bindings:["qkv","cache_keys","cache_values","rope_cos","rope_sin","params"],dispatch:{threads:"args.seqLen * workPerToken",workgroupSize:"workgroupSize"}}]},{id:"partial",priority:0,when:["legacyLayout","args.rotaryDim is defined","ranks.qkvT == 2","ranks.cacheKeysT == 1","ranks.cacheValuesT == 1",'(tensorDtypes.qkvT == "float32" or tensorDtypes.qkvT == "float16")','(tensorDtypes.cacheKeysT == "float32" or tensorDtypes.cacheKeysT == "float16")',"f16Ok(tensorDtypes.qkvT)","f16Ok(tensorDtypes.cacheKeysT)","args.numHeads > 0","args.numKvHeads > 0","args.headDim > 0","args.cacheLen > 0","args.seqLen > 0","args.rotaryDim > 0","args.rotaryDim % 2 == 0","args.rotaryDim <= args.headDim","dim(shapes.qkvT, 1) == (args.numHeads + 2 * args.numKvHeads) * args.headDim","numel(shapes.qkvT) >= args.seqLen * (args.numHeads + 2 * args.numKvHeads) * args.headDim","numel(shapes.cosT) >= (pastLen + args.seqLen) * (args.rotaryDim / 2)","numel(shapes.sinT) >= (pastLen + args.seqLen) * (args.rotaryDim / 2)","numel(shapes.cacheKeysT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim","numel(shapes.cacheValuesT) >= (args.layer + 1) * args.cacheLen * args.numKvHeads * args.headDim"],derive:{rotaryDim:"args.rotaryDim",rotaryHalf:"args.rotaryDim / 2",qPairs:"args.numHeads * (args.rotaryDim / 2)",kvPairs:"args.numKvHeads * (args.rotaryDim / 2)",kvPass:"args.numKvHeads * (args.headDim - args.rotaryDim)",workPerToken:"args.numHeads * (args.rotaryDim / 2) + args.numKvHeads * (args.rotaryDim / 2) + args.numKvHeads * (args.headDim - args.rotaryDim) + args.numKvHeads * args.headDim"},passes:[{id:"main",name:"LlamaPrefillPartialRopeCacheKv",shader:"prefill-partial-rope-cache-kv.wgsl.jinja",bindings:["qkv","cache_keys","cache_values","rope_cos","rope_sin","params"],dispatch:{threads:"args.seqLen * workPerToken",workgroupSize:"workgroupSize"}}]},{cases:[{suffix:"",mropeGate:["not mrope"],positions:[],storageFloor:"8",bindingFloor:"9"},{suffix:"_mrope",mropeGate:["mropeContract"],positions:["rope_positions"],storageFloor:"9",bindingFloor:"10"}],variants:[{id:"qk_norm{suffix}",priority:1,when:["packedNormLayout","commonNormArgs","commonShape","commonNormDtypes","weightsShape","{mropeGate}"],derive:{rotaryDim:"effectiveRotaryDim",assembledInputs:!1},passes:[{id:"main",name:"LlamaPrefillQkNormRopeCacheKv",shader:"prefill-qk-norm-rope-cache-kv.wgsl.jinja",bindings:["qkv","weights","cache_keys","cache_values","rope_cos","rope_sin","{positions}","params"],dispatch:{x:"totalHeads",y:"min(args.seqLen, DISPATCH_FOLD_WIDTH)",z:"ceilDiv(args.seqLen, DISPATCH_FOLD_WIDTH)"}}]},{id:"qk_norm_projected{suffix}",priority:2,when:["projectedNormLayout","commonNormArgs","commonShape","commonNormDtypes","weightsShape","gateShape","ranks.projectionsT == 1","tensorDtypes.projectionsT == tensorDtypes.qkvT","args.qgProjectionOffset is defined","args.kProjectionOffset is defined","args.vProjectionOffset is defined","args.kProjectionOffset >= args.qgProjectionOffset + args.seqLen * 2 * qHidden","args.vProjectionOffset >= args.kProjectionOffset + args.seqLen * kvHidden","numel(shapes.projectionsT) >= args.vProjectionOffset + args.seqLen * kvHidden","numel(shapes.projectionsT) * dtypeBytes(dtypes.A) <= device.limits.maxStorageBufferBindingSize","numel(shapes.projectionsT) * dtypeBytes(dtypes.A) <= device.limits.maxBufferSize","device.limits.maxStorageBuffersPerShaderStage >= {storageFloor}","device.limits.maxBindingsPerBindGroup >= {bindingFloor}","{mropeGate}"],derive:{rotaryDim:"effectiveRotaryDim",qgOffset:"args.qgProjectionOffset",kOffset:"args.kProjectionOffset",vOffset:"args.vProjectionOffset",assembledInputs:!0},passes:[{id:"main",name:"LlamaPrefillAssembleQkNormRopeCacheKv",shader:"prefill-qk-norm-rope-cache-kv.wgsl.jinja",bindings:["projections","qkv","gate","weights","cache_keys","cache_values","rope_cos","rope_sin","{positions}","params"],dispatch:{x:"totalHeads",y:"min(args.seqLen, DISPATCH_FOLD_WIDTH)",z:"ceilDiv(args.seqLen, DISPATCH_FOLD_WIDTH)"}}]},{id:"qk_norm_separate{suffix}",priority:1,when:["separateNormLayout","commonNormArgs","commonShape","commonNormDtypes","weightsShape","gateShape","ranks.qgT == 2","ranks.kT == 2","ranks.vT == 2","tensorDtypes.qgT == tensorDtypes.qkvT","tensorDtypes.kT == tensorDtypes.qkvT","tensorDtypes.vT == tensorDtypes.qkvT","numel(shapes.qgT) >= args.seqLen * 2 * qHidden","numel(shapes.kT) >= args.seqLen * kvHidden","numel(shapes.vT) >= args.seqLen * kvHidden","{mropeGate}"],derive:{rotaryDim:"effectiveRotaryDim",qSize:"qHidden",kvSize:"kvHidden",assembledInputs:!1},passes:[{id:"assemble",name:"LlamaPrefillAssembleQkv",shader:"prefill-assemble-qkv.wgsl.jinja",bindings:["qg","k_in","v_in","qkv","gate"],dispatch:{threads:"args.seqLen * qkvStride",workgroupSize:"workgroupSize"}},{id:"main",name:"LlamaPrefillQkNormRopeCacheKv",shader:"prefill-qk-norm-rope-cache-kv.wgsl.jinja",bindings:["qkv","weights","cache_keys","cache_values","rope_cos","rope_sin","{positions}","params"],dispatch:{x:"totalHeads",y:"min(args.seqLen, DISPATCH_FOLD_WIDTH)",z:"ceilDiv(args.seqLen, DISPATCH_FOLD_WIDTH)"}}]}]}]},assets:[["prefill-assemble-qkv.wgsl.jinja",'{% include "ops/_shared/flat-index-2d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const Q_SIZE:u32={{ qSize }}u;const KV_SIZE:u32={{ kvSize }}u;const HEAD_DIM:u32={{ headDim }}u;const QKV_STRIDE:u32={{ qkvStride }}u;const SEQ_LEN:u32={{ seqLen }}u;fn store_act(value:f32)->{{ activationScalar }} {return {{ activationScalar }}(value);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(workgroupSize, "g", "SEQ_LEN * QKV_STRIDE") }} let t=g/QKV_STRIDE;let c=g%QKV_STRIDE;if(c<Q_SIZE){let head=c/HEAD_DIM;let d=c%HEAD_DIM;let qg_base=t*2u*Q_SIZE+head*2u*HEAD_DIM+d;qkv[g]=store_act(f32(qg[qg_base]));gate[t*Q_SIZE+c]=store_act(f32(qg[qg_base+HEAD_DIM]));}else if(c<Q_SIZE+KV_SIZE){qkv[g]=store_act(f32(k_in[t*KV_SIZE+(c-Q_SIZE)]));}else{qkv[g]=store_act(f32(v_in[t*KV_SIZE+(c-Q_SIZE-KV_SIZE)]));} }'],["prefill-partial-rope-cache-kv.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const LAYER:u32={{ layer }}u;const CACHE_LEN:u32={{ cacheLen }}u;const NUM_KV_HEADS:u32={{ numKvHeads }}u;const HEAD_DIM:u32={{ headDim }}u;const ROTARY_DIM:u32={{ rotaryDim }}u;const ROTARY_HALF:u32={{ rotaryHalf }}u;const Q_HIDDEN:u32={{ qHidden }}u;const KV_HIDDEN:u32={{ kvHidden }}u;const QKV_STRIDE:u32={{ qkvStride }}u;const Q_PAIRS:u32={{ qPairs }}u;const KV_PAIRS:u32={{ kvPairs }}u;const KV_PASS:u32={{ kvPass }}u;const PASS_PER_HEAD:u32={{ headDim }}u-{{ rotaryDim }}u;const WORK_PER_TOKEN:u32={{ workPerToken }}u;fn rope_index(pos:u32,d:u32)->u32{return pos*ROTARY_HALF+d;}
{% include "ops/_shared/kv-cache-index.wgsl.jinja" %}fn store_cache(value:f32)->{{ cacheScalar }} {return {{ cacheScalar }}(value);}fn store_act(value:f32)->{{ activationScalar }} {return {{ activationScalar }}(value);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){let g=gid.x+gid.y*{{ DISPATCH_FOLD_WIDTH }}u*{{ workgroupSize }}u;let token=g/WORK_PER_TOKEN;if(token>=params.seq_len){return;}let pos=params.past_len+token;if(pos>=CACHE_LEN){return;}let local=g%WORK_PER_TOKEN;let row=token*QKV_STRIDE;if(local<Q_PAIRS){let head=local/ROTARY_HALF;let d=local%ROTARY_HALF;let base=row+head*HEAD_DIM;let rope=rope_index(pos,d);let c=rope_cos[rope];let s=rope_sin[rope];let first=f32(qkv[base+d]);let second=f32(qkv[base+d+ROTARY_HALF]);qkv[base+d]=store_act(first*c-second*s);qkv[base+d+ROTARY_HALF]=store_act(second*c+first*s);return;}var after=local-Q_PAIRS;if(after<KV_PAIRS){let head=after/ROTARY_HALF;let d=after%ROTARY_HALF;let base=row+Q_HIDDEN+head*HEAD_DIM;let rope=rope_index(pos,d);let c=rope_cos[rope];let s=rope_sin[rope];let first=f32(qkv[base+d]);let second=f32(qkv[base+d+ROTARY_HALF]);cache_keys[cache_index(pos,head,d)]=store_cache(first*c-second*s);cache_keys[cache_index(pos,head,d+ROTARY_HALF)]=store_cache(second*c+first*s);return;}after=after-KV_PAIRS;if(after<KV_PASS){let head=after/PASS_PER_HEAD;let d=ROTARY_DIM+(after%PASS_PER_HEAD);let base=row+Q_HIDDEN+head*HEAD_DIM;cache_keys[cache_index(pos,head,d)]=store_cache(f32(qkv[base+d]));return;}let v_local=after-KV_PASS;let head=v_local/HEAD_DIM;let dim=v_local%HEAD_DIM;let vbase=row+Q_HIDDEN+KV_HIDDEN+head*HEAD_DIM;cache_values[cache_index(pos,head,dim)]=store_cache(f32(qkv[vbase+dim]));}`],["prefill-qk-norm-rope-cache-kv.wgsl.jinja",`{% include "ops/_shared/tree-fold-inplace.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const LAYER:u32={{ layer }}u;const CACHE_LEN:u32={{ cacheLen }}u;const SEQ_LEN:u32={{ seqLen }}u;const NUM_HEADS:u32={{ numHeads }}u;const NUM_KV_HEADS:u32={{ numKvHeads }}u;const TOTAL_HEADS:u32=NUM_HEADS+NUM_KV_HEADS;const HEAD_DIM:u32={{ headDim }}u;const ROTARY_DIM:u32={{ rotaryDim }}u;const ROTARY_HALF:u32=ROTARY_DIM/2u;const Q_HIDDEN:u32={{ qHidden }}u;const KV_HIDDEN:u32={{ kvHidden }}u;const QKV_STRIDE:u32={{ qkvStride }}u;
{% if assembledInputs %}const QG_OFFSET:u32={{ qgOffset }}u;const K_OFFSET:u32={{ kOffset }}u;const V_OFFSET:u32={{ vOffset }}u;
{% endif %}const RMS_EPS:f32={{ rmsEps }};const W_Q_NORM:u32={{ qNormOffset }}u;const W_K_NORM:u32={{ kNormOffset }}u;const WORKGROUP_SIZE:u32={{ workgroupSize }}u;var<workgroup>partials:array<f32,{{ workgroupSize }}>;
{% include "ops/_shared/kv-cache-index.wgsl.jinja" %}{% if mrope %}const MROPE_SECTION_H:u32={{ mropeSectionH }}u;const MROPE_SECTION_W:u32={{ mropeSectionW }}u;fn mrope_row(token:u32,pair:u32)->u32{let lane=pair%3u;var axis=0u;if(lane==1u&&pair<3u*MROPE_SECTION_H){axis=1u;}else if(lane==2u&&pair<3u*MROPE_SECTION_W){axis=2u;}return rope_positions[axis*SEQ_LEN+token];}
{% endif %}fn source_value(token:u32,head:u32,dim:u32,is_k:bool)->f32{
{% if assembledInputs %}if(is_k){return f32(projections[K_OFFSET+token*KV_HIDDEN+head*HEAD_DIM+dim]);}return f32(projections[QG_OFFSET+token*2u*Q_HIDDEN+head*2u*HEAD_DIM+dim]);
{% else %}let region=select(0u,Q_HIDDEN,is_k);return f32(qkv[token*QKV_STRIDE+region+head*HEAD_DIM+dim]);
{% endif %}}fn normalized_checkpoint(value:f32,scale:f32,norm_offset:u32,dim:u32)->{{ activationScalar }} {return {{ activationScalar }}(value*scale*f32(weights[norm_offset+dim]));}fn store_cache(value:f32)->{{ cacheScalar }} {return {{ cacheScalar }}(value);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(workgroup_id)wid:vec3<u32>,@builtin(local_invocation_id)lid3:vec3<u32>){let token=wid.y+wid.z*{{ DISPATCH_FOLD_WIDTH }}u;let head_index=wid.x;if(token>=SEQ_LEN||head_index>=TOTAL_HEADS){return;}let is_k=head_index>=NUM_HEADS;let head=select(head_index,head_index-NUM_HEADS,is_k);let lid=lid3.x;let row=token*QKV_STRIDE;let norm_offset=select(W_Q_NORM,W_K_NORM,is_k);var sum=0.0;for(var dim=lid;dim<HEAD_DIM;dim=dim+WORKGROUP_SIZE){let value=source_value(token,head,dim,is_k);sum=fma(value,value,sum);
{% if assembledInputs %}if(!is_k){let source=QG_OFFSET+token*2u*Q_HIDDEN+head*2u*HEAD_DIM+HEAD_DIM+dim;gate[token*Q_HIDDEN+head*HEAD_DIM+dim]={{ activationScalar }}(f32(projections[source]));}
{% endif %}}partials[lid]=sum;workgroupBarrier();{{ wgsl_tree_fold(["partials"]) }} let scale=inverseSqrt(partials[0]/f32(HEAD_DIM)+RMS_EPS);let pos=params.past_len+token;let cache_valid=token<params.seq_len&&pos<CACHE_LEN;let qkv_base=row+select(0u,Q_HIDDEN,is_k)+head*HEAD_DIM;for(var dim=lid;dim<ROTARY_HALF;dim=dim+WORKGROUP_SIZE){let first_checkpoint=normalized_checkpoint(source_value(token,head,dim,is_k),scale,norm_offset,dim);let second_dim=dim+ROTARY_HALF;let second_checkpoint=normalized_checkpoint(source_value(token,head,second_dim,is_k),scale,norm_offset,second_dim);let first=f32(first_checkpoint);let second=f32(second_checkpoint);var out_first=first;var out_second=second;if(cache_valid){
{% if mrope %}let rope=mrope_row(token,dim)*ROTARY_HALF+dim;
{% else %}let rope=pos*ROTARY_HALF+dim;
{% endif %}let c=rope_cos[rope];let s=rope_sin[rope];out_first=first*c-second*s;out_second=second*c+first*s;}if(!is_k){qkv[qkv_base+dim]={{ activationScalar }}(out_first);qkv[qkv_base+second_dim]={{ activationScalar }}(out_second);}else{
{% if writeRotatedK %}qkv[qkv_base+dim]=select(first_checkpoint,{{ activationScalar }}(out_first),cache_valid);qkv[qkv_base+second_dim]=select(second_checkpoint,{{ activationScalar }}(out_second),cache_valid);
{% else %}qkv[qkv_base+dim]=first_checkpoint;qkv[qkv_base+second_dim]=second_checkpoint;
{% endif %}if(cache_valid){cache_keys[cache_index(pos,head,dim)]=store_cache(out_first);cache_keys[cache_index(pos,head,second_dim)]=store_cache(out_second);} } }for(var dim=ROTARY_DIM+lid;dim<HEAD_DIM;dim=dim+WORKGROUP_SIZE){let checkpoint=normalized_checkpoint(source_value(token,head,dim,is_k),scale,norm_offset,dim);qkv[qkv_base+dim]=checkpoint;if(is_k&&cache_valid){cache_keys[cache_index(pos,head,dim)]=store_cache(f32(checkpoint));} }if(is_k){let vbase=row+Q_HIDDEN+KV_HIDDEN+head*HEAD_DIM;for(var dim=lid;dim<HEAD_DIM;dim=dim+WORKGROUP_SIZE){
{% if assembledInputs %}let source=V_OFFSET+token*KV_HIDDEN+head*HEAD_DIM+dim;let checkpoint={{ activationScalar }}(f32(projections[source]));qkv[vbase+dim]=checkpoint;
{% else %}let checkpoint=qkv[vbase+dim];
{% endif %}if(cache_valid){cache_values[cache_index(pos,head,dim)]=store_cache(f32(checkpoint));} } } }`],["prefill-rope-cache-kv.wgsl.jinja",`{% include "ops/_shared/flat-index-2d.wgsl.jinja" %}{{ env.wgsl.resourceDeclarations }} const LAYER:u32={{ layer }}u;const CACHE_LEN:u32={{ cacheLen }}u;const NUM_KV_HEADS:u32={{ numKvHeads }}u;const HEAD_DIM:u32={{ headDim }}u;const HEAD_DIM_HALF:u32={{ headDimHalf }}u;const Q_HIDDEN:u32={{ qHidden }}u;const KV_HIDDEN:u32={{ kvHidden }}u;const QKV_STRIDE:u32={{ qkvStride }}u;const Q_PAIRS:u32={{ qPairs }}u;const KV_PAIRS:u32={{ kvPairs }}u;const WORK_PER_TOKEN:u32={{ workPerToken }}u;fn rope_index(pos:u32,d:u32)->u32{return pos*HEAD_DIM_HALF+d;}
{% include "ops/_shared/kv-cache-index.wgsl.jinja" %}fn load_act(value:{{ activationScalar }})->f32{return f32(value);}fn store_act(value:f32)->{{ activationScalar }} {return {{ activationScalar }}(value);}fn store_cache(value:f32)->{{ cacheScalar }} {return {{ cacheScalar }}(value);}@compute@workgroup_size({{ workgroupSize }})fn main(@builtin(global_invocation_id)gid:vec3<u32>){ {{ flat_index_2d(workgroupSize, "g", "") }} let token=g/WORK_PER_TOKEN;if(token>=params.seq_len){return;}let pos=params.past_len+token;if(pos>=CACHE_LEN){return;}let local=g%WORK_PER_TOKEN;let row=token*QKV_STRIDE;if(local<Q_PAIRS){let head=local/HEAD_DIM_HALF;let d=local%HEAD_DIM_HALF;let base=row+head*HEAD_DIM;let rope=rope_index(pos,d);let c=rope_cos[rope];let s=rope_sin[rope];let first=load_act(qkv[base+d]);let second=load_act(qkv[base+d+HEAD_DIM_HALF]);qkv[base+d]=store_act(first*c-second*s);qkv[base+d+HEAD_DIM_HALF]=store_act(second*c+first*s);return;}let after_q=local-Q_PAIRS;if(after_q<KV_PAIRS){let head=after_q/HEAD_DIM_HALF;let d=after_q%HEAD_DIM_HALF;let base=row+Q_HIDDEN+head*HEAD_DIM;let rope=rope_index(pos,d);let c=rope_cos[rope];let s=rope_sin[rope];let first=load_act(qkv[base+d]);let second=load_act(qkv[base+d+HEAD_DIM_HALF]);let out_first=first*c-second*s;let out_second=second*c+first*s;qkv[base+d]=store_act(out_first);qkv[base+d+HEAD_DIM_HALF]=store_act(out_second);cache_keys[cache_index(pos,head,d)]=store_cache(out_first);cache_keys[cache_index(pos,head,d+HEAD_DIM_HALF)]=store_cache(out_second);return;}let v_local=after_q-KV_PAIRS;let head=v_local/HEAD_DIM;let dim=v_local%HEAD_DIM;let vbase=row+Q_HIDDEN+KV_HIDDEN+head*HEAD_DIM;cache_values[cache_index(pos,head,dim)]=store_cache(load_act(qkv[vbase+dim]));}`]]}],["com.xenova.StridedCopy",{manifest:{domain:"com.xenova",name:"StridedCopy",sinceVersion:1,inputs:{srcT:{onnx:"Src",dtype:"S"}},outputs:{dstT:{onnx:"Dst",dtype:"D"}},args:{rows:{kind:"u32"},srcStride:{kind:"u32"},srcStart:{kind:"u32",required:!1},dstStride:{kind:"u32"},dstStart:{kind:"u32",required:!1},copyCols:{kind:"u32"}},typeConstraints:{S:["float32","float16","uint32","int32"],D:["float32","float16","uint32","int32"]},derive:{colChunk:4096,workgroupSize:64},bindings:{s:{arg:"srcT",elementType:"$sourceElement"},d:{arg:"dstT",elementType:"$destElement"},p:{struct:[{name:"rows",type:"u32"},{name:"copyCols",type:"u32"},{name:"srcStride",type:"u32"},{name:"srcStart",type:"u32",default:0},{name:"dstStride",type:"u32"},{name:"dstStart",type:"u32",default:0}]}},variants:[{id:"vec4_f16",priority:10,when:['dtypes.S == "f16"','dtypes.D == "f16"','device.features.has("shader-f16")',"args.srcStride % 4 == 0","(args.srcStart if args.srcStart else 0) % 4 == 0","args.dstStride % 4 == 0","(args.dstStart if args.dstStart else 0) % 4 == 0","args.copyCols % 4 == 0"],derive:{vectorized:!0,directScalarCopy:!1,sourceElement:'"vec4<f16>"',destElement:'"vec4<f16>"',destScalar:'"f16"',storageItemWidth:4},passes:[{id:"main",name:"StridedCopy",shader:"strided-copy.wgsl.jinja",bindings:["s","d","p"],dispatch:{workgroups:"args.rows * ceil(args.copyCols / (colChunk * storageItemWidth))"}}]},{id:"scalar",priority:0,when:["f16Ok(dtypes.S) and f16Ok(dtypes.D)"],derive:{vectorized:!1,directScalarCopy:"dtypes.S == dtypes.D",sourceElement:"dtypes.S",destElement:"dtypes.D",destScalar:"dtypes.D",storageItemWidth:1},passes:[{id:"main",name:"StridedCopy",shader:"strided-copy.wgsl.jinja",bindings:["s","d","p"],dispatch:{workgroups:"args.rows * ceil(args.copyCols / (colChunk * storageItemWidth))"}}]}]},assets:[["strided-copy.wgsl.jinja",`{{ env.wgsl.resourceDeclarations }} const COL_CHUNK:u32={{ colChunk }}u;const STORAGE_ITEM_WIDTH:u32={{ storageItemWidth }}u;@compute@workgroup_size({{ workgroupSize }},1,1)fn main(@builtin(workgroup_id)wg:vec3<u32>,@builtin(local_invocation_id)lid:vec3<u32>){let copyCols=p.copyCols/STORAGE_ITEM_WIDTH;let colTiles=max((copyCols+COL_CHUNK-1u)/COL_CHUNK,1u);let flat=wg.x+wg.y*{{ DISPATCH_FOLD_WIDTH }}u;if(flat>=p.rows*colTiles){return;}let r=flat/colTiles;let srcRow=(r*p.srcStride+p.srcStart)/STORAGE_ITEM_WIDTH;let dstRow=(r*p.dstStride+p.dstStart)/STORAGE_ITEM_WIDTH;let col_base=(flat-r*colTiles)*COL_CHUNK;let col_end=min(col_base+COL_CHUNK,copyCols);var i:u32=col_base+lid.x;loop{if(i>=col_end){break;}
{% if vectorized or directScalarCopy %}d[dstRow+i]=s[srcRow+i];
{% else %}d[dstRow+i]={{ destScalar }}(f32(s[srcRow+i]));
{% endif %}i=i+{{ workgroupSize }}u;} }`]]}]]);var _o=new Map,nv=new Map;function rf(e){let t=nv.get(e);if(t===void 0)return;if(Array.isArray(t.assets)){let s=new Map(_o);for(let[a,i]of t.assets)s.set(a,i);return{manifest:t.manifest,assets:zr(s)}}let r=t.assets;if(_o.size===0)return{manifest:t.manifest,assets:r};let n=_o;return{manifest:t.manifest,assets:Object.freeze({readText(s){if(r.has(s))return r.readText(s);let a=n.get(s);return a===void 0?r.readText(s):a},has(s){return r.has(s)||n.has(s)}})}}var sv=()=>{throw new Error("This feature is not included in this bundle.")},nf=sv;function ns(e,t={}){let r=sf(e);return new ts(r.manifest,{...r.renderedSourceAudit===void 0?{}:{renderedSourceAudit:r.renderedSourceAudit},...t,assets:r.assets})}function sf(e){let t=rf(e);if(t)return t;let r=tf.get(e);if(r){let n=new Map(ef);for(let[s,a]of r.assets)n.set(s,a);return{manifest:r.manifest,assets:zr(n)}}return nf(e)}function Pe(e,t=new Set){Ha(e,t,!0)}function Ha(e,t,r){if(!(e==null||typeof e!="object")&&!t.has(e)&&(t.add(e),!(ArrayBuffer.isView(e)||e instanceof ArrayBuffer))){if(uv(e)){r&&e.destroy();return}if(ov(e)){r&&e.destroy();return}if(e instanceof Map){for(let n of e.values())Ha(n,t,r);r&&e.clear();return}if(e instanceof Set){for(let n of e.values())Ha(n,t,r);r&&e.clear();return}for(let n of Object.keys(e))Ha(e[n],t,r)}}function uv(e){let t=e;return Array.isArray(t.shape)&&typeof t.dtype=="string"&&typeof t.destroy=="function"&&t.buffer!=null&&typeof t.buffer.destroy=="function"}function ov(e){let t=e;return typeof t.destroy=="function"&&typeof t.mapAsync=="function"&&typeof t.getMappedRange=="function"}function ct(e,t){if(e.length===1)throw e[0];if(e.length>1)throw new AggregateError(e,t)}async function Kt(e,t){try{await e.queueIdle(),Pe(t)}catch{}}var hn=`<think>
`,lv="<think>",Va="</think>",Ur=`<think>

</think>

`;var ko="webgpu-template-policy-probe",cv="webgpu-template-policy-probe-next",dv="webgpu-template-policy-probe-result",af=`<tool_call>
<function=webgpu_probe>
<parameter=x>
1
</parameter>
</function>
</tool_call>`,Kr=Object.freeze({preserve_thinking:!0});function $a(e,t){return e.endsWith(hn)?hn:e.endsWith(Ur)?Ur:t?"":null}function of(e,t,r){let n=[...t.map(s=>`reasoning
</think>

${s}`),...r?["unfinished reasoning"]:[]];return e===hn?n:e===Ur?t:[...n.map(s=>hn+s),...t.map(s=>Ur+s)]}function wr(e,t,r,n,s){return e.render({...s,messages:r,tools:null,bos_token:t.bos_token,eos_token:t.eos_token,add_generation_prompt:n})}function Ao({prelude:e,rawText:t}){return!(e===hn||e===""&&t.includes(lv))||t.includes(Va)?e+t:e+t+(t.endsWith(`
`)?"":`
`)+`</think>

`}var lf=e=>({content:Ao(e)});function uf(e,t,r,n,s){let a=new WeakMap,i=o=>{if(a.has(o))return a.get(o);let l=wr(e,t,[{role:"user",content:ko}],!0,o),c=$a(l,s);return a.set(o,c),c},u=o=>{let l=i(o.templateArgs),c=o.position==="after_last_user"?n:r;return l===null||c===null?{content:o.visibleText}:c({prelude:l,rawText:o.rawText})};return Object.freeze({templateArgs:Kr,assistantContent:o=>u(o).content,assistantMessageFields:o=>{let{content:l,...c}=u(o);return c}})}function cf(e,t,r,n,{openPrelude:s=!1}={}){let a=uf(e,t,r,null,s),i={role:"user",content:ko},u={role:"user",content:cv},o=Object.freeze({...Kr,enable_thinking:!0}),l=Object.freeze({...Kr,enable_thinking:!1});try{if(!wr(e,t,[i],!0,o).endsWith(hn)||!wr(e,t,[i],!0,l).endsWith(Ur)||s&&$a(wr(e,t,[i],!0,Kr),!1)!==null)return;for(let d of[o,l,Kr]){let p=wr(e,t,[i],!0,d),f=$a(p,s);if(f===null)return;for(let g of of(f,["answer"],!0)){let m={visibleText:g,rawText:g,templateArgs:d},h={...a.assistantMessageFields(m),role:"assistant",content:a.assistantContent(m)};if(!wr(e,t,[i,h,u],!0,d).startsWith(p+g))return}}let c=n.find(d=>fv(e,t,d,s))??null;return uf(e,t,r,c,s)}catch{return}}function fv(e,t,r,n){let s={role:"user",content:ko},a={role:"tool",content:dv};try{for(let i of[!0,!1,void 0]){let u=i===void 0?Kr:{...Kr,enable_thinking:i},o=wr(e,t,[s],!0,u),l=$a(o,n);if(l===null)return!1;for(let c of of(l,[af,`checking

${af}`,"answer"],!1)){let d={...r({prelude:l,rawText:c}),role:"assistant"};if(!wr(e,t,[s,d,a],!0,u).startsWith(o+c))return!1}}return!0}catch{return!1}}var pv=({prelude:e,rawText:t})=>{if(e===Ur)return{reasoning_content:"",content:t};let r=t.indexOf(Va);return r===-1?{reasoning_content:t,content:""}:{reasoning_content:t.slice(0,r),content:t.slice(r+Va.length).replace(/^\n\n/,"")}},df=e=>({reasoning_content:"",content:Ao(e)});function ff(e,t){return cf(e,t,df,[df,pv,lf],{openPrelude:!0})}var gv={"qwen-thinking-optional":ff};function pf(e,t){return mv(e,t)?.policy}function mv(e,t){for(let[r,n]of Object.entries(gv)){let s=n(e,t);if(s)return{name:r,policy:s}}}async function gf(e,t){let r=Math.max(1,Math.floor(t.baseDepth)),n=Math.max(r,Math.floor(t.maxDepth)),s=vv(t.minSampleTokens,24,2),a=.97,i=5,u=[r];for(;u.length<i&&u.at(-1)<n;){let d=u.at(-1);u.push(Math.min(n,d*2))}let o=[],l=null,c=0;for(let d of u){let p=Math.max(s,d+1),f=await e(d,p);if(!Number.isFinite(f.tokensPerSecond)||f.tokensPerSecond<=0)throw new Error(`decode calibration at depth ${d} returned invalid throughput`);if(f.tokenIds.length!==p)throw new Error(`decode calibration at depth ${d} produced ${f.tokenIds.length}/${p} tokens`);let g=l===null||hv(l,f.tokenIds);if(o.push({depth:d,sampleTokens:p,tokensPerSecond:f.tokensPerSecond,outputMatchesReference:g}),!g)return yo(o,a,"output-mismatch");if((l===null||f.tokenIds.length>l.length)&&(l=f.tokenIds),c>0&&f.tokensPerSecond<=c/a)return yo(o,a,"saturated");c=f.tokensPerSecond}return yo(o,a,"max-depth")}function yo(e,t,r){let n=e.filter(i=>i.outputMatchesReference),s=Math.max(...n.map(i=>i.tokensPerSecond));return{depth:n.find(i=>i.tokensPerSecond>=s*t)?.depth??n[0]?.depth??1,samples:e,totalSampleTokens:e.reduce((i,u)=>i+u.sampleTokens,0),stoppedBecause:r}}function hv(e,t){let r=Math.min(e.length,t.length);for(let n=0;n<r;++n)if(e[n]!==t[n])return!1;return!0}function vv(e,t,r){return Number.isFinite(e)?Math.max(r,Math.floor(e)):t}async function mf(e,t=6){let r=Math.max(1,Math.floor(t)),n=e.tensorFromTypedArray("uint32",[1,1],new Uint32Array([0]));try{await e.readTensor(n);let s=[];for(let a=0;a<r;++a){let i=performance.now();await e.readTensor(n),s.push(performance.now()-i)}return s.sort((a,i)=>a-i),s[s.length>>1]}finally{n.destroy()}}function hf(e){let t=globalThis;return typeof t.ImageBitmap=="function"&&e instanceof t.ImageBitmap||typeof t.VideoFrame=="function"&&e instanceof t.VideoFrame}var vf="https://huggingface.co";function wv(e){return typeof e=="string"&&/^https?:/i.test(e)}function bv(e){return wv(e)||e.startsWith("/")||e.startsWith(".")}function So(e,t={}){let{revision:r="main",file:n}=t,s=n?n.slice(n.lastIndexOf(".")).toLowerCase():"";if(s&&e.toLowerCase().endsWith(s))return e;if(bv(e))return n?`${_v(e)}/${n}`:e;let a=`${vf}/${e}/resolve/${r}`;return n?`${a}/${n}`:a}function _v(e){return e.endsWith("/")?e.slice(0,-1):e}function kv(e){try{return new URL(e).origin===vf}catch{return!1}}function wf(e){return(t,r={})=>{if(!kv(t))return globalThis.fetch(t,r);let n=new Headers(r.headers);return n.set("Authorization",`Bearer ${e}`),globalThis.fetch(t,{...r,headers:n})}}var Av=new Set(["mozilla","brave","chrome","graphics","generic renderer","webkit webgl","webgl","webgpu","angle","unknown","subsystem opengl es translator"]),yv=new Set(["apple","apple gpu","intel","intel graphics","hd graphics","intel hd graphics","intel hd graphics family","intel uhd graphics","intel arc graphics","intel iris","intel iris graphics","intel iris plus","intel iris plus graphics","intel iris pro","intel iris pro graphics","amd","amd radeon","amd radeon graphics","amd radeon pro graphics","amd unknown","nvidia","nvidia tegra","nvidia graphics device","nvidia geforce gpu","adreno","adreno gpu","mali","powervr"]),nx=Math.max(...[...Av].map(e=>e.length)),sx=Math.max(...[...yv].map(e=>e.length));var Sv="shader-f16",xv="subgroups",Tv="chromium-experimental-subgroup-matrix",Cv="timestamp-query",qv="packed_4x8_integer_dot_product";function bf(e){let t=yt(e),r=t.adapterInfo,n=new Set(t.features),s=new Set(t.wgslLanguageFeatures),a={};for(let i of Wn){let u=t.limits[i];typeof u=="number"&&(a[i]=u)}return{vendor:r.vendor??"",architecture:r.architecture??"",device:r.device??"",description:r.description??"",isFallbackAdapter:r.isFallbackAdapter===!0,subgroupMinSize:r.subgroupMinSize??null,subgroupMaxSize:r.subgroupMaxSize??null,subgroupMatrixConfigs:r.subgroupMatrixConfigs??null,features:t.features,wgslLanguageFeatures:t.wgslLanguageFeatures,capabilities:{shaderF16:n.has(Sv),subgroups:n.has(xv),subgroupMatrix:n.has(Tv),timestampQuery:n.has(Cv),dp4a:s.has(qv)},limits:a}}function Za(e){let t=bf(e);return{vendor:t.vendor,architecture:t.architecture,device:t.device,description:t.description,isFallbackAdapter:t.isFallbackAdapter,subgroupMinSize:t.subgroupMinSize??void 0,subgroupMaxSize:t.subgroupMaxSize??void 0,features:{shaderF16:t.capabilities.shaderF16,subgroups:t.capabilities.subgroups,subgroupMatrix:t.capabilities.subgroupMatrix,timestampQuery:t.capabilities.timestampQuery}}}function xo(e){return e.fetch??(e.accessToken?wf(e.accessToken):void 0)}async function _f(e){let t=globalThis.navigator?.gpu;if(!t)return{ok:!1,reason:"WebGPU isn't available in this browser. Try a recent Chrome or Edge."};let r=null;try{r=await t.requestAdapter({powerPreference:"high-performance"})}catch{}if(!r)return{ok:!1,reason:"No WebGPU adapter is available on this device."};try{let n=await e(r);return n?{ok:!1,reason:n}:{ok:!0}}catch{return{ok:!0}}}var Bv="hello",Xa=class{#e;#t;model;tokenizer;#r;#n;#s;#i;eosTokenIds;#o;generationState;#a;#u;#l;#c;#f;#d;#p;#w;#g;#m=null;#h=new Set;#y=null;#C=new WeakMap;#_=null;#A=!1;#q=Promise.resolve();destroyed=!1;#O=null;#B=!0;#v=null;#I;lost=new Promise(t=>{this.#I=t});chatTemplateArgs={};constructor(t){this.#e=t.runtime,this.#t=t.ownsRuntime,this.model=t.model,this.tokenizer=t.tokenizer,this.#r=t.chatTemplate,this.#n=t.tokenizerConfig,this.#s=t.chatTemplatePolicy===null?void 0:t.chatTemplatePolicy??pf(t.chatTemplate,t.tokenizerConfig),this.#i=new an(t.tokenizer),this.eosTokenIds=t.eosTokenIds,this.generationState=t.generationState,this.#o=t.defaultMaxNewTokens,this.#a=t.prefixSnapshotStore??null,this.#u=t.prefixSnapshotIdentity??null,this.#d=t.vision??null,this.#p=t.createGenerationState??null,this.#w=t.disposeGenerationState??null,this.#g=t.generationConfig??Object.freeze({do_sample:!1});let r=t.tokenizer.token_to_id?.bind(t.tokenizer);this.#l=r?.("<think>")??null,this.#c=r?.("</think>")??null,this.#f=r?Dc(this.#F(),r):null;let n=new WeakRef(this);t.runtime.host?.device?.lost?.then(s=>{let a=n.deref();a&&a.#M(s)})}#M(t){if(!this.#B||this.#v)return;this.#v=t;let r=new Gr(t.reason,t.message);for(let n of this.#h)n.deviceLost(r);this.#I(r)}#S(){if(this.destroyed)throw new Error("The model has been disposed");if(this.#v)throw new Gr(this.#v.reason,this.#v.message)}get generation_config(){return this.#g}get thinkOpenTokenId(){return this.#l}get thinkCloseTokenId(){return this.#c}get runtime(){return this.#e}acquireGenerationLease(){if(this.#S(),this.#A)throw new Rr("The model is already generating");if(this.#m)throw new Error("The model's cache is held by a session; generate through the session, or dispose it first");return this.#E()}#E(){this.#A=!0;let t;this.#q=new Promise(n=>{t=n});let r=!0;return()=>{r&&(r=!1,this.#A=!1,t())}}get lastAssistantContent(){return this.#_?.content??null}get lastAssistantMessage(){return this.#_}get contextLength(){return this.generationState.cache.maxLength}get contextFull(){return this.generationState.cache.get_seq_length()>=this.generationState.cache.maxLength}#k(){return Object.freeze({...this.#s?.templateArgs??{},...this.chatTemplateArgs})}#b(t,r,n,s=null){let a=this.#f?.inMarkup===!0;return this.#r.render({...n,messages:t.map(i=>typeof i.content=="string"?a?Lc(i):i:{...i,content:i.content.map(u=>u.type==="image"?{type:"image"}:u)}),tools:s,bos_token:this.#n.bos_token,eos_token:this.#n.eos_token,add_generation_prompt:r})}#F(){try{return this.#b([{role:"user",content:"probe"}],!0,this.#k(),Gc)}catch{return null}}#x(t){let r=[];for(let n of t)if(typeof n.content!="string")for(let s of n.content){if(s.type!=="image")continue;if(!this.#d)throw new Error("This model has no vision tower; it cannot take image messages");let a=s.image,i=hf(a),u=i?this.#C.get(a):void 0;u||(u=this.#d.planImage(a),i&&this.#C.set(a,u)),r.push(u)}return this.#d?{imageTokenId:this.#d.imageTokenId,images:r}:void 0}async#P(t,r,n,s){let a=this.generationState.cache;if(!a.captureRewindPoint)return 0;let i=Mn(t);if(i.length===0)return 0;let u=Js(this.tokenizer,l=>this.#b(l,!1,r),i);if(u.length===0||u.length>=n.length)return 0;for(let l=0;l<u.length;++l)if(u[l]!==n[l])return 0;if(await this.#G(a,u))return this.#i.commit(u,[],u.length),u.length;for await(let l of this.streamTokens({suffixIds:[...u],maxNewTokens:1,eosTokenId:this.eosTokenIds,stopOnEos:!1},s));return this.generationState.cache.get_seq_length()!==u.length?(this.resetCache(),0):(this.#i.commit(u,[],u.length),await a.captureRewindPoint(),await this.#D(a,u),u.length)}#R(t,r){if(!this.#a||this.#u===null)return null;let n=t.prefixSnapshotLayout?.(r.length);return n?`${this.#u}|layout=${n}|ids=${r.join(",")}`:null}async#G(t,r){let n=this.#R(t,r);if(n===null||!t.importPrefixSnapshot)return!1;let s=await this.#a?.load(n);return!s||s.length!==r.length||!await t.importPrefixSnapshot(s)?!1:t.get_seq_length()!==r.length?(t.truncate(0),!1):!0}async#D(t,r){let n=this.#R(t,r);if(n===null||!t.exportPrefixSnapshot)return;let s;try{s=await t.exportPrefixSnapshot()}catch{return}!s||s.length!==r.length||this.#a?.save({identity:n,...s}).catch(()=>{})}async primeSystemPrefix(t,r={}){let n=this.acquireGenerationLease();try{if(this.generationState.cache.get_seq_length()!==0)throw new Error("primeSystemPrefix needs an empty cache; call it before the first turn");let s=this.#k(),a=Mn(t);if(a.length===0)return 0;let i=this.#b([...a,{role:"user",content:Bv}],!0,s),u=this.tokenizer.encode(i,{add_special_tokens:!1}).ids;return await this.#P(t,s,u,r)}finally{n()}}renderPrompt(t,r=!0){return this.#b(t,r,this.#k())}encodePrompt(t){return this.tokenizer.encode(this.renderPrompt(t),{add_special_tokens:!1}).ids}async*generate(t,r={}){let n=this.acquireGenerationLease();this.#_=null;let s=this.#k(),a=[],i="",u=!1;try{this.generationState.cache.get_seq_length()!==this.#i.cacheLength&&this.resetCache();let o=r.maxNewTokens??this.#o,l=r.eosTokenId??this.eosTokenIds,c=this.#b(t,!0,s),d=this.#x(t),p=this.#i.prepare(c,d),{inputIds:f,prefixLength:g}=p;if(g!==this.#i.cacheLength){let k=this.generationState.cache,S=Ys(k,g);S>0?(k.truncate(S),this.#i.rewind(S),g=S):(this.resetCache(),g=0),f=p.fullyEncodedIds}g===0&&(g=await this.#P(t,s,p.fullyEncodedIds,r),g>0&&(f=p.fullyEncodedIds));let m=f.slice(g);m.length===0&&(this.resetCache(),f=p.fullyEncodedIds,m=f.slice());let h=this.generationState.cache.maxLength-this.generationState.cache.get_seq_length()-m.length;if(h<1)throw new Error(m.length>=this.generationState.cache.maxLength?`message is too long for the context window (${m.length} tokens; window is ${this.generationState.cache.maxLength})`:`context window is full (${this.generationState.cache.maxLength} tokens) \u2014 clear the chat to continue`);let v=Math.min(o,h),b=Xs(k=>this.tokenizer.decode(k,{skip_special_tokens:!0})),_=f.length-m.length,C=d?p.imageSpans.filter(k=>k.start>=_).map(k=>d.images[k.image]):[];for await(let k of this.streamTokens({suffixIds:m,maxNewTokens:v,eosTokenId:l,stopOnEos:!0,...C.length>0?{images:C}:{}},r)){if(r.signal?.aborted||this.destroyed)return;a.push(k);let S=b.push(k);i=b.text,yield{token:k,delta:S,text:i}}let y=b.flush();y!==""&&(i=b.text,yield{token:null,delta:y,text:i});let E=Jt(this.tokenizer,a,{skip_special_tokens:!1});this.#_=bu(this.#s,{visibleText:i,rawText:E,templateArgs:s}),this.#i.commit(f,a,this.generationState.cache.get_seq_length(),d&&{imageTokenId:d.imageTokenId,imageSpans:p.imageSpans}),u=!0}catch(o){throw this.#v?new Gr(this.#v.reason,this.#v.message,{cause:o}):o}finally{u||this.resetCache(),n()}}async complete(t,r={}){let n="";for await(let s of this.generate(t,r))n=s.text;return n}async benchmarkFixedTokenIds(t,r,n,s){let a=this.acquireGenerationLease();try{return await this.#L(t,r,n,s)}finally{a()}}benchmarkMessages(t,r,n){let s=this.#x(t),a=new an(this.tokenizer).prepare(this.renderPrompt(t),s);return this.benchmarkFixedTokenIds(a.inputIds,r,n,s?.images)}async calibrateDecodePipeline(t,r,n=[0]){let s=this.acquireGenerationLease();try{return await gf(async(a,i)=>{let u=await this.#L(n,i,r(a));return{tokensPerSecond:u.decodeTps,tokenIds:u.ids}},t)}finally{s()}}async#L(t,r,n,s){this.resetCache();let a=[],i=0,u=0,o=performance.now();try{for await(let l of this.streamTokens({suffixIds:[...t],maxNewTokens:r,eosTokenId:n.eosTokenId??this.eosTokenIds,stopOnEos:!1,...s&&s.length>0?{images:s}:{}},n))a.length===0&&(i=performance.now()),a.push(l);u=performance.now()}finally{this.resetCache()}return{ttftMs:a.length>0?i-o:NaN,decodeTps:a.length>1?(a.length-1)/((u-i)/1e3):0,tokens:a.length,ids:a}}deviceInfo(){return Za(this.#e.device)}create_session(t={}){this.#S();let r=this.#m===null,n=this.generationState;if(!r){if(!this.#p)throw new Error("The model holds one session at a time; dispose the current one first");n=this.#p()}let s=n.cache.rewindPointCapacity,a;try{a=new aa({host:this.#y??=this.#Q(),state:n,ledger:r?this.#i:new an(this.tokenizer),options:t,release:()=>{if(this.#h.delete(a),!r){this.#T(n);return}s!==void 0&&n.cache.setRewindPointCapacity?.(s),this.#m=null}})}catch(i){throw r||this.#T(n),i}return r&&(this.#m=a),this.#h.add(a),a}#Q(){return{tokenizer:this.tokenizer,eosTokenIds:this.eosTokenIds,thinkOpenTokenId:this.#l,thinkCloseTokenId:this.#c,toolCallDelimiters:this.#f,defaultMaxNewTokens:this.#o,generationConfig:this.#g,templateArgs:t=>Object.freeze({...this.#k(),...t}),renderPrompt:(t,r,n,s)=>this.#b(t,r,n,s??null),planImages:t=>this.#x(t),historyMessage:t=>bu(this.#s,t),streamTokens:(t,{decodePipelineDepth:r,...n})=>this.streamTokens({...n,generationState:t},r!==void 0?{decodePipelineDepth:r}:{}),acquireLease:()=>{if(this.#S(),this.#A)throw new Rr("The model is already generating; one turn runs at a time");return this.#E()},restoreSystemPrefix:(t,r)=>this.#G(t.cache,r),persistSystemPrefix:(t,r)=>this.#D(t.cache,r)}}reset(){this.destroyed||(this.#m?this.#m.reset():this.resetCache())}resetCache(){this.generationState.cache.truncate(0),this.#i.clear(),this.#_=null}#T(t){this.#w?this.#w(t):t.cache.dispose?.()}dispose(){return this.#O??=this.#j()}async#j(){this.destroyed=!0;let t=[];for(let n of await Promise.allSettled([...this.#h].map(s=>s.dispose())))n.status==="rejected"&&t.push(n.reason);this.#v||await this.#q;let r=n=>{try{n()}catch(s){t.push(s)}};r(()=>this.#T(this.generationState)),r(()=>this.model.dispose()),this.#B=!1,this.#t&&await this.#e.destroy().catch(n=>t.push(n)),ct(t,"The model failed to release everything it held")}};async function*kf(e){let t=[],r=()=>{let s=e.submit();return{unit:s,result:Rv(s.result)}},n=[];try{for(;t.length<e.depth&&e.shouldSubmit();)t.push(r());for(;t.length>0;){let s=t[0];e.shouldSubmit()&&t.length-1<e.depth&&t.push(r()),t.shift();let a=await s.result;if(yield*e.accept(s.unit,a))return}}catch(s){throw n.push(s),s}finally{let s=await Promise.allSettled(t.map(({result:a})=>a));if(n.length>0){let a=s.flatMap(i=>i.status==="rejected"?[i.reason]:[]);if(a.length>0)throw new AggregateError([...n,...a],"Pipelined generation and in-flight cleanup both failed")}}}var Nr=64,To=new WeakMap;async function Iv(e){let t=To.get(e);t!==void 0&&(To.delete(e),await t)}var Ev=4;function Pv(e){return Math.max(e+1,Math.min(Nr,e*Ev))}function ss(e,t,r=t){let n=e.decode_pipeline_depth;return typeof n!="number"||!Number.isFinite(n)||t<=1?t:Math.max(1,Math.min(Math.floor(n),r))}async function Co(e,t,r){for(let n of r){for await(let s of e.streamTokenIdsFromCache({input_ids:[new Array(n).fill(0)],generation_state:{cache:t},max_new_tokens:2,stop_on_eos:!1}));t.truncate(0)}}async function*Af(e,t,r){if(t.length===0)throw new Error("generation requires at least one input token");if(e.sampling===void 0&&un(r)!==null)throw new Error("this model decodes greedily only; it cannot sample (do_sample)");let{maxNewTokens:n,eosTokenId:s,stopOnEos:a,onPrefillDone:i,onPrefillProgress:u,signal:o}=ea(r,e.eosTokenId!==void 0?{eosTokenId:e.eosTokenId}:{}),l=e.cache,c=y=>a&&ta(y,s);await Iv(l),o?.throwIfAborted();let d=l.get_seq_length(),p=o?await l.mutableStateCheckpoint?.()??null:null;p?.capture();let f;try{f=await e.prefill(t,d,{signal:o,onProgress:u&&(y=>u({done:y,total:t.length}))})}catch(y){try{p?.restore()}catch(E){throw new AggregateError([y,E],"Prefill and its rollback both failed")}throw y}if(l.advance(t.length),i?.({tokens:t.length,cache_length:l.get_seq_length()}),o?.throwIfAborted(),n<=0||c(f))return;yield f;let g=1;if(g>=n)return;let m={value:null},h=async y=>m.value??=await e.beginDecode(y,n-g),v=!1,b=async function*(y){return l.advance(1),c(y)?!0:o?.aborted?(v=!0,!0):(yield y,g+=1,g>=n)},_=l.get_seq_length();if(_>=l.maxLength)return;let C=[];try{if(e.pipelineDepth>1){let k=await l.mutableStateCheckpoint?.()??null;if(k){let O=Pv(e.pipelineDepth),j=await l.alternateMutableStateCheckpoint?.()??null,F=j?[k,j]:[k],K=await h(_),Z=Math.min(n-g,l.maxLength-_),Q=[f],N=0,q=0,x=!1;for(;N<Z;){let B=N,U=j?Z:Math.min(Z,B+O),M=[];try{yield*kf({depth:e.pipelineDepth,shouldSubmit:()=>N<U,submit:()=>{N%O===0&&F[N/O%F.length].capture();let G=K.submitStep(N===B?Q[B]:null,_+N);return N+=1,{result:G}},accept:async function*(G,$){return l.advance(1),q+=1,c($)?(x=!0,!0):o?.aborted?(v=!0,!0):(yield $,g+=1,Q.push($),!1)}})}catch(G){throw M.push(...G instanceof AggregateError?G.errors:[G]),G}finally{try{if(q<N){let G=Math.floor(Math.max(0,q-1)/O),$=G*O;F[G%F.length].restore();let X=Q.slice($,q);if(K.replayKnownInputs){let H=K.replayKnownInputs(X,_+$);x?(To.set(l,H),H.catch(()=>{})):await H}else for(let H=0;H<X.length;++H)await K.step(X[H],_+$+H)}}catch(G){throw M.length>0?new AggregateError([...M,G],"Pipelined decode and cleanup both failed"):G}}if(x)return;v&&o.throwIfAborted()}return}let S=n-g,A=_,w=0,I=!0,z=await h(_);yield*kf({depth:e.pipelineDepth,shouldSubmit:()=>w<S&&A<l.maxLength,submit:()=>{let O=z.submitStep(I?f:null,A);return A+=1,w+=1,I=!1,{result:O}},accept:(O,j)=>b(j)}),v&&o.throwIfAborted();return}let y=f,E=_;for(;g<n&&E<l.maxLength;){let S=await(await h(E)).step(y,E);if(yield*b(S))break;y=S,E=l.get_seq_length()}v&&o.throwIfAborted()}catch(y){throw C.push(y),y}finally{try{await m.value?.finish()}catch(y){throw C.length>0?new AggregateError([...C,y],"Token decode and session cleanup both failed"):y}}}function Rv(e){let t=e instanceof Promise?e:new Promise((r,n)=>{e.then(r,n)});return t.catch(()=>{}),t}var vn=4,Ya=class extends Xa{#e;#t=0;constructor(t){super(t)}streamTokens(t,r){return this.model.streamTokenIdsFromCache({input_ids:[t.suffixIds],generation_state:t.generationState??this.generationState,max_new_tokens:t.maxNewTokens,eos_token_id:t.eosTokenId,stop_on_eos:t.stopOnEos,decode_pipeline_depth:r.decodePipelineDepth??this.#e,...t.images?{images:t.images}:{},...t.signal?{signal:t.signal}:{},...t.onPrefillProgress?{on_prefill_progress:t.onPrefillProgress}:{},...t.sampling?{do_sample:!0,...t.sampling}:{}})}get decodePipelineDepth(){return this.#e}get readbackLatencyMs(){return this.#t}async tuneDecodePipeline(t={}){if(this.#t=await mf(this.runtime),t.decodePipelineDepth!==void 0)return this.#e=ss({decode_pipeline_depth:t.decodePipelineDepth},vn,Nr),this.#e;let r=this.contextLength,n=Math.min(Nr,r-1);if(n<=vn)return this.#e=vn,vn;let s=await this.calibrateDecodePipeline({baseDepth:vn,maxDepth:n,minSampleTokens:Math.min(24,r)},a=>({decodePipelineDepth:a}));return this.#e=s.depth,s.depth}async benchmark(t,r={}){let n=ss({decode_pipeline_depth:r.decodePipelineDepth??this.#e},vn,Nr);return{...await this.benchmarkMessages(t,r.maxNewTokens??128,{decodePipelineDepth:n}),depth:n}}};var Ja=class{#e;constructor(t){this.#e=t}create_session(t={}){return this.#e.create_session(t)}get generation_config(){return this.#e.generation_config}get contextLength(){return this.#e.contextLength}get lost(){return this.#e.lost}deviceInfo(){return this.#e.deviceInfo()}memoryStats(){return this.#e.runtime.memoryStats()}dispose(){return this.#e.dispose()}};var Gv=class{constructor(e){this.trie=this._build_trie(e)}_build_trie(e){let t=Object.create(null);for(let r of e){let n=t;for(let s=0;s<r.length;++s){let a=r[s];n=n[a]??=Object.create(null)}n.end=r}return t}split(e){let t=[],r=e.length,n=0,s=0;for(;s<r;){let a=this.trie,i=null,u=s;for(;u<r&&(a=a[e[u]]);)a.end&&(i=a.end),++u;i?(s>n&&t.push(e.slice(n,s)),t.push(i),s+=i.length,n=s):++s}return n<r&&t.push(e.slice(n)),t}},yf=Gv,Dv=class{constructor(e){this.content=e.content,this.id=e.id,this.single_word=e.single_word??!1,this.lstrip=e.lstrip??!1,this.rstrip=e.rstrip??!1,this.special=e.special??!1,this.normalized=e.normalized??!this.special}},Lv=Dv,If=(()=>{let e=[...Array.from({length:94},(s,a)=>a+33),...Array.from({length:12},(s,a)=>a+161),...Array.from({length:82},(s,a)=>a+174)],t=e.slice(),r=0;for(let s=0;s<256;++s)e.includes(s)||(e.push(s),t.push(256+r),r+=1);let n=t.map(s=>String.fromCharCode(s));return Object.fromEntries(e.map((s,a)=>[s,n[a]]))})(),Ov=e=>Object.fromEntries(Object.entries(e).map(([t,r])=>[r,t])),Mv=Ov(If),Sf=".,!?\u2026\u3002\uFF0C\u3001\u0964\u06D4\u060C",Fv=new Map([["(?i:'s|'t|'re|'ve|'m|'ll|'d)","(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))"],["(?i:[sdmt]|ll|ve|re)","(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])"],["[^\\r\\n\\p{L}\\p{N}]?+","[^\\r\\n\\p{L}\\p{N}]?"],["[^\\s\\p{L}\\p{N}]++","[^\\s\\p{L}\\p{N}]+"],["(?>\\p{Nd}{510})","(?:\\p{Nd}{510})"],["\\p{Nd}{3}+","(?:\\p{Nd}{3})+"],["\\G",""],[` ?[^(\\s|[${Sf}])]+`,` ?[^\\s${Sf}]+`]]),ei="\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E",Bo=e=>e.replace(/ \./g,".").replace(/ \?/g,"?").replace(/ \!/g,"!").replace(/ ,/g,",").replace(/ \' /g,"'").replace(/ n't/g,"n't").replace(/ 'm/g,"'m").replace(/ 's/g,"'s").replace(/ 've/g,"'ve").replace(/ 're/g,"'re"),ti=(e,t=!0)=>{if(e.Regex!==void 0){let r=e.Regex.replace(/\\([#&~])/g,"$1");r=r.replace(/\\A/g,"^").replace(/\\z/g,"$").replace(/\\Z/g,"(?=\\r?\\n?$)");for(let[n,s]of Fv)r=r.replaceAll(n,s);try{return new RegExp(r,"gu")}catch(n){if(!(n instanceof SyntaxError)||!n.message.toLowerCase().includes("invalid property name"))throw n;let s=!1,a=r.replace(/(\\[pP])\{([^}=]+)\}/g,(i,u,o)=>{try{return new RegExp(`\\p{${o}}`,"u"),`${u}{${o}}`}catch{return s=!0,`${u}{Script=${o}}`}});if(!s)throw n;try{return new RegExp(a,"gu")}catch{throw n}}}else if(e.String!==void 0){let r=Qv(e.String);return new RegExp(t?r:`(${r})`,"gu")}else return console.warn("Unknown pattern type:",e),null},Qv=e=>e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),jv=(e,t,r)=>{let n=[],s=0;for(;s<e.length;){if(n.push(e[s]),(t.get(e[s])??r)!==r){++s;continue}for(;++s<e.length&&(t.get(e[s])??r)===r;)t.get(n.at(-1))!==r&&(n[n.length-1]+=e[s])}return n},Wv=e=>e>=19968&&e<=40959||e>=13312&&e<=19903||e>=131072&&e<=173791||e>=173824&&e<=177983||e>=177984&&e<=178207||e>=178208&&e<=183983||e>=63744&&e<=64255||e>=194560&&e<=195103,zv=e=>Number.isInteger(e)||typeof e=="bigint",Kv=e=>{let t=0;for(let r of e)++t;return t},Uv=e=>Ef(e.toLowerCase()),It=(...e)=>Array.prototype.concat.apply([],e),Io=e=>new Map(Object.entries(e)),Nv=(e,t)=>{let r=[],n=0;for(let s of e.matchAll(t)){let a=s[0];n<s.index&&r.push(e.slice(n,s.index)),a.length>0&&r.push(a),n=s.index+a.length}return n<e.length&&r.push(e.slice(n)),r},Ef=e=>e.replace(/\p{M}/gu,""),xf=(e,t,r=[])=>{if(!e||Array.isArray(e)||typeof e!="object")return`${t} must be a valid object`;for(let n of r)if(!(n in e))return`${t} must contain a "${n}" property`;return null},Hv=e=>e.match(/\S+/g)||[],$v=class{constructor(){let e=function(...t){return e._call(...t)};return Object.setPrototypeOf(e,new.target.prototype)}},as=$v,Vv=class extends as{constructor(e){super(),this.config=e}_call(e){return this.normalize(e)}},rr=Vv,Zv=class extends rr{tokenize_chinese_chars(e){let t=[];for(let r=0;r<e.length;++r){let n=e[r],s=n.charCodeAt(0);Wv(s)?(t.push(" "),t.push(n),t.push(" ")):t.push(n)}return t.join("")}strip_accents(e){return e.normalize("NFD").replace(/\p{Mn}/gu,"")}is_control(e){switch(e){case"	":case`
`:case"\r":return!1;default:return/^\p{Cc}|\p{Cf}|\p{Co}|\p{Cs}$/u.test(e)}}clean_text(e){let t=[];for(let r of e){let n=r.charCodeAt(0);n===0||n===65533||this.is_control(r)||(/^\s$/.test(r)?t.push(" "):t.push(r))}return t.join("")}normalize(e){return this.config.clean_text&&(e=this.clean_text(e)),this.config.handle_chinese_chars&&(e=this.tokenize_chinese_chars(e)),this.config.lowercase?(e=e.toLowerCase(),this.config.strip_accents!==!1&&(e=this.strip_accents(e))):this.config.strip_accents&&(e=this.strip_accents(e)),e}},Xv=Zv,Yv=class extends rr{constructor(e){super(e),this.charsmap=e.precompiled_charsmap??null}normalize(e){return e=e.replace(/[\u0001-\u0008\u000B\u000E-\u001F\u007F\u008F\u009F]/gm,""),e=e.replace(/[\u0009\u000A\u000C\u000D\u00A0\u1680\u2000-\u200F\u2028\u2029\u202F\u205F\u2581\u3000\uFEFF\uFFFD]/gm," "),e.includes("\uFF5E")?e=e.split("\uFF5E").map(r=>r.normalize("NFKC")).join("\uFF5E"):e=e.normalize("NFKC"),e}},Jv=Yv,ew=class extends rr{constructor(e){super(e),this.normalizers=(e.normalizers??[]).map(t=>Pf(t))}normalize(e){return this.normalizers.reduce((t,r)=>r?r.normalize(t):t,e)}},tw=ew,rw=class extends rr{normalize(e){let t=ti(this.config.pattern??{});return t===null?e:e.replaceAll(t,this.config.content??"")}},nw=rw,sw=class extends rr{constructor(){super(...arguments),this.form="NFC"}normalize(e){return e=e.normalize(this.form),e}},ri=sw,aw=class extends ri{constructor(){super(...arguments),this.form="NFC"}},iw=aw,uw=class extends ri{constructor(){super(...arguments),this.form="NFD"}},ow=uw,lw=class extends ri{constructor(){super(...arguments),this.form="NFKC"}},cw=lw,dw=class extends ri{constructor(){super(...arguments),this.form="NFKD"}},fw=dw,pw=class extends rr{normalize(e){return this.config.strip_left&&this.config.strip_right?e=e.trim():(this.config.strip_left&&(e=e.trimStart()),this.config.strip_right&&(e=e.trimEnd())),e}},gw=pw,mw=class extends rr{normalize(e){return Ef(e)}},hw=mw,vw=class extends rr{normalize(e){return e.toLowerCase()}},ww=vw,bw=class extends rr{normalize(e){return e=this.config.prepend+e,e}},_w=bw;function kw(e){if(e===null)return null;switch(e.type){case"BertNormalizer":return new Xv(e);case"Precompiled":return new Jv(e);case"Sequence":return new tw(e);case"Replace":return new nw(e);case"NFC":return new iw(e);case"NFD":return new ow(e);case"NFKC":return new cw(e);case"NFKD":return new fw(e);case"Strip":return new gw(e);case"StripAccents":return new hw(e);case"Lowercase":return new ww(e);case"Prepend":return new _w(e);default:throw new Error(`Unknown Normalizer type: ${e.type}`)}}var Pf=kw,Aw=class extends as{pre_tokenize(e,t){return(Array.isArray(e)?e.map(r=>this.pre_tokenize_text(r,t)):this.pre_tokenize_text(e,t)).flat()}_call(e,t){return this.pre_tokenize(e,t)}},Et=Aw,yw=class extends Et{constructor(e){super(),this.config=e,this.add_prefix_space=this.config.add_prefix_space??!1,this.trim_offsets=this.config.trim_offsets??!1,this.use_regex=this.config.use_regex??!0,this.pattern=/'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu,this.byte_encoder=If,this.text_encoder=new TextEncoder}pre_tokenize_text(e,t){return this.add_prefix_space&&!e.startsWith(" ")&&(e=" "+e),(this.use_regex?e.match(this.pattern)||[]:[e]).map(n=>Array.from(this.text_encoder.encode(n),s=>this.byte_encoder[s]).join(""))}},Sw=yw,xw=class extends Et{pre_tokenize_text(e,t){return e.match(/\w+|[^\w\s]+/g)||[]}},Tw=xw,Cw=class extends Et{constructor(e){super(),this.replacement=e.replacement??"\u2581",this.str_rep=e.str_rep||this.replacement,this.prepend_scheme=e.prepend_scheme??"always"}pre_tokenize_text(e,t){let{section_index:r=void 0}=t??{},n=e.replaceAll(" ",this.str_rep);return!n.startsWith(this.replacement)&&(this.prepend_scheme==="always"||this.prepend_scheme==="first"&&r===0)&&(n=this.str_rep+n),[n]}},qw=Cw,Bw=class extends Et{constructor(e){super(),this.config=e,this.pattern=ti(this.config.pattern??{},this.config.invert??!0)}pre_tokenize_text(e){return this.pattern===null?[]:this.config.invert?e.match(this.pattern)||[]:this.config.behavior?.toLowerCase()==="removed"?e.split(this.pattern).filter(t=>t):Nv(e,this.pattern)}},Iw=Bw,Ew=class extends Et{constructor(e){super(),this.config=e,this.pattern=new RegExp(`[^${ei}]+|[${ei}]+`,"gu")}pre_tokenize_text(e){return e.match(this.pattern)||[]}},Pw=Ew,Rw=class extends Et{constructor(e){super(),this.config=e;let t=`[^\\d]+|\\d${this.config.individual_digits?"":"+"}`;this.pattern=new RegExp(t,"gu")}pre_tokenize_text(e){return e.match(this.pattern)||[]}},Gw=Rw,Dw=class extends Et{constructor(){super(),this.pattern=new RegExp(`[^\\s${ei}]+|[${ei}]`,"gu")}pre_tokenize_text(e,t){return e.trim().match(this.pattern)||[]}},Lw=Dw,Ow=class extends Et{constructor(e){super(),this.config=e,this.pattern=ti(this.config.pattern??{}),this.content=this.config.content??""}pre_tokenize_text(e){return this.pattern===null?[e]:[e.replaceAll(this.pattern,this.config.content??"")]}},Mw=Ow,Fw=class extends Et{constructor(e){super(),this.tokenizers=(e.pretokenizers??[]).map(t=>Rf(t))}pre_tokenize_text(e,t){return this.tokenizers.reduce((r,n)=>n?n.pre_tokenize(r,t):r,[e])}},Qw=Fw,jw=class extends Et{pre_tokenize_text(e){return Hv(e)}},Ww=jw,zw=class extends Et{constructor(e){super(),this.config=e,this._length=e.length}pre_tokenize_text(e){let t=[];for(let r=0;r<e.length;r+=this._length)t.push(e.slice(r,r+this._length));return t}},Kw=zw;function Uw(e){if(e===null)return null;switch(e.type){case"BertPreTokenizer":return new Lw;case"Sequence":return new Qw(e);case"Whitespace":return new Tw;case"WhitespaceSplit":return new Ww;case"Metaspace":return new qw(e);case"ByteLevel":return new Sw(e);case"Split":return new Iw(e);case"Punctuation":return new Pw(e);case"Digits":return new Gw(e);case"Replace":return new Mw(e);case"FixedLength":return new Kw(e);default:throw new Error(`Unknown PreTokenizer type: ${e.type}`)}}var Rf=Uw,Nw=class extends as{constructor(e){super(),this.config=e,this.vocab=[],this.tokens_to_ids=new Map,this.unk_token_id=void 0,this.unk_token=void 0,this.end_of_word_suffix=void 0,this.fuse_unk=this.config.fuse_unk??!1}_call(e){let t=this.encode(e);return this.fuse_unk&&(t=jv(t,this.tokens_to_ids,this.unk_token_id)),t}},ni=Nw,Hw=class extends ni{constructor(e){super(e),this.max_input_chars_per_word=100,this.tokens_to_ids=Io(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.max_input_chars_per_word=e.max_input_chars_per_word??100,this.vocab=new Array(this.tokens_to_ids.size);for(let[t,r]of this.tokens_to_ids)this.vocab[r]=t}encode(e){let t=[];for(let r of e){let n=[...r];if(n.length>this.max_input_chars_per_word){t.push(this.unk_token);continue}let s=!1,a=0,i=[];for(;a<n.length;){let u=n.length,o=null;for(;a<u;){let l=n.slice(a,u).join("");if(a>0&&(l=this.config.continuing_subword_prefix+l),this.tokens_to_ids.has(l)){o=l;break}--u}if(o===null){s=!0;break}i.push(o),a=u}s?t.push(this.unk_token):t.push(...i)}return t}},Tf=Hw,Cf=class Gf{constructor(t,r){this.is_leaf=t,this.children=r}static default(){return new Gf(!1,new Map)}},$w=class{constructor(){this.root=Cf.default()}extend(e){for(let t of e)this.push(t)}push(e){let t=this.root;for(let r of e){let n=t.children.get(r);n===void 0&&(n=Cf.default(),t.children.set(r,n)),t=n}t.is_leaf=!0}*common_prefix_search(e){let t=this.root;if(t===void 0)return;let r="";for(let n of e){if(r+=n,t=t.children.get(n),t===void 0)return;t.is_leaf&&(yield r)}}},Vw=$w,qo=class Df{constructor(t,r,n,s,a){this.token_id=t,this.node_id=r,this.pos=n,this.length=s,this.score=a,this.prev=null,this.backtrace_score=0}clone(){let t=new Df(this.token_id,this.node_id,this.pos,this.length,this.score);return t.prev=this.prev,t.backtrace_score=this.backtrace_score,t}},Zw=class{constructor(e,t,r){this.chars=Array.from(e),this.len=this.chars.length,this.bos_token_id=t,this.eos_token_id=r,this.nodes=[],this.begin_nodes=Array.from({length:this.len+1},()=>[]),this.end_nodes=Array.from({length:this.len+1},()=>[]);let n=new qo(this.bos_token_id??0,0,0,0,0),s=new qo(this.eos_token_id??0,1,this.len,0,0);this.nodes.push(n.clone()),this.nodes.push(s.clone()),this.begin_nodes[this.len].push(s),this.end_nodes[0].push(n)}insert(e,t,r,n){let s=this.nodes.length,a=new qo(n,s,e,t,r);this.begin_nodes[e].push(a),this.end_nodes[e+t].push(a),this.nodes.push(a)}viterbi(){let e=this.len,t=0;for(;t<=e;){if(this.begin_nodes[t].length==0)return[];for(let i of this.begin_nodes[t]){i.prev=null;let u=0,o=null;for(let l of this.end_nodes[t]){let c=l.backtrace_score+i.score;(o===null||c>u)&&(o=l.clone(),u=c)}if(o!==null)i.prev=o,i.backtrace_score=u;else return[]}++t}let r=[],s=this.begin_nodes[e][0].prev;if(s===null)return[];let a=s.clone();for(;a.prev!==null;)r.push(a.clone()),a=a.clone().prev.clone();return r.reverse(),r}piece(e){return this.chars.slice(e.pos,e.pos+e.length).join("")}tokens(){return this.viterbi().map(t=>this.piece(t))}token_ids(){return this.viterbi().map(t=>t.token_id)}},Xw=Zw;function Yw(e){if(e.length===0)throw new Error("Array must not be empty");let t=e[0],r=0;for(let n=1;n<e.length;++n)e[n]<t&&(t=e[n],r=n);return[t,r]}var Jw=class extends ni{constructor(e,t){super(e);let r=e.vocab.length;this.vocab=new Array(r),this.scores=new Array(r);for(let n=0;n<r;++n)[this.vocab[n],this.scores[n]]=e.vocab[n];this.unk_token_id=e.unk_id,this.unk_token=this.vocab[e.unk_id],this.tokens_to_ids=new Map(this.vocab.map((n,s)=>[n,s])),this.bos_token=" ",this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.unk_token=this.vocab[this.unk_token_id],this.min_score=Yw(this.scores)[0],this.unk_score=this.min_score-10,this.scores[this.unk_token_id]=this.unk_score,this.trie=new Vw,this.trie.extend(this.vocab),this.fuse_unk=!0}populate_nodes(e){let t=e.chars,r=1,n=0;for(;n<t.length;){let s=!1,a=[],i=t.slice(n).join(""),u=this.trie.common_prefix_search(i);for(let o of u){a.push(o);let l=this.tokens_to_ids.get(o),c=this.scores[l],d=Kv(o);e.insert(n,d,c,l),!s&&d===r&&(s=!0)}s||e.insert(n,r,this.unk_score,this.unk_token_id),n+=r}}tokenize(e){let t=new Xw(e,this.bos_token_id,this.eos_token_id);return this.populate_nodes(t),t.tokens()}encode(e){let t=[];for(let r of e){let n=this.tokenize(r);t.push(...n)}return t}},qf=Jw,eb=class{constructor(e=(r,n)=>r>n,t=1/0){this._heap=[],this._comparator=e,this._max_size=t}get size(){return this._heap.length}is_empty(){return this.size===0}peek(){return this._heap[0]}push(...e){return this.extend(e)}extend(e){for(let t of e)if(this.size<this._max_size)this._heap.push(t),this._sift_up();else{let r=this._smallest();this._comparator(t,this._heap[r])&&(this._heap[r]=t,this._sift_up_from(r))}return this.size}pop(){let e=this.peek(),t=this.size-1;return t>0&&this._swap(0,t),this._heap.pop(),this._sift_down(),e}replace(e){let t=this.peek();return this._heap[0]=e,this._sift_down(),t}_parent(e){return(e+1>>>1)-1}_left(e){return(e<<1)+1}_right(e){return e+1<<1}_greater(e,t){return this._comparator(this._heap[e],this._heap[t])}_swap(e,t){let r=this._heap[e];this._heap[e]=this._heap[t],this._heap[t]=r}_sift_up(){this._sift_up_from(this.size-1)}_sift_up_from(e){for(;e>0&&this._greater(e,this._parent(e));)this._swap(e,this._parent(e)),e=this._parent(e)}_sift_down(){let e=0;for(;this._left(e)<this.size&&this._greater(this._left(e),e)||this._right(e)<this.size&&this._greater(this._right(e),e);){let t=this._right(e)<this.size&&this._greater(this._right(e),this._left(e))?this._right(e):this._left(e);this._swap(e,t),e=t}}_smallest(){return 2**Math.floor(Math.log2(this.size))-1}},tb=eb,rb=class{constructor(e){this.capacity=e,this.cache=new Map}get(e){if(!this.cache.has(e))return;let t=this.cache.get(e);return this.cache.delete(e),this.cache.set(e,t),t}put(e,t){this.cache.has(e)&&this.cache.delete(e),this.cache.set(e,t),this.cache.size>this.capacity&&this.cache.delete(this.cache.keys().next().value)}clear(){this.cache.clear()}},nb=rb,sb=class extends ni{constructor(e){super(e),this.tokens_to_ids=Io(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.vocab=new Array(this.tokens_to_ids.size);for(let[r,n]of this.tokens_to_ids)this.vocab[n]=r;let t=Array.isArray(e.merges[0]);this.merges=t?e.merges:e.merges.map(r=>r.split(" ",2)),this.bpe_ranks=new Map(this.merges.map((r,n)=>[JSON.stringify(r),n])),this.end_of_word_suffix=e.end_of_word_suffix,this.continuing_subword_suffix=e.continuing_subword_suffix??null,this.byte_fallback=this.config.byte_fallback??!1,this.byte_fallback&&(this.text_encoder=new TextEncoder),this.ignore_merges=this.config.ignore_merges??!1,this.max_length_to_cache=256,this.cache_capacity=1e4,this.cache=new nb(this.cache_capacity)}clear_cache(){this.cache.clear()}bpe(e){if(e.length===0)return[];let t=this.cache.get(e);if(t!==void 0)return t;let r=Array.from(e);this.end_of_word_suffix&&(r[r.length-1]+=this.end_of_word_suffix);let n=[];if(r.length>1){let s=new tb((u,o)=>u.score<o.score),a={token:r[0],bias:0,prev:null,next:null},i=a;for(let u=1;u<r.length;++u){let o={bias:u/r.length,token:r[u],prev:i,next:null};i.next=o,this.add_node(s,i),i=o}for(;!s.is_empty();){let u=s.pop();if(u.deleted||!u.next||u.next.deleted)continue;if(u.deleted=!0,u.next.deleted=!0,u.prev){let l={...u.prev};u.prev.deleted=!0,u.prev=l,l.prev?l.prev.next=l:a=l}let o={token:u.token+u.next.token,bias:u.bias,prev:u.prev,next:u.next.next};o.prev?(o.prev.next=o,this.add_node(s,o.prev)):a=o,o.next&&(o.next.prev=o,this.add_node(s,o))}for(let u=a;u!==null;u=u.next)n.push(u.token)}else n=r;if(this.continuing_subword_suffix)for(let s=0;s<n.length-1;++s)n[s]+=this.continuing_subword_suffix;return e.length<this.max_length_to_cache&&this.cache.put(e,n),n}add_node(e,t){let r=this.bpe_ranks.get(JSON.stringify([t.token,t.next.token]));r!==void 0&&(t.score=r+t.bias,e.push(t))}encode(e){let t=[];for(let r of e){if(this.ignore_merges&&this.tokens_to_ids.has(r)){t.push(r);continue}let n=this.bpe(r);for(let s of n)if(this.tokens_to_ids.has(s))t.push(s);else if(this.byte_fallback){let a=Array.from(this.text_encoder.encode(s)).map(i=>`<0x${i.toString(16).toUpperCase().padStart(2,"0")}>`);a.every(i=>this.tokens_to_ids.has(i))?t.push(...a):this.unk_token!=null&&t.push(this.unk_token)}else this.unk_token!=null&&t.push(this.unk_token)}return t}},Bf=sb,ab=class extends ni{constructor(e,t){super(e);let r=e.vocab;this.tokens_to_ids=Io(t.target_lang?r[t.target_lang]:r),this.bos_token=t.bos_token,this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t.eos_token,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.pad_token=t.pad_token,this.pad_token_id=this.tokens_to_ids.get(this.pad_token),this.unk_token=t.unk_token,this.unk_token_id=this.tokens_to_ids.get(this.unk_token),this.vocab=new Array(this.tokens_to_ids.size);for(let[n,s]of this.tokens_to_ids)this.vocab[s]=n}encode(e){return e}},ib=ab;function ub(e,t){switch(e.type){case"WordPiece":return new Tf(e);case"Unigram":return new qf(e,t.eos_token);case"BPE":return new Bf(e);default:if(e.vocab)return Array.isArray(e.vocab)?new qf(e,t.eos_token):Object.hasOwn(e,"continuing_subword_prefix")&&Object.hasOwn(e,"unk_token")?Object.hasOwn(e,"merges")?new Bf(e):new Tf(e):new ib(e,{target_lang:t.target_lang,bos_token:t.bos_token,eos_token:t.eos_token,pad_token:t.pad_token,unk_token:t.unk_token});throw new Error(`Unknown TokenizerModel type: ${e?.type}`)}}var ob=ub,lb=class extends as{constructor(e){super(),this.config=e}_call(e,...t){return this.post_process(e,...t)}},is=lb,cb=class extends is{post_process(e,t=null,r=!0){let n=t===null?this.config.single:this.config.pair,s=[],a=[];for(let i of n)"SpecialToken"in i?r&&(s.push(i.SpecialToken.id),a.push(i.SpecialToken.type_id)):"Sequence"in i&&(i.Sequence.id==="A"?(s=It(s,e),a=It(a,new Array(e.length).fill(i.Sequence.type_id))):i.Sequence.id==="B"&&(s=It(s,t),a=It(a,new Array(t.length).fill(i.Sequence.type_id))));return{tokens:s,token_type_ids:a}}},db=cb,fb=class extends is{post_process(e,t=null){return{tokens:e,tokens_pair:t}}},pb=fb,gb=class extends is{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t=null,r=!0){r&&(e=It([this.cls[0]],e,[this.sep[0]]));let n=new Array(e.length).fill(0);if(t){let s=[],a=r?[this.sep[0]]:[];e=It(e,s,t,a),n=It(n,new Array(t.length+s.length+a.length).fill(1))}return{tokens:e,token_type_ids:n}}},mb=gb,hb=class extends is{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t,r=!0){r&&(e=It([this.cls[0]],e,[this.sep[0]]));let n=new Array(e.length).fill(0);if(t){let s=r?[this.sep[0]]:[],a=r?[this.sep[0]]:[];e=It(e,s,t,a),n=It(n,new Array(t.length+s.length+a.length).fill(1))}return{tokens:e,token_type_ids:n}}},vb=hb,wb=class extends is{constructor(e){super(e),this.processors=(e.processors??[]).map(t=>Lf(t))}post_process(e,t=null,r=!0){let n={tokens:e,tokens_pair:t};for(let s of this.processors)n=s.post_process(n.tokens,n.tokens_pair,r);return n}},bb=wb;function _b(e){if(e===null)return null;switch(e.type){case"TemplateProcessing":return new db(e);case"ByteLevel":return new pb(e);case"BertProcessing":return new mb(e);case"RobertaProcessing":return new vb(e);case"Sequence":return new bb(e);default:throw new Error(`Unknown PostProcessor type: ${e.type}`)}}var Lf=_b,kb=class extends as{constructor(e){super(),this.config=e,this.added_tokens=[],this.end_of_word_suffix=null,this.trim_offsets="trim_offsets"in e?e.trim_offsets:!1}_call(e){return this.decode(e)}decode(e){return this.decode_chain(e).join("")}},Ut=kb,Ab=class extends Ut{constructor(e){super(e),this.byte_decoder=Mv,this.text_decoder=new TextDecoder("utf-8",{fatal:!1,ignoreBOM:!0}),this.end_of_word_suffix=null}convert_tokens_to_string(e){let t=e.join(""),r=new Uint8Array([...t].map(n=>this.byte_decoder[n]));return this.text_decoder.decode(r)}decode_chain(e){let t=[],r=[];for(let n of e)this.added_tokens.find(s=>s.content===n)!==void 0?(r.length>0&&(t.push(this.convert_tokens_to_string(r)),r=[]),t.push(n)):r.push(n);return r.length>0&&t.push(this.convert_tokens_to_string(r)),t}},yb=Ab,Sb=class extends Ut{constructor(e){super(e),this.cleanup=e.cleanup}decode_chain(e){return e.map((t,r)=>{if(r!==0){let n=this.config.prefix;n&&t.startsWith(n)?t=t.replace(n,""):t=" "+t}return this.cleanup&&(t=Bo(t)),t})}},xb=Sb,Tb=class extends Ut{constructor(e){super(e),this.replacement=e.replacement??"\u2581"}decode_chain(e){let t=[];for(let r=0;r<e.length;++r){let n=e[r].replaceAll(this.replacement," ");r==0&&n.startsWith(" ")&&(n=n.substring(1)),t.push(n)}return t}},Cb=Tb,qb=class extends Ut{constructor(e){super(e),this.suffix=e.suffix??""}decode_chain(e){return e.map((t,r)=>t.replaceAll(this.suffix,r===e.length-1?"":" "))}},Bb=qb,Ib=class extends Ut{constructor(e){super(e),this.pad_token=e.pad_token??"",this.word_delimiter_token=e.word_delimiter_token??"",this.cleanup=e.cleanup}convert_tokens_to_string(e){if(e.length===0)return"";let t=[e[0]];for(let s=1;s<e.length;++s)e[s]!==t.at(-1)&&t.push(e[s]);let n=t.filter(s=>s!==this.pad_token).join("");return this.cleanup&&(n=Bo(n).replaceAll(this.word_delimiter_token," ").trim()),n}decode_chain(e){return[this.convert_tokens_to_string(e)]}},Eb=Ib,Pb=class extends Ut{constructor(e){super(e),this.decoders=(e.decoders??[]).map(t=>Of(t))}decode_chain(e){return this.decoders.reduce((t,r)=>r.decode_chain(t),e)}},Rb=Pb,Gb=class extends Ut{decode_chain(e){let t=ti(this.config.pattern),r=this.config.content??"";return t===null?e:e.map(n=>n.replaceAll(t,r))}},Db=Gb,Lb=class extends Ut{decode_chain(e){return[e.join("")]}},Ob=Lb,Mb=class extends Ut{constructor(e){super(e),this.content=e.content??"",this.start=e.start??0,this.stop=e.stop??0}decode_chain(e){return e.map(t=>{let r=0;for(let s=0;s<this.start&&t[s]===this.content;++s){r=s+1;continue}let n=t.length;for(let s=0;s<this.stop;++s){let a=t.length-s-1;if(t[a]===this.content){n=a;continue}else break}return t.slice(r,n)})}},Fb=Mb,Qb=class extends Ut{constructor(e){super(e),this.text_decoder=new TextDecoder}decode_chain(e){let t=[],r=[];for(let n of e){let s=null;if(n.length===6&&n.startsWith("<0x")&&n.endsWith(">")){let a=parseInt(n.slice(3,5),16);isNaN(a)||(s=a)}if(s!==null)r.push(s);else{if(r.length>0){let a=this.text_decoder.decode(Uint8Array.from(r));t.push(a),r=[]}t.push(n)}}if(r.length>0){let n=this.text_decoder.decode(Uint8Array.from(r));t.push(n),r=[]}return t}},jb=Qb;function Wb(e){if(e===null)return null;switch(e.type){case"ByteLevel":return new yb(e);case"WordPiece":return new xb(e);case"Metaspace":return new Cb(e);case"BPEDecoder":return new Bb(e);case"CTC":return new Eb(e);case"Sequence":return new Rb(e);case"Replace":return new Db(e);case"Fuse":return new Ob(e);case"Strip":return new Fb(e);case"ByteFallback":return new jb(e);default:throw new Error(`Unknown Decoder type: ${e.type}`)}}var Of=Wb,zb=class{constructor(e,t){let r=xf(e,"Tokenizer",["model","decoder","post_processor","pre_tokenizer","normalizer"]);if(r)throw new Error(r);let n=xf(t,"Config");if(n)throw new Error(n);this.tokenizer=e,this.config=t,this.normalizer=Pf(this.tokenizer.normalizer),this.pre_tokenizer=Rf(this.tokenizer.pre_tokenizer),this.model=ob(this.tokenizer.model,this.config),this.post_processor=Lf(this.tokenizer.post_processor),this.decoder=Of(this.tokenizer.decoder),this.special_tokens=[],this.all_special_ids=[],this.added_tokens=[];let s=[],a=[];this.added_tokens_map=new Map;for(let i of this.tokenizer.added_tokens){let u=new Lv(i);if(this.added_tokens.push(u),this.model.tokens_to_ids.set(u.content,u.id),this.model.vocab[u.id]=u.content,u.special&&(this.special_tokens.push(u.content),this.all_special_ids.push(u.id)),this.added_tokens_map.set(u.content,u),u.normalized&&this.normalizer!==null){let o=this.normalizer(u.content);a.push(o),this.added_tokens_map.set(o,u)}else s.push(u.content)}(this.config.additional_special_tokens??[]).forEach(i=>{this.special_tokens.includes(i)||this.special_tokens.push(i)}),this.decoder&&(this.decoder.added_tokens=this.added_tokens,this.decoder.end_of_word_suffix=this.model.end_of_word_suffix),this.splitter_unnormalized=new yf(s),this.splitter_normalized=new yf(a),this.remove_space=this.config.remove_space,this.clean_up_tokenization_spaces=this.config.clean_up_tokenization_spaces??!0,this.do_lowercase_and_remove_accent=this.config.do_lowercase_and_remove_accent??!1}encode(e,{text_pair:t=null,add_special_tokens:r=!0,return_token_type_ids:n=null}={}){let{tokens:s,token_type_ids:a}=this.tokenize_helper(e,{text_pair:t,add_special_tokens:r}),i=s.map(o=>this.added_tokens_map.get(o)?.id??this.model.tokens_to_ids.get(o)??this.model.unk_token_id),u={ids:i,tokens:s,attention_mask:new Array(i.length).fill(1)};return n&&a&&(u.token_type_ids=a),u}decode(e,t={}){if(!Array.isArray(e)||e.length===0||!zv(e[0]))throw Error("token_ids must be a non-empty array of integers.");let r=e.map(s=>this.model.vocab[Number(s)]??this.model.unk_token);t.skip_special_tokens&&(r=r.filter(s=>!this.special_tokens.includes(s)));let n=this.decoder?this.decoder(r):r.join(" ");return this.decoder&&this.decoder.end_of_word_suffix&&(n=n.replaceAll(this.decoder.end_of_word_suffix," "),t.skip_special_tokens&&(n=n.trim())),(t.clean_up_tokenization_spaces??this.clean_up_tokenization_spaces)&&(n=Bo(n)),n}tokenize(e,{text_pair:t=null,add_special_tokens:r=!1}={}){return this.tokenize_helper(e,{text_pair:t,add_special_tokens:r}).tokens}encode_text(e){if(e===null)return null;let t=this.splitter_unnormalized.split(e);return t.forEach((r,n)=>{let s=this.added_tokens_map.get(r);s&&(s.lstrip&&n>0&&(t[n-1]=t[n-1].trimEnd()),s.rstrip&&n<t.length-1&&(t[n+1]=t[n+1].trimStart()))}),t.flatMap((r,n)=>{if(r.length===0)return[];if(this.added_tokens_map.has(r))return[r];if(this.remove_space===!0&&(r=r.trim().split(/\s+/).join(" ")),this.do_lowercase_and_remove_accent&&(r=Uv(r)),this.normalizer!==null&&(r=this.normalizer(r)),r.length===0)return[];let s=this.splitter_normalized.split(r);return s.forEach((a,i)=>{let u=this.added_tokens_map.get(a);u&&(u.lstrip&&i>0&&(s[i-1]=s[i-1].trimEnd()),u.rstrip&&i<s.length-1&&(s[i+1]=s[i+1].trimStart()))}),s.flatMap(a=>{if(a.length===0)return[];if(this.added_tokens_map.has(a))return[a];let i=this.pre_tokenizer!==null?this.pre_tokenizer(a,{section_index:n}):[a];return this.model(i)})})}tokenize_helper(e,{text_pair:t=null,add_special_tokens:r=!0}){let n=this.encode_text(e),s=this.encode_text(t||null);return this.post_processor?this.post_processor(n,s,r):{tokens:It(n??[],s??[])}}token_to_id(e){return this.model.tokens_to_ids.get(e)}id_to_token(e){return this.model.vocab[e]}get_added_tokens_decoder(){let e=new Map;for(let t of this.added_tokens)e.set(t.id,t);return e}get_vocab(e=!0){let t=new Map;for(let r=0;r<this.model.vocab.length;++r){let n=this.model.vocab[r];(e||!this.added_tokens_map.has(n))&&t.set(n,r)}return t}},Mf=zb;function Eo(e){return typeof e=="string"?e.trim():""}function us(e,t){try{return e[t]}catch{return}}function Qf(e){let t=us(e,"constructor");return typeof t=="object"||typeof t=="function"?Eo(us(t,"name")):""}function Kb(e){let t=Eo(us(e,"name"));if(t&&t!=="Error")return t;let r=Qf(e);return r&&r!=="Error"&&r!=="Object"?r:""}function nr(e){if(e===null)return"null thrown";if(e===void 0)return"undefined thrown";if(typeof e=="object"){let t=us(e,"message"),r=typeof t=="string"?t:"",n=Kb(e);if(r.trim())return n&&!r.startsWith(n)?`${n}: ${r}`:r;let s=n||Qf(e)||"unknown error",a=us(e,"cause"),i=a===void 0?"":nr(a);if(i&&i!==s)return`${s} (no message; caused by ${i})`;if(!(e instanceof Error)){let u=Ff(e);if(u&&u!=="[object Object]"&&u!==s)return`${s}: ${u}`}return`${s} (no message)`}return Ff(e)||`${typeof e} thrown`}function Ff(e){try{return Eo(String(e))}catch{return""}}var Ub=pr,Nb=Object.freeze({"subgroup-size-control":["subgroups"],"chromium-experimental-subgroup-matrix":["subgroups"]});function zf(e,t=[]){let r=new Set(Kf(t));for(let[n,s]of Object.entries(Nb))s.some(a=>r.has(a))&&r.add(n);return Ub.filter(n=>e.features.has(n)&&!r.has(n)).map(n=>n)}function Kf(e){return typeof e=="string"?e.split(",").map(t=>t.trim()).filter(Boolean):Array.isArray(e)?[...e]:e instanceof Set?[...e]:[]}function Hb(e,t){if(!t||Object.keys(t).length===0)return e;let r=e,n={...r};for(let[s,a]of Object.entries(t)){if(!Object.hasOwn(r,s))throw new Error(`limit override names an unknown limit: ${s}`);let i=r[s];if(typeof i=="number"){if(a>i)throw new Error(`limit override ${s}=${a} exceeds what the adapter reports (${i}); overrides only lower a limit`);n[s]=a}}return n}function Uf(e,t=!1){return async r=>{e.pushErrorScope("validation");let n=null;try{let a={layout:"auto",compute:{module:e.createShaderModule({code:r}),entryPoint:"main"}};t?e.createComputePipeline(a):await e.createComputePipelineAsync(a)}catch(s){n=s instanceof Error?s.message:String(s)}try{let s=await e.popErrorScope();n===null&&s!==null&&(n=s.message)}catch(s){n===null&&(n=s instanceof Error?s.message:String(s))}return n}}function Nf(e,t=!1){let r=$b(e.lost);return async n=>{if(r.failure!==null)return r.failure;let s=null,a=0;try{for(let l of Po)e.pushErrorScope(l),a+=1;let o={layout:"auto",compute:{module:e.createShaderModule({code:n}),entryPoint:"main"}};t?e.createComputePipeline(o):await e.createComputePipelineAsync(o)}catch(u){s=nr(u)}let i=[];for(let u=a-1;u>=0;u-=1)try{i.push(e.popErrorScope())}catch(o){s??=nr(o)}for(let u of await Promise.allSettled(i))u.status==="rejected"?s??=nr(u.reason):u.value!==null&&(s??=u.value.message);return s??r.failure}}function $b(e){let t={failure:null};return e?.then(r=>{t.failure=`WebGPU device lost during capability probe: ${r.message}`},r=>{t.failure=`WebGPU device loss state unavailable: ${nr(r)}`}),t}function Hf(e,t,r,n,s,a,i,u){let o=t.info,l=Vb(r?.wgslLanguageFeatures??globalThis.navigator?.gpu?.wgslLanguageFeatures,s),c=n?n.min:o.subgroupMinSize,d=n?n.max:o.subgroupMaxSize,p=u===void 0?ln(o.subgroupMatrixConfigs):u;return{adapterInfo:{vendor:o.vendor,architecture:o.architecture,device:o.device,description:o.description,isFallbackAdapter:o.isFallbackAdapter===!0,...typeof c=="number"?{subgroupMinSize:c}:{},...typeof d=="number"?{subgroupMaxSize:d}:{},...p!==null?{subgroupMatrixConfigs:p}:{}},features:on(e.features),wgslLanguageFeatures:zn(l),wgslCompileFailures:Iu(i),limits:Hb(la(e.limits),a)}}function Vb(e,t){let r=Kf(t);if(r.length===0||e===null||e===void 0)return e;let n=new Set(r);return Array.from(zn(e)).filter(s=>!n.has(s))}function $f(e){return{maxBufferSize:Number(e.limits.maxBufferSize),maxStorageBufferBindingSize:Number(e.limits.maxStorageBufferBindingSize),maxStorageBuffersPerShaderStage:Number(e.limits.maxStorageBuffersPerShaderStage),maxComputeWorkgroupStorageSize:Number(e.limits.maxComputeWorkgroupStorageSize),...jf(e,"maxComputeInvocationsPerWorkgroup"),...jf(e,"maxComputeWorkgroupSizeX")}}function jf(e,t){let r=Number(e.limits[t]);return Number.isFinite(r)?{[t]:r}:{}}function Wf(e){return e instanceof Error?e.message:String(e)}var Po=["validation","out-of-memory","internal"];function Zb(e){return t=>{e.push(nr(t.error))}}function Vf({adapter:e,device:t,deviceInfo:r,destroy:n,gpu:s,syncPipelineCreation:a,diagnosticSink:i,rejectUncapturedErrors:u}){let o=[];u&&t.addEventListener("uncapturederror",Zb(o));let l=()=>{if(o.length===0)return;let[b,..._]=o.splice(0);throw new Error(`WebGPU dropped work after an error no error scope captured: ${b}`+(_.length>0?`
(${_.length} more uncaptured error(s) followed, usually its cascade)`:""))},c=new WeakMap,d=b=>{if(i)try{i(b)}catch{}},p=a?async b=>{for(let S of Po)t.pushErrorScope(S);let _,C;try{_=t.createComputePipeline(b)}catch(S){C=S}let y=[],E;for(let S=Po.length-1;S>=0;S-=1)try{y.push(t.popErrorScope())}catch(A){E??=A}let k=[];for(let S of await Promise.allSettled(y))S.status==="rejected"?E??=S.reason:S.value!==null&&k.push(S.value);if(C!==void 0)throw C;if(E!==void 0)throw E;if(k.length>0)throw new Error(`createComputePipeline failed: ${k.map(S=>S.message).join(`
`)}`);return _}:b=>t.createComputePipelineAsync(b),f=async b=>{try{return await p(b)}catch(_){if(i){let C=b.compute.module,y=c.get(C),E=Wf(_),k=C.getCompilationInfo;if(typeof k=="function")try{let S=await k.call(C);if(S.messages.length>0){let A=S.messages.map(w=>`${w.type} ${w.lineNum}:${w.linePos}: ${w.message}`);E=`${E}
${A.join(`
`)}`}}catch{}d({stage:"build",message:E,...b.label||y?.label?{label:b.label??y?.label}:{},...y?{shaderSource:y.source}:{}})}throw _}},g=0,m=0,h=new Map;return{gpu:s,adapter:e,device:t,deviceInfo:r,createShaderModule:(b,_)=>{try{let C=t.createShaderModule({code:b,label:_});return c.set(C,{source:b,..._?{label:_}:{}}),C}catch(C){throw d({stage:"build",message:Wf(C),..._?{label:_}:{},shaderSource:b}),C}},createComputePipeline:f,createBuffer:b=>{let _=r.limits?.maxBufferSize,C=Number(b.size);if(typeof _=="number"&&C>_)throw new Error(`WebGPU buffer${b.label?` "${b.label}"`:""} of ${ya(C,1)} exceeds this device's maxBufferSize of ${ya(_,1)}. This allocation is too large for this GPU/browser.`);let y=t.createBuffer(b);g+=C,g>m&&(m=g);let E=(b.label??"(unlabelled)").replace(/\d+/g,"#"),k=h.get(E);k?(k.bytes+=C,k.count+=1):h.set(E,{bytes:C,count:1});let S=y.destroy.bind(y),A=!0;return y.destroy=()=>{if(A){A=!1,g-=C;let w=h.get(E);w.bytes-=C,w.count-=1,w.count===0&&h.delete(E)}S()},y},memory:{get liveBytes(){return g},get peakBytes(){return m},markPeak(){m=g},census(b=8){return[...h.entries()].map(([_,{bytes:C,count:y}])=>({label:_,bytes:C,count:y})).sort((_,C)=>C.bytes-_.bytes).slice(0,b)}},writeBuffer:(b,_,C)=>t.queue.writeBuffer(b,_,C),submit:async(b,_)=>{t.queue.submit(b),_.wait&&(await t.queue.onSubmittedWorkDone(),l())},mapRead:async(b,_,C)=>{await b.mapAsync(GPUMapMode.READ,_,C);let y=b.getMappedRange(_,C).slice(0);return b.unmap(),l(),y},destroy:u?async()=>{await n(),l()}:n}}var Do=[{componentType:"f16",resultComponentType:"f32",M:8,N:8,K:8},{componentType:"f32",resultComponentType:"f32",M:8,N:8,K:8},{componentType:"f16",resultComponentType:"f16",M:8,N:8,K:8}];function Zf(e,t){return e.componentType===t.componentType&&e.resultComponentType===t.resultComponentType&&e.M===t.M&&e.N===t.N&&e.K===t.K}function Lo(e){return Do.some(t=>Zf(e,t))}function Ro(e){return e==="i8"?"i32":e==="u8"?"u32":e}function Go(e,t){return e==="i8"||e==="u8"?Math.ceil(t/4):t}function Xb(e){let{componentType:t,resultComponentType:r,M:n,N:s,K:a}=e,i=Go(t,a),u=Go(t,s),o=Go(r,s);return`${t==="f16"||r==="f16"?`enable f16;
`:""}enable subgroups;
enable chromium_experimental_subgroup_matrix;
diagnostic(off, chromium.subgroup_matrix_uniformity);
@group(0) @binding(0) var<storage, read> lhs_values: array<${Ro(t)}>;
@group(0) @binding(1) var<storage, read> rhs_values: array<${Ro(t)}>;
@group(0) @binding(2) var<storage, read_write> output: array<${Ro(r)}>;
@compute @workgroup_size(128, 1, 1)
fn main(@builtin(local_invocation_index) tid: u32) {
  let lhs = subgroupMatrixLoad<subgroup_matrix_left<${t}, ${a}, ${n}>, row_major>(&lhs_values, 0u, ${i}u);
  let rhs = subgroupMatrixLoad<subgroup_matrix_right<${t}, ${s}, ${a}>, row_major>(&rhs_values, 0u, ${u}u);
  var accumulator: subgroup_matrix_result<${r}, ${s}, ${n}>;
  let product = subgroupMatrixMultiplyAccumulate(lhs, rhs, accumulator);
  let bank = subgroupBroadcastFirst(tid);
  subgroupMatrixStore<row_major>(&output, bank * ${n*o}u, product, ${o}u);
}
`}function Yb(e){if(!Lo(e))return Xb(e);let t=e.componentType,r=e.resultComponentType;return`${t==="f16"?`enable f16;
`:""}enable subgroups;
enable chromium_experimental_subgroup_matrix;
diagnostic(off, chromium.subgroup_matrix_uniformity);
@group(0) @binding(0) var<storage, read_write> output: array<${r}>;
var<workgroup> lhs_values: array<${t}, 64>;
var<workgroup> rhs_values: array<${t}, 64>;
@compute @workgroup_size(128, 1, 1)
fn main(@builtin(local_invocation_index) tid: u32) {
  if (tid < 64u) {
    lhs_values[tid] = ${t==="f16"?"1.0h":"1.0f"};
    rhs_values[tid] = ${t==="f16"?"1.0h":"1.0f"};
  }
  workgroupBarrier();
  let lhs = subgroupMatrixLoad<subgroup_matrix_left<${t}, 8, 8>, row_major>(&lhs_values, 0u, 8u);
  let rhs = subgroupMatrixLoad<subgroup_matrix_right<${t}, 8, 8>, row_major>(&rhs_values, 0u, 8u);
  var accumulator: subgroup_matrix_result<${r}, 8, 8>;
  let product = subgroupMatrixMultiplyAccumulate(lhs, rhs, accumulator);
  let bank = subgroupBroadcastFirst(tid);
  subgroupMatrixStore<row_major>(&output, bank * 64u, product, 8u);
}
`}async function Xf(e,t,r){let n=on(e),s=ln(t);if(!n.has("chromium-experimental-subgroup-matrix")||!n.has("subgroups"))return s;let a=Do[0],i=s?.some(d=>d.componentType===a.componentType&&d.resultComponentType===a.resultComponentType&&d.M===8&&d.N===8&&d.K===8),u=s?.some(d=>d.componentType==="f16"&&d.resultComponentType==="f16"&&d.M===8&&d.N===8&&d.K===8),o=s===null?[...Do]:s.filter(d=>!Lo(d));u&&!i&&o.push(a);let l=s?.filter(Lo)??[],c=[];for(let d of o)if(!c.some(p=>Zf(p,d))&&(c.push(d),!((d.componentType==="f16"||d.resultComponentType==="f16")&&!n.has("shader-f16"))))try{await r(Yb(d))===null&&l.push(d)}catch{}return ln(l)}function Jb(e){return nr(e)}function Yf(e,t){if(e)try{e(t)}catch{}}function e0(e,t){e.addEventListener("uncapturederror",r=>{t?Yf(t,{stage:"execute",message:`WebGPU uncaptured error: ${Jb(r.error)}`}):console.error("WebGPU uncaptured error:",r.error)}),t&&e.lost.then(r=>{r.reason!=="destroyed"&&Yf(t,{stage:"execute",message:`WebGPU device lost (${r.reason}): ${r.message}`})})}async function Jf(e={}){let t=globalThis.navigator?.gpu;if(!t)throw new Error("WebGPU is not available in this browser context");let r=await t.requestAdapter({powerPreference:e.powerPreference??"high-performance"});if(!r)throw new Error("No WebGPU adapter was returned");let n=await r.requestDevice({requiredFeatures:zf(r,e.disabledFeatures),requiredLimits:e.requiredLimits??$f(r),label:e.label??"webgpu-ml-runtime",...e.dawnToggles?{dawnToggles:e.dawnToggles}:{}});e0(n,e.diagnosticSink);let s=await w2(Uf(n)),a=await Xf(n.features,r.info.subgroupMatrixConfigs,Nf(n));return Vf({gpu:t,adapter:r,device:n,deviceInfo:Hf(n,r,t,null,e.disabledFeatures,null,s,a),diagnosticSink:e.diagnosticSink,destroy:()=>n.destroy()})}function t0(e){return e===null||typeof e!="object"?!1:"destroy"in e&&"mapAsync"in e&&"getMappedRange"in e&&!("buffer"in e)}function wn(e){return t0(e)?e:e.buffer}function Oo(e){return wn(e.type==="uniform"?e.buffer:e.tensor)}function br(e,t,r,n){let s=wn(e),a=e,i=a.byteOffset??0;if(!Number.isSafeInteger(i)||i<0)throw new Error(`${n} tensor view has an invalid byteOffset ${i}`);t??=i;for(let[o,l]of[["offset",t],["byteLength",r]])if(!Number.isSafeInteger(l)||l<0||l%4!==0)throw new Error(`${n} ${o} must be a non-negative multiple of 4, got ${l}`);let u=t+r;if(!Number.isSafeInteger(u))throw new Error(`${n} byte range exceeds safe integer storage`);if(typeof a.byteLength=="number"){if(!Number.isSafeInteger(a.byteLength)||a.byteLength<0)throw new Error(`${n} tensor view has an invalid byteLength ${a.byteLength}`);let o=Ze(a.byteLength,4);if(t<i||u>i+o)throw new Error(`${n} range [${t}, ${u}] exceeds tensor view [${i}, ${i+o}]`)}if(typeof s.size=="number"&&u>s.size)throw new Error(`${n} range [${t}, ${u}] exceeds backing buffer size ${s.size}`);return{buffer:s,offset:t}}var si=class{pipelines=new Map;maxPipelineEntries=Number.POSITIVE_INFINITY;#e=new Map;#t=new Map;physicalPipelines=new Map;pipelineLayouts=new Map;bindGroupLayouts=new Map;bindGroups=new Map;maxBindGroupEntries=4096;bufferIds=new WeakMap;#r=1;captureShaders=!1;capturedShaders=new Map;compileCount=0;compileTotalMs=0;#n=0;#s;constructor(t){this.#s=t}async buildPipeline(t){let r=this.pipelines.get(t.cacheKey);if(r)return this.maxPipelineEntries!==Number.POSITIVE_INFINITY&&(this.pipelines.delete(t.cacheKey),this.pipelines.set(t.cacheKey,r)),r;let n=this.#e.get(t.cacheKey);if(n)return n.promise;let s=this.#n,a=this.#i(t,s),i={generation:s,promise:a};this.#e.set(t.cacheKey,i);try{return await a}finally{this.#e.get(t.cacheKey)===i&&this.#e.delete(t.cacheKey)}}async#i({cacheKey:t,name:r,op:n,kernelId:s,entryPoint:a,layoutSignature:i,layoutFactory:u,sourceFactory:o},l){let c=await o();if(l!==this.#n)throw new Error(`Pipeline build ${t} was invalidated by clearPrograms()`);if(this.captureShaders){let p=s??r;this.capturedShaders.has(p)||this.capturedShaders.set(p,{name:r,source:c,...n===void 0?{}:{op:n},...s===void 0?{}:{kernelId:s}})}let d;if(i){let p=r0(c,a,i);d=this.physicalPipeline(p,c,a,i)??await this.#u({key:p,source:c,entryPoint:a,layoutSignature:i,generation:l,name:r,layoutFactory:u})}else d=await this.#l({source:c,entryPoint:a,name:r,layoutFactory:u});return l===this.#n&&(this.pipelines.set(t,d),this.#o()),d}#o(){for(;this.pipelines.size>this.maxPipelineEntries;){let t=this.pipelines.entries().next();if(t.done===!0)return;let[r,n]=t.value;this.pipelines.delete(r);let s=!1;for(let a of this.pipelines.values())if(a===n){s=!0;break}s||this.#a(n)}}#a(t){for(let[r,n]of this.physicalPipelines){let s=n.findIndex(a=>a.pipeline===t);if(!(s<0)){n.splice(s,1),n.length===0&&this.physicalPipelines.delete(r);return}}}async#u({key:t,source:r,entryPoint:n,layoutSignature:s,generation:a,name:i,layoutFactory:u}){let o=this.#c(t,r,n,s,a);if(o)return o.promise;let l=this.#l({source:r,entryPoint:n,name:i,layoutFactory:u}),c={key:t,source:r,entryPoint:n,layoutSignature:s,generation:a,promise:l};this.#f(c);try{let d=await l;return a===this.#n&&this.setPhysicalPipeline(t,r,n,s,d),d}finally{this.#d(c)}}async#l({source:t,entryPoint:r,name:n,layoutFactory:s}){let a=performance.now(),i=this.#s.createShaderModule(t,n),u=s(),o=await this.#s.createComputePipeline({label:n,layout:u,compute:{module:i,entryPoint:r}});return this.compileCount+=1,this.compileTotalMs+=performance.now()-a,o}cachedBindGroups({name:t,cacheKey:r,pipeline:n,bindings:s}){return{bindGroup:this.buildGroupBindGroup(t,r,n,s)}}buildGroupBindGroup(t,r,n,s){let a=T2(r,s,u=>this.bufferId(Oo(u))),i=this.bindGroups.get(a);if(i===void 0&&(i=this.createBindGroup(t,n,s),this.bindGroups.set(a,i),this.bindGroups.size>this.maxBindGroupEntries)){let u=this.bindGroups.keys().next().value;u!==void 0&&this.bindGroups.delete(u)}return i}createBindGroup(t,r,n){return this.#s.device.createBindGroup({label:`${t}-bind-group`,layout:r.getBindGroupLayout(0),entries:n.map((s,a)=>{let i={buffer:Oo(s),offset:s.offset??0};return s.size!==void 0&&s.size>0&&(i.size=s.size),{binding:s.binding??a,resource:i}})})}bufferId(t){let r=this.bufferIds.get(t);return r===void 0&&(r=this.#r++,this.bufferIds.set(t,r)),r}pipelineLayout(t,r,n=0){let s=ep(t,r,n),a=this.pipelineLayouts.get(s.signature);if(a)return a;let i=this.#s.device.createPipelineLayout({label:`${t}-layout`,bindGroupLayouts:[this.bindGroupLayout(t,s.entries)],...n>0?{immediateSize:n}:{}});return this.pipelineLayouts.set(s.signature,i),i}pipelineLayoutSignature(t,r,n=0){return ep(t,r,n).signature}bindGroupLayout(t,r){let n=r.map(u=>({binding:u.binding,visibility:GPUShaderStage.COMPUTE,buffer:{type:u.type}})),s=n.map(u=>`${u.binding}:${u.buffer.type}`).sort().join("|"),a=this.bindGroupLayouts.get(s);if(a)return a;let i=this.#s.device.createBindGroupLayout({label:`${t}-bgl0`,entries:n});return this.bindGroupLayouts.set(s,i),i}clearBindGroups(){let t=this.bindGroups.size;return this.bindGroups.clear(),t}clearPrograms(){let t=this.pipelines.size;return this.#n+=1,this.#e.clear(),this.#t.clear(),this.bindGroups.clear(),this.pipelines.clear(),this.physicalPipelines.clear(),this.pipelineLayouts.clear(),this.bindGroupLayouts.clear(),this.captureShaders||this.capturedShaders.clear(),t}getRenderedShaders(){return[...this.capturedShaders.values()]}physicalPipeline(t,r,n,s){return this.physicalPipelines.get(t)?.find(i=>i.entryPoint===n&&i.layoutSignature===s&&i.source===r)?.pipeline}#c(t,r,n,s,a){return this.#t.get(t)?.find(u=>u.generation===a&&u.entryPoint===n&&u.layoutSignature===s&&u.source===r)}#f(t){let r=this.#t.get(t.key);r?r.push(t):this.#t.set(t.key,[t])}#d(t){let r=this.#t.get(t.key);if(!r)return;let n=r.indexOf(t);n<0||(r.splice(n,1),r.length===0&&this.#t.delete(t.key))}setPhysicalPipeline(t,r,n,s,a){let i=this.physicalPipelines.get(t),u={source:r,entryPoint:n,layoutSignature:s,pipeline:a};i?i.push(u):this.physicalPipelines.set(t,[u])}};function ep(e,t,r=0){let n=t.map((u,o)=>typeof u=="string"?{binding:o,type:u}:{binding:u.binding??o,type:u.type}),s=new Set;for(let u of n){if(!Number.isInteger(u.binding)||u.binding<0)throw new Error(`pipeline layout ${e} has invalid binding index ${u.binding}`);if(s.has(u.binding))throw new Error(`pipeline layout ${e} has duplicate binding index ${u.binding}`);if(typeof u.type!="string"||u.type.length===0)throw new Error(`pipeline layout ${e} binding ${u.binding} is missing a buffer type`);s.add(u.binding)}let a=n.map(u=>`${u.binding}:${u.type}`).sort();return{signature:r>0?`${a.join("|")}|imm=${r}`:a.join("|"),entries:n}}function r0(e,t,r){return`${za(e)}|entry=${t}|layout=${r}`}var ai=class{pool=new Map;bytes=0;maxBytes;constructor(t=64*1024*1024){this.maxBytes=t}acquire(t,r){let n=this.pool.get(r);return n&&n.length>0?(this.bytes-=r,n.pop()):t.createBuffer({label:"tensor-readback",size:r,usage:GPUBufferUsage.COPY_DST|GPUBufferUsage.MAP_READ})}release(t,r){if(this.bytes+t>this.maxBytes){r.destroy();return}let n=this.pool.get(t);n||(n=[],this.pool.set(t,n)),n.push(r),this.bytes+=t}discard(t){try{t.destroy()}catch{}}clear(){let t=0;for(let r of this.pool.values())for(let n of r)n.destroy(),t++;return this.pool.clear(),this.bytes=0,t}};var ui=64*1024,n0=4;function Pt(e){return e!==void 0&&e.kind===void 0}var Nt=class{runtime;dtype;shape;byteOffset;buffer;size;byteLength;layout;destroyed;constructor({runtime:t,dtype:r,shape:n,buffer:s,byteOffset:a=0,layout:i}){if(!Number.isSafeInteger(a)||a<0)throw new Error(`tensor byteOffset must be a nonnegative integer, got ${a}`);if(this.runtime=t,this.dtype=r,this.shape=n,this.byteOffset=a,this.buffer=s,this.size=we(n),i!==void 0&&(ca(r,"WebGpuTensor"),this.layout=i),this.byteLength=et(r,this.size,i),!Number.isSafeInteger(a+this.byteLength)||typeof s.size=="number"&&Ze(a+this.byteLength,4)>s.size)throw new Error(`tensor range (${a}+${this.byteLength} bytes) exceeds backing buffer capacity`);this.destroyed=!1}destroy(){this.destroyed||(this.buffer.destroy(),this.destroyed=!0)}},ii=class{host;pipelines;readback;destroyed;mappedTensorUploadsAvailable;constructor({host:t}){this.host=t,this.pipelines=new si(t),this.readback=new ai,this.destroyed=!1,this.mappedTensorUploadsAvailable=!0}get device(){return this.host.deviceInfo}async destroy(){this.destroyed||(this.destroyed=!0,this.clearBindGroupCache(),this.clearReadbackPool(),await this.beforeHostDestroy(),await this.host.destroy())}async beforeHostDestroy(){}clearReadbackPool(){return this.readback.clear()}clearBindGroupCache(){return this.pipelines.clearBindGroups()}tensorFromTypedArray(t,r,n,s={}){let a=we(r);if(n.length!==a)throw new Error(`tensor data length ${n.length} does not match shape element count ${a}`);let i=n;if(s.layout==="packed"){if(ca(t,"tensorFromTypedArray"),!(n instanceof Uint32Array)&&!(n instanceof Uint8Array))throw new Error("A packed uint8 tensor uploads from a Uint32Array (widened) or a Uint8Array (bytes)");i=Kc(n)}else if(!s0(t,n)){let l=Ge[t]?.arrayCtor;throw new Error(l===void 0?`unsupported tensor dtype ${t}`:`${t} tensor data must be a ${l.name}, not ${n.constructor.name}`)}let u={label:"tensor",size:nt(i.byteLength),usage:GPUBufferUsage.STORAGE|GPUBufferUsage.COPY_DST|GPUBufferUsage.COPY_SRC};if(this.mappedTensorUploadsAvailable&&i.byteLength>=ui){let l=null;try{l=this.host.createBuffer({...u,mappedAtCreation:!0});let c=l.getMappedRange();return new Uint8Array(c).set(new Uint8Array(i.buffer,i.byteOffset,i.byteLength)),l.unmap(),new Nt({runtime:this,dtype:t,shape:r,buffer:l,layout:s.layout})}catch{try{l?.destroy()}catch{}this.mappedTensorUploadsAvailable=!1}}let o=this.host.createBuffer(u);try{let l=i;if(i.byteLength%4!==0){let c=new Uint8Array(nt(i.byteLength));c.set(new Uint8Array(i.buffer,i.byteOffset,i.byteLength)),l=c}return(i.byteLength>0||s.layout!=="packed")&&this.host.writeBuffer(o,0,l),new Nt({runtime:this,dtype:t,shape:r,buffer:o,layout:s.layout})}catch(l){try{o.destroy()}catch{}throw l}}empty(t,r,n="tensor-output",s={}){if(!Mo(t))throw new Error(`Unsupported dtype: ${t}`);let a=et(t,we(r),s.layout),i=this.host.createBuffer({label:n,size:nt(a),usage:GPUBufferUsage.STORAGE|GPUBufferUsage.COPY_SRC|GPUBufferUsage.COPY_DST});return new Nt({runtime:this,dtype:t,shape:r,buffer:i,layout:s.layout})}readTensor(t){if(t.byteLength===0){let f=Ge[t.dtype]?.arrayCtor;return f?Promise.resolve(new f(0)):Promise.reject(new Error(`Unsupported dtype: ${t.dtype}`))}let r=t.byteOffset??0,n=Math.floor(r/4)*4,s=r-n,a=nt(s+t.byteLength),i=br({buffer:t.buffer},n,a,"readback source"),u=this.readback.acquire(this.host,a),o=this.host.device.createCommandEncoder({label:"readTensor"});o.copyBufferToBuffer(i.buffer,i.offset,u,0,a),this.host.device.queue.submit([o.finish()]);let{dtype:l,byteLength:c,layout:d,size:p}=t;return(async()=>{let f;try{f=await this.host.mapRead(u,0,a)}catch(m){throw this.readback.discard(u),m}if(this.readback.release(a,u),(s!==0||a!==c)&&(f=f.slice(s,s+c)),d==="packed")return Uc(new Uint32Array(f),p);let g=Ge[l]?.arrayCtor;if(!g)throw new Error(`Unsupported dtype: ${l}`);return new g(f)})()}async runProgram(t,r={}){await this.runProgramSequence([t],r)}async runProgramSequence(t,r={}){let n=await this.prepareProgramSteps(t,!1);for(let s of t)this.#e(s);await this.executePreparedProgramSequence(n,r)}#e(t){let r=`program ${t.name??t.kind??"compute"}`;if(this.destroyed)throw new Error(`${r}: runtime has been destroyed`);if(tp(t))this.#t(t.src,`${r} source`),this.#t(t.dst,`${r} destination`);else if(rp(t))this.#t(t.dst,`${r} destination`);else if(Array.isArray(t.bindings))for(let n of t.bindings)n.type!=="uniform"&&this.#t(n.tensor,`${r} binding ${n.binding}`)}#t(t,r){if(t instanceof Nt){if(t.destroyed)throw new Error(`${r}: tensor has been destroyed`);if(t.runtime.destroyed)throw new Error(`${r}: tensor's owning runtime has been destroyed`)}}async prepareProgramSteps(t,r){if(!Array.isArray(t)||t.length===0)throw new Error("prepareProgramSequence requires at least one program");let n=new Array(t.length),s=0,a=!1,i=[],u=async()=>{for(;!a;){let o=s++;if(o>=t.length)return;try{n[o]=await this.#r(t[o],r)}catch(l){i.push({index:o,error:l}),a=!0}}};if(await Promise.all(Array.from({length:Math.min(n0,t.length)},u)),i.length>0)throw i.sort((o,l)=>o.index-l.index),i[0].error;return n}async#r(t,r){if(this.#e(t),tp(t))return i0(t);if(rp(t))return a0(t);let{name:n,op:s,kernelId:a,source:i,entryPoint:u="main",cacheKey:o=n,bindings:l,dispatchWorkgroups:c,profile:d,submitAfter:p,immediateSize:f,immediateData:g}=t;if(typeof i!="string"||i.length===0)throw new Error("program requires WGSL source");if(!Array.isArray(l)||l.length===0)throw new Error("program requires bindings");if(!Array.isArray(c)||c.length!==3)throw new Error("program requires a 3D dispatchWorkgroups array");if(f!==void 0&&f>0&&g===void 0)throw new Error(`program ${n} reserves ${f} immediate bytes but supplied no immediate data`);let m=this.pipelines.pipelineLayoutSignature(n,l,f),h=await this.pipelines.buildPipeline({name:n,...s===void 0?{}:{op:s},...a===void 0?{}:{kernelId:a},entryPoint:u,cacheKey:o,layoutSignature:m,layoutFactory:()=>this.pipelines.pipelineLayout(n,l,f),sourceFactory:()=>i});this.#e(t);let v=r?this.pipelines.cachedBindGroups({name:n,cacheKey:o,pipeline:h,bindings:l}).bindGroup:this.pipelines.createBindGroup(n,h,l);return{pipeline:h,bindGroup:v,...g?{immediateData:g}:{},dispatchWorkgroups:c,name:n,cacheKey:o,profile:d,...p?{submitAfter:!0}:{}}}async executePreparedProgramSequence(t,r={}){if(!Array.isArray(t)||t.length===0)throw new Error("executePreparedProgramSequence requires at least one prepared step");await this.runMergedSteps(t,r.wait??!1)}createUniformU32(t,r){let n=new Uint32Array(t),s=this.host.createBuffer({label:r,size:n.byteLength,usage:GPUBufferUsage.UNIFORM|GPUBufferUsage.COPY_DST});return this.host.writeBuffer(s,0,n),s}async runMergedSteps(t,r){let n=os(t),s=n.map(a=>{let i=this.host.device.createCommandEncoder({label:"compute-dispatch"});return bn(i,a),i.finish()});for(let a=0;a+1<s.length;++a)await this.host.submit([s[a]],{wait:!0});await this.host.submit([s[s.length-1]],{wait:r||Fo(n[n.length-1])})}};function s0(e,t){let r=Ge[e]?.arrayCtor;return r!==void 0&&t instanceof r}function Mo(e){return Ge[e]!==void 0}function tp(e){return e.kind==="copy"}function rp(e){return e.kind==="clear"}function a0(e){if(!e.dst)throw new Error("clear step requires dst (buffer or tensor)");let t=e.dstOffset??0,r=e.byteLength;if(!Number.isInteger(t)||t<0||t%4!==0)throw new Error(`clear step offset must be a non-negative multiple of 4, got ${t}`);if(!Number.isInteger(r)||r<0||r%4!==0)throw new Error(`clear step byteLength must be a non-negative multiple of 4, got ${r}`);return{kind:"clear",dst:wn(e.dst),dstOffset:t,byteLength:r,name:e.name??"clear"}}function i0(e){if(!e.src||!e.dst)throw new Error("copy step requires src and dst (buffer or tensor)");let t=e.byteLength;if(t<=0)throw new Error("copy step requires positive integer byteLength");let r=br(e.src,e.srcOffset,t,"copy source"),n=br(e.dst,e.dstOffset,t,"copy destination");return{kind:"copy",src:r.buffer,dst:n.buffer,srcOffset:r.offset,dstOffset:n.offset,byteLength:t,name:e.name??"copy"}}function os(e){let t=[],r=0;for(let n=0;n<e.length;++n){let s=e[n];Pt(s)&&s.submitAfter&&!bt(s.dispatchWorkgroups)&&(t.push(e.slice(r,n+1)),r=n+1)}return r<e.length&&t.push(e.slice(r)),t}function Fo(e){let t=e[e.length-1];return Pt(t)&&t.submitAfter===!0&&!bt(t.dispatchWorkgroups)}function bn(e,t,r){let n=null,s=t.findIndex(c=>Pt(c)&&!bt(c.dispatchWorkgroups)),a=s,i=!1;for(let c=s+1;c<t.length;++c){let d=t[c];Pt(d)?!bt(d.dispatchWorkgroups)&&i&&(a=c,i=!1):i=!0}let u=null,o=null,l=()=>{n!==null&&(n.end(),n=null),u=null,o=null};for(let c=0;c<t.length;++c){let d=t[c];if(d.kind==="copy"){l(),d.byteLength>0&&e.copyBufferToBuffer(d.src,d.srcOffset,d.dst,d.dstOffset,d.byteLength);continue}if(d.kind==="clear"){l(),d.byteLength>0&&e.clearBuffer(d.dst,d.dstOffset,d.byteLength);continue}if(!bt(d.dispatchWorkgroups)){if(n===null){let p=c===s?r?.beginningOfSequenceWriteIndex:void 0,f=c===a?r?.endOfSequenceWriteIndex:void 0,g=p!==void 0||f!==void 0;n=e.beginComputePass({label:"compute-sequence-pass",...g?{timestampWrites:{querySet:r.querySet,...p!==void 0?{beginningOfPassWriteIndex:p}:{},...f!==void 0?{endOfPassWriteIndex:f}:{}}}:{}})}d.pipeline!==u&&(n.setPipeline(d.pipeline),u=d.pipeline),d.bindGroup!==o&&(n.setBindGroup(0,d.bindGroup),o=d.bindGroup),d.immediateData&&oi(n,d.immediateData),n.dispatchWorkgroups(d.dispatchWorkgroups[0],d.dispatchWorkgroups[1],d.dispatchWorkgroups[2])}}l()}function oi(e,t){let r=e;if(typeof r.setImmediates!="function")throw new Error("this WebGPU implementation reports the immediate_address_space language feature but the compute pass encoder has no setImmediates()");r.setImmediates(0,t)}function bt(e){return e[0]===0||e[1]===0||e[2]===0}var li=class extends ii{profileSession=null;pendingProfileRuns=[];#e=new Map;#t=Promise.resolve();#r=!1;get captureShaders(){return this.pipelines.captureShaders}set captureShaders(t){this.pipelines.captureShaders=t}getRenderedShaders(){return this.pipelines.getRenderedShaders()}async beforeHostDestroy(){await this.#t,this.#l()}clearKernelCaches(){return this.pipelines.clearPrograms()}tensorFromWriter(t,r,n){let s=Ge[t];if(!s)throw new Error(`Unsupported dtype: ${t}`);let a=we(r),i=a*s.storageByteSize,u={label:"tensor",size:nt(i),usage:GPUBufferUsage.STORAGE|GPUBufferUsage.COPY_DST|GPUBufferUsage.COPY_SRC,mappedAtCreation:!0};if(this.mappedTensorUploadsAvailable&&i>=ui){let l=null;try{l=this.host.createBuffer(u)}catch{this.mappedTensorUploadsAvailable=!1}if(l){let c;try{c=new s.arrayCtor(l.getMappedRange(),0,a)}catch{this.mappedTensorUploadsAvailable=!1;try{l.destroy()}catch{}l=null}if(l){try{n(c)}catch(d){try{l.unmap()}catch{}try{l.destroy()}catch{}throw d}try{l.unmap()}catch(d){this.mappedTensorUploadsAvailable=!1;try{l.destroy()}catch{}throw d}return new Nt({runtime:this,dtype:t,shape:r,buffer:l})}}}let o=new s.arrayCtor(a);return n(o),this.tensorFromTypedArray(t,r,o)}beginMappedTensorUpload(t,r,n="tensor"){let s=Ge[t];if(!s)throw new Error(`Unsupported dtype: ${t}`);let a=[...r],i=we(a)*s.storageByteSize;if(!this.mappedTensorUploadsAvailable||i<ui)return null;let u;try{u=this.host.createBuffer({label:n,size:nt(i),usage:GPUBufferUsage.STORAGE|GPUBufferUsage.COPY_DST|GPUBufferUsage.COPY_SRC,mappedAtCreation:!0})}catch{return this.mappedTensorUploadsAvailable=!1,null}let o;try{let g=u.getMappedRange();if(g.byteLength<i)throw new Error(`Mapped tensor range has ${g.byteLength} bytes; expected at least ${i}`);o=new Uint8Array(g,0,i)}catch{this.mappedTensorUploadsAvailable=!1;try{u.destroy()}catch{}return null}let l="open",c=null,d=!1,p=()=>{d||(d=!0,u.destroy())},f=g=>{if(l!=="open")throw new Error(`Cannot ${g} mapped tensor upload after it was ${l}`)};return{destination:o,commit:()=>{f("commit");try{u.unmap()}catch(g){l="aborted",this.mappedTensorUploadsAvailable=!1;try{p()}catch{}throw g}try{c=new Nt({runtime:this,dtype:t,shape:a,buffer:u})}catch(g){l="aborted";try{p()}catch{}throw g}return l="committed",c},rollback:g=>{if(g!==c)throw new Error("Cannot roll back a tensor that was not returned by this mapped upload");if(l!=="rolled back"){if(l!=="committed")throw new Error(`Cannot roll back mapped tensor upload after it was ${l}`);c.destroy(),l="rolled back"}},abort:()=>{if(l==="aborted")return;f("abort"),l="aborted";let g;try{u.unmap()}catch(m){this.mappedTensorUploadsAvailable=!1,g=m}try{p()}catch(m){if(g===void 0)throw m}if(g!==void 0)throw g}}}allocateWeightsBuffer({byteLength:t,dtype:r,shape:n,label:s="weights"}){if(!Mo(r))throw new Error(`Unsupported dtype: ${r}`);if(!Number.isSafeInteger(t)||t<0)throw new Error(`byteLength must be a nonnegative integer, got ${t}`);let a=et(r,we(n));if(t<a)throw new Error(`weights tensor requires ${a} bytes, but byteLength is ${t}`);let i=this.host.createBuffer({label:s,size:nt(t),usage:GPUBufferUsage.STORAGE|GPUBufferUsage.COPY_DST|GPUBufferUsage.COPY_SRC});return new Nt({runtime:this,dtype:r,shape:n,buffer:i})}writeWeightsRange(t,r,n){if(!st(t))throw new Error("writeWeightsRange expects a GPU tensor");if(!Number.isSafeInteger(r)||r<0)throw new Error(`byteOffset must be a nonnegative integer, got ${r}`);if(r+n.byteLength>t.byteLength)throw new Error(`write range [${r}, ${r+n.byteLength}] exceeds tensor byteLength ${t.byteLength}`);let s=br(t,(t.byteOffset??0)+r,n.byteLength,"weight write");n.byteLength>0&&this.host.writeBuffer(s.buffer,s.offset,n)}async copyBufferToBuffer({src:t,dst:r,srcOffset:n,dstOffset:s,byteLength:a,wait:i=!1}){let u=br(t,n,a,"copy source"),o=br(r,s,a,"copy destination");if(a===0)return;let l=this.host.device.createCommandEncoder({label:"copyBufferToBuffer"});l.copyBufferToBuffer(u.buffer,u.offset,o.buffer,o.offset,a),await this.host.submit([l.finish()],{wait:i})}clearBuffers(t){let r=null;for(let{dst:n,dstOffset:s=0,byteLength:a}of t){if(!Number.isInteger(s)||s<0||s%4!==0)throw new Error(`clear buffer offset must be a non-negative multiple of 4, got ${s}`);if(!Number.isInteger(a)||a<0||a%4!==0)throw new Error(`clear buffer byteLength must be a non-negative multiple of 4, got ${a}`);a!==0&&(r??=this.host.device.createCommandEncoder({label:"clear-buffers"}),r.clearBuffer(wn(n),s,a))}r&&this.host.device.queue.submit([r.finish()])}async queueIdle(){await this.host.device.queue.onSubmittedWorkDone()}async prepareProgramSequence(t){return this.prepareProgramSteps(t,!0)}async executePreparedProgramSequence(t,r={}){if(!this.profileSession)return super.executePreparedProgramSequence(t,r);if(!Array.isArray(t)||t.length===0)throw new Error("executePreparedProgramSequence requires at least one prepared step");let n=r.wait??!1;await this.#i();try{await this._runSteps(t,{wait:n,mergePass:!1})}finally{this.#a()}}preparedProgramSequenceRequiresCompletionFences(t){return t.some(r=>Pt(r)&&r.submitAfter===!0&&!bt(r.dispatchWorkgroups))}#n=8;#s=[];async#i(){for(;this.#n<=0;)await new Promise(t=>this.#s.push(t));this.#n-=1}#o(){return this.#n<=0?!1:(this.#n-=1,!0)}#a(){this.#n+=1,this.#s.shift()?.()}enqueuePreparedProgramSequence(t){if(!Array.isArray(t)||t.length===0)throw new Error("enqueuePreparedProgramSequence requires at least one prepared step");let r=os(t);if(this.profileSession&&r.length===1&&this.#o()){let n=this._runSteps(t,{wait:!1,mergePass:!1});this.pendingProfileRuns.push(n.finally(()=>this.#a()));return}for(let n of r){let s=this.host.device.createCommandEncoder({label:"compute-dispatch"});bn(s,n),this.host.device.queue.submit([s.finish()])}}async flushDeferredProfile(){let t=this.pendingProfileRuns;this.pendingProfileRuns=[],await Promise.all(t)}async measurePreparedSequenceGpuMs(t,r=1,n){if(this.destroyed||!this.host.device.features.has("timestamp-query")||this.#r||!t.some(i=>Pt(i)&&!bt(i.dispatchWorkgroups)))return null;let s=this.#t,a;this.#t=new Promise(i=>{a=i}),await s;try{if(this.destroyed||this.#r)return null;let i=this.#e.get(2)??this.#c(2),u=os(t);if(u.length>1)return await this.#u(u,r,i,n);let o=this.host.device.createCommandEncoder({label:"gpu-time-measure"});if(t.some(d=>d.kind==="clear"||d.kind==="copy"))for(let d=0;d<r;++d)bn(o,t,{querySet:i.querySet,...d===0?{beginningOfSequenceWriteIndex:0}:{},...d===r-1?{endOfSequenceWriteIndex:1}:{}});else{let d=o.beginComputePass({label:"gpu-time-pass",timestampWrites:{querySet:i.querySet,beginningOfPassWriteIndex:0,endOfPassWriteIndex:1}}),p=!1,f=null,g=null;for(let m=0;m<r;++m)for(let h of t)!Pt(h)||bt(h.dispatchWorkgroups)||(h.pipeline!==f&&(d.setPipeline(h.pipeline),f=h.pipeline),h.bindGroup!==g&&(d.setBindGroup(0,h.bindGroup),g=h.bindGroup),h.immediateData&&oi(d,h.immediateData),d.dispatchWorkgroups(h.dispatchWorkgroups[0],h.dispatchWorkgroups[1],h.dispatchWorkgroups[2]),p=!0);if(d.end(),!p)return null}o.resolveQuerySet(i.querySet,0,2,i.resolveBuffer,0),o.copyBufferToBuffer(i.resolveBuffer,0,i.readbackBuffer,0,16);let l=this.host.submit([o.finish()],{wait:!0});try{n?.()}catch{}await l;let c;try{c=await this.readTimestampBuffer(i.readbackBuffer,2)}catch{return this.#r=!0,this.#l(),null}return Number(c[1]-c[0])/1e6}finally{a()}}async measurePreparedSequenceGpuSamples(t,r,n){if(this.destroyed||!this.host.device.features.has("timestamp-query")||this.#r||!Number.isSafeInteger(r)||r<=0||!Number.isSafeInteger(n)||n<=1||this.preparedProgramSequenceRequiresCompletionFences(t)||!Qo(t))return null;let s=this.#t,a;this.#t=new Promise(i=>{a=i}),await s;try{if(this.destroyed||this.#r)return null;let i=n*2,u=this.#e.get(i)??this.#c(i);for(let c=0;c<n;++c)for(let d=0;d<r;++d){let p=this.host.device.createCommandEncoder({label:"gpu-time-sample-batch"});bn(p,t,{querySet:u.querySet,...d===0?{beginningOfSequenceWriteIndex:c*2}:{},...d===r-1?{endOfSequenceWriteIndex:c*2+1}:{}}),this.host.device.queue.submit([p.finish()])}let o=this.host.device.createCommandEncoder({label:"gpu-time-sample-batch-resolve"});o.resolveQuerySet(u.querySet,0,i,u.resolveBuffer,0),o.copyBufferToBuffer(u.resolveBuffer,0,u.readbackBuffer,0,i*8),await this.host.submit([o.finish()],{wait:!1});let l;try{l=await this.readTimestampBuffer(u.readbackBuffer,i)}catch{return this.#r=!0,this.#l(),null}return Array.from({length:n},(c,d)=>Number(l[d*2+1]-l[d*2])/1e6)}finally{a()}}async#u(t,r,n,s){if(!Number.isInteger(r)||r<=0)return null;let a=t.findIndex(Qo),i=-1;for(let l=t.length-1;l>=0;--l)if(Qo(t[l])){i=l;break}if(a<0||i<0)return null;let u=[];for(let l=0;l<r;++l)for(let c=0;c<t.length;++c){let d=this.host.device.createCommandEncoder({label:"gpu-time-measure-boundary"}),p=l===0&&c===a,f=l===r-1&&c===i;bn(d,t[c],{querySet:n.querySet,...p?{beginningOfSequenceWriteIndex:0}:{},...f?{endOfSequenceWriteIndex:1}:{}}),l===r-1&&c===t.length-1&&(d.resolveQuerySet(n.querySet,0,2,n.resolveBuffer,0),d.copyBufferToBuffer(n.resolveBuffer,0,n.readbackBuffer,0,16)),u.push(d.finish())}for(let l=0;l<u.length;++l){let c=this.host.submit([u[l]],{wait:!0});if(l===u.length-1)try{s?.()}catch{}await c}let o;try{o=await this.readTimestampBuffer(n.readbackBuffer,2)}catch{return this.#r=!0,this.#l(),null}return Number(o[1]-o[0])/1e6}#l(){for(let t of this.#e.values())t.querySet.destroy(),t.resolveBuffer.destroy(),t.readbackBuffer.destroy();this.#e.clear()}#c(t){let r=this.createTimestampResources(t);return this.#e.set(t,r),r}pipelineCompileStats(){return{count:this.pipelines.compileCount,totalMs:this.pipelines.compileTotalMs}}memoryStats(){let t=this.host.memory;return t?{liveBytes:t.liveBytes,peakBytes:t.peakBytes}:null}memoryCensus(t=8){return this.host.memory?.census(t)??null}markMemoryPeak(){this.host.memory?.markPeak()}async _runSteps(t,{wait:r=!1,mergePass:n}){let s=!!this.profileSession;if(!s&&n){await this.runMergedSteps(t,r);return}let a=os(t),i=t.map((f,g)=>!Pt(f)||bt(f.dispatchWorkgroups)?-1:g).filter(f=>f>=0),u=4096,o=2,l=u/o,c=s?Array.from({length:Math.ceil(i.length/l)},(f,g)=>this.createTimestampResources(Math.min(l,i.length-g*l)*o)):[],d=0,p=[];for(let f=0;f<a.length;++f){let g=this.host.device.createCommandEncoder({label:s?"profiled-compute-dispatch":"compute-dispatch"});for(let m of a[f]){if(m.kind==="copy"){m.byteLength>0&&g.copyBufferToBuffer(m.src,m.srcOffset,m.dst,m.dstOffset,m.byteLength);continue}if(m.kind==="clear"){m.byteLength>0&&g.clearBuffer(m.dst,m.dstOffset,m.byteLength);continue}if(bt(m.dispatchWorkgroups))continue;let h=g.beginComputePass({label:s?`${m.name??"compute"}-profile-pass`:"compute-pass",...c.length>0?{timestampWrites:{querySet:c[Math.floor(d/l)].querySet,beginningOfPassWriteIndex:d%l*2,endOfPassWriteIndex:d%l*2+1}}:{}});h.setPipeline(m.pipeline),h.setBindGroup(0,m.bindGroup),m.immediateData&&oi(h,m.immediateData),h.dispatchWorkgroups(m.dispatchWorkgroups[0],m.dispatchWorkgroups[1],m.dispatchWorkgroups[2]),h.end(),d+=1}if(f===a.length-1)for(let m=0;m<c.length;++m){let h=Math.min(l,i.length-m*l)*o,v=c[m];g.resolveQuerySet(v.querySet,0,h,v.resolveBuffer,0),g.copyBufferToBuffer(v.resolveBuffer,0,v.readbackBuffer,0,h*8)}p.push(g.finish())}for(let f=0;f+1<p.length;++f)await this.host.submit([p[f]],{wait:!0});if(!s||c.length===0){await this.host.submit([p[p.length-1]],{wait:r||Fo(a[a.length-1])});return}try{await this.host.submit([p[p.length-1]],{wait:r});let f=(await Promise.all(c.map((g,m)=>this.readTimestampBuffer(g.readbackBuffer,Math.min(l,i.length-m*l)*o)))).flat();for(let g=0;g<i.length;++g){let m=t[i[g]];Pt(m)&&this.profileSession?.record(m,f[g*2],f[g*2+1])}}finally{for(let f of c)f.querySet.destroy(),f.resolveBuffer.destroy(),f.readbackBuffer.destroy()}}createTimestampResources(t){let r=t*8,n=this.host.device.createQuerySet({type:"timestamp",count:t}),s=this.host.createBuffer({label:"timestamp-resolve",size:r,usage:GPUBufferUsage.QUERY_RESOLVE|GPUBufferUsage.COPY_SRC}),a=this.host.createBuffer({label:"timestamp-readback",size:r,usage:GPUBufferUsage.COPY_DST|GPUBufferUsage.MAP_READ});return{querySet:n,resolveBuffer:s,readbackBuffer:a}}async readTimestampBuffer(t,r){let n=await this.host.mapRead(t,0,r*8),s=new BigUint64Array(n);return Array.from(s,a=>a)}};function Qo(e){return e.some(t=>Pt(t)&&!bt(t.dispatchWorkgroups))}async function np(e={}){let t=e.host??await Jf(e);return new li({host:t})}function sr(e){return e instanceof Error?e.message:String(e)}function ci(e){return e.byteOffset===0&&e.byteLength===e.buffer.byteLength&&e.buffer instanceof ArrayBuffer?e.buffer:e.slice().buffer}function ls(e,t){if(e.length===0)return new Uint8Array(0);if(e.length===1&&e[0].byteLength===t)return e[0];let r=new Uint8Array(t),n=0;for(let s of e)r.set(s,n),n+=s.byteLength;return r}function Qe(e){let t=e&32768?-1:1,r=e>>>10&31,n=e&1023;return r===31?n===0?t*(1/0):NaN:r===0?t*2**-14*(n/1024):t*2**(r-15)*(1+n/1024)}var jo=null;function sp(){if(!jo){let e=new Float32Array(65536);for(let t=0;t<e.length;++t)e[t]=Qe(t);jo=e}return jo}function Hr(e){let t=new Uint16Array(e.length);if(e instanceof Float32Array){let r=new Uint32Array(e.buffer,e.byteOffset,e.length);for(let n=0;n<e.length;++n){let s=r[n],a=(s>>>23&255)-112;if(a>0&&a<31){let i=s&8388607,u=i&8191,o=i>>>13;(u>4096||u===4096&&(o&1)===1)&&(o+=1,o===1024&&(o=0,a+=1)),t[n]=a>=31?s>>>16&32768|31743:s>>>16&32768|a<<10|o}else t[n]=ip(s)}}else for(let r=0;r<e.length;++r)t[r]=Wo(e[r]);return t}var ap=new Float32Array(1),u0=new Uint32Array(ap.buffer);function Wo(e){return Number.isNaN(e)?32256:(ap[0]=e,ip(u0[0]))}function ip(e){let t=e>>>16&32768,r=e>>>23&255,n=e&8388607;if(r===255)return n!==0?32256:t|31744;let s=r-127+15;if(s>=31)return t|31743;if(s<=0){if(s<-10)return t;let l=n|8388608,c=14-s,d=1<<c-1,p=l&(1<<c)-1,f=l>>>c;return(p>d||p===d&&(f&1)===1)&&(f+=1),t|f}let a=4096,i=n&8191,u=n>>>13,o=s;return(i>a||i===a&&(u&1)===1)&&(u+=1,u===1024&&(u=0,o+=1)),o>=31?t|31743:t|o<<10|u}var o0=new Set(["F32","BF16","F16"]),l0=new Set(["float32","float16"]);var c0=(()=>{let e=new Uint16Array([258]);return new Uint8Array(e.buffer)[0]===2})();function d0(e){return typeof e=="string"&&o0.has(e)}function f0(e){return typeof e=="string"&&l0.has(e)}function di(e){if(!d0(e))throw new Error(`Unsupported source dtype: ${e}`)}function p0(e){if(!f0(e))throw new Error(`Unsupported target dtype: ${e}`)}function cs(e,t){if(e==="F32")return g0(t);if(e==="BF16")return fp(t);if(e==="F16")return gp(t);throw new Error(`Unsupported source dtype for float32 copy: ${e}`)}function up(e,t){di(e),p0(t)}function op(e){if(e==="F32")return 4;if(e==="BF16"||e==="F16")return 2;throw new Error(`Unsupported source dtype: ${e}`)}function lp(e){if(e==="float32")return 4;if(e==="float16")return 2;throw new Error(`Unsupported target dtype: ${e}`)}function cp(e){return new Uint8Array(e.buffer,e.byteOffset,e.byteLength)}function g0(e){let t=e.byteLength/4;return e.byteOffset%4===0?new Float32Array(e.buffer,e.byteOffset,t):m0(e,Float32Array,4)}function dp(e,t){if(e==="BF16"&&t==="float32")return fp;if(e==="BF16"&&t==="float16")return w0;if(e==="F16"&&t==="float32")return gp;if(e==="F32"&&t==="float16")return _0;throw new Error(`No converter for ${e} -> ${t}`)}function m0(e,t,r){let n=new t(e.byteLength/r);return new Uint8Array(n.buffer).set(e),n}function fp(e){let t=pp(e),r=new Float32Array(t);return h0(e,r)}function h0(e,t,r=0){let n=pp(e);v0(t.length,r,n,"BF16 -> F32");let s=new Uint32Array(t.buffer,t.byteOffset,t.length);if(c0&&e.byteOffset%2===0){let a=new Uint16Array(e.buffer,e.byteOffset,n);for(let i=0;i<n;++i)s[r+i]=a[i]<<16}else for(let a=0;a<n;++a){let i=e[a*2],u=e[a*2+1];s[r+a]=(u<<8|i)<<16}return t}function pp(e){if(e.byteLength%2!==0)throw new Error(`BF16 byte length ${e.byteLength} is not divisible by 2`);return e.byteLength/2}function v0(e,t,r,n){if(!Number.isSafeInteger(t)||t<0)throw new Error(`${n} destination offset must be a non-negative safe integer, got ${t}`);if(t>e||r>e-t)throw new Error(`${n} destination range [${t}, ${t+r}) exceeds length ${e}`)}function w0(e){let t=e.byteLength/2,r=new Uint16Array(t);return b0(e,r),r}function b0(e,t,r=0,n=1){let s=e.byteLength/2,a=y0();if(e.byteOffset%2===0){let i=new Uint16Array(e.buffer,e.byteOffset,s);for(let u=0,o=r;u<s;++u,o+=n)t[o]=a[i[u]]}else for(let i=0,u=r;i<s;++i,u+=n){let o=e[i*2],l=e[i*2+1];t[u]=a[l<<8|o]}return t}function _0(e){let t=e.byteLength/4;if(e.byteOffset%4===0)return Hr(new Float32Array(e.buffer,e.byteOffset,t));let r=new Float32Array(t);return new Uint8Array(r.buffer).set(e),Hr(r)}function gp(e){let t=e.byteLength/2,r=new Float32Array(t),n=sp();if(e.byteOffset%2===0){let s=new Uint16Array(e.buffer,e.byteOffset,t);for(let a=0;a<t;++a)r[a]=n[s[a]]}else{let s=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let a=0;a<t;++a)r[a]=n[s.getUint16(a*2,!0)]}return r}var mp=new Uint32Array(1),k0=new Float32Array(mp.buffer);function A0(e){return mp[0]=e<<16>>>0,Wo(k0[0])}var zo=null;function y0(){if(zo)return zo;let e=new Uint16Array(65536);for(let t=0;t<65536;++t)e[t]=A0(t);return zo=e,e}var T=Object.freeze({F32:0,F16:1,Q4_0:2,Q4_1:3,Q5_0:6,Q5_1:7,Q8_0:8,Q8_1:9,Q2_K:10,Q3_K:11,Q4_K:12,Q5_K:13,Q6_K:14,Q8_K:15,IQ2_XXS:16,IQ2_XS:17,IQ3_XXS:18,IQ1_S:19,IQ4_NL:20,IQ3_S:21,IQ2_S:22,IQ4_XS:23,I8:24,I16:25,I32:26,I64:27,F64:28,IQ1_M:29,BF16:30,TQ1_0:34,TQ2_0:35,MXFP4:39,NVFP4:40,Q1_0:41,Q2_0:42,PRISM_PQ2_0:142,PRISM_PTQ1_0:143}),_r=Object.freeze(Object.fromEntries(Object.entries(T).map(([e,t])=>[t,e])));var ye=class extends Error{constructor(t){super(t),this.name="GGUFError"}};var Je=256,Ko=128,S0=18,x0=64,hp=18,kn=32,T0=18,C0=32,vp=20,q0=32,B0=22,I0=32,E0=24,$r=32,Ht=34,P0=32,R0=36,G0=292,Uo=Je,ds=210,No=Je,_n=144,Ho=Je,fs=176,D0=Je,L0=66,wp=54,O0=32,M0=17,F0=64,Q0=36,bp=50,_p=56;function kr(e,t){if(!Number.isInteger(t)||t<0)throw new ye(`Invalid tensor element count: ${t}`);if(e===T.F32)return t*4;if(e===T.F16||e===T.BF16)return t*2;if(e===T.I8)return t;if(e===T.I16)return t*2;if(e===T.I32)return t*4;if(e===T.I64||e===T.F64)return t*8;if(e===T.Q1_0){if(t%Ko!==0)throw new ye(`Q1_0 tensor element count must be divisible by ${Ko}`);return t/Ko*S0}if(e===T.Q4_0)return xe(e,t,kn,T0);if(e===T.PRISM_PQ2_0)return xe(e,t,128,34);if(e===T.PRISM_PTQ1_0)return xe(e,t,128,28);if(e===T.Q2_0)return xe(e,t,x0,hp);if(e===T.Q4_1)return xe(e,t,C0,vp);if(e===T.Q5_0)return xe(e,t,q0,B0);if(e===T.Q5_1)return xe(e,t,I0,E0);if(e===T.Q8_0)return xe(e,t,$r,Ht);if(e===T.Q8_1)return xe(e,t,P0,R0);if(e===T.Q2_K)return xe(e,t,Je,84);if(e===T.Q3_K)return xe(e,t,Je,110);if(e===T.Q4_K)return xe(e,t,No,_n);if(e===T.Q5_K)return xe(e,t,Ho,fs);if(e===T.Q6_K)return xe(e,t,Uo,ds);if(e===T.Q8_K)return xe(e,t,Je,G0);if(e===T.IQ4_NL)return xe(e,t,32,18);if(e===T.IQ4_XS)return xe(e,t,Je,136);if(e===T.IQ2_XXS)return xe(e,t,Je,66);if(e===T.IQ2_XS)return xe(e,t,Je,74);if(e===T.IQ2_S)return xe(e,t,Je,82);if(e===T.IQ3_XXS)return xe(e,t,Je,98);if(e===T.IQ3_S)return xe(e,t,Je,110);if(e===T.TQ2_0)return xe(e,t,D0,L0);if(e===T.TQ1_0)return xe(e,t,Je,wp);if(e===T.MXFP4)return xe(e,t,O0,M0);if(e===T.NVFP4)return xe(e,t,F0,Q0);if(e===T.IQ1_S)return xe(e,t,Je,bp);if(e===T.IQ1_M)return xe(e,t,Je,_p);throw new ye(`Unsupported GGML tensor type ${_r[e]??e}`)}function xe(e,t,r,n){if(t%r!==0)throw new ye(`${_r[e]??e} tensor element count ${t} must be divisible by ${r}`);return t/r*n}function kp(e){if(e.byteLength%Ht!==0)throw new ye(`Q8_0 byte length ${e.byteLength} is not a multiple of ${Ht}`);let t=e.byteLength/Ht,r=new Float32Array(t*$r),n=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let s=0;s<t;++s){let a=s*Ht,i=Qe(n.getUint16(a,!0)),u=s*$r;for(let o=0;o<32;++o)r[u+o]=n.getInt8(a+2+o)*i}return r}function Ap(e){if(e.byteLength%ds!==0)throw new ye(`Q6_K byte length ${e.byteLength} is not a multiple of ${ds}`);let t=e.byteLength/ds,r=new Float32Array(t*Uo),n=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let s=0;s<t;++s){let a=s*ds,i=Qe(n.getUint16(a+208,!0)),u=s*Uo;for(let o=0;o<2;++o){let l=a+o*64,c=a+128+o*32,d=a+192+o*8,p=u+o*128;for(let f=0;f<32;++f){let g=f>>4,m=e[c+f],h=(e[l+f]&15|(m>>0&3)<<4)-32,v=(e[l+f+32]&15|(m>>2&3)<<4)-32,b=(e[l+f]>>4|(m>>4&3)<<4)-32,_=(e[l+f+32]>>4|(m>>6&3)<<4)-32;r[p+f]=i*n.getInt8(d+g+0)*h,r[p+f+32]=i*n.getInt8(d+g+2)*v,r[p+f+64]=i*n.getInt8(d+g+4)*b,r[p+f+96]=i*n.getInt8(d+g+6)*_}}}return r}function ps(e,t,r){if(e<4)return[t[r+e]&63,t[r+e+4]&63];let n=t[r+e+4]&15|t[r+e-4]>>6<<4,s=t[r+e+4]>>4|t[r+e-0]>>6<<4;return[n,s]}function yp(e){if(e.byteLength%_n!==0)throw new ye(`Q4_K byte length ${e.byteLength} is not a multiple of ${_n}`);let t=e.byteLength/_n,r=new Float32Array(t*No),n=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let s=0;s<t;++s){let a=s*_n,i=Qe(n.getUint16(a,!0)),u=Qe(n.getUint16(a+2,!0)),o=a+4,l=a+16,c=s*No,d=0;for(let p=0;p<256;p+=64){let[f,g]=ps(d+0,e,o),m=i*f,h=u*g,[v,b]=ps(d+1,e,o),_=i*v,C=u*b;for(let y=0;y<32;++y)r[c++]=m*(e[l+y]&15)-h;for(let y=0;y<32;++y)r[c++]=_*(e[l+y]>>4)-C;l+=32,d+=2}}return r}function Sp(e){if(e.byteLength%fs!==0)throw new ye(`Q5_K byte length ${e.byteLength} is not a multiple of ${fs}`);let t=e.byteLength/fs,r=new Float32Array(t*Ho),n=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let s=0;s<t;++s){let a=s*fs,i=Qe(n.getUint16(a,!0)),u=Qe(n.getUint16(a+2,!0)),o=a+4,l=a+16,c=a+48,d=s*Ho,p=0,f=1,g=2;for(let m=0;m<256;m+=64){let[h,v]=ps(p+0,e,o),b=i*h,_=u*v,[C,y]=ps(p+1,e,o),E=i*C,k=u*y;for(let S=0;S<32;++S){let A=e[l+S]&f?16:0;r[d++]=b*((e[c+S]&15)+A)-_}for(let S=0;S<32;++S){let A=e[l+S]&g?16:0;r[d++]=E*((e[c+S]>>4)+A)-k}c+=32,p+=2,f<<=2,g<<=2}}return r}var ET=Int8Array.from([-127,-104,-83,-65,-49,-35,-22,-10,1,13,25,38,53,69,89,113]);var PT=Int8Array.from([0,1,2,3,4,6,8,12,0,-1,-2,-3,-4,-6,-8,-12]),RT=new DataView(new ArrayBuffer(4));var j0=new Set([T.Q2_0,T.TQ1_0,T.PRISM_PQ2_0,T.PRISM_PTQ1_0,T.IQ2_XXS,T.IQ2_XS,T.IQ2_S,T.IQ3_XXS,T.IQ3_S,T.TQ2_0,T.IQ1_S,T.IQ1_M]);function fi(e){return j0.has(e)}function xp(e,t){let r=new DataView(t.buffer,t.byteOffset,t.byteLength);switch(e){case T.PRISM_PQ2_0:case T.PRISM_PTQ1_0:{let n=e===T.PRISM_PTQ1_0;return s=>{let a=Qe(r.getUint16((s>>2)*(n?28:34)+(n?26:0),!0));return[a,a]}}case T.Q2_0:return n=>{let s=Qe(r.getUint16((n>>1)*hp,!0));return[s,s]};case T.TQ1_0:return n=>{let s=Qe(r.getUint16((n>>3)*wp+52,!0));return[s,s]};case T.IQ2_XXS:return n=>{let s=n>>3,a=n&7,i=s*66,u=Qe(r.getUint16(i,!0)),o=r.getUint32(i+2+8*a+4,!0)>>>0,l=u*(.5+(o>>>28))*.25;return[l,l]};case T.IQ2_XS:case T.IQ2_S:{let n=e===T.IQ2_XS?74:82,s=e===T.IQ2_XS?66:74;return a=>{let i=a>>3,u=a&7,o=i*n,l=Qe(r.getUint16(o,!0)),c=t[o+s+u];return[l*(.5+(c&15))*.25,l*(.5+(c>>4))*.25]}}case T.IQ3_XXS:return n=>{let s=n>>3,a=n&7,i=s*98,u=Qe(r.getUint16(i,!0)),o=r.getUint32(i+2+64+4*a,!0)>>>0,l=u*(.5+(o>>>28))*.5;return[l,l]};case T.IQ3_S:return n=>{let s=n>>3,a=n&7,i=s*110,u=Qe(r.getUint16(i,!0)),o=t[i+106+(a>>1)],l=(a&1)===0?u*(1+2*(o&15)):u*(1+2*(o>>4));return[l,l]};case T.TQ2_0:return n=>{let s=Qe(r.getUint16((n>>3)*66+64,!0));return[s,s]};case T.IQ1_S:return n=>{let s=n>>3,a=n&7,i=s*bp,u=Qe(r.getUint16(i,!0)),o=r.getUint16(i+34+2*a,!0),l=u*(2*(o>>12&7)+1)/8;return[l,l]};case T.IQ1_M:return n=>{let s=n>>3,a=n&7,u=s*_p+48,o=[r.getUint16(u,!0),r.getUint16(u+2,!0),r.getUint16(u+4,!0),r.getUint16(u+6,!0)],l=Qe((o[0]>>12|o[1]>>8&240|o[2]>>4&3840|o[3]&61440)&65535),c=l*(2*(o[a>>1]>>6*(a%2)+0&7)+1),d=l*(2*(o[a>>1]>>6*(a%2)+3&7)+1);return[c/8,d/8]};default:throw new ye(`iqBlockScales: type ${e} is not an exact-repack IQ type`)}}var Tp={8:kp,12:yp,13:Sp,14:Ap},$o=new Set([2,3,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,29,34,35,39,40,41,42,142,143]);var Bp=1179993927,W0=32,z0=new TextDecoder("utf-8",{fatal:!0}),Te=Object.freeze({UINT8:0,INT8:1,UINT16:2,INT16:3,UINT32:4,INT32:5,FLOAT32:6,BOOL:7,STRING:8,ARRAY:9,UINT64:10,INT64:11,FLOAT64:12});function ar(e,t,r,n){let s=e[t];if(typeof s=="number")return s;if(n!==void 0)return n;throw new Error(`${r} GGUF: missing numeric metadata "${t}"`)}function pi(e){return typeof e=="number"?e:void 0}function K0(e){if(typeof e=="number"||Array.isArray(e)&&e.every(t=>typeof t=="number"))return e}function gi(e){let t=[];for(let s of["tokenizer.ggml.eos_token_id","tokenizer.ggml.eot_token_id"]){let a=K0(e[s]);for(let i of typeof a=="number"?[a]:a??[])t.includes(i)||t.push(i)}let r=e["tokenizer.ggml.tokens"],n=Array.isArray(r)?r.indexOf("<|im_end|>"):-1;return n>=0&&!t.includes(n)&&t.push(n),t}var Cp={[Te.UINT8]:1,[Te.INT8]:1,[Te.BOOL]:1,[Te.UINT16]:2,[Te.INT16]:2,[Te.UINT32]:4,[Te.INT32]:4,[Te.FLOAT32]:4,[Te.UINT64]:8,[Te.INT64]:8,[Te.FLOAT64]:8};function Ip(e){let r=new DataView(e.buffer,e.byteOffset,e.byteLength),n=e.byteLength,s=0,a=c=>s+c<=n,i=()=>{let c=r.getUint32(s+4,!0)*4294967296+r.getUint32(s,!0);return s+=8,c};if(!a(24))return{need:262168};if(r.getUint32(0,!0)!==Bp)return{end:0};s=8;let u=i(),o=i(),l=()=>{if(!a(8))return!1;let c=i();return a(c)?(s+=c,!0):!1};for(let c=0;c<o;c++){if(!l()||!a(4))return{need:s+64+262144};let d=r.getUint32(s,!0);if(s+=4,d===Te.STRING){if(!l())return{need:s+64+262144}}else if(d===Te.ARRAY){if(!a(12))return{need:s+12+262144};let p=r.getUint32(s,!0);s+=4;let f=i(),g=Cp[p];if(g!==void 0){if(!a(f*g))return{need:s+f*g+262144};s+=f*g}else if(p===Te.STRING){let m=s;for(let h=0;h<f;h++)if(!l()){let v=h>0?(s-m)/h:16;return{need:Math.ceil(s+(f-h)*v*1.25)+262144}}}else return{end:s}}else{let p=Cp[d];if(p===void 0)return{end:s};if(!a(p))return{need:s+p+262144};s+=p}}for(let c=0;c<u;c++){if(!l()||!a(4))return{need:s+(u-c)*96+262144};let d=r.getUint32(s,!0);if(s+=4,!a(d*8+12))return{need:s+(u-c)*96+262144};s+=d*8+12}return{end:s}}function Ep(e){if(e.has("token_embd.weight")){let r=e.info("token_embd.weight").shape[0];if(Number.isInteger(r)&&r>0)return r}let t=e.metadata["tokenizer.ggml.tokens"];if(Array.isArray(t)&&t.length>0)return t.length}var $t=class{#e;#t;version;tensorCount;metadataCount;metadata;alignment;dataStart;constructor(t,r={}){this.#e=H0(t);let n=new Vo(this.#e);if(n.u32()!==Bp)throw new ye("Invalid GGUF magic");this.version=n.u32(),this.tensorCount=gs(n.u64(),"tensor count"),this.metadataCount=gs(n.u64(),"metadata count"),this.metadata={};for(let u=0;u<this.metadataCount;++u){let o=n.string();this.metadata[o]=n.metadataValue()}let a=new Map;for(let u=0;u<this.tensorCount;++u){let o=n.string(),l=Array.from({length:n.u32()},()=>gs(n.u64(),`${o} dimension`)),c=n.u32(),d=gs(n.u64(),`${o} offset`),p=l.reduce((f,g)=>f*g,1);a.set(o,{name:o,dims:l,shape:[...l].reverse(),type:c,typeName:_r[c]??`UNKNOWN_${c}`,offset:d,elementCount:p,byteLength:kr(c,p),dataOffsets:[0,0]})}let i=this.metadata["general.alignment"]??W0;if(typeof i!="number"||!Number.isInteger(i)||i<=0)throw new ye(`Invalid general.alignment: ${i}`);if(this.alignment=i,this.dataStart=Ze(n.offset,i),!r.headerOnly){if(this.dataStart>this.#e.byteLength)throw new ye("GGUF header exceeds file size");for(let u of a.values()){let o=this.dataStart+u.offset,l=o+u.byteLength;if(l>this.#e.byteLength)throw new ye(`Tensor ${u.name} exceeds file size`);u.dataOffsets=[o,l]}}this.#t=a}names(){return[...this.#t.keys()]}has(t){return this.#t.has(t)}info(t){return N0(this.#r(t))}tensorBytes(t){let r=this.#r(t),[n,s]=r.dataOffsets;return this.#e.subarray(n,s)}tensorByteRange(t,r,n){let[s]=this.#r(t).dataOffsets;return this.#e.subarray(s+r,s+n)}async streamTensorRange(t,r,n,s,a={}){let i=this.#r(t);Jo(i,r,n),qp(a.signal);let[u]=i.dataOffsets;await s(this.#e.subarray(u+r,u+n)),qp(a.signal)}async streamTensors(t,r){for(let n of t)await r(n,this.tensorBytes(n))}#r(t){let r=this.#t.get(t);if(!r)throw new ye(`Unknown tensor: ${t}`);return r}};function Pp(e,t){if(!fi(e))throw new ye(`repackIQToExactInt8: type ${e} is not an exact-repack IQ type`);let r=Vr(e,t),n=r.length/32,s=xp(e,t),a=new Uint32Array(n*8),i=new Float32Array(n*2);for(let u=0;u<n;++u){let[o,l]=s(u);i[u*2]=o,i[u*2+1]=l;for(let c=0;c<8;++c){let d=c<4?o:l,p=0;for(let f=0;f<4;++f){let g=u*32+c*4+f,m=d!==0?Math.round(r[g]/d):0;m>127?m=127:m<-127&&(m=-127),p|=(m&255)<<f*8}a[u*8+c]=p>>>0}}return{bits:a,scales:i}}function Vr(e,t){let r=Tp[e];if(r)return r(t);if($o.has(e))throw new ye(`CPU dequantizer for ${_r[e]??e} is not in this bundle (built for the shipped model's weight formats \u2014 load that model's own GGUF, or rebuild)`);return Rp(e)?U0(e,t):null}function Rp(e){return e===T.I8||e===T.I16||e===T.I32||e===T.I64||e===T.F64}function U0(e,t){let r=e===T.I8?1:e===T.I16?2:e===T.I32?4:8;if(t.byteLength%r!==0)throw new ye(`${_r[e]} byte length ${t.byteLength} is not divisible by ${r}`);let n=new Float32Array(t.byteLength/r),s=new DataView(t.buffer,t.byteOffset,t.byteLength);for(let a=0;a<n.length;++a){let i=a*r;e===T.I8?n[a]=s.getInt8(i):e===T.I16?n[a]=s.getInt16(i,!0):e===T.I32?n[a]=s.getInt32(i,!0):e===T.I64?n[a]=Number(s.getBigInt64(i,!0)):n[a]=s.getFloat64(i,!0)}return n}function mi(e){return $o.has(e)}function Zo(e){return mi(e)||e===T.F32||e===T.F16||e===T.BF16||e===T.I8||e===T.I16||e===T.I32||e===T.I64||e===T.F64}function Gp(e){if(e===T.F32)return"F32";if(e===T.F16)return"F16";if(e===T.BF16)return"BF16";throw new ye(`Unsupported GGUF scalar type ${_r[e]??e}; quantized types dequantize separately`)}function Xo(e){return mi(e)||Rp(e)?"F32":Gp(e)}function Yo(e,t){return Vr(e,t)??cs(Gp(e),t)}function N0(e){return{name:e.name,dims:[...e.dims],shape:[...e.shape],type:e.type,typeName:e.typeName,offset:e.offset,elementCount:e.elementCount,byteLength:e.byteLength,dataOffsets:[...e.dataOffsets]}}function Jo(e,t,r){if(!Number.isSafeInteger(t)||!Number.isSafeInteger(r)||t<0||r<t)throw new RangeError(`Invalid byte range [${t}, ${r}) for tensor ${e.name}`);if(r>e.byteLength)throw new RangeError(`Byte range [${t}, ${r}) exceeds tensor ${e.name} size ${e.byteLength}`)}function qp(e){if(e?.aborted)throw e.reason??new Error("aborted")}var Vo=class{bytes;view;offset;constructor(t){this.bytes=t,this.view=new DataView(t.buffer,t.byteOffset,t.byteLength),this.offset=0}u8(){this.#t(1);let t=this.view.getUint8(this.offset);return this.offset+=1,t}i8(){this.#t(1);let t=this.view.getInt8(this.offset);return this.offset+=1,t}u16(){this.#t(2);let t=this.view.getUint16(this.offset,!0);return this.offset+=2,t}i16(){this.#t(2);let t=this.view.getInt16(this.offset,!0);return this.offset+=2,t}u32(){this.#t(4);let t=this.view.getUint32(this.offset,!0);return this.offset+=4,t}i32(){this.#t(4);let t=this.view.getInt32(this.offset,!0);return this.offset+=4,t}u64(){this.#t(8);let t=this.view.getBigUint64(this.offset,!0);return this.offset+=8,t}i64(){this.#t(8);let t=this.view.getBigInt64(this.offset,!0);return this.offset+=8,t}f32(){this.#t(4);let t=this.view.getFloat32(this.offset,!0);return this.offset+=4,t}f64(){this.#t(8);let t=this.view.getFloat64(this.offset,!0);return this.offset+=8,t}length(t){this.#t(8);let r=this.view.getUint32(this.offset+4,!0);if(r>2097151)throw new ye(`${t} ${this.view.getBigUint64(this.offset,!0)} exceeds JavaScript safe integer range`);let n=r*2**32+this.view.getUint32(this.offset,!0);return this.offset+=8,n}string(){let t=this.length("string length");this.#t(t);let r=this.bytes.subarray(this.offset,this.offset+t);return this.offset+=t,z0.decode(r)}metadataValue(){let t=this.u32();return this.#e(t)}#e(t){switch(t){case Te.UINT8:return this.u8();case Te.INT8:return this.i8();case Te.UINT16:return this.u16();case Te.INT16:return this.i16();case Te.UINT32:return this.u32();case Te.INT32:return this.i32();case Te.FLOAT32:return this.f32();case Te.BOOL:return!!this.u8();case Te.STRING:return this.string();case Te.UINT64:return gs(this.u64(),"metadata uint64");case Te.INT64:return $0(this.i64(),"metadata int64");case Te.FLOAT64:return this.f64();case Te.ARRAY:{let r=this.u32();if(r===Te.ARRAY)throw new ye("Nested metadata arrays are not supported");let n=this.length("metadata array length"),s=new Array(n);for(let a=0;a<n;a++)s[a]=this.#e(r);return s}default:throw new ye(`Unsupported GGUF metadata type ${t}`)}}#t(t){if(this.offset+t>this.bytes.byteLength)throw new ye("Unexpected end of GGUF file")}};function H0(e){if(e instanceof Uint8Array)return e;if(ArrayBuffer.isView(e))return new Uint8Array(e.buffer,e.byteOffset,e.byteLength);if(e instanceof ArrayBuffer)return new Uint8Array(e);throw new ye("GGUF input must be an ArrayBuffer or Uint8Array")}function gs(e,t){if(e>BigInt(Number.MAX_SAFE_INTEGER))throw new ye(`${t} ${e} exceeds JavaScript safe integer range`);return Number(e)}function $0(e,t){if(e>BigInt(Number.MAX_SAFE_INTEGER)||e<BigInt(Number.MIN_SAFE_INTEGER))throw new ye(`${t} ${e} exceeds JavaScript safe integer range`);return Number(e)}var Rt=class extends Error{constructor(t){super(t),this.name="DirectByteReadUnavailableError"}};var V0=1e4,Dp=3,nl=class extends Error{};async function Lp(e,t,r,n,s){for(let a=1;;a++){ir(e);let i=new AbortController,u=new nl(`${n} delivered nothing for ${r} ms`),o,l=()=>{clearTimeout(o),o=setTimeout(()=>i.abort(u),r)},c=()=>i.abort(e?.reason);e?.addEventListener("abort",c,{once:!0});let d=0;l();try{return await s(i.signal,t&&(p=>{d+=p,t(p)}),l)}catch(p){if(i.signal.reason!==u||e?.aborted)throw p;if(a>=Dp)throw new Error(`${n} stalled ${Dp} times`,{cause:u});d!==0&&t?.(-d)}finally{clearTimeout(o),e?.removeEventListener("abort",c)}}}function Z0(e,t){let r=e.url||t;try{let n=new URL(r);return n.hash="",n.search="",n.toString()}catch{return r}}function Op(e){try{let t=new URL(e);if(t.hostname!=="huggingface.co")return;let r=t.pathname.split("/").filter(Boolean),n=r.indexOf("resolve");if(n>=0&&/^[0-9a-f]{40,64}$/i.test(r[n+1]??""))return r[n+1].toLowerCase();let s=r.indexOf("resolve-cache");return s>=0?r.slice(s+1).find(a=>/^[0-9a-f]{40,64}$/i.test(a))?.toLowerCase():void 0}catch{return}}function el(e,t){let r=Z0(e,t),n=e.headers.get("etag")?.trim();if(n&&!/^W\//i.test(n))return`http|${r}|etag=${n}`;let s=Op(r)??Op(t);return s?`http|${r}|revision=${s}`:void 0}async function jp(e,t,r){let n=null;try{n=await t(e,{method:"HEAD",signal:r})}catch{}if(n?.ok){let a=n.headers.get("content-length"),i=a===null?null:Number(a);if(a!==null&&!Number.isFinite(i))throw new Error(`Invalid content-length header: ${a}`);return{size:i,acceptsRanges:(n.headers.get("accept-ranges")??"").toLowerCase().includes("bytes"),resourceIdentity:el(n,e)}}let s=await t(e,{headers:{Range:"bytes=0-0"},signal:r});if(s.status===206){await s.arrayBuffer();let a=Number(s.headers.get("content-range")?.split("/").pop());return{size:Number.isFinite(a)?a:null,acceptsRanges:!0,resourceIdentity:el(s,e)}}if(s.ok){await s.body?.cancel().catch(()=>{});let a=s.headers.get("content-length"),i=a===null?null:Number(a);return{size:Number.isFinite(i??NaN)?i:null,acceptsRanges:!1,resourceIdentity:el(s,e)}}throw new Error(`HEAD ${e} failed${n?` (${n.status} ${n.statusText})`:""} and the range probe returned ${s.status}`)}async function wi(e,t={}){return(await sl(e,t))?.size??null}async function sl(e,t={}){let r=t.fetch??((a,i)=>fetch(a,i)),n=AbortSignal.timeout(t.timeoutMs??4e3),s=t.signal?AbortSignal.any([t.signal,n]):n;try{let{size:a,resourceIdentity:i}=await jp(e,r,s);return{size:a,resourceIdentity:i}}catch{return null}}async function al(e,t,{requireRangeRequests:r=!1,knownSize:n=void 0,knownAcceptsRanges:s=void 0,knownResourceIdentity:a=void 0}={}){let i=t.fetch??globalThis.fetch;if(typeof i!="function")throw new Error("No fetch implementation available; pass options.fetch");let u=n,o=s,l=a;if(u===void 0){let h=await jp(e,i,t.signal);u=h.size,o=h.acceptsRanges,l=h.resourceIdentity}else o=o!==!1;if(r&&!o)throw new Error(`Range requests are required for ${e}, but the server did not advertise Accept-Ranges: bytes`);let c=t.stallTimeoutMs??V0,d=e,p=async(h,v,b)=>{let _={headers:{Range:`bytes=${h}-${v-1}`},signal:b};if(d!==e){let y=await i(d,_);if(y.status!==401&&y.status!==403&&y.status!==410)return y;await hi(y),d=e}let C=await i(e,_);return C.status===206&&C.redirected&&C.url&&C.url!==e&&(d=C.url),C},f=(h,v)=>`Range ${h}-${v-1} of ${e}`,g=!0,m={url:e,size:u??null,acceptsRanges:o,resourceIdentity:l,writeTensorAcceptsEphemeralBytes:!0,async readRange(h,v,b={}){if(h===v)return new Uint8Array(0);Wp(h,v);let _=b.signal??t.signal;return Lp(_,b.onByteProgress??null,c,f(h,v),async(C,y,E)=>{if(o){let A=await p(h,v,C);if(A.status!==206&&A.status!==200)throw new Error(`${f(h,v)} failed: ${A.status}`);if(A.status===200&&r)throw new Error(`${f(h,v)} returned 200 instead of 206; refusing full-response fallback`);let w=await tl(A,{expectedLength:A.status===206?v-h:null,onByteProgress:A.status===200?Fp(h,v,y):y,onChunk:E});if(A.status===200)return w.subarray(h,v);if(w.byteLength!==v-h)throw new Error(`Range ${h}-${v-1} returned ${w.byteLength} bytes`);return w}let k=await i(e,{signal:C});if(!k.ok)throw new Error(`GET ${e} failed: ${k.status}`);return(await tl(k,{onByteProgress:Fp(h,v,y),onChunk:E})).subarray(h,v)})},async readRangeInto(h,v,b,_={}){if(Y0(h,v,b),b.byteLength===0)return;let C=_.signal??t.signal;if(ir(C),!g)throw new Rt("Direct response-body reads were disabled after a non-streaming response");try{await Lp(C,_.onByteProgress??null,c,f(h,v),async(y,E,k)=>{if(o){let A=await p(h,v,y);if(A.status!==206&&A.status!==200)throw new Error(`${f(h,v)} failed: ${A.status}`);if(A.status===200){if(r)throw await hi(A),new Error(`${f(h,v)} returned 200 instead of 206; refusing full-response fallback`);await Mp(A,h,v,b,{signal:y,onByteProgress:E,onChunk:k});return}await X0(A,b,{signal:y,onByteProgress:E,onChunk:k});return}let S=await i(e,{signal:y});if(!S.ok)throw new Error(`GET ${e} failed: ${S.status}`);await Mp(S,h,v,b,{signal:y,onByteProgress:E,onChunk:k})})}catch(y){throw y instanceof Rt&&(g=!1,delete m.readRangeInto),y}},async readAll(h={}){let v=h.signal??t.signal,b=h.onProgress??t.onProgress,_=await i(e,{signal:v});if(!_.ok)throw new Error(`GET ${e} failed: ${_.status}`);return tl(_,{expectedLength:Number.isFinite(u)?u:null,onProgress:b,progressTotal:Number.isFinite(u)?u:null,fromCache:!1})},async readTensor(){return null},async writeTensor(){},async close(){}};return m}async function tl(e,{expectedLength:t=null,onByteProgress:r=null,onProgress:n=null,progressTotal:s=null,fromCache:a=!1,yieldEveryBytes:i=null,onChunk:u=null}={}){if(!e.body?.getReader){let f=await e.arrayBuffer(),g=new Uint8Array(f);return rl({onByteProgress:r,onProgress:n,loaded:g.byteLength,delta:g.byteLength,total:s??t,fromCache:a}),g}let o=e.body.getReader(),l=s??t??null,c=0;if(typeof t=="number"&&Number.isFinite(t)&&t>0){let f=new Uint8Array(t),g=[],m=0;for(;;){let{done:h,value:v}=await o.read();if(h)break;v&&(u?.(),c+v.byteLength<=f.byteLength&&g.length===0?f.set(v,c):(g.length===0&&g.push(f.subarray(0,c)),g.push(v)),c+=v.byteLength,rl({onByteProgress:r,onProgress:n,loaded:c,delta:v.byteLength,total:l,fromCache:a}),i&&(m+=v.byteLength)>=i&&(m=0,await vi()))}return g.length===0?c===t?f:f.subarray(0,c):ls(g,c)}let d=[],p=0;for(;;){let{done:f,value:g}=await o.read();if(f)break;g&&(u?.(),d.push(g),c+=g.byteLength,rl({onByteProgress:r,onProgress:n,loaded:c,delta:g.byteLength,total:l,fromCache:a}),i&&(p+=g.byteLength)>=i&&(p=0,await vi()))}return ls(d,c)}async function X0(e,t,{signal:r,onByteProgress:n=null,onChunk:s=null}={}){if(ir(r),!e.body?.getReader)throw await hi(e),new Rt("Direct response-body reads require a streaming response body");let a=e.body.getReader(),i=0;try{for(;;){ir(r);let{done:u,value:o}=await a.read();if(ir(r),u)break;if(o){if(s?.(),i+o.byteLength>t.byteLength)throw Qp(t.byteLength,i+o.byteLength);t.set(o,i),i+=o.byteLength,n?.(o.byteLength),ir(r)}}}catch(u){throw await a.cancel(u).catch(()=>{}),u}if(i!==t.byteLength)throw Qp(t.byteLength,i)}async function Mp(e,t,r,n,{signal:s,onByteProgress:a=null,onChunk:i=null}={}){if(ir(s),!e.body?.getReader)throw await hi(e),new Rt("Direct response-range reads require a streaming response body");let u=e.body.getReader(),o=0,l=0;try{for(;;){ir(s);let{done:c,value:d}=await u.read();if(ir(s),c)break;if(!d)continue;i?.();let p=o,f=o+d.byteLength,g=Math.max(t,p),m=Math.min(r,f);if(m>g){let h=g-p,v=m-g;n.set(d.subarray(h,h+v),g-t),l+=v,a?.(v)}o=f}}catch(c){throw await u.cancel(c).catch(()=>{}),c}if(l!==n.byteLength)throw new Error(`Full response has ${o} bytes; range [${t}, ${r}) is unavailable`)}function Fp(e,t,r){if(!r)return null;let n=0;return s=>{let a=n,i=n+s;n=i;let u=Math.max(0,Math.min(t,i)-Math.max(e,a));u>0&&r(u)}}async function hi(e){try{await e.body?.cancel()}catch{}}function vi(){return new Promise(e=>setTimeout(e,0))}function rl({onByteProgress:e,onProgress:t,loaded:r,delta:n,total:s,fromCache:a}){e?.(n),t?.({loaded:r,total:s??null,fromCache:a})}function Y0(e,t,r){Wp(e,t);let n=t-e;if(r.byteLength!==n)throw new RangeError(`Destination has ${r.byteLength} bytes; expected ${n} for range [${e}, ${t})`)}function Wp(e,t){if(!Number.isSafeInteger(e)||!Number.isSafeInteger(t)||e<0||t<e)throw new RangeError(`Invalid byte range [${e}, ${t})`)}function Qp(e,t){return new Error(`Response returned ${t} bytes; expected ${e}`)}function ir(e){if(e?.aborted)throw e.reason??new Error("aborted")}async function An(e,t={},r={}){if(t.source)return t.source;if(typeof e=="string"&&/^https?:/i.test(e))return al(e,t,r);if(typeof e=="string"&&e.startsWith("file:"))return ol(new URL(e),t);if(e instanceof URL){if(e.protocol==="http:"||e.protocol==="https:")return al(e.toString(),t,r);if(e.protocol==="file:")return ol(e,t);throw new Error(`Unsupported URL protocol: ${e.protocol}`)}if(typeof e=="string")return ol(e,t);throw new Error("url must be a string or URL")}var d_=32<<20,f_=4,Up=65536;async function ol(e,t={}){let{open:r,stat:n}=await Promise.resolve().then(()=>(ul(),il)),s=e instanceof URL?(await Promise.resolve().then(()=>(Kp(),zp))).fileURLToPath(e):e,a=await n(s),i=await r(s,"r"),u=async(o,l,c,d={})=>{p_(o,l,c,a.size);let p=d.signal??t.signal,f=d.onProgress??t.onProgress,g=c.byteLength,m=0,h=async(y,E)=>{let k=0;for(;k<E;){Np(p);let{bytesRead:S}=await i.read(c,y+k,E-k,o+y+k);if(S===0)throw new Error(`Short read at ${o}: expected ${g} bytes, got ${y+k}`);k+=S,m+=S,d.onByteProgress?.(S),f?.({loaded:m,total:g,fromCache:!1,range:[o,l],inFlight:m<g})}},v=g>=d_?f_:1,b=Math.ceil(g/v/Up)*Up,_=[];for(let y=0;y<g;y+=b)_.push(h(y,Math.min(b,g-y)));let C=(await Promise.allSettled(_)).find(y=>y.status==="rejected");if(C)throw C.reason;Np(p)};return{url:e.toString(),size:a.size,acceptsRanges:!0,writeTensorAcceptsEphemeralBytes:!0,async readRange(o,l,c={}){if(o===l)return new Uint8Array(0);let d=l-o,p=new Uint8Array(d);return await u(o,l,p,c),p},readRangeInto:u,async readAll(o={}){let l=o.onProgress??t.onProgress,c=new Uint8Array(a.size),d=0,p=1<<22;for(;d<a.size;){let f=Math.min(p,a.size-d),{bytesRead:g}=await i.read(c,d,f,d);if(g===0)break;d+=g,l?.({loaded:d,total:a.size,fromCache:!1})}if(d!==a.size)throw new Error(`Short read on ${s}: read ${d}/${a.size} bytes`);return c},async readTensor(){return null},async writeTensor(){},async close(){await i.close()}}}function p_(e,t,r,n){if(!Number.isSafeInteger(e)||!Number.isSafeInteger(t)||e<0||t<e)throw new RangeError(`Invalid byte range [${e}, ${t})`);if(t>n)throw new RangeError(`Byte range [${e}, ${t}) exceeds source size ${n}`);let s=t-e;if(r.byteLength!==s)throw new RangeError(`Destination has ${r.byteLength} bytes; expected ${s} for range [${e}, ${t})`)}function Np(e){if(e?.aborted)throw e.reason??new Error("aborted")}async function Hp(e,t,r=""){let n=e.cacheStorage??globalThis.caches;return n?.open?n.open((e.cacheName??t)+r):null}var g_="gguf-v1";async function ll(e,t={}){let r=m_(e),n=await v_(r,t);if(n)return n;let s=await An(e,{fetch:t.fetch,signal:t.signal});try{let{gguf:a,bytes:i}=await cl(s,t.signal);return await w_(r,i,t,s.size??null),a}finally{await s.close()}}async function cl(e,t){if(!e.acceptsRanges||!e.readRange){if(!e.readAll)throw new Error(`GGUF source ${e.url} supports neither ranges nor readAll()`);let i=await e.readAll({signal:t});return{gguf:new $t(i,{headerOnly:!0}),bytes:i}}let r=64*1024*1024,n=e.size,s=n!=null?Math.min(n,r):r,a=await e.readRange(0,Math.min(4*1024*1024,s),{signal:t});for(;;){let i=Ip(a);if("end"in i||a.byteLength>=s)break;let u=Math.min(s,Math.max(i.need,Math.ceil(a.byteLength*1.25),a.byteLength+1024*1024)),o=await e.readRange(a.byteLength,u,{signal:t});if(o.byteLength===0)break;a=ls([a,o],a.byteLength+o.byteLength)}return{gguf:new $t(a,{headerOnly:!0}),bytes:a}}function m_(e){return e instanceof URL?e.toString():String(e)}var h_="-headers";async function $p(e){return Hp(e,g_,h_)}async function v_(e,t){if(t.force||t.cache===!1)return null;let r=await $p(t);if(!r)return null;let n=await r.match(e);if(!n)return null;let s=Number(n.headers.get("x-gguf-source-size")),a=Number.isFinite(s)?await wi(e,{fetch:t.fetch,signal:t.signal}):void 0;if(a===void 0||a!==null&&a!==s)return await r.delete(e).catch(()=>{}),null;try{return new $t(new Uint8Array(await n.arrayBuffer()),{headerOnly:!0})}catch{return null}}async function w_(e,t,r,n){if(r.cache===!1||t.byteLength>32*1024*1024)return;let s=await $p(r);if(!s)return;let a=new Response(ci(t),{headers:{"content-type":"application/octet-stream","content-length":String(t.byteLength),...n!==null&&Number.isFinite(n)?{"x-gguf-source-size":String(n)}:{}}});try{await s.put(e,a)}catch(i){typeof console<"u"&&console.warn(`gguf header cache write failed: ${sr(i)}`)}}var Vt="chunks",Ar="meta";function bi(e,{cache:t,cacheKey:r=e.url,force:n=!1}){let s={...e,writeTensorAcceptsEphemeralBytes:t.acceptsEphemeralBytes===!0?!0:void 0,retainsWrittenTensors:!0,async readTensor(a,i){if(n)return null;try{return await t.get(r,a,i)}catch(u){return typeof console<"u"&&console.warn(`byte-range cache read failed: ${sr(u)}`),null}},async writeTensor(a,i,u){try{let o=u.byteOffset===0&&u.byteLength===u.buffer.byteLength?u:new Uint8Array(u);await t.put(r,a,i,o)}catch(o){typeof console<"u"&&console.warn(`byte-range cache write failed: ${sr(o)}`)}}};return e.readRangeInto?s.readRangeInto=async(a,i,u,o)=>{let l=e.readRangeInto;if(!l)throw delete s.readRangeInto,new Rt("The wrapped source disabled direct response-body reads");try{await l.call(e,a,i,u,o)}finally{e.readRangeInto||delete s.readRangeInto}}:delete s.readRangeInto,s}async function b_(e,t,r,n,s={}){let a=typeof r=="string"?r:r.toString();if(!/^https?:/i.test(a))return!1;let i=await wi(a,s);if(i===null||i===n)return!1;try{await e.delete?.(t)}catch(u){typeof console<"u"&&console.warn(`stale byte-range cache purge failed: ${sr(u)}`)}return!0}var Vp=new Map;function __(e,t,r){let n=0,s=e.length-1,a=-1;for(;n<=s;){let i=n+s>>1;e[i][0]<=t?(a=i,n=i+1):s=i-1}return a>=0&&e[a][1]>=r?e[a]:null}function ws(e){if(typeof indexedDB>"u")return null;let t=new Map,r=null,n=new Set,s=async(i,u,o,l)=>new Promise((c,d)=>{let p=i.transaction(Vt,"readonly").objectStore(Vt).get([u,o,l]);p.onsuccess=()=>{let f=p.result;if(!f){c(null);return}f.arrayBuffer().then(g=>c(new Uint8Array(g)),d)},p.onerror=()=>d(p.error)}),a=(i,u)=>{let o=t.get(u);return o||(o=new Promise((l,c)=>{let d=i.transaction(Vt,"readonly").objectStore(Vt).getAllKeys(IDBKeyRange.bound([u],[u,[],[]]));d.onsuccess=()=>l(d.result.map(([,p,f])=>[p,f]).sort((p,f)=>p[0]-f[0])),d.onerror=()=>c(d.error)}),t.set(u,o)),o};return{acceptsEphemeralBytes:!0,async get(i,u,o){let l=await vs(e),c=await s(l,i,u,o);if(c)return c;if(r&&r.key===i&&r.begin<=u&&o<=r.end)return r.bytes.subarray(u-r.begin,o-r.begin);let d=__(await a(l,i),u,o);if(!d)return null;let p=`${i}:${d[0]}:${d[1]}`;if(n.has(p))return null;let f;try{f=await s(l,i,d[0],d[1])}catch{return n.add(p),null}return f?(r={key:i,begin:d[0],end:d[1],bytes:f},f.subarray(u-d[0],o-d[0])):null},async put(i,u,o,l){let c=await vs(e);return t.delete(i),new Promise((d,p)=>{let f=c.transaction(Vt,"readwrite").objectStore(Vt).put(new Blob([ci(l)]),[i,u,o]);f.onsuccess=()=>d(),f.onerror=()=>p(f.error)})},async getMeta(i){let u=await vs(e);return new Promise((o,l)=>{let c=u.transaction(Ar,"readonly").objectStore(Ar).get(i);c.onsuccess=()=>o(c.result??null),c.onerror=()=>l(c.error)})},async putMeta(i,u){let o=await vs(e);return new Promise((l,c)=>{let d=o.transaction(Ar,"readwrite").objectStore(Ar).put(u,i);d.onsuccess=()=>l(),d.onerror=()=>c(d.error)})},async delete(i){let u=await vs(e);t.delete(i),r?.key===i&&(r=null);let o=IDBKeyRange.bound([i],[i,[],[]]);return await new Promise((l,c)=>{let d=u.transaction(Vt,"readwrite").objectStore(Vt).delete(o);d.onsuccess=()=>l(),d.onerror=()=>c(d.error)}),new Promise((l,c)=>{let d=u.transaction(Ar,"readwrite").objectStore(Ar).delete(i);d.onsuccess=()=>l(),d.onerror=()=>c(d.error)})}}}function vs(e){let t=Vp.get(e);if(t)return t;let r=new Promise((n,s)=>{let a=indexedDB.open(e,2);a.onupgradeneeded=()=>{let i=a.result;i.objectStoreNames.contains(Vt)||i.createObjectStore(Vt),i.objectStoreNames.contains(Ar)||i.createObjectStore(Ar)},a.onsuccess=()=>n(a.result),a.onerror=()=>s(a.error),a.onblocked=()=>s(new Error("indexedDB open blocked"))});return Vp.set(e,r),r}async function dl(e,t,r,n,s={}){try{let a=n(await e.getMeta(t));if(!a)return null;if(!s.verifyResourceIdentity)return await b_(e,t,r,a.size,s)?null:a;let i=typeof r=="string"?r:r.toString();if(!/^https?:/i.test(i))return{...a,resourceIdentity:void 0};let u=await sl(i,s);if(u&&(u.size!==null&&u.size!==a.size||u.resourceIdentity!==void 0&&u.resourceIdentity!==a.resourceIdentity)){try{await e.delete?.(t)}catch(o){typeof console<"u"&&console.warn(`stale byte-range cache purge failed: ${sr(o)}`)}return null}return u?.resourceIdentity===a.resourceIdentity?a:{...a,resourceIdentity:void 0}}catch(a){return typeof console<"u"&&console.warn(`byte-range meta cache read failed: ${sr(a)}`),null}}var bs=class{#e=[];covered(t,r){for(let n of this.#e){if(n.begin>t)break;if(r<=n.end)return!0}return!1}add(t,r){if(r<=t)return 0;let n=0;for(;n<this.#e.length&&this.#e[n].end<t;)n+=1;let s=n,a=t,i=r,u=0;for(;s<this.#e.length&&this.#e[s].begin<=r;){let o=this.#e[s];u+=Math.max(0,Math.min(r,o.end)-Math.max(t,o.begin)),a=Math.min(a,o.begin),i=Math.max(i,o.end),s+=1}return this.#e.splice(n,s-n,{begin:a,end:i}),r-t-u}read(t,r,n){let s=0,a=0;return{received:i=>{if(s=Math.min(r-t,s+i),s<=a)return;let u=this.add(t+a,t+s);a=s,u>0&&n?.(u)},finish:()=>{let i=this.add(t,r);i>0&&n?.(i)}}}};function _s(e,{maxBytes:t,maxGap:r}){let n=[...e].sort((i,u)=>i.begin-u.begin),s=[],a=null;for(let i of n){if(!a){a={begin:i.begin,end:i.end,tensors:[i]};continue}let u=i.begin-a.end,o=i.end-a.begin;u<=r&&o<=t?(a.end=Math.max(a.end,i.end),a.tensors.push(i)):(s.push(a),a={begin:i.begin,end:i.end,tensors:[i]})}return a&&s.push(a),s}var k_=24<<20,A_=1<<20,y_=96<<20,S_=32;async function _i(e,t,r,{concurrency:n=S_,chunkMaxBytes:s=k_,chunkMaxGap:a=A_,byteBudget:i=y_,signal:u,onProgress:o,onByteProgress:l,directDestinations:c,coalesceAgainst:d,countedChunks:p}={}){if(t.length===0)return;let f=e.readRangeInto&&e.writeTensorAcceptsEphemeralBytes===!0&&c?.size?c:void 0,g={maxBytes:s,maxGap:a},m=d?.length&&e.retainsWrittenTensors?d:void 0,h=m?x_(t,m,g):T_(t,g,f),v=h.reduce((E,k)=>E+(k.end-k.begin),0),b=0,_=new Map,C=(E={})=>{if(!o)return;let k=b;for(let S of _.values())k+=S;o({loaded:k,total:v,...E})},y=async E=>{let{begin:k,end:S,tensors:A}=h[E],w=S-k,I=p?.read(k,S,l),z=I?.received??(W=>l?.(W)),O=A.length===1?A[0]:void 0,j=e.readRangeInto,F=O?.begin===k&&O.end===S&&j&&e.writeTensorAcceptsEphemeralBytes===!0?c?.get(O.name):void 0;if(F){let W=F();if(W){if(W.bytes.byteLength!==w)throw await Zp(W),new Error(`Direct destination for ${O.name} has ${W.bytes.byteLength} bytes; expected ${w}`);let P=!1;try{let R=await e.readTensor(k,S),Y=R?.byteLength===w;Xp(u),Y?(W.bytes.set(R),P=!0):(_.set(E,0),await j.call(e,k,S,W.bytes,{signal:u,onByteProgress:ne=>{_.set(E,(_.get(E)??0)+ne),z(ne),C({fromCache:!1,range:[k,S],inFlight:!0})}}),P=!0,await Promise.resolve().then(()=>e.writeTensor(k,S,W.bytes)).catch(()=>{})),Y&&!p&&l?.(w),I?.finish(),Xp(u),await r({begin:k,end:S,bytes:W.bytes,tensors:A,directDestination:W}),_.delete(E),b+=w,C({fromCache:Y,range:[k,S]});return}catch(R){if(await Zp(W),R instanceof Rt&&!P)_.delete(E);else throw R}}}let K=await C_(A.map(W=>Promise.resolve().then(()=>e.readTensor(W.begin,W.end)))),Z=K.every((W,P)=>W!=null&&W.byteLength===A[P].end-A[P].begin),Q=A[0].begin,N=A.reduce((W,P)=>Math.max(W,P.end),A[0].end),q=N-Q,x,B=k,U=S;if(Z){B=Q,U=N;let P=(A.length===1?A[0]:void 0)?K[0]:void 0;if(P?.byteLength===q)x=P;else{x=new Uint8Array(q);for(let R=0;R<A.length;++R)x.set(K[R],A[R].begin-Q)}}else _.set(E,0),x=await e.readRange(k,S,{signal:u,onByteProgress:W=>{_.set(E,(_.get(E)??0)+W),z(W),C({fromCache:!1,range:[k,S],inFlight:!0})}});Z&&!p&&l?.(w),I?.finish();let M=Promise.resolve().then(()=>r({begin:B,end:U,bytes:x,tensors:A})),G=Z?Promise.resolve():Promise.resolve().then(()=>e.writeTensor(k,S,x)).catch(()=>{}),$=!1,X,H=M.catch(W=>{$=!0,X=W});if(await Promise.all([H,G]),$)throw X;_.delete(E),b+=w,C({fromCache:Z,range:[k,S]})};await new Promise((E,k)=>{let S=0,A=0,w=0,I=h.length,z=!1,O=!1,j,F=new Set,K=()=>{},Z=()=>{z||!O||F.size!==0||(z=!0,K(),k(j))},Q=x=>{O||(O=!0,j=x),Z()},N=(x,B,U,M)=>{if(F.delete(x),A--,w-=B,U){Q(M);return}if(O){Z();return}if(--I===0){z=!0,K(),E();return}q()},q=()=>{if(!(z||O)){if(u?.aborted){Q(u.reason??new Error("aborted"));return}for(;S<h.length&&A<n;){let x=h[S].end-h[S].begin;if(A>0&&w+x>i)break;let B=S++;A++,w+=x;let U=y(B);F.add(U),U.then(()=>N(U,x,!1),M=>N(U,x,!0,M))}}};if(u){let x=()=>Q(u.reason??new Error("aborted"));u.aborted?x():(u.addEventListener("abort",x,{once:!0}),K=()=>{u.removeEventListener("abort",x)})}q()})}function x_(e,t,r){let n=_s(t,r),s=n.map(l=>l.begin),a=l=>{let c=0,d=s.length-1,p=-1;for(;c<=d;){let g=c+d>>1;s[g]<=l.begin?(p=g,c=g+1):d=g-1}let f=p===-1?void 0:n[p];return f&&l.end<=f.end?f:void 0},i=new Map,u=[];for(let l of e){let c=a(l);if(!c){u.push(l);continue}let d=i.get(c);d?d.push(l):i.set(c,[l])}let o=[...i].map(([l,c])=>({begin:l.begin,end:l.end,tensors:c.sort((d,p)=>d.begin-p.begin)}));return o.push(..._s(u,r)),o.sort((l,c)=>l.begin-c.begin)}function T_(e,t,r){if(!r)return _s(e,t);let n=[...e].sort((u,o)=>u.begin-o.begin),s=[],a=[],i=()=>{a.length!==0&&(s.push(..._s(a,t)),a=[])};for(let u of n){if(!r.has(u.name)){a.push(u);continue}i(),s.push({begin:u.begin,end:u.end,tensors:[u]})}return i(),s}async function Zp(e){try{await e.abort()}catch{}}function Xp(e){if(e?.aborted)throw e.reason??new Error("aborted")}async function C_(e){let t=new Array(e.length),r=!1,n;if(await Promise.all(e.map((s,a)=>s.then(i=>{t[a]=i},i=>{r||(r=!0,n=i)}))),r)throw n;return t}var q_="gguf-cache-v1",B_=32<<20;function I_(e){let t=e;return t?.headerBytes&&Number.isFinite(t.size)&&(t.resourceIdentity===void 0||typeof t.resourceIdentity=="string")?t:null}var yr=class e{#e;#t;onByteProgress=null;#r=null;#n=new bs;constructor(t,r){this.#e=t,this.#t=r}get supportsDirectDestinations(){return this.#t.readRangeInto&&this.#t.writeTensorAcceptsEphemeralBytes===!0?!0:void 0}get resourceIdentity(){return this.#t.resourceIdentity}static async open(t,r={}){let n=r.cacheKey??(typeof t=="string"?t:t.toString()),s=r.cache===!1||r.source?null:r.chunkCache??ws(r.cacheName??q_),a=!!r.force,i={source:r.source,fetch:r.fetch,signal:r.signal},u=s&&!a?await dl(s,n,t,I_,{fetch:r.fetch,signal:r.signal,verifyResourceIdentity:!0}):null;if(s&&u){let l=new $t(u.headerBytes,{headerOnly:!0}),c=await An(t,i,{knownSize:u.size,knownAcceptsRanges:u.acceptsRanges??!0,knownResourceIdentity:u.resourceIdentity});return new e(l,bi(c,{cache:s,cacheKey:n,force:a}))}let o=await An(t,i);try{let{gguf:l,bytes:c}=await cl(o,r.signal);return s&&o.size!=null&&o.acceptsRanges&&c.byteLength<=B_&&s.putMeta(n,{size:o.size,headerBytes:c,acceptsRanges:o.acceptsRanges,resourceIdentity:o.resourceIdentity}).catch(()=>{}),new e(l,s?bi(o,{cache:s,cacheKey:n,force:a}):o)}catch(l){throw await o.close(),l}}get metadata(){return this.#e.metadata}names(){return this.#e.names()}has(t){return this.#e.has(t)}info(t){return this.#e.info(t)}planReads(t=[]){let r=t.filter(n=>this.#e.has(n)).map(n=>{let s=this.#e.info(n),a=this.#e.dataStart+s.offset;return{name:n,begin:a,end:a+s.byteLength}});this.#r=r.length>0?r.sort((n,s)=>n.begin-s.begin):null,this.#n=new bs}async tensorBytes(t,r={}){let n=this.#e.info(t),s=this.#e.dataStart+n.offset;return this.#s(s,s+n.byteLength,r.signal)}async tensorByteRange(t,r,n,s={}){let a=this.#e.dataStart+this.#e.info(t).offset;return this.#s(a+r,a+n,s.signal)}async streamTensorRange(t,r,n,s,a={}){let i=this.#e.info(t);Jo(i,r,n);let u=this.#e.dataStart+i.offset,o=u+r,l=u+n;if(!Number.isSafeInteger(o)||!Number.isSafeInteger(l))throw new RangeError(`Absolute byte range for tensor ${t} is outside the safe integer range`);await _i(this.#t,[{name:t,begin:o,end:l}],({bytes:c,directDestination:d})=>s(c,d),{signal:a.signal,onByteProgress:this.onByteProgress??void 0,countedChunks:this.#n,directDestinations:a.directDestination?new Map([[t,a.directDestination]]):void 0})}async#s(t,r,n){let s=this.#n.read(t,r,u=>this.onByteProgress?.(u)),a=await this.#t.readTensor(t,r);if(a)return s.finish(),a;let i=await this.#t.readRange(t,r,{signal:n,onByteProgress:s.received});return s.finish(),await this.#t.writeTensor(t,r,i),i}async streamTensors(t,r,n={}){let s=t.map(a=>{let i=this.#e.info(a),u=this.#e.dataStart+i.offset;return{name:a,begin:u,end:u+i.byteLength}});await _i(this.#t,s,async a=>{for(let i of a.tensors)await r(i.name,a.bytes.subarray(i.begin-a.begin,i.end-a.begin),a.directDestination)},{concurrency:n.concurrency,chunkMaxBytes:n.chunkMaxBytes,chunkMaxGap:n.chunkMaxGap,byteBudget:n.byteBudget,signal:n.signal,onByteProgress:this.onByteProgress??void 0,directDestinations:n.directDestinations,coalesceAgainst:this.#r??void 0,countedChunks:this.#n})}async close(){await this.#t.close()}};var Yp="(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",E_="(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",P_="(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",R_="(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}+| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",G_="[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",tg=new Set(["llama3","llama-v3","llama-bpe","falcon3","falcon-h1","pixtral","midm-2.0","lfm2","jina-v5-nano"]),D_=new Set(["gpt-4o","llama4","kanana2","talkie","minimax-m2"]),L_=new Set(["qwen2","deepseek-r1-qwen","kormo","f2llmv2","megrez","stablelm2","hunyuan","solar-open"]),O_=new Set(["qwen35"]),M_=new Set(["default","gpt-2","phi-2","jina-es","jina-de","gigachat","jina-v2-es","jina-v2-de","a.x-4.0","mellum","modern-bert","jina-v1-en","jina-v2-code","roberta-bpe","exaone4"]);function pl(e){let t=e["tokenizer.ggml.pre"];return typeof t=="string"&&t.length>0?t:null}function F_(e){let t=pl(e);return t!==null&&tg.has(t)}function yn(...e){return{type:"Sequence",pretokenizers:[...e.map(t=>({type:"Split",pattern:{Regex:t},behavior:"Isolated",invert:!1})),{type:"ByteLevel",add_prefix_space:!1,trim_offsets:!1,use_regex:!1}]}}var Q_=`{% for message in messages %}<|im_start|>{{ message['role'] }}
{{ message['content'] }}<|im_end|>
{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
{% endif %}`;function rg(e){let t=sg(e,"tokenizer.ggml.tokens"),r=e["tokenizer.ggml.merges"]??[],n={};if(t.forEach((a,i)=>{n[a]=i}),j_(e))return{version:"1.0",truncation:null,padding:null,added_tokens:fl(e,t),normalizer:{type:"Replace",pattern:{String:" "},content:"\u2581"},pre_tokenizer:{type:"Split",pattern:{String:" "},behavior:"MergedWithPrevious",invert:!1},post_processor:null,decoder:{type:"Sequence",decoders:[{type:"Replace",pattern:{String:"\u2581"},content:" "},{type:"ByteFallback"},{type:"Fuse"}]},model:{type:"BPE",dropout:null,unk_token:Sn(t,e["tokenizer.ggml.unknown_token_id"])??null,continuing_subword_prefix:null,end_of_word_suffix:null,fuse_unk:!0,byte_fallback:!0,ignore_merges:!1,vocab:n,merges:eg(r)}};let s=F_(e)||pl(e)==="minicpm5";return{version:"1.0",truncation:null,padding:null,added_tokens:fl(e,t),normalizer:null,pre_tokenizer:W_(e),post_processor:K_(),decoder:z_(),model:{type:"BPE",dropout:null,unk_token:Sn(t,e["tokenizer.ggml.unknown_token_id"])??null,continuing_subword_prefix:null,end_of_word_suffix:null,fuse_unk:!1,byte_fallback:!!e["tokenizer.ggml.byte_fallback"],ignore_merges:s,vocab:n,merges:eg(r)}}}function j_(e){let t=e["tokenizer.ggml.model"];return typeof t!="string"?!1:t==="llama"||t==="spm"||t.startsWith("gemma")||t.startsWith("llama")}function ng(e){let t=sg(e,"tokenizer.ggml.tokens"),r=e["tokenizer.ggml.bos_token_id"],n=e["tokenizer.ggml.eos_token_id"],s=e["tokenizer.ggml.unknown_token_id"],a=e["tokenizer.ggml.padding_token_id"],i=Object.fromEntries(fl(e,t).map(u=>[u.id,{content:u.content,single_word:u.single_word,lstrip:u.lstrip,rstrip:u.rstrip,normalized:u.normalized,special:u.special}]));return{tokenizer_class:"Qwen2Tokenizer",model_max_length:U_(e)??e["general.context_length"]??32768,bos_token:Sn(t,r),eos_token:Sn(t,n),unk_token:Sn(t,s),pad_token:Sn(t,a),bos_token_id:Number.isInteger(r)?r:void 0,eos_token_id:Number.isInteger(n)?n:void 0,eos_token_ids:gi(e),unk_token_id:Number.isInteger(s)?s:void 0,pad_token_id:Number.isInteger(a)?a:void 0,chat_template:gl(e,t),added_tokens_decoder:i}}function gl(e,t=H_(e)){let r=e["tokenizer.chat_template"];if(typeof r=="string")return r;let n=e["tokenizer.ggml.chat_template"];return typeof n=="string"?n:t.includes("<|im_start|>")&&t.includes("<|im_end|>")?Q_:null}function W_(e){let t=pl(e);if(t===null)return e["general.architecture"]==="qwen3"?yn(Yp):Jp();if(O_.has(t))return yn(E_);if(t==="minicpm5")return yn("\\p{N}{1,3}",R_);if(L_.has(t)||t==="qwen3")return yn(Yp);if(tg.has(t))return yn(P_);if(D_.has(t))return yn(G_);if(M_.has(t))return Jp();throw new Error(`Unsupported tokenizer.ggml.pre "${t}": this build has no split regex for it, and the GPT-2 fallback would tokenize the prompt differently from the reference implementation.`)}function Jp(){return{type:"ByteLevel",add_prefix_space:!1,trim_offsets:!0,use_regex:!0}}function z_(){return{type:"ByteLevel",add_prefix_space:!1,trim_offsets:!1,use_regex:!1}}function K_(){return{type:"ByteLevel",add_prefix_space:!1,trim_offsets:!1,use_regex:!1}}function fl(e,t){let r=Array.isArray(e["tokenizer.ggml.token_type"])?e["tokenizer.ggml.token_type"]:[],n=new Set;for(let s of[e["tokenizer.ggml.bos_token_id"],e["tokenizer.ggml.eos_token_id"],e["tokenizer.ggml.unknown_token_id"],e["tokenizer.ggml.padding_token_id"]])Number.isInteger(s)&&n.add(s);for(let s=0;s<r.length;++s)N_(r[s])&&n.add(s);return[...n].filter(s=>s>=0&&s<t.length).sort((s,a)=>s-a).map(s=>({id:s,content:t[s],single_word:!1,lstrip:!1,rstrip:!1,normalized:!1,special:!0}))}function U_(e){let t=e["general.architecture"];return typeof t=="string"?e[`${t}.context_length`]:void 0}function N_(e){return Number.isInteger(e)&&e!==1&&e!==6}function eg(e){return Array.isArray(e)?e.map(t=>Array.isArray(t)?t.join(" "):String(t)):[]}function Sn(e,t){return typeof t=="number"&&Number.isInteger(t)&&t>=0&&t<e.length?e[t]:void 0}function sg(e,t){let r=e[t];if(!Array.isArray(r)||r.some(n=>typeof n!="string"))throw new Error(`GGUF metadata ${t} must be a string array`);return r}function H_(e){let t=e["tokenizer.ggml.tokens"];return Array.isArray(t)&&t.every(r=>typeof r=="string")?t:[]}var $_="webgpu-prefix-snapshots",ag="prefix-snapshot-slot",ig=3,V_=0;function Z_(e){if(typeof e!="object"||e===null)return!1;let t=e;return t.version===ig&&typeof t.chunkKey=="string"&&typeof t.identity=="string"&&typeof t.length=="number"&&typeof t.layout=="string"&&Array.isArray(t.sizes)&&t.sizes.every(r=>Number.isInteger(r)&&r>0)}function X_(){let e=globalThis.crypto?.randomUUID?.();return e?`prefix-snapshot:${e}`:`prefix-snapshot:${Date.now().toString(36)}:${(++V_).toString(36)}:${Math.random().toString(36).slice(2)}`}function Y_(e){let t=async()=>{let r=await e.getMeta(ag);return Z_(r)?r:null};return{async load(r){try{let n=await t();if(!n||n.identity!==r)return null;let s=[],a=0;for(let i of n.sizes){let u=await e.get(n.chunkKey,a,a+i);if(!u||u.byteLength!==i)return null;s.push(u),a+=i}return{identity:r,length:n.length,layout:n.layout,chunks:s}}catch(n){return ki("prefix snapshot read failed",n),null}},async save(r){let n=X_();try{let s=await t(),a=0,i=[];for(let o of r.chunks)await e.put(n,a,a+o.byteLength,o),i.push(o.byteLength),a+=o.byteLength;let u={version:ig,chunkKey:n,identity:r.identity,length:r.length,layout:r.layout,sizes:i};if(await e.putMeta(ag,u),s&&s.chunkKey!==n)try{await e.delete?.(s.chunkKey)}catch(o){ki("previous prefix snapshot cleanup failed",o)}}catch(s){ki("prefix snapshot write failed",s);try{await e.delete?.(n)}catch(a){ki("prefix snapshot cleanup failed",a)}}}}}function ug(e=$_){let t=ws(e);return t?Y_(t):null}function ki(e,t){typeof console<"u"&&console.warn(`${e}: ${t instanceof Error?t.message:String(t)}`)}function ml(e,t,r,n="main"){let s=t??e.defaultModelId;return s.toLowerCase().endsWith(".gguf")?!/^https?:/i.test(s)&&typeof location<"u"?new URL(s,location.href).href:s:So(s,{revision:n,file:r??e.defaultFile(s)})}function og(e,t,r={}){let n=t??e.defaultModelId,s=n.toLowerCase().endsWith(".gguf"),a=r.mmproj!==void 0?r.mmproj:s?null:e.defaultMmproj?.(n)??null;if(a===null)return null;if(!e.vision)throw new Error(`this model takes no vision projector (mmproj "${a}")`);if(/^[a-z][a-z0-9+.-]*:/i.test(a))return a;if(s){let i=ml(e,n,r.file,r.revision??"main");return i.slice(0,i.lastIndexOf("/")+1)+a}return So(n,{revision:r.revision??"main",file:a})}function J_(e){let t=e.metadata,r=ng(t),n=rg(t),s=new Mf(n,r),a=typeof r.chat_template=="string"?r.chat_template:gl(t);if(!a)throw new Error("GGUF metadata is missing a chat_template");let i=ek(r,s);return{tokenizer:s,chatTemplate:new bd(a),tokenizerConfig:r,eosTokenIds:i}}function ek(e,t){let r=new Set,n=e.eos_token_id;Pr(n)&&r.add(n);let s=e.eos_token_ids;if(Array.isArray(s))for(let a of s)Pr(a)&&r.add(a);if(typeof e.eos_token=="string"){let a=t.token_to_id(e.eos_token);Pr(a)&&r.add(a)}if(r.size===0)throw new Error("GGUF metadata has no eos token id");return[...r]}function tk(e){let t=s=>{let a=e[`general.sampling.${s}`];return Pr(a)?a:void 0},r={temperature:t("temp"),top_k:t("top_k"),top_p:t("top_p"),min_p:t("min_p")},n=Object.fromEntries(Object.entries(r).filter(([,s])=>s!==void 0));return Object.freeze({do_sample:Object.keys(n).length>0,...n})}var rk=Object.freeze({do_sample:!1});function Ai(){return typeof requestAnimationFrame=="function"?new Promise(e=>requestAnimationFrame(()=>e())):new Promise(e=>setTimeout(e,0))}function lg(e,t=null,r={}){return _f(async n=>{let s=ml(e,t,r.file,r.revision??"main"),a=og(e,t,r),i={fetch:xo(r),signal:r.signal},[u,o]=await Promise.all([ll(s,i),a===null?null:ll(a,i)]);return cg([u,o])??e.unsupportedReason(u,n,o)})}function cg(e){let t=new Set;for(let r of e)for(let n of r?.names()??[]){let{type:s,typeName:a}=r.info(n);[0,1,8,12,13,14].includes(s)||t.add(a??String(s))}return t.size===0?null:`This model uses weight formats this build does not include: ${[...t].join(", ")}`}function nk(e,t){if(!e)return null;let{features:r}=Za(t.device),n=[r.shaderF16?"f16":"no-f16",r.subgroups?"sg":"no-sg",r.subgroupMatrix?"sgmat":"no-sgmat"].join("+");return`gguf|${e}|${n}`}async function dg(e,t,r=null,n={}){let s=n.onProgress??(()=>{}),a=ml(e,r,n.file,n.revision??"main"),i=og(e,r,n),u=xo(n);s({status:"init",message:"Requesting WebGPU device"});let o=n.runtime??await np(n.runtimeOptions??{}),l=!n.runtime;o.captureShaders=!0;let c=async()=>{},d=null,p=null,f=null,g=(m,h)=>{e.disposeGenerationState?e.disposeGenerationState(m,h):h.cache.dispose?.()};try{s({status:"init",message:"Reading model headers"});let m={fetch:u,signal:n.signal,cache:n.cache,force:n.force,cacheName:n.cacheName},[h,v]=await Promise.allSettled([yr.open(a,m),i===null?Promise.resolve(null):yr.open(i,m)]);if(h.status==="rejected"||v.status==="rejected")throw h.status==="fulfilled"&&await h.value.close().catch(()=>{}),v.status==="fulfilled"&&await v.value?.close().catch(()=>{}),h.status==="rejected"?h.reason:v.reason;let b=h.value,_=v.value;c=_?async()=>{await Promise.all([b.close(),_.close()])}:()=>b.close(),s({status:"init",message:"Checking device support"});let C=cg([b,_])??await e.unsupportedReason(b,o.device,_);if(C)throw new Error(C);s({status:"tokenizer",message:"Loading tokenizer"}),await Ai();let{tokenizer:y,chatTemplate:E,tokenizerConfig:k,eosTokenIds:S}=J_(b);await Ai();let A=_?[b,_]:[b],w=A.reduce((Q,N)=>Q+N.names().reduce((q,x)=>q+N.info(x).byteLength,0),0),I=0,z=Q=>{I+=Q,s({status:"weights",kind:"bytes",loaded:I,total:w||null,fraction:w?Math.min(1,I/w):void 0,message:"Streaming weights"})};for(let Q of A)Q.onByteProgress=z;s({status:"weights",message:"Preparing weight upload"});let O=d=await e.loadModel(o,b,Q=>{!Pr(Q.total)||Q.total<=0||s({status:"weights",kind:"tensors",loaded:Q.processed,total:Q.total,message:"Uploading to GPU"})},_,n.signal);await c(),n.signal?.throwIfAborted();let j=p=e.createGenerationState(O,n.maxLength),F=n.primeSystemPrompt,K=f=t({runtime:o,ownsRuntime:l,model:O,tokenizer:y,chatTemplate:E,tokenizerConfig:k,eosTokenIds:S,generationState:j,defaultMaxNewTokens:e.defaultMaxNewTokens,prefixSnapshotStore:n.prefixSnapshotStore===void 0?ug():n.prefixSnapshotStore,prefixSnapshotIdentity:nk(b.resourceIdentity&&(e.vision?`${b.resourceIdentity}|multimodal`:b.resourceIdentity),o),vision:e.vision?.(O)??null,generationConfig:O.canSample?.()?tk(b.metadata):rk,createGenerationState:()=>e.createGenerationState(O,n.maxLength),disposeGenerationState:Q=>g(O,Q)});n.chatTemplateArgs&&(K.chatTemplateArgs={...n.chatTemplateArgs}),s({status:"weights",kind:"tensors",message:F?"Warming up kernels and caching system prompt":"Compiling kernels (warmup)"}),await Ai();let Z=typeof O.warmRemainingPrefill=="function";if(await O.warmup(j.cache,Z?{deferRemainingPrefill:!0}:void 0),n.signal?.throwIfAborted(),n.decodePipelineDepth===void 0&&s({status:"weights",kind:"tensors",message:"Tuning decode pipeline"}),await K.tuneDecodePipeline({decodePipelineDepth:n.decodePipelineDepth}),n.signal?.throwIfAborted(),F!==void 0){s({status:"weights",kind:"tensors",message:"Caching system prompt"}),await Ai();let Q=await K.primeSystemPrefix([{role:"system",content:F}]);Q>0&&j.cache.get_seq_length()!==Q&&typeof console<"u"&&console.warn(`system-prompt caching primed ${Q} tokens but the cache holds ${j.cache.get_seq_length()} \u2014 something reset it after priming, so the first turn will prefill the prompt again`),n.signal?.throwIfAborted()}return s({status:"ready",message:"Ready",fraction:1}),Z&&O.warmRemainingPrefill(j.cache).catch(()=>{}),K}catch(m){if(await c().catch(()=>{}),f)await f.dispose().catch(()=>{});else if(d){let h=d,v=p;for(let b of[()=>v&&g(h,v),()=>h.dispose()])try{b()}catch{}}throw l&&await o.destroy().catch(()=>{}),m}}function Sr(e,t,r="Config"){let n=e[t];if(typeof n!="number"||!Number.isInteger(n)||n<=0)throw new Error(`${r}.${t} must be a positive integer`);return n}function fg(e,t,r="Config"){for(let n of t)Sr(e,n,r)}function sk(e){if(!(e.rope_theta===void 0&&e.rope_scaling==null))return{rope_type:e.rope_scaling?.rope_type??e.rope_scaling?.type??"default",rope_theta:e.rope_theta}}var ur=class{vocab_size;hidden_size;intermediate_size;num_hidden_layers;num_attention_heads;num_key_value_heads;hidden_act;max_position_embeddings;initializer_range;rms_norm_eps;use_cache;pad_token_id;bos_token_id;eos_token_id;pretraining_tp;tie_word_embeddings;rope_parameters;attention_bias;attention_dropout;mlp_bias;head_dim;model_type;constructor(t={}){this.vocab_size=Sr(t,"vocab_size","LlamaConfig"),this.hidden_size=Sr(t,"hidden_size","LlamaConfig"),this.intermediate_size=Sr(t,"intermediate_size","LlamaConfig"),this.num_hidden_layers=Sr(t,"num_hidden_layers","LlamaConfig"),this.num_attention_heads=Sr(t,"num_attention_heads","LlamaConfig"),this.num_key_value_heads=t.num_key_value_heads??this.num_attention_heads,this.hidden_act=t.hidden_act,this.max_position_embeddings=Sr(t,"max_position_embeddings","LlamaConfig"),this.initializer_range=t.initializer_range,this.rms_norm_eps=t.rms_norm_eps,this.use_cache=t.use_cache,this.pad_token_id=t.pad_token_id,this.bos_token_id=t.bos_token_id,this.eos_token_id=t.eos_token_id,this.pretraining_tp=t.pretraining_tp,this.tie_word_embeddings=t.tie_word_embeddings,this.rope_parameters=t.rope_parameters??sk(t),this.attention_bias=t.attention_bias,this.attention_dropout=t.attention_dropout??0,this.mlp_bias=t.mlp_bias,this.head_dim=t.head_dim??Math.floor(this.hidden_size/this.num_attention_heads),this.model_type="llama",this.validate()}validate(){if(ak(this,"LlamaConfig",["vocab_size","hidden_size","intermediate_size","num_hidden_layers","num_attention_heads","num_key_value_heads","max_position_embeddings","head_dim"]),this.hidden_act!=="silu")throw new Error(`Unsupported Llama hidden_act for V1: ${this.hidden_act}`);if(this.attention_bias||this.mlp_bias)throw new Error("Llama megakernel V1 does not support attention_bias or mlp_bias");if(this.attention_dropout!==0)throw new Error("Llama megakernel V1 only supports attention_dropout=0");if(this.rope_parameters?.rope_type!=="default")throw new Error("Llama megakernel V1 only supports default RoPE")}toJSON(){return{...this,rope_parameters:{...this.rope_parameters}}}};function ks(e){if(!e||typeof e!="object")throw new Error("Llama-like config must be an object");if(e instanceof ur)return e;let t=e;if(t.model_type==="llama")return new ur(t);if(t.model_type){if(typeof t.validate!="function")throw new Error("Llama-like configs with model_type must be constructed by their Config class");return e}return new ur(t)}function Gt(e){return Array.isArray(e.layer_types)&&e.layer_types.some(t=>t==="conv")}function dt(e){return Array.isArray(e.layer_types)&&e.layer_types.some(t=>t==="linear_attention")}function Zr(e){let t=e.layer_types,r=[];for(let n=0;n<e.num_hidden_layers;++n){let s=t?.[n];s!=="conv"&&s!=="linear_attention"&&r.push(n)}return r}function yi(e){if(!Array.isArray(e.layer_types))throw new Error("linear attention config requires layer_types");return e.layer_types}function Fe(e,t){if(typeof e!="number")throw new Error(`${t} must be a number`);return e}function Si(e){let t=e.num_experts;return typeof t!="number"||t<=0?null:{numExperts:t,topK:Fe(e.num_experts_per_tok,"config.num_experts_per_tok"),moeIntermediateSize:Fe(e.moe_intermediate_size,"config.moe_intermediate_size"),numDenseLayers:Fe(e.num_dense_layers,"config.num_dense_layers"),normTopkProb:e.norm_topk_prob!==!1,routedScale:typeof e.routed_scaling_factor=="number"?e.routed_scaling_factor:1,useExpertBias:!!e.use_expert_bias}}function xn(e,t){return e!==null&&t>=e.numDenseLayers}function ak(e,t,r){if(fg(e,r,t),e.hidden_size!==e.num_attention_heads*e.head_dim)throw new Error(`${t} hidden_size must equal num_attention_heads * head_dim for V1`);if(e.num_attention_heads%e.num_key_value_heads!==0)throw new Error(`${t} num_attention_heads must be divisible by num_key_value_heads`);if(e.head_dim%2!==0)throw new Error(`${t} head_dim must be even for RoPE`)}function xi(e){let t=e.hidden_size,r=e.intermediate_size,n=e.num_attention_heads*e.head_dim,s=e.num_key_value_heads*e.head_dim,a=[{id:"embed_tokens",name:"model.embed_tokens.weight",length:e.vocab_size*t}];for(let i=0;i<e.num_hidden_layers;++i){let u=`L${i}.attn.qkv`,o=`L${i}.mlp.gate_up`;a.push({id:`layers.${i}.q_proj`,name:`model.layers.${i}.self_attn.q_proj.weight`,length:n*t,packedGroup:u},{id:`layers.${i}.k_proj`,name:`model.layers.${i}.self_attn.k_proj.weight`,length:s*t,packedGroup:u},{id:`layers.${i}.v_proj`,name:`model.layers.${i}.self_attn.v_proj.weight`,length:s*t,packedGroup:u},{id:`layers.${i}.o_proj`,name:`model.layers.${i}.self_attn.o_proj.weight`,length:t*n},{id:`layers.${i}.gate_proj`,name:`model.layers.${i}.mlp.gate_proj.weight`,length:r*t,packedGroup:o},{id:`layers.${i}.up_proj`,name:`model.layers.${i}.mlp.up_proj.weight`,length:r*t,packedGroup:o},{id:`layers.${i}.down_proj`,name:`model.layers.${i}.mlp.down_proj.weight`,length:t*r},{id:`layers.${i}.input_layernorm`,name:`model.layers.${i}.input_layernorm.weight`,length:t},{id:`layers.${i}.post_attention_layernorm`,name:`model.layers.${i}.post_attention_layernorm.weight`,length:t})}return a.push({id:"model_norm",name:"model.norm.weight",length:t},{id:"lm_head",name:"lm_head.weight",length:e.vocab_size*t}),a}function hl(e,t,r=0){if(t%2!==0||t<=0||r<0||r+t>e)throw new Error(`interleaved rope needs an even rotaryDim with offset + rotaryDim <= headDim; got ${t} at ${r} of ${e}`);let n=t/2,s=new Uint32Array(e);for(let a=0;a<e;++a)s[a]=a;for(let a=0;a<n;++a)s[r+a]=r+2*a,s[r+a+n]=r+2*a+1;return s}function ik(e,t,r,n){let s=e.length/(t*r);if(!Number.isInteger(s))throw new Error(`projection of ${e.length} values is not [${t} * ${r}, hidden]`);let a=new Float32Array(e.length);for(let i=0;i<t;++i){let u=i*r*s;for(let o=0;o<r;++o)a.set(e.subarray(u+n[o]*s,u+(n[o]+1)*s),u+o*s)}return a}function pg(e,t){let r=hl(t.head_dim,t.rotary_dim);return e.map(n=>{let s=n.id.endsWith(".q_proj")?t.num_attention_heads:n.id.endsWith(".k_proj")?t.num_key_value_heads:0;if(s===0)return n;if(n.transform)throw new Error(`${n.name} already carries a transform; cannot also permute rope lanes`);return{...n,transform:a=>ik(a,s,t.head_dim,r)}})}var Ti=/^blk\.\d+\.attn_(q|k)\.weight$/;function mg(e,t){let r=hl(t.head_dim,t.rotary_dim),n=l=>l.endsWith(".attn_q.weight")?t.num_attention_heads:t.num_key_value_heads,s=async(l,c)=>gg(await e.tensorBytes(l,c),e.info(l),n(l),t.head_dim,r),a=new Map,i=(l,c)=>{if(c?.signal)return s(l,c);let d=a.get(l);return d||(d=s(l),a.set(l,d)),d},u=async(l,c,d,p,f={})=>{let g=e.info(l);if(ok(g,c,d),!Ti.test(l)){if(e.streamTensorRange){await e.streamTensorRange(l,c,d,p,f);return}Tn(f.signal);let h=await e.tensorByteRange(l,c,d,f);Tn(f.signal),await p(h),Tn(f.signal);return}Ci(g.byteLength,g,n(l),t.head_dim,r),Tn(f.signal);let m=(await i(l,f)).slice(c,d);Tn(f.signal),await p(m),Tn(f.signal)};return{get metadata(){return e.metadata},get supportsDirectDestinations(){return e.supportsDirectDestinations},names:()=>e.names(),has:l=>e.has(l),info:l=>e.info(l),tensorBytes:(l,c)=>Ti.test(l)?i(l,c):e.tensorBytes(l,c),tensorByteRange:async(l,c,d,p)=>Ti.test(l)?(await i(l,p)).subarray(c,d):e.tensorByteRange?e.tensorByteRange(l,c,d,p):(await e.tensorBytes(l,p)).subarray(c,d),...e.streamTensorRange||e.tensorByteRange?{streamTensorRange:u}:{},streamTensors:async(l,c,d)=>{let p=new Map;for(let g of l){if(!Ti.test(g))continue;let m=e.info(g),h=n(g);Ci(m.byteLength,m,h,t.head_dim,r),p.set(g,{info:m,heads:h})}let f=l.filter(g=>!p.has(g));f.length>0&&await e.streamTensors(f,c,d),p.size>0&&await e.streamTensors([...p.keys()],async(g,m,h)=>{let v=p.get(g);if(!h){await c(g,gg(m,v.info,v.heads,t.head_dim,r));return}if(!uk(m,h.bytes))throw new Error(`${g}: direct rope destination does not exactly alias the streamed bytes`);ck(m,v.info,v.heads,t.head_dim,r),await c(g,m,h)},d)}}}function uk(e,t){return e.buffer===t.buffer&&e.byteOffset===t.byteOffset&&e.byteLength===t.byteLength}function ok(e,t,r){if(!Number.isSafeInteger(t)||!Number.isSafeInteger(r)||t<0||r<t)throw new RangeError(`Invalid byte range [${t}, ${r}) for tensor ${e.name}`);if(r>e.byteLength)throw new RangeError(`Byte range [${t}, ${r}) exceeds tensor ${e.name} size ${e.byteLength}`)}function Tn(e){if(e?.aborted)throw e.reason??new Error("aborted")}function Ci(e,t,r,n,s){if(t.shape.length!==2)throw new Error(`${t.name}: rope permutation needs a rank-2 tensor, got rank ${t.shape.length}`);if(!Number.isInteger(r)||r<=0||!Number.isInteger(n)||n<=0)throw new Error(`${t.name}: rope permutation needs positive integer heads/head_dim, got ${r}/${n}`);let[a,i]=t.shape;if(a!==r*n)throw new Error(`${t.name}: ${a} rows != ${r} heads * ${n} head_dim`);if(s.length!==n)throw new Error(`${t.name}: rope permutation has ${s.length} lanes, expected head_dim ${n}`);let u=new Uint8Array(n);for(let c=0;c<n;++c){let d=s[c];if(d>=n)throw new Error(`${t.name}: rope permutation source lane ${d} is outside head_dim ${n}`);if(u[d])throw new Error(`${t.name}: rope permutation source lane ${d} is duplicated`);u[d]=1}let o=kr(t.type,i),l=o*a;if(l!==t.byteLength)throw new Error(`${t.name}: metadata says ${t.byteLength} B, but ${a} rows x ${o} B = ${l} B`);if(e!==t.byteLength)throw new Error(`${t.name}: received ${e} B, expected ${t.byteLength} B`);return{rows:a,rowBytes:o}}function lk(e){let t=new Uint8Array(e.length),r=[];for(let n=0;n<e.length;++n){if(t[n])continue;let s=[],a=n;do t[a]=1,s.push(a),a=e[a];while(a!==n);s.length>1&&r.push(s)}return r}function gg(e,t,r,n,s){let{rowBytes:a}=Ci(e.byteLength,t,r,n,s),i=new Uint8Array(e.byteLength);for(let u=0;u<r;++u){let o=u*n*a;for(let l=0;l<n;++l){let c=o+s[l]*a;i.set(e.subarray(c,c+a),o+l*a)}}return i}function ck(e,t,r,n,s){let{rowBytes:a}=Ci(e.byteLength,t,r,n,s),i=lk(s),u=new Uint8Array(a);for(let o=0;o<r;++o){let l=o*n*a;for(let c of i){let d=l+c[0]*a;u.set(e.subarray(d,d+a));for(let p=0;p+1<c.length;++p){let f=l+c[p]*a,g=l+c[p+1]*a;e.copyWithin(f,g,g+a)}e.set(u,l+c[c.length-1]*a)}}return e}var dk=64*1024,As=class{runtime;tensor;constructor({runtime:t,tensor:r}){this.runtime=t,this.tensor=r}writeTensor({sourceDtype:t,targetDtype:r,sourceBytes:n,targetElementOffset:s}){up(t,r);let a=op(t),i=lp(r);if(n.byteLength%a!==0)throw new Error(`source byte length ${n.byteLength} not divisible by ${a}`);let u=n.byteLength/a,o=s*i;if(t==="F32"&&r==="float32")return this.runtime.writeWeightsRange(this.tensor,o,n),s+u;if(t==="F16"&&r==="float16")return this.runtime.writeWeightsRange(this.tensor,o,n),s+u;let l=Math.max(1,Math.floor(dk/i)),c=dp(t,r);for(let d=0;d<u;d+=l){let p=Math.min(l,u-d),f=n.subarray(d*a,(d+p)*a),g=c(f),m=o+d*i;this.runtime.writeWeightsRange(this.tensor,m,cp(g))}return s+u}};function kg(e){e&&e.depth++===0&&(e.since=performance.now())}function Ag(e){e&&--e.depth===0&&(e.hostMs+=performance.now()-e.since)}var qi=class{specs;handlers;pending;#e=new Map;#t=new Map;#r=!1;constructor(t=[]){this.specs=t.map(xg),this.handlers=new Map,this.pending=new Set}prepare({reader:t,context:r={}}){for(let n of this.#e.values())if(n.state==="running"||n.state==="committing"||n.state==="rolling-back")throw new Error(`Cannot prepare WeightPlan while derived task ${n.spec.name} is ${n.state}`);this.handlers.clear(),this.pending.clear(),this.#e.clear(),this.#t.clear(),this.#r=!1;for(let n of this.specs)n.type==="tensor"?this.#n(n,t,r):n.type==="group"&&this.#s(n,t,r);for(let n of this.specs)n.type==="derived"&&this.#i(n,t,r);return this}names(){return[...this.handlers.keys()]}derivedTaskNames(){return[...this.#e.keys()]}directDestinations(){let t=new Map;for(let[r,n]of this.handlers)n.openDirect&&t.set(r,n.openDirect);return t}async onChunk({bytes:t,tensors:r,directDestination:n},s={}){if(n){let i=r.length===1?r[0]:void 0;if(!i||i.offset!==0||i.length!==t.byteLength||t!==n.bytes)throw await vl(n),new Error("A direct weight destination requires one exact, unsliced tensor chunk");let u=this.handlers.get(i.name);if(!u?.receiveDirect)throw await vl(n),new Error(`WeightPlan has no direct handler for tensor: ${i.name}`);await fk(u,n,s);return}let a=[];for(let i of r){let u=this.handlers.get(i.name);if(!u)continue;let o=t.subarray(i.offset,i.offset+i.length);a.push(hg(u,o,s))}if(a.length){let i=!1,u;if(await Promise.all(a.map(o=>o.catch(l=>{i||(i=!0,u=l)}))),i)throw u}}async readSequential(t,r={}){for(let n of this.names()){let s=await t.read(n),a=this.handlers.get(n);a&&await hg(a,s,r)}}assertComplete(){if(this.pending.size>0){let t=[...this.pending].slice(0,5).join(", ");throw new Error(`WeightPlan incomplete - ${this.pending.size} tensor(s) never arrived (first: ${t})`)}}async finishDerivedTasks(){let t=[...this.#e.values()],r=!1,n;if(await Promise.all(t.map(async s=>{try{if(s.state==="metadata")return;if(s.state==="waiting")throw new Error(`Derived weight task ${s.spec.name} is missing completed dependencies: ${[...s.remaining].join(", ")}`);if(await s.promise,s.state==="failed")throw new Error(`Derived weight task ${s.spec.name} failed`);if(s.state==="aborted")throw new Error(`Derived weight task ${s.spec.name} was aborted`)}catch(a){r||(r=!0,n=a)}})),r)throw n}async abortDerivedTasks(){this.#r=!0;let t=[...this.#e.values()],r=!1,n,s=a=>{r||(r=!0,n=a)};if(await Promise.all(t.map(async a=>{if(!(a.state==="metadata"||a.state==="published"||a.state==="retained-fallback"||a.state==="aborted")){if(a.state==="waiting"){a.state="aborted";return}if(a.state==="running"&&!a.abortStarted){a.abortStarted=!0;try{await a.spec.abort?.(a.context)}catch(i){s(i)}}try{await a.promise}catch(i){s(i)}}})),r)throw n}#n(t,r,n){let s=wg(r,t.name);if(!s){_g(t,n);return}bg(t,s);let a=this.#u(t.name,async(i,u)=>{await hk(t,i,{...n,...u,description:s}),this.pending.delete(t.name),this.#o([t.name],u)},t.progressLabel??t.name);t.directDestination&&!Bi(n)&&(a.openDirect=()=>t.directDestination?.({...n,description:s})??null,a.receiveDirect=async(i,u)=>{await vk(t,i,{...n,...u,description:s}),this.pending.delete(t.name),this.#o([t.name],u)})}#s(t,r,n){let s={},a=t.names.filter(o=>{let l=wg(r,o);return l&&(s[o]=l),!l});if(a.length>0){_g(t,n,a);return}for(let o of t.names)bg(t,s[o],o);let i={},u=t.names.length;for(let o of t.names)this.#u(o,async(l,c)=>{if(this.pending.delete(o),i[o]=l,--u===0)try{await wk(t,i,{...n,...c,descriptions:s}),this.#o(t.names,c)}finally{for(let d of t.names)delete i[d]}},t.progressLabel??o)}#i(t,r,n){if(this.#e.has(t.name))throw new Error(`Duplicate derived weight task: ${t.name}`);if(new Set(t.dependencies).size!==t.dependencies.length)throw new Error(`Derived weight task ${t.name} has duplicate dependencies`);let a={},i=t.dependencies.filter(o=>!this.handlers.has(o)||!r.has(o)?!0:(a[o]=r.describe(o),!1));if(i.length>0)throw new Error(`Derived weight task ${t.name} has missing dependencies: ${i.join(", ")}`);let u={spec:t,context:{...n,descriptions:a},descriptions:a,remaining:new Set(t.dependencies),state:Bi(n)?"metadata":"waiting",promise:null,transaction:null,abortStarted:!1};if(this.#e.set(t.name,u),u.state!=="metadata")for(let o of t.dependencies){let l=this.#t.get(o);l?l.push(u):this.#t.set(o,[u])}}#o(t,r){let n=new Set;for(let s of t)for(let a of this.#t.get(s)??[])a.state==="waiting"&&(a.remaining.delete(s),n.add(a));for(let s of n)s.remaining.size===0&&this.#a(s,r)}#a(t,r){if(t.state!=="waiting")return;if(this.#r){t.state="aborted";return}t.state="running";let n={...t.context,...r,descriptions:t.descriptions},s;try{s=t.spec.start(n)}catch(i){s=Promise.reject(i)}let a=(async()=>{try{let i=await s;if(i===null){t.state="retained-fallback";return}if(mk(t.spec.name,i),t.transaction=i,this.#r){t.state="rolling-back",await Tg(t),t.state="aborted";return}t.state="committing";try{await i.commit()}catch(u){throw t.state="rolling-back",await vg(t,u),t.state="failed",u}t.transaction=null,t.state="published"}catch(i){throw t.transaction&&t.state!=="published"&&t.state!=="aborted"&&(t.state="rolling-back",await vg(t,i)),t.state="failed",i}})();t.promise=a,a.catch(()=>{})}#u(t,r,n=t){if(this.handlers.has(t))throw new Error(`Duplicate WeightPlan handler for tensor: ${t}`);let s={name:t,progressLabel:n,received:!1,receive:r};return this.handlers.set(t,s),this.pending.add(t),s}};function yg(e){return xg({type:"tensor",required:!0,...e})}async function Sg(e,{source:t=null,reader:r,context:n={},concurrency:s,chunkMaxBytes:a,onProgress:i=null,signal:u}={}){let o=e instanceof qi?e:new qi(e);if(!r)throw new Error("WeightPlan execution requires a reader");let l=[],c={hostMs:0,depth:0,since:0},d=performance.now(),p=null;try{o.prepare({reader:r,context:n});let f={...n,timing:c,onTensorProgress:pk(o,i)};t?await t.streamAll(g=>o.onChunk(g,f),{concurrency:s,chunkMaxBytes:a,names:o.names(),onProgress:g=>{p=g.loaded,i?.({...g,phase:"bytes"})},signal:u,directDestinations:t.supportsDirectDestinations===!0?o.directDestinations():void 0}):await o.readSequential(r,f),o.assertComplete(),await o.finishDerivedTasks(),i?.({phase:"loaded",bytes:p,wallMs:performance.now()-d,hostMs:c.hostMs})}catch(f){l.push(f);try{await o.abortDerivedTasks()}catch{}}try{await gk(t,r)}catch(f){l.push(f)}if(l.length===1)throw l[0];if(l.length>1)throw new AggregateError(l,"WeightPlan execution and I/O cleanup both failed");return o}async function hg(e,t,r){if(e.received)throw new Error(`WeightPlan received tensor ${e.name} more than once`);e.received=!0,kg(r.timing);try{await e.receive(t,r)}finally{Ag(r.timing)}r.onTensorProgress?.({name:e.name,label:e.progressLabel})}async function fk(e,t,r){if(e.received)throw await vl(t),new Error(`WeightPlan received tensor ${e.name} more than once`);e.received=!0,kg(r.timing);try{await e.receiveDirect?.(t,r)}finally{Ag(r.timing)}r.onTensorProgress?.({name:e.name,label:e.progressLabel})}async function vl(e){try{await e.abort()}catch{}}function pk(e,t){if(!t)return null;let r=0,n=e.names().length;return({name:s,label:a})=>{t({phase:"tensor",processed:++r,total:n,name:s,label:a})}}async function gk(e,t){let r=new Set,n=e?[t,e]:[t],s=[];for(let a of n){let i=a.closeTarget??a;if(!r.has(i)){r.add(i);try{await a.close()}catch(u){s.push(u)}}}if(s.length===1)throw s[0];if(s.length>1)throw new AggregateError(s,"Multiple WeightPlan I/O handles failed to close")}function xg(e){if(!e||typeof e!="object")throw new Error("WeightPlan spec must be an object");if(e.type==="tensor"){if(typeof e.name!="string"||e.name.length===0)throw new Error("WeightPlan tensor spec requires a name");return{required:!0,...e}}if(e.type==="group"){if(!Array.isArray(e.names)||e.names.length===0)throw new Error("WeightPlan group spec requires a non-empty names array");for(let t of e.names)if(typeof t!="string"||t.length===0)throw new Error("WeightPlan group tensor names must be non-empty strings");return{required:!0,...e}}if(e.type==="derived"){if(typeof e.name!="string"||e.name.length===0)throw new Error("Derived weight task requires a name");if(!Array.isArray(e.dependencies)||e.dependencies.length===0)throw new Error(`Derived weight task ${e.name} requires non-empty dependencies`);for(let t of e.dependencies)if(typeof t!="string"||t.length===0)throw new Error(`Derived weight task ${e.name} dependencies must be non-empty strings`);if(typeof e.start!="function")throw new Error(`Derived weight task ${e.name} requires start()`);return{...e,type:"derived"}}throw new Error(`Unknown WeightPlan spec type: ${e.type}`)}function mk(e,t){if(!t||typeof t!="object"||typeof t.commit!="function"||typeof t.rollback!="function")throw new Error(`Derived weight task ${e} returned an invalid transaction`)}async function Tg(e){let t=e.transaction;e.transaction=null,t&&await t.rollback()}async function vg(e,t){try{await Tg(e)}catch{}}function Bi(e){return e.rt?.metadataOnly===!0}async function hk(e,t,r){let n=e.transform&&!Bi(r)?await e.transform(t,r):t,s=e.upload?await e.upload(n,r):n;e.run&&await e.run(s,r),await e.assign?.(s,r),await e.cleanup?.(s,r)}async function vk(e,t,r){let n=!1,s;try{s=await t.commit(),n=!0,e.run&&await e.run(s,r),await e.assign?.(s,r),await e.cleanup?.(s,r)}catch(a){if(n)try{await t.rollback(s)}catch{}else try{await t.abort()}catch{}throw a}}async function wk(e,t,r){let n=e.transform&&!Bi(r)?await e.transform(t,r):t,s=e.upload?await e.upload(n,r):n;e.run&&await e.run(s,r),await e.assign?.(s,r),await e.cleanup?.(s,r)}function wg(e,t){return e.has(t)?e.describe(t):null}function bg(e,t,r=e.type==="tensor"?e.name:"group"){if(e.sourceDtype&&t.dtype!==e.sourceDtype)throw new Error(`${r} has dtype ${t.dtype}; expected ${e.sourceDtype}`);if(e.shape){let n=bk(e.shape,t.elementCount,t);if(e.reshape){let s=n.reduce((a,i)=>a*i,1);if(s!==t.elementCount)throw new Error(`${r} has ${t.elementCount} elements; expected ${s} for reshape [${n}]`)}else if(n.length!==t.shape.length||n.some((s,a)=>s!==t.shape[a]))throw new Error(`${r} has shape [${t.shape}]; expected [${n}]`)}}function _g(e,t,r=e.type==="tensor"?[e.name]:[...e.names]){if(e.required===!1){e.missing?.(r,t);return}throw new Error(`Missing required ${zt(r.length,"tensor")}: ${r.join(", ")}`)}function bk(e,t,r){return typeof e=="function"?e({valueLength:t,description:r}):Array.isArray(e)?e:r.shape?r.shape:[t]}function xr(e){return Math.max(0,Math.floor(Math.min(e.maxBufferSize,e.maxStorageBufferBindingSize)))}function Tr({byteLimit:e,maxBlocks:t,bytesForBlocks:r}){if(!Number.isFinite(e)||e<0)throw new Error(`pack shard byte limit must be a finite non-negative number, got ${e}`);if(!Number.isSafeInteger(t)||t<0)throw new Error(`pack shard block count must be a non-negative safe integer, got ${t}`);if(r(t)<=e)return t;let n=0,s=t;for(;s-n>1;){let a=n+Math.floor((s-n)/2);r(a)<=e?n=a:s=a}return n}function Cr(e,t,r){if(!Number.isSafeInteger(t)||t<0)throw new Error(`${r} shard capacity must be a non-negative safe integer, got ${t}`);let n=[],s=new Map,a=new Set;for(let c of e){if(a.has(c.id))throw new Error(`Duplicate ${r} packed weight id: ${c.id}`);if(a.add(c.id),!Number.isSafeInteger(c.blockCount)||c.blockCount<=0)throw new Error(`${r} packed weight ${c.id} has invalid block count ${c.blockCount}`);let d=c.unitKey===void 0?`id:${c.id}`:`unit:${c.unitKey}`,p=s.get(d);if(p||(p={entries:[],blockCount:0},s.set(d,p),n.push(p)),p.entries.push(c),p.blockCount+=c.blockCount,!Number.isSafeInteger(p.blockCount))throw new Error(`${r} allocation unit containing ${c.id} exceeds safe integer range`)}let i=[],u=new Map,o=0,l=()=>{o!==0&&(i.push(o),o=0)};for(let c=0;c<n.length;++c){let d=n[c];o>0&&o+d.blockCount>t&&l();let p=i.length;for(let f of d.entries)u.set(f.id,{shardIndex:p,unitIndex:c,blockOffset:o,blockCount:f.blockCount}),o+=f.blockCount;o>t&&l()}return l(),{shardBlockCounts:i,placements:u}}var qg="top";function Cg(e){let t=/^layers\.(\d+)\./.exec(e);return t?`layers.${t[1]}`:qg}function _k(e){return e==="top"?qg:`layers.${e}`}function Ii({entries:e,elementBytes:t,limits:r}){if(!Number.isSafeInteger(t)||t<=0)throw new Error(`dense weight element size must be a positive integer, got ${t}`);let n=[],s=[],a=0;for(let p of e){if(!Number.isSafeInteger(p.length)||p.length<0)throw new Error(`dense weight ${p.id} has invalid element count ${p.length}`);if(p.length===0){s.push(p.id);continue}n.push({id:p.id,blockCount:p.length,unitKey:Cg(p.id)}),a+=p.length}let i=xr(r),u=i>0?Tr({byteLimit:i,maxBlocks:a,bytesForBlocks:p=>p*t}):a,o=Cr(n,u,"dense weights"),l=o.shardBlockCounts.length>0?[...o.shardBlockCounts]:[0],c=new Map,d=new Map;for(let p of n){let f=o.placements.get(p.id);if(!f)throw new Error(`dense weight ${p.id} is missing from its own shard plan`);c.set(p.id,{shardIndex:f.shardIndex,elementOffset:f.blockOffset}),d.set(p.unitKey,f.shardIndex)}for(let p of s){let f=Cg(p),g=d.get(f)??0;d.set(f,g),c.set(p,{shardIndex:g,elementOffset:0})}return{shardElementCounts:l,placements:c,shardIndexByUnit:d}}function Bg(e,t){let r=e.denseShards;if(!r||r.length<=1)return 0;let n=_k(t),s=e.denseShardIndex?.[n];if(s===void 0)throw new Error(`no dense weight shard holds ${n}; the loader and the emit disagree on the dense layout`);if(s<0||s>=r.length)throw new Error(`dense weight shard ${s} for ${n} is outside the ${r.length}-shard plan`);return s}function Cn(e,t){let r=e.find(n=>n.id==="lm_head");return r?!t(r.name):!1}function Eg({runtime:e,entries:t,stateDict:r,tieWordEmbeddings:n=!1,modelName:s="model"}){let a=Dt(),i=[],u=0;for(let l of t){let c=r[l.name];if(c===void 0&&l.id==="lm_head"&&n){if(a.top.embed_tokens===void 0)throw new Error(`${s}: tied lm_head requires embed_tokens to come first in entries`);ut(a,l.id,a.top.embed_tokens);continue}if(c===void 0)throw new Error(`Missing ${s} state dict weight: ${l.name}`);let d=kk(c,l.name);if(l.transform&&(d=l.transform(d,l.shape)),d.length!==l.length)throw new Error(`Weight ${l.name} length ${d.length} does not match expected ${l.length}`);ut(a,l.id,u),i.push(d),u+=d.length}let o=Sk(i,u,Float32Array);return{weights:e.tensorFromTypedArray("float32",[o.length],o),offsets:a}}function wl(e=8*1024*1024){let t=0;return r=>(t+=r,t<e?Promise.resolve():(t=0,vi()))}async function bl({runtime:e,config:t,entries:r,reader:n,source:s=null,canUseFloat16:a=or,canUseQ8LmHead:i=_l,canUseQ4LmHead:u=ys,modelName:o="model",onProgress:l=null,signal:c,skipDenseIds:d}){let p=e.device.features.has("shader-f16")&&a(t),f=p?"float16":"float32",g=p?2:4,m=Cn(r,B=>n.has(B)),h=Dt(),v=[],b=[],_=new Set,C=null;for(let B of r){if(d?.has(B.id))continue;if(B.id==="lm_head"&&m){if(!_.has("embed_tokens"))throw new Error(`${o}: tied lm_head requires embed_tokens to come first in entries`);C=B,v.push({entry:B,tied:!0});continue}let U=n.describe(B.name);if(di(U.dtype),U.elementCount!==B.length)throw new Error(`${o} tensor ${B.name} length ${U.elementCount} does not match expected ${B.length}`);b.push({id:B.id,length:B.length}),_.add(B.id),v.push({entry:B,tied:!1,sourceDtype:U.dtype,sourceShape:U.shape,elementOffset:0,shardIndex:0})}let y=Ii({entries:b,elementBytes:g,limits:e.device.limits});for(let B of v){if(B.tied)continue;let U=y.placements.get(B.entry.id);if(!U)throw new Error(`${o}: dense weight ${B.entry.id} has no place in the shard plan`);B.elementOffset=U.elementOffset,B.shardIndex=U.shardIndex,ut(h,B.entry.id,U.elementOffset)}C&&ut(h,C.id,h.top.embed_tokens);let E=y.shardElementCounts.length>1,k=y.shardElementCounts.map((B,U)=>e.allocateWeightsBuffer({byteLength:B*g,dtype:f,shape:[B],label:E?`${o}-weights.${U}`:`${o}-weights`})),S=k[0],A=k.map(B=>new As({runtime:e,tensor:B})),w=p&&u(t),I=p&&!w&&i(t),z=w||I,O=null,j=null,F=v.filter(B=>!B.tied),K=0,Z=wl(),Q=[];for(let B of F)Q.push(yg({name:B.entry.name,sourceDtype:B.sourceDtype,run:async U=>{let M=B.sourceDtype,G=U;if(B.entry.transform){let $=cs(B.sourceDtype,U),X=B.entry.transform($,B.sourceShape);M="F32",G=new Uint8Array(X.buffer,X.byteOffset,X.byteLength)}if(A[B.shardIndex].writeTensor({sourceDtype:M,targetDtype:f,sourceBytes:G,targetElementOffset:B.elementOffset}),z&&(B.entry.id==="embed_tokens"||B.entry.id==="lm_head")){let $=cs(B.sourceDtype,U);B.entry.id==="embed_tokens"?O=$:j=$}K+=1,l?.({processed:K,total:F.length,name:B.entry.name}),await Z(U.byteLength)}}));try{await Sg(Q,{reader:n,source:s,signal:c})}catch(B){throw await Kt(e,k),B}for(let B of v)B.tied&&z&&B.entry.id==="lm_head"&&(j=O);z&&j&&await Z(1/0);let N=w&&j?Ak(j,t.vocab_size,t.hidden_size):null,q=I&&j?yk(j,t.vocab_size,t.hidden_size):null,x=e.device.features.has("shader-f16");return{weights:S,offsets:h,denseShards:k,denseShardIndex:Object.fromEntries(y.shardIndexByUnit),lmHeadQ4:N?e.tensorFromTypedArray("uint32",[N.bits.length],N.bits):null,lmHeadQ4Scales:N?x?e.tensorFromTypedArray("float16",[N.scales.length],Hr(N.scales)):e.tensorFromTypedArray("float32",[N.scales.length],N.scales):null,lmHeadQ8:q?e.tensorFromTypedArray("uint32",[q.packed.length],q.packed):null,lmHeadQ8Scales:q?e.tensorFromTypedArray("float32",[q.scales.length],q.scales):null,packs:{},cacheDtype:f}}function kk(e,t){if(e instanceof Float32Array)return e;if(xk(e))return Float32Array.from(e);if(Array.isArray(e))return Float32Array.from(e.flat(1/0));throw new Error(`Weight ${t} must be a Float32Array or array`)}function Dt(){return{top:{},layers:[]}}function ut(e,t,r){let n=/^layers\.(\d+)\.(.+)$/.exec(t);if(!n){e.top[t]=r;return}let s=Number(n[1]);(e.layers[s]??={})[n[2]]=r}function qr(e,t){let r=/^layers\.(\d+)\.(.+)$/.exec(t);return r?e.layers[Number(r[1])]?.[r[2]]:e.top[t]}function or(e){return!e.prismHadamard&&e.hidden_size%4===0&&e.intermediate_size%4===0}function _l(e){return!e.prismHadamard&&e.hidden_size%4===0}function ys(e){return!e.prismHadamard&&e.hidden_size%32===0}function Ak(e,t,r){let n=r/32,s=t*n,a=new Uint32Array(s*4),i=new Float32Array(s);for(let u=0;u<t;++u)for(let o=0;o<n;++o){let l=u*r+o*32,c=0,d=0;for(let m=0;m<32;++m){let h=e[l+m];Math.abs(h)>c&&(c=Math.abs(h),d=h)}let p=d/-8,f=p!==0?1/p:0,g=u*n+o;i[g]=p;for(let m=0;m<4;++m){let h=0;for(let v=0;v<4;++v){let b=m*4+v,_=Ig(Math.round(e[l+b]*f)+8),C=Ig(Math.round(e[l+b+16]*f)+8);h|=((_|C<<4)&255)<<v*8}a[g*4+m]=h>>>0}}return{bits:a,scales:i}}function Ig(e){return e<0?0:e>15?15:e}function yk(e,t,r){let n=r/4,s=new Uint32Array(t*n),a=new Float32Array(t);for(let i=0;i<t;++i){let u=i*r,o=0;for(let c=0;c<r;++c)o=Math.max(o,Math.abs(e[u+c]));let l=o>0?o/127:1;a[i]=l;for(let c=0;c<n;++c){let d=u+c*4,p=Ei(e[d]/l),f=Ei(e[d+1]/l),g=Ei(e[d+2]/l),m=Ei(e[d+3]/l);s[i*n+c]=p&255|(f&255)<<8|(g&255)<<16|(m&255)<<24}}return{packed:s,scales:a}}function Pg(e,t,r){let n=null;return e.tensorByteRange?async(s,a)=>e.tensorByteRange(t,s*r,(s+a)*r):async(s,a)=>(n??=await e.tensorBytes(t),n.subarray(s*r,(s+a)*r))}function Sk(e,t,r){let n=new r(t),s=0;for(let a of e)n.set(a,s),s+=a.length;return n}function Ei(e){return Math.max(-127,Math.min(127,Math.round(e)))}function xk(e){return ArrayBuffer.isView(e)&&!(e instanceof DataView)&&"length"in e}var mC=BigInt(1e8),hC=new TextDecoder("utf-8",{fatal:!0});function Gg(e,t,r){if(!Number.isInteger(t)||t<0)throw new Error(`cache advance expects a non-negative integer, got ${t}`);let n=e+t;if(n>r)throw new Error(`cache advance overflows maxLength ${r}: ${e} + ${t}`);return n}function Dg(e,t,r){if(!Number.isInteger(t)||t<0||t>r)throw new Error(`cache truncate expects an integer in [0, ${r}], got ${t}`);if(t>e)throw new Error(`cache truncate cannot extend length from ${e} to ${t}; use advance()`);return t}var Lg=e=>e.gridHeight*e.gridWidth,Pi=e=>e.start+Lg(e),Ck=e=>Lg(e)-Math.max(e.gridHeight,e.gridWidth),Ri=class{#e=[];get spans(){return this.#e}get isEmpty(){return this.#e.length===0}addImageSpan(t){for(let[n,s]of[["start",t.start],["gridHeight",t.gridHeight],["gridWidth",t.gridWidth]])if(!Number.isSafeInteger(s)||s<(n==="start"?0:1))throw new Error(`rope image span ${n} must be a ${n==="start"?"non-negative":"positive"} integer, got ${s}`);let r=this.#e.at(-1);if(r&&t.start<Pi(r))throw new Error(`rope image span at row ${t.start} overlaps the span ending at row ${Pi(r)}`);this.#e.push({start:t.start,gridHeight:t.gridHeight,gridWidth:t.gridWidth})}spanAt(t){for(let r of this.#e){if(t<r.start)return null;if(t<Pi(r))return r}return null}delta(t){let r=0;for(let n of this.#e){if(Pi(n)>t)break;r-=Ck(n)}return r}fill(t,r,n){if(n.length<3*r)throw new Error(`rope positions need ${3*r} slots, got ${n.length}`);for(let s=0;s<r;++s){let a=t+s,i=this.spanAt(a);if(i===null){let l=a+this.delta(a);n[s]=l,n[r+s]=l,n[2*r+s]=l;continue}let u=i.start+this.delta(i.start),o=a-i.start;n[s]=u,n[r+s]=u+Math.floor(o/i.gridWidth),n[2*r+s]=u+o%i.gridWidth}return n}splits(t){let r=this.spanAt(t);return r!==null&&r.start!==t}boundaryAtMost(t){let r=this.spanAt(t);return r===null?t:r.start}truncate(t){if(this.splits(t))throw new Error(`cannot truncate rope positions to ${t}: it splits the image span starting at row ${this.spanAt(t).start}`);this.#e=this.#e.filter(r=>r.start<t)}clear(){this.#e=[]}};var Gi=256,kl=16*1024*1024,Og=4;function qk(e,t){for(;t;)[e,t]=[t,e%t];return e}function Sl(e,t,r,n){let s=t*r*n;if(s<=0)return e;let a=Gi/qk(s,Gi);return Ze(e,a)}function Li(e,t){return Ze(e*t,Gi/4)}function xl(e,t){return e.num_hidden_layers*t*Li(e.hidden_size,Fe(e.conv_L_cache,"config.conv_L_cache"))}var Br=class e{runtime;config;batchSize;maxLength;seqLength;kvLayers;keyStates;valueStates;convStates;linearConvStates;linearRecurrentStates;mutableStateCheckpoint;alternateMutableStateCheckpoint;linearConvStatesBacking=null;linearRecurrentStatesBacking=null;kvSlotByLayer;standaloneStateBuffersPadded=!1;mutableStateCheckpointStorage=null;mutableStateCheckpointPromise=null;alternateCheckpointStorage=null;alternateCheckpointPromise=null;rewindPoints=[];rewindPointLimit=1;rewindPointClock=0;ropePositions=new Ri;constructor({runtime:t,config:r,batchSize:n,maxLength:s,keyStates:a,valueStates:i,convStates:u=null,linearConvStates:o=null,linearRecurrentStates:l=null,seqLength:c=0}){this.runtime=t,this.config=ks(r),this.batchSize=n,this.maxLength=s,this.seqLength=c;let d=[...Zr(this.config)];if(a.shape[0]!==d.length)throw new Error(`KV cache backing has ${a.shape[0]} packed layer(s) but the config derives ${d.length} KV layer(s)`);this.kvSlotByLayer=new Int32Array(this.config.num_hidden_layers),this.kvSlotByLayer.fill(-1);for(let p=0;p<d.length;++p){let f=d[p];if(!Number.isInteger(f)||f<0||f>=this.config.num_hidden_layers)throw new Error(`KV cache layer ${f} is outside [0, ${this.config.num_hidden_layers})`);if(this.kvSlotByLayer[f]!==-1)throw new Error(`KV cache layer ${f} is duplicated`);this.kvSlotByLayer[f]=p}this.kvLayers=Object.freeze(d),this.keyStates=a,this.valueStates=i,this.convStates=u,this.linearConvStates=o,this.linearRecurrentStates=l,Ss(this)&&(this.mutableStateCheckpoint=()=>this.mutableStateCheckpointPromise??=this.buildMutableStateCheckpoint(),this.alternateMutableStateCheckpoint=()=>this.alternateCheckpointPromise??=this.buildAlternateMutableStateCheckpoint())}static allocate(t,r,n,s,a="float32"){let i=ks(r),o=[Zr(i).length,n,Sl(s??i.max_position_embeddings,i.num_key_value_heads,i.head_dim,ve(je(a))),i.num_key_value_heads,i.head_dim],l=Bk(t,i,n),c=new e({runtime:t,config:i,batchSize:n,maxLength:o[2],keyStates:t.empty(je(a),o,"llama-cache-keys"),valueStates:t.empty(je(a),o,"llama-cache-values"),convStates:Gt(i)?t.empty("float32",[xl(i,n)],"llama-cache-conv"):null,linearConvStates:l.linearConvStates,linearRecurrentStates:l.linearRecurrentStates});return c.linearConvStatesBacking=l.linearConvStatesBacking,c.linearRecurrentStatesBacking=l.linearRecurrentStatesBacking,c.standaloneStateBuffersPadded=!0,c.clearMutableState(),c}kvSlotForLayer(t){if(!Number.isInteger(t)||t<0||t>=this.config.num_hidden_layers)throw new Error(`KV cache layer ${t} is outside [0, ${this.config.num_hidden_layers})`);let r=this.kvSlotByLayer[t];if(r<0)throw new Error(`logical layer ${t} does not own KV cache state`);return r}keyStateForLayer(t){return this.kvStateForLayer(this.keyStates,t)}valueStateForLayer(t){return this.kvStateForLayer(this.valueStates,t)}convStateForLayer(t){if(!this.convStates)throw new Error("cache has no conv state");let r=this.config.hidden_size,n=Fe(this.config.conv_L_cache,"config.conv_L_cache"),s=Li(r,n)*ve(this.convStates.dtype);return Ts(this.convStates,[r*n],t*s)}kvStateForLayer(t,r){let n=this.kvSlotForLayer(r),s=this.batchSize*this.maxLength*this.config.num_key_value_heads*this.config.head_dim;return Ts(t,[s],n*s*ve(t.dtype))}get_seq_length(){return this.seqLength}advance(t){this.seqLength=Gg(this.seqLength,t,this.maxLength)}async buildMutableStateCheckpoint(){let t=await this.allocateCheckpointSlot();return this.mutableStateCheckpointStorage=t.storage,t.checkpoint}async buildAlternateMutableStateCheckpoint(){let t=await this.allocateCheckpointSlot();return this.alternateCheckpointStorage=t.storage,t.checkpoint}async allocateCheckpointSlot(){let t=this.mutableStateSource(),r=Pk(this.runtime,t);try{let n=await this.runtime.prepareProgramSequence(zg(t,r,"capture")),s=await this.runtime.prepareProgramSequence(zg(r,t,"restore"));return{storage:r,checkpoint:{capture:()=>this.runtime.enqueuePreparedProgramSequence(n),restore:()=>this.runtime.enqueuePreparedProgramSequence(s)}}}catch(n){throw xs(r),n}}async captureRewindPoint(){if(!Ss(this))return;let t=this.rewindPointAt(this.seqLength);if(t){t.lastUsed=++this.rewindPointClock;return}let r=this.rewindPoints.find(n=>n.length<0);!r&&this.rewindPoints.length<this.rewindPointLimit&&(r={...await this.allocateCheckpointSlot(),length:-1,lastUsed:0},this.rewindPoints.push(r)),r??=this.rewindPointToEvict(),r.checkpoint.capture(),r.length=this.seqLength,r.lastUsed=++this.rewindPointClock}get rewindPointCapacity(){return this.rewindPointLimit}setRewindPointCapacity(t){if(!Number.isInteger(t)||t<1)throw new Error(`rewind point capacity must be a positive integer, got ${t}`);for(this.rewindPointLimit=t;this.rewindPoints.length>t;){let r=this.rewindPoints.find(n=>n.length<0)??this.rewindPointToEvict();this.rewindPoints.splice(this.rewindPoints.indexOf(r),1),xs(r.storage)}}rewindPointAt(t){return this.rewindPoints.find(r=>r.length===t)}rewindPointToEvict(){let t=this.rewindPoints.filter(s=>s.length>=0);if(t.length===0)return;let r=t.reduce((s,a)=>a.length<s.length?a:s);return(t.length>1?t.filter(s=>s!==r):t).reduce((s,a)=>a.lastUsed<s.lastUsed?a:s)}dropRewindPoints(t){for(let r of this.rewindPoints)r.length>t&&(r.length=-1)}prefixSnapshotPlan(t){if(this.batchSize!==1||!Number.isInteger(t)||t<=0||t>this.maxLength)return null;let r=this.config.num_key_value_heads*this.config.head_dim,n=ve(je(this.keyStates.dtype)),s=t*r*n,a=this.maxLength*r*n;if(s%4!==0)return null;let i=[];for(let u of[this.keyStates,this.valueStates]){if((u.byteOffset??0)!==0)return null;for(let o=0;o<this.kvLayers.length;++o)i.push({base:u,byteOffset:o*a,byteLength:s})}for(let u of Fg(this.mutableStateSource())){let o=u.dst,c=(u.dstOffset??0)-(o.byteOffset??0);if(c<0||c+u.byteLength>o.byteLength)return null;i.push({base:o,byteOffset:c,byteLength:u.byteLength})}return i}prefixSnapshotChunks(t){let r=this.prefixSnapshotPlan(t);if(!r)return null;let n=[];for(let s of r)for(let a=0;a<s.byteLength;a+=kl)n.push({base:s.base,byteOffset:s.byteOffset+a,byteLength:Math.min(kl,s.byteLength-a)});return n}prefixSnapshotShapes(){let t=[];for(let s of[this.keyStates,this.valueStates]){let[a,i,,u,o]=s.shape;t.push(`${[a,i,u,o].join("x")}@${s.dtype}`)}let r=[this.convStates,...this.linearConvStates??[],...this.linearRecurrentStates??[]];for(let s of r)s&&t.push(`${s.shape.join("x")}@${s.dtype}`);let n=[];for(let s of t){let a=n[n.length-1];a&&a.shape===s?a.count+=1:n.push({shape:s,count:1})}return n.map(({shape:s,count:a})=>a>1?`${a}x[${s}]`:`[${s}]`).join(",")}prefixSnapshotLayout(t){let r=this.prefixSnapshotChunks(t);if(!r)return null;let n=r.map(s=>`${s.byteLength}@${s.base.dtype}`).join(",");return`v1|len=${t}|chunk=${kl}|shapes=${this.prefixSnapshotShapes()}|bytes=${n}`}async exportPrefixSnapshot(){let t=this.seqLength,r=this.prefixSnapshotLayout(t),n=this.prefixSnapshotChunks(t);if(!r||!n||Ss(this)&&!this.rewindPointAt(t)||!this.ropePositions.isEmpty)return null;let s=[];for(let a=0;a<n.length;a+=Og){let i=n.slice(a,a+Og).map(u=>this.runtime.readTensor(Ek(u)));for(let u of i)s.push(Ik(Kn(await u)))}return{length:t,layout:r,chunks:s}}async importPrefixSnapshot(t){let r=this.prefixSnapshotChunks(t.length),n=this.prefixSnapshotLayout(t.length);if(!r||!n||n!==t.layout||r.length!==t.chunks.length)return!1;for(let s=0;s<r.length;++s)if(r[s].byteLength!==t.chunks[s].byteLength)return!1;for(let s=0;s<r.length;++s)this.runtime.writeWeightsRange(r[s].base,r[s].byteOffset,t.chunks[s]);return this.dropRewindPoints(-1),this.seqLength=t.length,this.ropePositions.clear(),await this.captureRewindPoint(),!0}rewindableLengthAtMost(t){if(t<=0)return 0;if(!Ss(this)||t>=this.seqLength)return this.ropePositions.boundaryAtMost(Math.min(t,this.seqLength));let r=0;for(let n of this.rewindPoints)n.length>r&&n.length<=t&&!this.ropePositions.splits(n.length)&&(r=n.length);return r}mutableStateSource(){return{convStates:this.convStates,linearConvStates:this.linearConvStates,linearRecurrentStates:this.linearRecurrentStates,linearConvStatesBacking:this.linearConvStatesBacking,linearRecurrentStatesBacking:this.linearRecurrentStatesBacking,standaloneStateBuffersPadded:this.standaloneStateBuffersPadded}}clearMutableState(){let t=Fg(this.mutableStateSource());t.length>0&&this.runtime.clearBuffers(t)}canTruncateTo(t){return t!==this.seqLength&&this.ropePositions.splits(t)?!1:t===0||t===this.seqLength||!Ss(this)||this.rewindPointAt(t)!==void 0}truncate(t){if(Dg(this.seqLength,t,this.maxLength),!this.canTruncateTo(t))throw t!==this.seqLength&&this.ropePositions.splits(t)?new Error(`cache cannot rewind from ${this.seqLength} to ${t}: it would split a cached image span`):new Error(`cache with mutable conv/recurrent state cannot rewind from ${this.seqLength} to ${t}; reset to 0 instead`);if(this.ropePositions.truncate(t),t===0)this.clearMutableState(),this.dropRewindPoints(-1);else if(t!==this.seqLength){let r=this.rewindPointAt(t);r&&(r.checkpoint.restore(),r.lastUsed=++this.rewindPointClock),this.dropRewindPoints(t)}this.seqLength=t}dispose(){xs(this.mutableStateCheckpointStorage),xs(this.alternateCheckpointStorage);for(let t of this.rewindPoints.splice(0))xs(t.storage);this.ropePositions.clear(),Pe(this.keyStates),Pe(this.valueStates),Pe(this.convStates),Di(this.linearConvStates,this.linearConvStatesBacking),Di(this.linearRecurrentStates,this.linearRecurrentStatesBacking),this.convStates=null,this.linearConvStates=null,this.linearRecurrentStates=null,this.linearConvStatesBacking=null,this.linearRecurrentStatesBacking=null,this.standaloneStateBuffersPadded=!1,this.mutableStateCheckpointStorage=null,this.mutableStateCheckpointPromise=null,this.alternateCheckpointStorage=null,this.alternateCheckpointPromise=null}};function Bk(e,t,r){if(!dt(t))return{linearConvStates:null,linearRecurrentStates:null,linearConvStatesBacking:null,linearRecurrentStatesBacking:null};let n=yi(t),s=Fe(t.linear_conv_dim,"config.linear_conv_dim"),a=Fe(t.linear_conv_kernel_dim,"config.linear_conv_kernel_dim")-1,i=[r,Fe(t.linear_num_value_heads,"config.linear_num_value_heads"),Fe(t.linear_key_head_dim,"config.linear_key_head_dim"),Fe(t.linear_value_head_dim,"config.linear_value_head_dim")],u=Mg(e,n,"float32",[r,s,a],"llama-cache-linear-conv"),o=e.device.features.has("shader-f16")&&!1,l=Mg(e,n,o?"float16":"float32",i,"llama-cache-linear-recurrent");return{linearConvStates:u.states,linearRecurrentStates:l.states,linearConvStatesBacking:u.backing,linearRecurrentStatesBacking:l.backing}}function Mg(e,t,r,n,s){let a=t.reduce((m,h)=>m+(h==="linear_attention"?1:0),0);if(a===0)return{states:Array(t.length).fill(null),backing:null};let i=n.reduce((m,h)=>m*h,1),u=ve(je(r)),o=i*u,l=Ze(o,Gi),c=a*l,d=e.device.limits.maxBufferSize;if(Number.isFinite(d)&&d>0&&c>d)return{states:t.map((h,v)=>h==="linear_attention"?e.empty(je(r),n,`${s}-${v}`):null),backing:null};let p=e.empty(je(r),[c/u],s),f=0;return{states:t.map(m=>{if(m!=="linear_attention")return null;let h=Ts(p,n,f*l);return f+=1,h}),backing:p}}function Ts(e,t,r){let n=t.reduce((a,i)=>a*i,1),s=n*ve(e.dtype);if(r<0||r+s>e.byteLength)throw new Error("mutable-state tensor view exceeds its backing buffer");return{dtype:e.dtype,shape:[...t],byteOffset:(e.byteOffset??0)+r,size:n,byteLength:s,buffer:e.buffer,runtime:e.runtime,destroy:Or}}function Ik(e){let t=e;if(!t.buffer||t.byteLength===void 0)throw new Error("a prefix snapshot needs the bytes behind readTensor's result, but it returned no buffer");return new Uint8Array(t.buffer,t.byteOffset??0,t.byteLength)}function Ek(e){let t=ve(je(e.base.dtype));return Ts(e.base,[e.byteLength/t],e.byteOffset)}function Fg(e){let t=[];return Al(t,e.convStates,e.standaloneStateBuffersPadded),Qg(t,e.linearConvStates,e.linearConvStatesBacking,e.standaloneStateBuffersPadded),Qg(t,e.linearRecurrentStates,e.linearRecurrentStatesBacking,e.standaloneStateBuffersPadded),t}function Qg(e,t,r,n){if(r){Al(e,r,!0);return}if(Array.isArray(t))for(let s of t)Al(e,s,n)}function Al(e,t,r){t&&e.push({dst:t,dstOffset:t.byteOffset??0,byteLength:Ug(t,r,"clear")})}function Ug(e,t,r){if(e.byteLength%4===0)return e.byteLength;if(!t)throw new Error(`cannot ${r} ${e.byteLength}-byte mutable-state tensor safely: its backing allocation is not known to include 4-byte padding`);return Ze(e.byteLength,4)}function Ss(e){return e.convStates!==null||jg(e.linearConvStates)||jg(e.linearRecurrentStates)}function jg(e){return Array.isArray(e)&&e.some(t=>t!==null)}function Pk(e,t){let r=Wg(e,t.linearConvStates,t.linearConvStatesBacking,"llama-cache-checkpoint-linear-conv"),n=Wg(e,t.linearRecurrentStates,t.linearRecurrentStatesBacking,"llama-cache-checkpoint-linear-recurrent");return{convStates:Ng(e,t.convStates,"llama-cache-checkpoint-conv"),linearConvStates:r.states,linearRecurrentStates:n.states,linearConvStatesBacking:r.backing,linearRecurrentStatesBacking:n.backing,standaloneStateBuffersPadded:!0}}function Ng(e,t,r){return t?e.empty(t.dtype,t.shape,r):null}function Wg(e,t,r,n){if(!Array.isArray(t)){if(r!==null)throw new Error("mutable-state backing has no tensor views");return{states:null,backing:null}}if(!r)return{states:t.map((u,o)=>Ng(e,u,`${n}-${o}`)),backing:null};let s=e.empty(r.dtype,r.shape,n),a=r.byteOffset??0;return{states:t.map(u=>{if(!u)return null;if(u.buffer!==r.buffer||u.dtype!==r.dtype)throw new Error("mutable-state tensor views do not match their backing buffer");return Ts(s,u.shape,(u.byteOffset??0)-a)}),backing:s}}function zg(e,t,r){let n=[],s=e.standaloneStateBuffersPadded&&t.standaloneStateBuffersPadded;return yl(n,e.convStates,t.convStates,`llama-cache-${r}-conv`,s),Kg(n,e.linearConvStates,t.linearConvStates,e.linearConvStatesBacking,t.linearConvStatesBacking,`llama-cache-${r}-linear-conv`,s),Kg(n,e.linearRecurrentStates,t.linearRecurrentStates,e.linearRecurrentStatesBacking,t.linearRecurrentStatesBacking,`llama-cache-${r}-linear-recurrent`,s),n}function Kg(e,t,r,n,s,a,i){if(n||s){if(!n||!s)throw new Error("mutable-state checkpoint packed backings do not match");yl(e,n,s,a,!0);return}if(!Array.isArray(t)||!Array.isArray(r)||t.length!==r.length){if(t===null&&r===null)return;throw new Error("mutable-state checkpoint tensor arrays do not match")}for(let u=0;u<t.length;++u)yl(e,t[u],r[u],`${a}-${u}`,i)}function yl(e,t,r,n,s){if(!t||!r){if(t===null&&r===null)return;throw new Error("mutable-state checkpoint tensors do not match")}if(t.dtype!==r.dtype||t.byteLength!==r.byteLength)throw new Error("mutable-state checkpoint tensor shape or dtype changed");let a=Ug(t,s,"copy");e.push({kind:"copy",name:n,src:t,dst:r,srcOffset:t.byteOffset??0,dstOffset:r.byteOffset??0,byteLength:a})}function xs(e){e&&(Pe(e.convStates),Di(e.linearConvStates,e.linearConvStatesBacking),Di(e.linearRecurrentStates,e.linearRecurrentStatesBacking))}function Di(e,t){if(t){Pe(t);return}Pe(e)}var Oi=[64,128,256,512],Hg=512,$g=64;function Vg(e){for(let t of Oi)if(e<=t)return t;return Hg}function Zg(e){return e>0?[{kind:"replay",tokenCount:e}]:[]}function Xg({tokenCount:e,position:t,maxLength:r,minGraphTokens:n}){if(e<=0)return[];if(e<n||t+e>r)return Zg(e);let s=[],a=e,i=t;for(;a>0;){let u=Vg(a);if(i+u>r){s.push({kind:"replay",tokenCount:a});break}let o=Math.min(a,u);s.push({kind:"graph",blockLen:u,tokenCount:o}),a-=o,i+=o}return s}function Yg({tokenCount:e,position:t,maxLength:r,shortBlock:n,normalMinTokens:s,bucketMinTokens:a,bucketMaxTokens:i,bucketBlock:u,fitCache:o=!1}){if(e<=0)return[];if(n<=0||s<=n||a<=n||i<a||i>s||u<i)throw new Error(`invalid bucketed short-prefill geometry: short=${n}, normalMin=${s}, bucket=${a}..${i}->${u}`);if(e<n||t+e>r)return Zg(e);let l=[],c=e,d=t;for(;c>0;){if(c<n){l.push({kind:"replay",tokenCount:c});break}if(c>=a&&c<=i&&d+u<=r){l.push({kind:"graph",blockLen:u,tokenCount:c});break}let p=c<s?n:Vg(c);if(o&&d+p>r&&(p=Math.max(0,...[n,u,...Oi].filter(g=>g<=c&&d+g<=r))),p===0||d+p>r){l.push({kind:"replay",tokenCount:c});break}let f=Math.min(c,p);l.push({kind:"graph",blockLen:p,tokenCount:f}),c-=f,d+=f}return l}function Tl(e){let t=new Set;for(let r of e)r.kind==="graph"&&t.add(r.blockLen);return[...t].sort((r,n)=>r-n)}function Cl(e,t){let r=new Set,n=Math.min(t,2*Hg);for(let s=1;s<=n;++s)for(let a of Tl(e(s,0,t)))r.add(a);return[...r].sort((s,a)=>s-a)}var Jg=new Map;function Xr(e,t={}){let r=Jg.get(e);r||(r=ns(e),Jg.set(e,r));let n=r.manifest.conformance,s=t.conformance;if(n==="internal-lowering"&&s!==n)throw new Error(`${e}: this package is an internal tensor lowering, not the standard operator; graph callers must pass { conformance: "internal-lowering" } after performing the lowering explicitly`);if(s!==void 0&&n!==s)throw new Error(`${e}: graph caller expects conformance ${s}, but the manifest declares ${n??"standard"}`);return r}var Rk=new Set;function em(e){Rk.add(e)}function tm(e){let t=new Map;for(let w of e.tensors)t.set(w.id,w);let r=w=>t.get(w)?.view?.parent??w,n=new Map;for(let w of e.tensors)w.kind==="intermediate"&&!w.view&&n.set(w.id,w);let s=new Map,a=new Map,i=new Map,u=Dk(e,t),o=new Map,l=new Map,c=new Map;e.nodes.forEach((w,I)=>{for(let z of Object.values(w.inputs)){let O=r(z);n.has(O)&&(l.has(O)||l.set(O,I),c.set(O,I))}for(let z of Object.values(w.outputs)){let O=r(z);n.has(O)&&(o.has(O)||o.set(O,I),c.set(O,I))}});let d=w=>u.rootOf.get(w)??w,p=new Map,f=new Map,g=new Map;for(let w of n.values()){let I=d(w.id),z=o.get(w.id),O=c.get(w.id),j=l.get(w.id);if(z!==void 0){let F=p.get(I);(F===void 0||z<F)&&p.set(I,z)}if(O!==void 0){let F=f.get(I);(F===void 0||O>F)&&f.set(I,O)}if(j!==void 0){let F=g.get(I);(F===void 0||j<F)&&g.set(I,j)}}let m=new Set;for(let w of e.tensors){if(!w.view||w.kind==="intermediate")continue;let I=r(w.id);n.has(I)&&m.add(d(I))}for(let[w,I]of u.byRoot){let z=p.get(w),O=z!==void 0&&(g.get(w)??1/0)<z;(I.some(j=>!n.has(j))||z===void 0||O)&&m.add(w)}for(let w of m){let I=u.byRoot.get(w)??[w],z=t.get(I[0]),O=rm(a,Yr(z));i.set(O,z);for(let j of I)s.set(j,O)}let h=[],v=0,b=new Map,_=new Set;for(let w of n.values()){v+=Yr(w);let I=o.get(w.id),z=l.get(w.id);if(I===void 0&&z!==void 0)throw new Error(`graph "${e.name}": intermediate "${w.name}" (id ${w.id}) is read by node ${z} but no node produces it`);if(!u.rootOf.has(w.id)){if(I===void 0){h.push(w.name);continue}if(z!==void 0&&z<I)throw new Error(`graph "${e.name}": intermediate "${w.name}" (id ${w.id}) is read by node ${z} before it is produced by node ${I}; pooling cannot order that lifetime`)}let j=d(w.id);if(m.has(j)||_.has(j))continue;_.add(j);let F=f.get(j),K=b.get(F);K?K.push(j):b.set(F,[j])}let C=[],y=new Map;for(let w=0;w<e.nodes.length;++w){let I=b.get(w-1);if(I)for(let z of I)C.push(y.get(z));for(let z of Object.values(e.nodes[w].outputs)){let O=r(z),j=n.get(O);if(!j||s.has(O))continue;let F=d(O),K=y.get(F);K===void 0&&(K=Gk(C,a,i,j),y.set(F,K));for(let Z of u.byRoot.get(F)??[O])s.set(Z,K)}}let E=0;for(let w of a.values())E+=w;let k=new Map;for(let w of _){let I=p.get(w),z=f.get(w);k.set(I,(k.get(I)??0)+1),k.set(z+1,(k.get(z+1)??0)-1)}let S=0,A=0;for(let w of[...k.keys()].sort((I,z)=>I-z))S+=k.get(w),S>A&&(A=S);return{slots:s,slotByteLengths:a,slotAllocations:i,diagnostics:{intermediateCount:n.size,slotCount:a.size,peakLiveSlots:m.size+A,naiveBytes:v,pooledBytes:E,deadTensors:h}}}function rm(e,t){let r=e.size;return e.set(r,t),r}function Gk(e,t,r,n){let s=Yr(n),a=-1,i=1/0,u=-1,o=-1;for(let d=0;d<e.length;++d){let p=t.get(e[d]);p>=s&&p<i&&(i=p,a=d),p>o&&(o=p,u=d)}let l=a>=0?a:u;if(l<0){let d=rm(t,s);return r.set(d,n),d}let c=e[l];return e.splice(l,1),t.get(c)<s&&(t.set(c,s),r.set(c,n)),c}function Dk(e,t){let r=new Map,n=new Map,s=e.aliases??[];if(s.length===0)return{rootOf:r,byRoot:n};let a=new Map,i=l=>{let c=l;for(;a.get(c)!==c;)c=a.get(c);let d=l;for(;a.get(d)!==c;){let p=a.get(d);a.set(d,c),d=p}return c},u=l=>{a.has(l)||a.set(l,l)};for(let{from:l,to:c}of s){for(let[d,p]of[["from",l],["to",c]]){let f=t.get(p);if(!f)throw new Error(`graph "${e.name}": alias ${d} tensor id ${p} does not exist`);if(f.kind!=="intermediate"&&f.kind!=="output")throw new Error(`graph "${e.name}": alias ${d} tensor "${f.name}" is ${f.kind}; only intermediate/output tensors can share a buffer`)}u(l),u(c),a.set(i(l),i(c))}let o=new Map;for(let l of a.keys()){let c=i(l),d=o.get(c);d?d.push(l):o.set(c,[l])}for(let l of o.values()){let c=t.get(l[0]);for(let p of l){let f=t.get(p);if(f.dtype!==c.dtype)throw new Error(`graph "${e.name}": aliased tensors "${c.name}" (${c.dtype}) and "${f.name}" (${f.dtype}) must be the same dtype to share a buffer`);if(Yr(f)!==Yr(c))throw new Error(`graph "${e.name}": aliased tensors "${c.name}" (${Yr(c)} B) and "${f.name}" (${Yr(f)} B) must be the same byte length to share a buffer`)}let d=i(l[0]);n.set(d,l);for(let p of l)r.set(p,d)}return{rootOf:r,byRoot:n}}function Yr(e){return et(e.dtype,we(e.shape))}function ql(e,t,r){return nm(e,t,r).selected!==null}function nm(e,t,r){try{let n=e.explain({device:t},r);return{selected:n.selected,runnable:n.candidates.filter(s=>s.ok).map(s=>s.id)}}catch(n){return{selected:null,runnable:[],error:n instanceof Error?n.message:String(n)}}}function Bl(e,t){return{...e,features:[...e.features,...t]}}function Lk(e,t,r){let n=new Set(t.features),s=pr.filter(i=>!n.has(i));for(let i of s)if(ql(e,Bl(t,[i]),r))return[i];let a=[];for(let i of s)if(a.push(i),ql(e,Bl(t,a),r))return Ok(e,t,r,a);return[]}function Ok(e,t,r,n){let s=[...n];for(let a of n){let i=s.filter(u=>u!==a);i.length<s.length&&ql(e,Bl(t,i),r)&&(s=i)}return s}function Mi(e){let t=e/1048576;return`${t.toFixed(t<1e4?1:0)} MB`}function El(e,t){let r=ze(t),n=yt(r),s=[],a=[...e.tensors??[]],i=new Set,u=new Set,o=new Set,l=new Set;jd(()=>{for(let{op:p,request:f,id:g}of e.opRequests??[]){let m=g??p.manifest.id;o.add(m);let h=`${m}|${er(f)}`;if(i.has(h))continue;i.add(h);let{selected:v,runnable:b,error:_}=nm(p,n,f);for(let k of b)l.add(`${m}#${k}`);if(v!==null)continue;if(_!==void 0){let k=`${m}|fault`;if(u.has(k))continue;u.add(k),s.push({kind:"feature",op:m,missingFeatures:[],message:`${m} could not be evaluated for this device: ${_}`});continue}let C=Lk(p,n,f),y=`${m}|${C.join(",")}`;if(u.has(y))continue;u.add(y);let E=C.map(k=>`"${k}"`).join(", ");s.push({kind:"feature",op:m,missingFeatures:C,message:C.length>0?`${m} needs WebGPU ${zt(C.length,"feature")} ${E}, which this device lacks`:`${m} has no runnable variant on this device`})}});let c=r.limits.maxBufferSize;if(typeof c=="number"){let p=new Set;for(let f of a){let g=Il(f);g<=c||p.has(g)||(p.add(g),s.push({kind:"buffer",tensor:f.name,bytes:g,limitBytes:c,message:`tensor ${f.name} needs a ${Mi(g)} buffer, over this device's ${Mi(c)} maxBufferSize`}))}}let d=r.limits.maxStorageBufferBindingSize;if(typeof d=="number"){let p=new Set;for(let f of a){if(!f.boundWhole)continue;let g=Il(f);g<=d||p.has(g)||typeof c=="number"&&g>c||(p.add(g),s.push({kind:"binding",tensor:f.name,bytes:g,limitBytes:d,message:`tensor ${f.name} needs a ${Mi(g)} storage binding, over this device's ${Mi(d)} maxStorageBufferBindingSize`}))}}return{ok:s.length===0,problems:s,reason:s.length>0?Fk(s):null,opIds:[...o],selectableVariants:[...l],footprint:Mk(a)}}function Mk(e){let t=e.filter(n=>n.allocation);if(t.length===0)return null;let r=t.filter(n=>!n.released).map(n=>({name:n.name,bytes:Il(n)})).sort((n,s)=>s.bytes-n.bytes);return{totalBytes:r.reduce((n,s)=>n+s.bytes,0),buffers:r}}function Il(e){let t=e.byteLength??et(je(e.dtype),we(e.shape));if(!Number.isSafeInteger(t)||t<0)throw new Error(`support tensor ${e.name} has invalid byteLength ${t}`);let r=nt(t);if(!Number.isSafeInteger(r))throw new Error(`support tensor ${e.name} byte length exceeds safe integer storage`);return r}function Fk(e){let t=[...new Set(e.flatMap(a=>a.missingFeatures??[]))],r=e.filter(a=>a.kind==="buffer"||a.kind==="binding");if(t.length===0&&r.length===0)return`${e[0].message} \u2014 this build has no kernel for the quantization format combination this checkpoint uses. That is a coverage gap in the library, not a limitation of your GPU or browser, so switching either will not help.`;let n=t.length>0?`This GPU/browser is missing WebGPU ${zt(t.length,"feature")} ${t.map(a=>`"${a}"`).join(", ")} that the model needs`:r[0].message,s=t.length>0&&r.length>0?" Some weights also exceed the device's buffer limits.":"";return`${n} \u2014 it can't run here.${s} Try Chrome or Edge with up-to-date GPU drivers.`}function sm(e,t,r){let n=e.recordSupportOp;return n?(n.call(e,t,r),!0):!1}var Pl=new WeakMap;em(()=>{Pl=new WeakMap});var Rl=class{slots=new Map;capacities=null;capacitySlots=new Map;persistent=new Map;rt;constructor(t){this.rt=t}reserve(t){if(this.slots.size>0||this.capacitySlots.size>0)throw new Error("Collector scratch capacity must be planned before materialization");let r=new Map;for(let n of t){let s=new Map;for(let a of n){let i=s.get(a.dtype)??0;s.set(a.dtype,i+1);let u=r.get(a.dtype)??[];u[i]=Math.max(u[i]??0,a.byteLength),r.set(a.dtype,u)}}this.capacities=r}resourcesForPlan(t){if(t.length===0)return{};if(this.capacities){let s=new Map,a={};for(let i of t){let u=s.get(i.dtype)??0;s.set(i.dtype,u+1);let o=this.capacities.get(i.dtype)?.[u];if(o===void 0||o<i.byteLength)throw new Error(`No capacity for scratch ${i.id}`);let l=this.capacitySlots.get(i.dtype);l||(l=[],this.capacitySlots.set(i.dtype,l));let c=l[u];c||(c=this.rt.empty(i.dtype,[o/ve(i.dtype)],`webgpu-scratch-${i.id}`),l[u]=c),a[i.id]={runtime:c.runtime,dtype:i.dtype,shape:i.shape,size:we(i.shape),byteLength:i.byteLength,byteOffset:c.byteOffset??0,buffer:c.buffer,destroy:Or}}return a}let r=new Map,n={};for(let s of t){let a=JSON.stringify([s.dtype,s.shape]),i=r.get(a)??0;r.set(a,i+1);let u=this.slots.get(a);u||(u=[],this.slots.set(a,u));let o=u[i];o===void 0&&(o=this.rt.empty(s.dtype,s.shape,`webgpu-scratch-${s.id}`),u.push(o)),n[s.id]=o}return n}persistentResourcesForPlan(t,r){let n={};for(let s of r){let a=`${t}\0${s.id}`,i=this.persistent.get(a);i===void 0&&(i=this.rt.empty(s.dtype,s.shape,`webgpu-const-scratch-${s.id}`),this.persistent.set(a,i)),n[s.id]=i}return n}clearPersistent(){let t=[];for(let[r,n]of this.persistent)try{n.destroy(),this.persistent.delete(r)}catch(s){t.push(s)}ct(t,"Collector constant-scratch cleanup failed")}clear(){let t=[];for(let[r,n]of this.capacitySlots){for(let s=n.length-1;s>=0;--s){let a=n[s];try{a.destroy(),n.splice(s,1)}catch(i){t.push(i)}}n.length===0&&this.capacitySlots.delete(r)}for(let[r,n]of this.slots){for(let s=n.length-1;s>=0;--s){let a=n[s];try{a.destroy(),n.splice(s,1)}catch(i){t.push(i)}}n.length===0&&this.slots.delete(r)}ct(t,"Collector scratch cleanup failed")}};function Qk(e,t){let r=[e.manifest.id];for(let n of Object.keys(t.resources).sort()){let s=t.resources[n];if(s!==null&&typeof s=="object"&&"shape"in s&&"dtype"in s){let a=s;r.push(`${n}:[${a.shape.join(",")}]:${a.dtype}`)}else s==null?r.push(`${n}:-`):r.push(`${n}=${String(s)}`)}return r.push(`a:${JSON.stringify(t.args??{})}`),r.push(`t:${JSON.stringify(t.attrs??{})}`),r.push(`u:${JSON.stringify(t.tunables??{})}`),r.join("|")}var Fi=class{programs=[];constantPrograms=[];constantProgramsReady=null;cleanups=[];scratchPool;rt;timing;preparedCache;collectMs=0;flushMs=0;constructor(t,r={}){if(this.rt=t,this.scratchPool=new Rl(t),this.timing=r.timing===!0,r.cachePrepared){let n=Pl.get(t);n||(n=new Map,Pl.set(t,n)),this.preparedCache=n}else this.preparedCache=null}reserveScratchCapacities(t){this.scratchPool.reserve(t)}prepare(t,r){if(!this.preparedCache)return t.prepare(this.rt,r);let n=Qk(t,r),s=this.preparedCache.get(n);if(s)return{program:s.program,plan:s.plan,request:r};let a=t.prepare(this.rt,r);return this.preparedCache.set(n,{program:a.program,plan:a.plan}),a}add(t,r,n={}){if(sm(this.rt,t,r))return;let s=this.timing?performance.now():0,a=this.prepare(t,r),i=this.materialize(a,n);return this.pushMaterialized(i),this.cleanups.push(i.cleanup),this.timing&&(this.collectMs+=performance.now()-s),a}get programCount(){return this.programs.length}programSlice(t,r){return this.programs.slice(t,r)}addPrepared(t,r,n,s={}){let a=this.materialize({program:t,plan:r,request:n},s);this.pushMaterialized(a),this.cleanups.push(a.cleanup)}adoptMaterialized(t){for(let r of t)this.programs.push(r)}async buildSteps(){return this.rt.prepareProgramSequence(this.programs)}enqueue(t){this.rt.enqueuePreparedProgramSequence(t)}disposeBuild(){let t=[];try{this.clearBuildResources()}catch(r){t.push(r)}try{this.scratchPool.clearPersistent()}catch(r){t.push(r)}ct(t,"Program collector disposal failed")}async runConstantPrograms(){if(this.constantProgramsReady===null){if(this.constantPrograms.length===0)return;let t=[...this.constantPrograms];this.constantProgramsReady=this.rt.runProgramSequence(t,{wait:!0}).then(()=>{this.constantPrograms.splice(0,t.length),this.constantProgramsReady=null})}await this.constantProgramsReady,this.constantPrograms.length>0&&await this.runConstantPrograms()}async flush(t=!0){if(this.programs.length===0){this.clearBuildResources();return}let r=this.timing?performance.now():0;try{await this.rt.runProgramSequence(this.programs,{wait:t})}finally{this.timing&&(this.flushMs+=performance.now()-r),this.clearBuildResources()}}materialize(t,r){let n=r.constantPasses?.(t);if(n===void 0||n.size===0){let o=this.scratchPool.resourcesForPlan(t.plan.scratches);return Jn(this.rt,t,{scratchResources:o})}let s=Yn(t,n),a=t.plan.scratches.filter(o=>!s.has(o.id)),i=t.plan.scratches.filter(o=>s.has(o.id)),u={...this.scratchPool.resourcesForPlan(a),...this.scratchPool.persistentResourcesForPlan(r.constantScratchKey??"",i)};return Jn(this.rt,t,{scratchResources:u,constantPassIds:n})}pushMaterialized(t){for(let r of t.zeroInitClears)this.programs.push(r);for(let r of t.programs)this.programs.push(r);for(let r of t.constantPrograms??[])this.constantPrograms.push(r)}clearBuildResources(){this.programs=[],this.constantPrograms=[],this.constantProgramsReady=null;let t=[],r=0;for(let n of this.cleanups)try{n()}catch(s){this.cleanups[r++]=n,t.push(s)}this.cleanups.length=r;try{this.scratchPool.clear()}catch(n){t.push(n)}ct(t,"Program collector cleanup failed")}};function Cs(e,t,r){let n=e.tensors.find(a=>a.kind==="uniform"&&a.name===t);if(!n)throw new Error(`graphUniformPacker: graph "${e.name}" has no uniform "${t}"`);let s=[];if(e.nodes.forEach((a,i)=>{for(let[u,o]of Object.entries(a.inputs))o===n.id&&s.push(...jk(a.op,u,r?.[i]??null))}),s.length===0)throw new Error(`graphUniformPacker: no op reads uniform "${t}" in graph "${e.name}"`);return Wk(s,`${e.name}.${t}`,n.shape[0])}var am=new Map;function jk(e,t,r=null){let n=`${e}\0${t}\0${r??""}`,s=am.get(n);if(s)return s;let a=ns(e),i=[];for(let u of a.manifest.variants)if(!(r!==null&&u.id!==r))for(let o of u.passes)for(let l of o.bindings){if(l.buffer?.type!=="uniform"||!l.struct||l.arg!==t&&l.name!==t)continue;let c=l.struct.fields.map(d=>{if(d.type!=="u32"&&d.type!=="i32"&&d.type!=="f32")throw new Error(`${e}#${u.id}: uniform field ${d.name} has non-scalar type ${d.type} \u2014 step packers cover scalar structs only`);return{name:d.name,kind:d.type}});i.push({source:`${e}#${u.id}/${o.id}`,fields:c})}return am.set(n,i),i}function Wk(e,t,r){let n=e[0];for(let f of e)f.fields.length>n.fields.length&&(n=f);for(let f of e)f.fields.forEach((g,m)=>{let h=n.fields[m];if(g.name!==h.name||g.kind!==h.kind)throw new Error(`stepUniformPacker(${t}): ${f.source} declares field ${m} "${g.name}: ${g.kind}" but ${n.source} declares "${h.name}: ${h.kind}" \u2014 ops sharing one override buffer must agree on a common field prefix`)});let s=Math.max(4,Math.ceil(n.fields.length/4)*4);if(r!==void 0&&r!==s)throw new Error(`stepUniformPacker(${t}): g.uniform declares ${r*4} bytes but the struct (${n.fields.map(f=>f.name).join(", ")}) packs to ${s*4}`);let a=new ArrayBuffer(s*4),i=new Uint32Array(a),u=new Int32Array(a),o=new Float32Array(a),l=new Map(n.fields.map((f,g)=>[f.name,{...f,index:g}])),c=n.fields.filter(f=>!f.name.startsWith("_")).length,d=()=>n.fields.map(f=>f.name).join(", ");return Object.assign(f=>{let g=0;for(let m of Object.keys(f)){let h=l.get(m);if(!h)throw new Error(`stepUniformPacker(${t}): unknown field "${m}" (struct: ${d()})`);h.kind==="f32"?o[h.index]=f[m]:h.kind==="i32"?u[h.index]=f[m]:i[h.index]=f[m],g+=1}if(g!==c){let m=n.fields.filter(h=>!h.name.startsWith("_")&&!Object.hasOwn(f,h.name)).map(h=>h.name);throw new Error(`stepUniformPacker(${t}): missing field(s) ${m.join(", ")} (struct: ${d()})`)}return i},{fields:new Set(n.fields.map(f=>f.name))})}var im={passIds:new Set};function Gl(e,t){let r=new Set(e.plan.scratches.map(u=>u.id));if(r.size===0)return im;let n=new Set(e.plan.scratches.filter(u=>u.zeroInit===!0).map(u=>u.id)),s=new Set,a=new Set,i;for(let u=0;u<e.program.passes.length;u+=1){let o=e.program.passes[u];if(e.plan.passes[u].dispatchWorkgroups.some(p=>p===0))continue;if(o.submitAfter||o.viewAlias!==void 0){i=o.id;break}let c=[],d=!0;for(let p of o.bindingSpecs){let f=vr(p,r);if(p.buffer.type==="uniform"){t.get(p.arg??p.name)==="uniform"&&(d=!1);continue}if(p.buffer.type==="storage"){if(f===void 0||n.has(f)){d=!1;break}c.push(f);continue}if(f!==void 0){if(!a.has(f)){d=!1;break}continue}if(t.get(p.arg??p.name)!=="weight"){d=!1;break}}if(!d){i=o.id;break}s.add(o.id);for(let p of c)a.add(p)}if(s.size===0)return i===void 0?im:{passIds:new Set,stoppedBy:i};for(let u of e.program.passes)if(!s.has(u.id))for(let o of u.bindingSpecs){if(o.buffer.type!=="storage")continue;let l=vr(o,r);if(l!==void 0&&a.has(l))return{passIds:new Set,stoppedBy:u.id}}return{passIds:s,...i===void 0?{}:{stoppedBy:i}}}var Ll=new WeakMap,Dl=new Set;function zk(e,t){let r=Object.keys(e);if(r.length!==Object.keys(t).length)return!1;for(let n of r)if(e[n]!==t[n])return!1;return!0}function Kk(e,t){if(!e)return null;let r=Ll.get(e);if(!r)throw new Error(`graph "${t.name}": sibling compile has no reusable plans (support-record compiles cannot seed siblings)`);if(r.graph!==t)throw new Error(`graph "${t.name}": sibling compiled a DIFFERENT graph object ("${r.graph.name}") \u2014 siblings must share one emitted graph`);return r}function qn(e,t,r={},n={}){let s=n.pooling!==!1,a=n.constantInputPasses!==!1,i=Kk(n.sibling,e),u=i?.plan??tm(e),o=s?{plan:u,slotTensors:new Map,shared:n.scratch??null}:null,l=new Map,c=new Set,d=new Map,p=new Map,f=new Map,g=[],m=[];try{for(let w of e.tensors){if(w.view){let z=l.get(w.view.parent);if(!z){if(c.has(w.view.parent)){c.add(w.id);continue}throw new Error(`graph view "${w.name}" references unrealized parent id ${w.view.parent}`)}let O=i&&i.realized.get(w.view.parent)===z?i.realized.get(w.id):$k(z,w);l.set(w.id,O),w.kind!=="intermediate"&&d.set(w.name,O);continue}if(w.kind==="uniform"){let z=i?i.uniformById.get(w.id):t.createUniformU32(new Uint32Array(we(w.shape)),`graph-uniform-${w.name}`);i||m.push(z),p.set(w.id,z),f.set(w.name,z);continue}let I=i&&w.kind!=="weight"&&w.kind!=="state"?i.realized.get(w.id)??null:Hk(w,e.name,t,r,g,o);if(I===null){c.add(w.id);continue}l.set(w.id,I),w.kind!=="intermediate"&&d.set(w.name,I)}}catch(w){Qi(w,null,g,m)}let h;try{h=new Fi(t,{cachePrepared:!0})}catch(w){Qi(w,null,g,m)}let v=new Map(e.tensors.map(w=>[w.id,w])),b=null;if(n.opScratchCapacityPooling&&!t.recordSupportOp)try{b=i?.nodePlans??e.nodes.map(I=>{let z=Xr(I.op,{conformance:I.conformance}),O={};for(let[F,K]of Object.entries(I.inputs))O[F]=l.get(K)??p.get(K);for(let[F,K]of Object.entries(I.outputs))O[F]=l.get(K);let j=h.prepare(z,{resources:O,...I.args?{args:{...I.args}}:{},...I.attrs?{attrs:{...I.attrs}}:{},...I.tunables?{tunables:{...I.tunables}}:{}});return{program:j.program,plan:j.plan}});let w=b.map((I,z)=>{if(!a)return I.plan.scratches;let O=e.nodes[z],j=new Map;for(let[Z,Q]of Object.entries(O.inputs)){let N=v.get(Q)?.kind;N!==void 0&&j.set(Z,N)}let F=Gl({...I,request:{resources:{}}},j).passIds;if(F.size===0)return I.plan.scratches;let K=Yn({...I,request:{resources:{}}},F);return I.plan.scratches.filter(Z=>!K.has(Z.id))});h.reserveScratchCapacities(w)}catch(w){Qi(w,h,g,m)}let _=[],C=[],y=[],E=[],k=[];try{e.nodes.forEach((w,I)=>{let z=Xr(w.op,{conformance:w.conformance}),O={};for(let[x,B]of Object.entries(w.inputs))O[x]=l.get(B)??p.get(B);for(let[x,B]of Object.entries(w.outputs))O[x]=l.get(B);let j={resources:O,...w.args?{args:{...w.args}}:{},...w.attrs?{attrs:{...w.attrs}}:{},...w.tunables?{tunables:{...w.tunables}}:{}},F=h.programCount,K=new Map;for(let[x,B]of Object.entries(w.inputs)){let U=v.get(B)?.kind;U!==void 0&&K.set(x,U)}let Z=Dl,Q={constantPasses:x=>(Z=a?Gl(x,K).passIds:Dl,Z),constantScratchKey:String(I)};try{if(i){let x=i.nodePlans[I];Z=Q.constantPasses({...x,request:j});let B=i.nodeConstantPasses[I]??Dl;Z.size===0&&B.size===0&&zk(i.nodeResources[I],O)?h.adoptMaterialized(i.nodePrograms[I]):h.addPrepared(x.program,x.plan,j,{constantPasses:()=>Z,constantScratchKey:String(I)}),C.push(x),_.push(x.program.variant)}else{let x=b?.[I],B=x?{...x,request:j}:h.add(z,j,Q);x&&h.addPrepared(x.program,x.plan,j,Q),C.push(B?{program:B.program,plan:B.plan}:null),_.push(B?.program.variant??null)}}catch(x){let B=Object.entries(O).map(([U,M])=>`${U}=[${M?.shape??"?"}]`).join(" ");throw new Error(`graph node ${w.op} (args ${JSON.stringify(w.args??{})}; ${B}): ${x.message}`,{cause:x})}E.push(O),y.push(Z);let N=h.programSlice(F,h.programCount),q=Nk(e.name,w.op,w,v,I);for(let x of N){if(!("source"in x))continue;let B=x;B.profile={...B.profile,...q}}k.push(N)}),Uk(e,v,C)}catch(w){Qi(w,h,g,m)}let S=s?u.diagnostics:{...u.diagnostics,slotCount:u.diagnostics.intermediateCount,pooledBytes:u.diagnostics.naiveBytes},A={collector:h,tensors:l,allocation:S,nodeVariants:_,nodeConstantPasses:y,tensor(w){let I=d.get(w);if(!I)throw new Error(`graph "${e.name}" has no bound tensor named "${w}"`);return I},uniformBuffer(w){let I=f.get(w);if(!I)throw new Error(`graph "${e.name}" has no uniform named "${w}"`);return I},async runEager(){await h.runConstantPrograms(),await h.flush(!0)},buildSteps(w){if(!w)return h.runConstantPrograms().then(()=>h.buildSteps());let I=w.startNode??0,z=w.endNode??e.nodes.length;if(!Number.isInteger(I)||!Number.isInteger(z)||I<0||z<I||z>e.nodes.length)throw new Error(`graph "${e.name}": invalid node range [${I}, ${z}) for ${e.nodes.length} nodes`);let O=[];for(let j=I;j<z;++j)O.push(...k[j]);return h.runConstantPrograms().then(()=>O.length>0?t.prepareProgramSequence(O):Promise.resolve([]))},dispose(){Ll.delete(A),om(h,g,m)}};return C.every(w=>w!==null)&&Ll.set(A,{graph:e,plan:u,nodePlans:C,nodeConstantPasses:y,nodeResources:E,nodePrograms:k,realized:l,uniformById:p,uniformByName:f}),A}function Uk(e,t,r){let n=[],s=[];if(e.nodes.forEach((u,o)=>{let l=r[o];l&&(n.push(u),s.push(l.program.variant))}),n.length===0)return;let a=n.length===e.nodes.length?e:{...e,nodes:n},i=new Set;for(let u of n)for(let o of Object.values(u.inputs))t.get(o)?.kind==="uniform"&&i.add(o);for(let u of e.tensors)u.kind==="uniform"&&i.has(u.id)&&Cs(a,u.name,s);e.nodes.forEach((u,o)=>{let l=Object.entries(u.inputs).map(([g,m])=>({port:g,tensor:t.get(m)})).filter(g=>g.tensor?.kind==="uniform");if(l.length===0)return;let c=r[o];if(!c)return;let{program:d,plan:p}=c,f=new Map;d.passes.forEach((g,m)=>{if(g.repeatBaseId===void 0)return;let h=p.passes[m];h.dispatchWorkgroups.some(v=>v===0)||g.bindingSpecs.forEach((v,b)=>{if(v.buffer.type!=="uniform"||!v.struct)return;let _=l.find(({port:E})=>v.arg===E||v.name===E);if(!_)return;let C=g.bindings[b];if(C?.kind!=="uniform")return;let y=pa(C.struct,h.uniforms[v.name]);v.struct.fields.forEach((E,k)=>{if(E.internal)return;let S=`${_.tensor.id}\0${g.repeatBaseId}\0${v.name}\0${E.name}`,A=y[k],w=f.get(S);if(!w)f.set(S,{word:A,passId:g.id});else if(w.word!==A)throw new Error(`graph "${e.name}" node ${o} (${u.op}#${d.variant}) repeats pass "${g.repeatBaseId}" with different generated values for graph uniform "${_.tensor.name}" field ${E.name}:${E.type} (${w.passId} != ${g.id}); one caller-owned buffer cannot carry repeat-varying state. Move that field to a separate generated uniform binding.`)})})})})}function Qi(e,t,r,n){try{om(t,r,n)}catch(s){throw new AggregateError([e,s],"Model graph compilation and cleanup both failed")}throw e}function om(e,t,r){let n=[];try{e?.disposeBuild()}catch(s){n.push(s)}um(t,s=>s.destroy(),n),um(r,s=>s.destroy?.(),n),ct(n,"Model graph resource cleanup failed")}function um(e,t,r){for(let n=e.length-1;n>=0;--n)try{t(e[n]),e.splice(n,1)}catch(s){r.push(s)}}function Nk(e,t,r,n,s){let a=null;for(let l of Object.values(r.inputs)){let c=n.get(l);if(c?.kind==="weight"){a=c.name.replace(/\.[^.]+$/,"");break}}if(!a){let l=Object.values(r.outputs)[0];a=l!==void 0?n.get(l)?.name??t:t}let i=a.match(/(?:^|\.)(\d+)(?:\.|$)/),u=a.replace(/#\d+$/,""),o={phase:e,node:a,part:i?u.replace(/(^|\.)\d+(\.|$)/,(l,c,d)=>c&&d?".":""):u,graphNode:s};return i&&(o.layer=Number(i[1])),o}function Hk(e,t,r,n,s,a){if(a){let u=a.plan.slots.get(e.id);if(u!==void 0){let o=a.slotTensors.get(u);if(!o){let l=a.plan.slotByteLengths.get(u),c=a.plan.slotAllocations.get(u),d=()=>r.empty(c.dtype,c.shape,`graph-slot-${u}`);a.shared?o=a.shared.acquire(u,l,t,d):(o=d(),s.push(o)),a.slotTensors.set(u,o)}return Vk(o,e,a.plan.slotByteLengths.get(u))}if(e.kind==="intermediate")return null}switch(e.kind){case"weight":case"state":{let o=(e.kind==="weight"?n.weights:n.states)?.[e.name];if(!o)throw new Error(`graph ${e.kind} "${e.name}" has no binding`);return o}case"input":{let u=n.inputs?.[e.name];if(u===void 0)throw new Error(`graph input "${e.name}" has no binding`);if(st(u))return u;let o=r.tensorFromTypedArray(e.dtype,e.shape,u);return s.push(o),o}case"intermediate":case"stepInput":case"output":break}let i=r.empty(e.dtype,e.shape,`graph-${e.name}`);return s.push(i),i}function $k(e,t){let r=we(t.shape),n=t.view.elementOffset;if(!Number.isSafeInteger(n)||n<0)throw new Error(`graph view "${t.name}" needs a non-negative safe integer elementOffset`);if(e.layout==="packed"){if(t.dtype!=="uint8")throw new Error(`graph view "${t.name}": packed storage must retain uint8 dtype`);if(n%4!==0)throw new Error(`graph view "${t.name}": packed offsets must be word-aligned`);if(n+r>e.size)throw new Error(`graph view "${t.name}" exceeds its packed parent's logical range`)}let s=e.layout==="packed"?1:ve(t.dtype),a=(e.byteOffset??0)+n*s,i=et(t.dtype,r,e.layout);if(a+i>(e.byteOffset??0)+e.byteLength)throw new Error(`graph view "${t.name}" exceeds its parent's range (${a}+${i} > ${e.byteLength})`);return{runtime:e.runtime,dtype:t.dtype,shape:t.shape,size:r,byteLength:i,byteOffset:a,...e.layout?{layout:e.layout}:{},buffer:e.buffer,destroy:Or}}function Vk(e,t,r){let n=we(t.shape),s=et(t.dtype,n);if(s>r)throw new Error(`graph tensor "${t.name}" (${s} bytes) does not fit slot of ${r} bytes`);return{runtime:e.runtime,dtype:t.dtype,shape:t.shape,size:n,byteLength:s,byteOffset:e.byteOffset??0,buffer:e.buffer,destroy:Or}}var ji=128,lm=4*Uint32Array.BYTES_PER_ELEMENT,Zk=18;function Ol(e){return Math.ceil(e*Zk/Uint32Array.BYTES_PER_ELEMENT)*Uint32Array.BYTES_PER_ELEMENT}function Ml(e){return xr(e)}function Fl(e,t){if(!Number.isFinite(t)||t<0)throw new Error(`Q1 shard byte limit must be a finite non-negative number, got ${t}`);let r=Cr(e.map(s=>({id:s.id,blockCount:s.blockCount,...s.packedGroup===void 0?{}:{unitKey:s.packedGroup}})),Tr({byteLimit:t,maxBlocks:e.reduce((s,a)=>s+Math.max(0,a.blockCount),0),bytesForBlocks:s=>s*lm}),"Q1"),n=new Map;for(let[s,a]of r.placements)n.set(s,{shardIndex:a.shardIndex,groupIndex:a.unitIndex,blockOffset:a.blockOffset,blockCount:a.blockCount});return{shardBlockCounts:r.shardBlockCounts,locations:n}}function Xk(e){let t=e.device;return t.wgslLanguageFeatures?.has("immediate_address_space")===!0&&(t.limits?.maxImmediateSize??0)>=cm}var cm=16;function Wi(e){return e.replace(/[ \t]*\/\/.*$/gm,"").replace(/\n{2,}/g,`
`).trim()}function zi(e,t,r,n,s,a,i,u,o,l=[0,0]){if(u&&t.metadataStorage!==void 0&&t.metadataStorage!=="float")throw new Error(`transcode ${t.id} requires uint32 metadata storage`);if(u&&t.supportsF16Scales===!1)throw new Error(`transcode ${t.id} requires float32 scale storage`);let c=Math.ceil(i/t.workgroupSize),d=e.device.limits.maxComputeWorkgroupsPerDimension,p=!t.bakedWgsl&&Xk(e),f=[i,a,l[0],l[1]],g=p?null:e.createUniformU32(f,`${t.id}-params`),m=[{tensor:r,type:"read-only-storage",binding:0},{tensor:n,type:"storage",binding:1},{tensor:s,type:"storage",binding:2},...g?[{buffer:g,type:"uniform",binding:3}]:[]];o&&m.push({tensor:o,type:"read-only-storage",binding:4});let h={name:`com.xenova.transcode.${t.id}`,source:t.buildWgsl({f16Scales:u,immediateParams:p}),entryPoint:"main",cacheKey:`com.xenova.transcode:${t.id}:${t.metadataStorage==="atomicU32"?"au32":t.metadataStorage==="uint32"?"u32":u?"f16":"f32"}`+(p?":imm":""),bindings:m,...p?{immediateSize:cm,immediateData:new Uint32Array(f)}:{},dispatchWorkgroups:[Math.min(c,d),Math.ceil(c/d),1]},v=!1;return{program:h,release(){v||(v=!0,g?.destroy?.())}}}async function Ql(e,t){if(t.length!==0)try{await e.runProgramSequence(t.map(r=>r.program))}finally{for(let r of t)r.release()}}function Ki(e,t,r){return t.grid?Yk(e,t.grid,r):void 0}function Yk(e,t,r){let n=r.get(t.name);if(!n){let s=new Uint8Array(Math.ceil(t.data.byteLength/4)*4);s.set(t.data),n=e.tensorFromTypedArray("uint32",[s.byteLength/4],new Uint32Array(s.buffer)),r.set(t.name,n)}return n}var Jr=class{#e=[];track(t){this.#e.push(t)}destroyAll(){for(let t of this.#e)t.destroy();this.#e.length=0}};function Ui(e,t,r,n,s,a,i,u,o,l,c=[0,0]){let d=Ni(e,t,r,n,s,a,i,u,o,l,c);return e.runProgram(d.program).finally(()=>d.release())}function Ni(e,t,r,n,s,a,i,u,o,l,c=[0,0]){if(!t)throw new Error("transcode kernel for this weight format is not in this bundle (built for the shipped model's weight formats \u2014 load that model's own GGUF, or rebuild)");let d=r.byteLength%4===0&&r.byteOffset%4===0,p;if(d)p=new Uint32Array(r.buffer,r.byteOffset,r.byteLength/4);else{let m=new Uint8Array(Math.ceil(r.byteLength/4)*4);m.set(r),p=new Uint32Array(m.buffer)}let f=e.tensorFromTypedArray("uint32",[p.length],p);l?.track(f);let g=Ki(e,t,o);return zi(e,t,f,n,s,a,i,u,g,c)}var qe={Q8_0:{id:"q8_0_to_q8",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let base = bi * 34u;
  ss(ob, f16at(base));
  var q: array<u32, 8>; load32(base + 2u, &q);
  for (var w = 0u; w < 8u; w = w + 1u) { out_bits[ob * 8u + w] = q[w]; }
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let base = bi * 34u;
  ss(ob, f16at(base));
  var q: array<u32, 8>; load32(base + 2u, &q);
  for (var w = 0u; w < 8u; w = w + 1u) { out_bits[ob * 8u + w] = q[w]; }
}`},Q4_KNC:{id:"q4_k_to_q4knc",workgroupSize:64,metadataStorage:"uint32",supportsF16Scales:!1,bakedWgsl:!0,buildWgsl:e=>{if(e.f16Scales)throw new Error("transcode q4_k_to_q4knc requires uint32 metadata storage");return`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<u32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = bitcast<u32>(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let j = bi % 8u; let base = sb * 36u;
  let og = (ob / 8u) * 32u;
  let hi = (j & 1u) == 1u;
  let qsWordBase = base + 4u + (j >> 1u) * 8u;
  var qs: array<u32, 8>; load32(qsWordBase * 4u, &qs);
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let loByte = b8(&qs, 4u * w + k); let hiByte = b8(&qs, 4u * w + k + 16u);
      let nLo = select(loByte & 0x0Fu, loByte >> 4u, hi);
      let nHi = select(hiByte & 0x0Fu, hiByte >> 4u, hi);
      word = word | (((nLo | (nHi << 4u)) & 0xFFu) << (8u * k));
    }
    out_bits[og + 4u * j + w] = word;
  }
  if (j == 0u) {
    out_scales[(ob / 8u) * 5u] = src[base];
  }
  if ((j & 1u) == 0u) {
    let c = j >> 1u;
    let sm0 = getSM(base * 4u, j);
    let sm1 = getSM(base * 4u, j + 1u);
    let carrier = sm0.x | (sm0.y << 6u) | (sm1.x << 12u) | (sm1.y << 18u);
    out_scales[(ob / 8u) * 5u + 1u + c] = carrier;
  }
}`}},Q6_KN:{id:"q6_k_to_q6k",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn q6code(base: u32, e: u32) -> u32 {
  let h = e >> 7u; let l = e & 127u;
  let r = l >> 5u; let n = l & 31u;
  let qlb = gbyte(base + 64u * h + n + 32u * (r & 1u));
  let nib = select(qlb >> 4u, qlb & 15u, r < 2u);
  let hi2 = (gbyte(base + 128u + 32u * h + n) >> (2u * r)) & 3u;
  return nib | (hi2 << 4u);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let base = (bi / 8u) * 210u;
  let j = bi % 8u;
  let og = (ob / 8u) * 52u;
  var hiLo = 0u;
  var hiHi = 0u;
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let cLo = q6code(base, 32u * j + 4u * w + k);
      let cHi = q6code(base, 32u * j + 16u + 4u * w + k);
      word = word | (((cLo & 15u) | ((cHi & 15u) << 4u)) << (8u * k));
      hiLo = hiLo | ((cLo >> 4u) << (8u * k + 2u * w));
      hiHi = hiHi | ((cHi >> 4u) << (8u * k + 2u * w));
    }
    out_bits[og + 4u * j + w] = word;
  }
  out_bits[og + 32u + 2u * j] = hiLo;
  out_bits[og + 32u + 2u * j + 1u] = hiHi;
  if (j < 4u) { out_bits[og + 48u + j] = word32at(base + 192u + 4u * j); }
  if (j == 0u) { ss(ob / 8u, f16at(base + 208u)); }
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn q6code(base: u32, e: u32) -> u32 {
  let h = e >> 7u; let l = e & 127u;
  let r = l >> 5u; let n = l & 31u;
  let qlb = gbyte(base + 64u * h + n + 32u * (r & 1u));
  let nib = select(qlb >> 4u, qlb & 15u, r < 2u);
  let hi2 = (gbyte(base + 128u + 32u * h + n) >> (2u * r)) & 3u;
  return nib | (hi2 << 4u);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let base = (bi / 8u) * 210u;
  let j = bi % 8u;
  let og = (ob / 8u) * 52u;
  var hiLo = 0u;
  var hiHi = 0u;
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let cLo = q6code(base, 32u * j + 4u * w + k);
      let cHi = q6code(base, 32u * j + 16u + 4u * w + k);
      word = word | (((cLo & 15u) | ((cHi & 15u) << 4u)) << (8u * k));
      hiLo = hiLo | ((cLo >> 4u) << (8u * k + 2u * w));
      hiHi = hiHi | ((cHi >> 4u) << (8u * k + 2u * w));
    }
    out_bits[og + 4u * j + w] = word;
  }
  out_bits[og + 32u + 2u * j] = hiLo;
  out_bits[og + 32u + 2u * j + 1u] = hiHi;
  if (j < 4u) { out_bits[og + 48u + j] = word32at(base + 192u + 4u * j); }
  if (j == 0u) { ss(ob / 8u, f16at(base + 208u)); }
}`},Q5_KN:{id:"q5_k_to_q5k",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let j = bi % 8u; let base = sb * 44u;
  let og = (ob / 8u) * 44u;
  let hi = (j & 1u) == 1u;
  let qsWordBase = base + 12u + (j >> 1u) * 8u;
  var qs: array<u32, 8>; load32(qsWordBase * 4u, &qs);
  var qh: array<u32, 8>; load32((base + 4u) * 4u, &qh);
  var qhw = 0u;
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let lo_byte = b8(&qs, 4u * w + k); let hi_byte = b8(&qs, 4u * w + k + 16u);
      let nLo = select(lo_byte & 0x0Fu, lo_byte >> 4u, hi);
      let nHi = select(hi_byte & 0x0Fu, hi_byte >> 4u, hi);
      word = word | (((nLo | (nHi << 4u)) & 0xFFu) << (8u * k));
      let hLo = (b8(&qh, 4u * w + k) >> j) & 1u;
      let hHi = (b8(&qh, 4u * w + k + 16u) >> j) & 1u;
      qhw = qhw | (hLo << (4u * w + k)) | (hHi << (16u + 4u * w + k));
    }
    out_bits[og + 4u * j + w] = word;
  }
  out_bits[og + 32u + j] = qhw;
  if (j < 2u) {
    var sw = 0u; var mw = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let sm = getSM(base * 4u, 4u * j + k);
      sw = sw | (sm.x << (8u * k)); mw = mw | (sm.y << (8u * k));
    }
    out_bits[og + 40u + j] = sw;
    out_bits[og + 42u + j] = mw;
  }
  if (j == 0u) {
    let dd = unpack2x16float(src[base]);
    ss((ob / 8u) * 2u, dd.x); ss((ob / 8u) * 2u + 1u, dd.y);
  }
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let j = bi % 8u; let base = sb * 44u;
  let og = (ob / 8u) * 44u;
  let hi = (j & 1u) == 1u;
  let qsWordBase = base + 12u + (j >> 1u) * 8u;
  var qs: array<u32, 8>; load32(qsWordBase * 4u, &qs);
  var qh: array<u32, 8>; load32((base + 4u) * 4u, &qh);
  var qhw = 0u;
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let lo_byte = b8(&qs, 4u * w + k); let hi_byte = b8(&qs, 4u * w + k + 16u);
      let nLo = select(lo_byte & 0x0Fu, lo_byte >> 4u, hi);
      let nHi = select(hi_byte & 0x0Fu, hi_byte >> 4u, hi);
      word = word | (((nLo | (nHi << 4u)) & 0xFFu) << (8u * k));
      let hLo = (b8(&qh, 4u * w + k) >> j) & 1u;
      let hHi = (b8(&qh, 4u * w + k + 16u) >> j) & 1u;
      qhw = qhw | (hLo << (4u * w + k)) | (hHi << (16u + 4u * w + k));
    }
    out_bits[og + 4u * j + w] = word;
  }
  out_bits[og + 32u + j] = qhw;
  if (j < 2u) {
    var sw = 0u; var mw = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let sm = getSM(base * 4u, 4u * j + k);
      sw = sw | (sm.x << (8u * k)); mw = mw | (sm.y << (8u * k));
    }
    out_bits[og + 40u + j] = sw;
    out_bits[og + 42u + j] = mw;
  }
  if (j == 0u) {
    let dd = unpack2x16float(src[base]);
    ss((ob / 8u) * 2u, dd.x); ss((ob / 8u) * 2u + 1u, dd.y);
  }
}`},Q4_K:{id:"q4_k_to_kq4",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let s = bi % 8u; let base = sb * 36u;
  let dd = unpack2x16float(src[base]);
  let scm = getSM(base * 4u, s);
  let d1 = dd.x * f32(scm.x); let m1 = dd.y * f32(scm.y);
  ss(ob * 2u, d1); ss(ob * 2u + 1u, 8.0 * d1 - m1);
  let qsWordBase = base + 4u + (s >> 1u) * 8u; let hi = (s & 1u) == 1u;
  var qs: array<u32, 8>; load32(qsWordBase * 4u, &qs);
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var bb = 0u; bb < 4u; bb = bb + 1u) {
      let k = w * 4u + bb;
      let lo_byte = b8(&qs, k); let hi_byte = b8(&qs, k + 16u);
      let nLo = select(lo_byte & 0x0Fu, lo_byte >> 4u, hi);
      let nHi = select(hi_byte & 0x0Fu, hi_byte >> 4u, hi);
      word = word | (((nLo | (nHi << 4u)) & 0xFFu) << (bb * 8u));
    }
    out_bits[ob * 4u + w] = word;
  }
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let s = bi % 8u; let base = sb * 36u;
  let dd = unpack2x16float(src[base]);
  let scm = getSM(base * 4u, s);
  let d1 = dd.x * f32(scm.x); let m1 = dd.y * f32(scm.y);
  ss(ob * 2u, d1); ss(ob * 2u + 1u, 8.0 * d1 - m1);
  let qsWordBase = base + 4u + (s >> 1u) * 8u; let hi = (s & 1u) == 1u;
  var qs: array<u32, 8>; load32(qsWordBase * 4u, &qs);
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var bb = 0u; bb < 4u; bb = bb + 1u) {
      let k = w * 4u + bb;
      let lo_byte = b8(&qs, k); let hi_byte = b8(&qs, k + 16u);
      let nLo = select(lo_byte & 0x0Fu, lo_byte >> 4u, hi);
      let nHi = select(hi_byte & 0x0Fu, hi_byte >> 4u, hi);
      word = word | (((nLo | (nHi << 4u)) & 0xFFu) << (bb * 8u));
    }
    out_bits[ob * 4u + w] = word;
  }
}`}},Lt={Q4_K:{id:"q4_k_to_q8",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn rqa(x: vec4<f32>) -> vec4<f32> { return sign(x) * floor(abs(x) + vec4<f32>(0.5)); }
fn packQ8(ob: u32, w: u32, q: vec4<i32>) {
  out_bits[ob * 8u + w] = (u32(q.x) & 0xFFu) | ((u32(q.y) & 0xFFu) << 8u) | ((u32(q.z) & 0xFFu) << 16u) | ((u32(q.w) & 0xFFu) << 24u);
}
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { am = max(am, abs((*v)[w])); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { packQ8(ob, w, clamp(vec4<i32>(rqa((*v)[w] * inv)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; am = max(am, abs(vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]))); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; let c = vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]) * inv; packQ8(ob, w, clamp(vec4<i32>(rqa(c)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q4kval(qlw: u32, m: u32, hi: bool, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let p = dl * f32(nibble);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 144u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 16u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w];
    v[w] = vec4<f32>(q4kval(qlw, 0u, hi, dl, ml), q4kval(qlw, 1u, hi, dl, ml),
                     q4kval(qlw, 2u, hi, dl, ml), q4kval(qlw, 3u, hi, dl, ml));
  }
  writeQ8V(ob, &v);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn rqa(x: vec4<f32>) -> vec4<f32> { return sign(x) * floor(abs(x) + vec4<f32>(0.5)); }
fn packQ8(ob: u32, w: u32, q: vec4<i32>) {
  out_bits[ob * 8u + w] = (u32(q.x) & 0xFFu) | ((u32(q.y) & 0xFFu) << 8u) | ((u32(q.z) & 0xFFu) << 16u) | ((u32(q.w) & 0xFFu) << 24u);
}
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { am = max(am, abs((*v)[w])); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { packQ8(ob, w, clamp(vec4<i32>(rqa((*v)[w] * inv)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; am = max(am, abs(vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]))); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; let c = vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]) * inv; packQ8(ob, w, clamp(vec4<i32>(rqa(c)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q4kval(qlw: u32, m: u32, hi: bool, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let p = dl * f32(nibble);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 144u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 16u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w];
    v[w] = vec4<f32>(q4kval(qlw, 0u, hi, dl, ml), q4kval(qlw, 1u, hi, dl, ml),
                     q4kval(qlw, 2u, hi, dl, ml), q4kval(qlw, 3u, hi, dl, ml));
  }
  writeQ8V(ob, &v);
}`},Q6_K:{id:"q6_k_to_q8",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn rqa(x: vec4<f32>) -> vec4<f32> { return sign(x) * floor(abs(x) + vec4<f32>(0.5)); }
fn packQ8(ob: u32, w: u32, q: vec4<i32>) {
  out_bits[ob * 8u + w] = (u32(q.x) & 0xFFu) | ((u32(q.y) & 0xFFu) << 8u) | ((u32(q.z) & 0xFFu) << 16u) | ((u32(q.w) & 0xFFu) << 24u);
}
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { am = max(am, abs((*v)[w])); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { packQ8(ob, w, clamp(vec4<i32>(rqa((*v)[w] * inv)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; am = max(am, abs(vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]))); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; let c = vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]) * inv; packQ8(ob, w, clamp(vec4<i32>(rqa(c)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn i8at(j: u32) -> i32 { return (i32(gbyte(j)) << 24u) >> 24u; }
fn q6val(qlw: u32, qhw: u32, m: u32, useHigh: bool, qhShift: u32, sc: f32) -> f32 {
  let qln = (qlw >> (8u * m)) & 0xFFu;
  let nib = select(qln & 0x0Fu, qln >> 4u, useHigh);
  let q6 = nib | (((((qhw >> (8u * m)) & 0xFFu) >> qhShift) & 3u) << 4u);
  return sc * f32(i32(q6) - 32);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let q8blk = bi % 8u; let base = sb * 210u;
  let n = q8blk >> 2u; let sub = q8blk & 3u;
  let qlOff = base + n * 64u + (sub & 1u) * 32u;
  let qhBase = base + 128u + n * 32u;
  let scBase = base + 192u + n * 8u;
  let d = f16at(base + 208u);
  let useHigh = sub >= 2u; let qhShift = sub * 2u;
  let sc0 = d * f32(i8at(scBase + sub * 2u));
  let sc1 = d * f32(i8at(scBase + 1u + sub * 2u));
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(qhBase, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w]; let sc = select(sc0, sc1, w >= 4u);
    v[w] = vec4<f32>(q6val(qlw, qhw, 0u, useHigh, qhShift, sc), q6val(qlw, qhw, 1u, useHigh, qhShift, sc),
                     q6val(qlw, qhw, 2u, useHigh, qhShift, sc), q6val(qlw, qhw, 3u, useHigh, qhShift, sc));
  }
  writeQ8V(ob, &v);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn rqa(x: vec4<f32>) -> vec4<f32> { return sign(x) * floor(abs(x) + vec4<f32>(0.5)); }
fn packQ8(ob: u32, w: u32, q: vec4<i32>) {
  out_bits[ob * 8u + w] = (u32(q.x) & 0xFFu) | ((u32(q.y) & 0xFFu) << 8u) | ((u32(q.z) & 0xFFu) << 16u) | ((u32(q.w) & 0xFFu) << 24u);
}
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { am = max(am, abs((*v)[w])); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { packQ8(ob, w, clamp(vec4<i32>(rqa((*v)[w] * inv)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; am = max(am, abs(vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]))); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; let c = vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]) * inv; packQ8(ob, w, clamp(vec4<i32>(rqa(c)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn i8at(j: u32) -> i32 { return (i32(gbyte(j)) << 24u) >> 24u; }
fn q6val(qlw: u32, qhw: u32, m: u32, useHigh: bool, qhShift: u32, sc: f32) -> f32 {
  let qln = (qlw >> (8u * m)) & 0xFFu;
  let nib = select(qln & 0x0Fu, qln >> 4u, useHigh);
  let q6 = nib | (((((qhw >> (8u * m)) & 0xFFu) >> qhShift) & 3u) << 4u);
  return sc * f32(i32(q6) - 32);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let q8blk = bi % 8u; let base = sb * 210u;
  let n = q8blk >> 2u; let sub = q8blk & 3u;
  let qlOff = base + n * 64u + (sub & 1u) * 32u;
  let qhBase = base + 128u + n * 32u;
  let scBase = base + 192u + n * 8u;
  let d = f16at(base + 208u);
  let useHigh = sub >= 2u; let qhShift = sub * 2u;
  let sc0 = d * f32(i8at(scBase + sub * 2u));
  let sc1 = d * f32(i8at(scBase + 1u + sub * 2u));
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(qhBase, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w]; let sc = select(sc0, sc1, w >= 4u);
    v[w] = vec4<f32>(q6val(qlw, qhw, 0u, useHigh, qhShift, sc), q6val(qlw, qhw, 1u, useHigh, qhShift, sc),
                     q6val(qlw, qhw, 2u, useHigh, qhShift, sc), q6val(qlw, qhw, 3u, useHigh, qhShift, sc));
  }
  writeQ8V(ob, &v);
}`},Q5_K:{id:"q5_k_to_q8",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn rqa(x: vec4<f32>) -> vec4<f32> { return sign(x) * floor(abs(x) + vec4<f32>(0.5)); }
fn packQ8(ob: u32, w: u32, q: vec4<i32>) {
  out_bits[ob * 8u + w] = (u32(q.x) & 0xFFu) | ((u32(q.y) & 0xFFu) << 8u) | ((u32(q.z) & 0xFFu) << 16u) | ((u32(q.w) & 0xFFu) << 24u);
}
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { am = max(am, abs((*v)[w])); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { packQ8(ob, w, clamp(vec4<i32>(rqa((*v)[w] * inv)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; am = max(am, abs(vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]))); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; let c = vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]) * inv; packQ8(ob, w, clamp(vec4<i32>(rqa(c)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q5val(qlw: u32, qhw: u32, m: u32, hi: bool, sbk: u32, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let hibit = ((((qhw >> (8u * m)) & 0xFFu) >> sbk) & 1u) * 16u;
  let p = dl * f32(nibble + hibit);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 176u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 48u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(base + 16u, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w];
    v[w] = vec4<f32>(q5val(qlw, qhw, 0u, hi, sbk, dl, ml), q5val(qlw, qhw, 1u, hi, sbk, dl, ml),
                     q5val(qlw, qhw, 2u, hi, sbk, dl, ml), q5val(qlw, qhw, 3u, hi, sbk, dl, ml));
  }
  writeQ8V(ob, &v);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn rqa(x: vec4<f32>) -> vec4<f32> { return sign(x) * floor(abs(x) + vec4<f32>(0.5)); }
fn packQ8(ob: u32, w: u32, q: vec4<i32>) {
  out_bits[ob * 8u + w] = (u32(q.x) & 0xFFu) | ((u32(q.y) & 0xFFu) << 8u) | ((u32(q.z) & 0xFFu) << 16u) | ((u32(q.w) & 0xFFu) << 24u);
}
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { am = max(am, abs((*v)[w])); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { packQ8(ob, w, clamp(vec4<i32>(rqa((*v)[w] * inv)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var am = vec4<f32>(0.0);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; am = max(am, abs(vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]))); }
  let amax = max(max(am.x, am.y), max(am.z, am.w));
  let scale = div_exact(amax, 127.0, recip_exact(127.0));
  let inv = select(0.0, recip_exact(scale), scale > 0.0);
  ss(ob, scale);
  for (var w = 0u; w < 8u; w = w + 1u) { let b = w * 4u; let c = vec4<f32>((*vals)[b], (*vals)[b + 1u], (*vals)[b + 2u], (*vals)[b + 3u]) * inv; packQ8(ob, w, clamp(vec4<i32>(rqa(c)), vec4<i32>(-127), vec4<i32>(127))); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q5val(qlw: u32, qhw: u32, m: u32, hi: bool, sbk: u32, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let hibit = ((((qhw >> (8u * m)) & 0xFFu) >> sbk) & 1u) * 16u;
  let p = dl * f32(nibble + hibit);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 176u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 48u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(base + 16u, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w];
    v[w] = vec4<f32>(q5val(qlw, qhw, 0u, hi, sbk, dl, ml), q5val(qlw, qhw, 1u, hi, sbk, dl, ml),
                     q5val(qlw, qhw, 2u, hi, sbk, dl, ml), q5val(qlw, qhw, 3u, hi, sbk, dl, ml));
  }
  writeQ8V(ob, &v);
}`}},lr={Q4_K:{id:"q4_k_to_f32",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  for (var w = 0u; w < 8u; w = w + 1u) {
    let b = ob * 32u + w * 4u; let x = (*v)[w];
    out_bits[b] = bitcast<u32>(x.x); out_bits[b + 1u] = bitcast<u32>(x.y);
    out_bits[b + 2u] = bitcast<u32>(x.z); out_bits[b + 3u] = bitcast<u32>(x.w);
  }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  for (var l = 0u; l < 32u; l = l + 1u) { out_bits[ob * 32u + l] = bitcast<u32>((*vals)[l]); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q4kval(qlw: u32, m: u32, hi: bool, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let p = dl * f32(nibble);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 144u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 16u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w];
    v[w] = vec4<f32>(q4kval(qlw, 0u, hi, dl, ml), q4kval(qlw, 1u, hi, dl, ml),
                     q4kval(qlw, 2u, hi, dl, ml), q4kval(qlw, 3u, hi, dl, ml));
  }
  writeQ8V(ob, &v);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  for (var w = 0u; w < 8u; w = w + 1u) {
    let b = ob * 32u + w * 4u; let x = (*v)[w];
    out_bits[b] = bitcast<u32>(x.x); out_bits[b + 1u] = bitcast<u32>(x.y);
    out_bits[b + 2u] = bitcast<u32>(x.z); out_bits[b + 3u] = bitcast<u32>(x.w);
  }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  for (var l = 0u; l < 32u; l = l + 1u) { out_bits[ob * 32u + l] = bitcast<u32>((*vals)[l]); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q4kval(qlw: u32, m: u32, hi: bool, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let p = dl * f32(nibble);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 144u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 16u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w];
    v[w] = vec4<f32>(q4kval(qlw, 0u, hi, dl, ml), q4kval(qlw, 1u, hi, dl, ml),
                     q4kval(qlw, 2u, hi, dl, ml), q4kval(qlw, 3u, hi, dl, ml));
  }
  writeQ8V(ob, &v);
}`},Q5_K:{id:"q5_k_to_f32",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  for (var w = 0u; w < 8u; w = w + 1u) {
    let b = ob * 32u + w * 4u; let x = (*v)[w];
    out_bits[b] = bitcast<u32>(x.x); out_bits[b + 1u] = bitcast<u32>(x.y);
    out_bits[b + 2u] = bitcast<u32>(x.z); out_bits[b + 3u] = bitcast<u32>(x.w);
  }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  for (var l = 0u; l < 32u; l = l + 1u) { out_bits[ob * 32u + l] = bitcast<u32>((*vals)[l]); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q5val(qlw: u32, qhw: u32, m: u32, hi: bool, sbk: u32, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let hibit = ((((qhw >> (8u * m)) & 0xFFu) >> sbk) & 1u) * 16u;
  let p = dl * f32(nibble + hibit);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 176u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 48u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(base + 16u, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w];
    v[w] = vec4<f32>(q5val(qlw, qhw, 0u, hi, sbk, dl, ml), q5val(qlw, qhw, 1u, hi, sbk, dl, ml),
                     q5val(qlw, qhw, 2u, hi, sbk, dl, ml), q5val(qlw, qhw, 3u, hi, sbk, dl, ml));
  }
  writeQ8V(ob, &v);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  for (var w = 0u; w < 8u; w = w + 1u) {
    let b = ob * 32u + w * 4u; let x = (*v)[w];
    out_bits[b] = bitcast<u32>(x.x); out_bits[b + 1u] = bitcast<u32>(x.y);
    out_bits[b + 2u] = bitcast<u32>(x.z); out_bits[b + 3u] = bitcast<u32>(x.w);
  }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  for (var l = 0u; l < 32u; l = l + 1u) { out_bits[ob * 32u + l] = bitcast<u32>((*vals)[l]); }
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q5val(qlw: u32, qhw: u32, m: u32, hi: bool, sbk: u32, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let hibit = ((((qhw >> (8u * m)) & 0xFFu) >> sbk) & 1u) * 16u;
  let p = dl * f32(nibble + hibit);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 176u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 48u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(base + 16u, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w];
    v[w] = vec4<f32>(q5val(qlw, qhw, 0u, hi, sbk, dl, ml), q5val(qlw, qhw, 1u, hi, sbk, dl, ml),
                     q5val(qlw, qhw, 2u, hi, sbk, dl, ml), q5val(qlw, qhw, 3u, hi, sbk, dl, ml));
  }
  writeQ8V(ob, &v);
}`},Q6_K:{id:"q6_k_to_f32",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  for (var w = 0u; w < 8u; w = w + 1u) {
    let b = ob * 32u + w * 4u; let x = (*v)[w];
    out_bits[b] = bitcast<u32>(x.x); out_bits[b + 1u] = bitcast<u32>(x.y);
    out_bits[b + 2u] = bitcast<u32>(x.z); out_bits[b + 3u] = bitcast<u32>(x.w);
  }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  for (var l = 0u; l < 32u; l = l + 1u) { out_bits[ob * 32u + l] = bitcast<u32>((*vals)[l]); }
}
fn i8at(j: u32) -> i32 { return (i32(gbyte(j)) << 24u) >> 24u; }
fn q6val(qlw: u32, qhw: u32, m: u32, useHigh: bool, qhShift: u32, sc: f32) -> f32 {
  let qln = (qlw >> (8u * m)) & 0xFFu;
  let nib = select(qln & 0x0Fu, qln >> 4u, useHigh);
  let q6 = nib | (((((qhw >> (8u * m)) & 0xFFu) >> qhShift) & 3u) << 4u);
  return sc * f32(i32(q6) - 32);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let q8blk = bi % 8u; let base = sb * 210u;
  let n = q8blk >> 2u; let sub = q8blk & 3u;
  let qlOff = base + n * 64u + (sub & 1u) * 32u;
  let qhBase = base + 128u + n * 32u;
  let scBase = base + 192u + n * 8u;
  let d = f16at(base + 208u);
  let useHigh = sub >= 2u; let qhShift = sub * 2u;
  let sc0 = d * f32(i8at(scBase + sub * 2u));
  let sc1 = d * f32(i8at(scBase + 1u + sub * 2u));
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(qhBase, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w]; let sc = select(sc0, sc1, w >= 4u);
    v[w] = vec4<f32>(q6val(qlw, qhw, 0u, useHigh, qhShift, sc), q6val(qlw, qhw, 1u, useHigh, qhShift, sc),
                     q6val(qlw, qhw, 2u, useHigh, qhShift, sc), q6val(qlw, qhw, 3u, useHigh, qhShift, sc));
  }
  writeQ8V(ob, &v);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn writeQ8V(ob: u32, v: ptr<function, array<vec4<f32>, 8>>) {
  for (var w = 0u; w < 8u; w = w + 1u) {
    let b = ob * 32u + w * 4u; let x = (*v)[w];
    out_bits[b] = bitcast<u32>(x.x); out_bits[b + 1u] = bitcast<u32>(x.y);
    out_bits[b + 2u] = bitcast<u32>(x.z); out_bits[b + 3u] = bitcast<u32>(x.w);
  }
}
fn writeQ8(ob: u32, vals: ptr<function, array<f32, 32>>) {
  for (var l = 0u; l < 32u; l = l + 1u) { out_bits[ob * 32u + l] = bitcast<u32>((*vals)[l]); }
}
fn i8at(j: u32) -> i32 { return (i32(gbyte(j)) << 24u) >> 24u; }
fn q6val(qlw: u32, qhw: u32, m: u32, useHigh: bool, qhShift: u32, sc: f32) -> f32 {
  let qln = (qlw >> (8u * m)) & 0xFFu;
  let nib = select(qln & 0x0Fu, qln >> 4u, useHigh);
  let q6 = nib | (((((qhw >> (8u * m)) & 0xFFu) >> qhShift) & 3u) << 4u);
  return sc * f32(i32(q6) - 32);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let q8blk = bi % 8u; let base = sb * 210u;
  let n = q8blk >> 2u; let sub = q8blk & 3u;
  let qlOff = base + n * 64u + (sub & 1u) * 32u;
  let qhBase = base + 128u + n * 32u;
  let scBase = base + 192u + n * 8u;
  let d = f16at(base + 208u);
  let useHigh = sub >= 2u; let qhShift = sub * 2u;
  let sc0 = d * f32(i8at(scBase + sub * 2u));
  let sc1 = d * f32(i8at(scBase + 1u + sub * 2u));
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(qhBase, &qh);
  var v: array<vec4<f32>, 8>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w]; let sc = select(sc0, sc1, w >= 4u);
    v[w] = vec4<f32>(q6val(qlw, qhw, 0u, useHigh, qhShift, sc), q6val(qlw, qhw, 1u, useHigh, qhShift, sc),
                     q6val(qlw, qhw, 2u, useHigh, qhShift, sc), q6val(qlw, qhw, 3u, useHigh, qhShift, sc));
  }
  writeQ8V(ob, &v);
}`}},jl={Q8_0_TO_Q4:{id:"q8_0_to_q4req",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn packNibbles(ob: u32, nib: ptr<function, array<u32, 32>>) {
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let e = w * 4u + k;
      word = word | ((((*nib)[e] | ((*nib)[e + 16u] << 4u)) & 0xFFu) << (8u * k));
    }
    out_bits[ob * 4u + w] = word;
  }
}
fn writeQ4Sym(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var amax = 0.0; var maxv = 0.0;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let v = (*vals)[j];
    if (abs(v) > amax) { amax = abs(v); maxv = v; }
  }
  let d = maxv / -8.0;
  let id = select(0.0, recip_exact(d), d != 0.0);
  ss(ob, d);
  var q: array<u32, 32>;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let x = (*vals)[j] * id + 8.5;
    q[j] = min(15u, u32(max(x, 0.0)));
  }
  packNibbles(ob, &q);
}
fn writeQ4Affine(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var vmin = (*vals)[0]; var vmax = (*vals)[0];
  for (var j = 1u; j < 32u; j = j + 1u) {
    vmin = min(vmin, (*vals)[j]);
    vmax = max(vmax, (*vals)[j]);
  }
  let d1 = div_exact(vmax - vmin, 15.0, recip_exact(15.0));
  let id = select(0.0, recip_exact(d1), d1 != 0.0);
  ss(ob * 2u, d1);
  ss(ob * 2u + 1u, vmin + 8.0 * d1);
  var q: array<u32, 32>;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let x = ((*vals)[j] - vmin) * id + 0.5;
    q[j] = min(15u, u32(max(x, 0.0)));
  }
  packNibbles(ob, &q);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let base = bi * 34u;
  let d = f16at(base);
  var vals: array<f32, 32>;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let b = gbyte(base + 2u + j);
    vals[j] = d * f32((i32(b) << 24u) >> 24u);
  }
  writeQ4Sym(ob, &vals);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn packNibbles(ob: u32, nib: ptr<function, array<u32, 32>>) {
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let e = w * 4u + k;
      word = word | ((((*nib)[e] | ((*nib)[e + 16u] << 4u)) & 0xFFu) << (8u * k));
    }
    out_bits[ob * 4u + w] = word;
  }
}
fn writeQ4Sym(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var amax = 0.0; var maxv = 0.0;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let v = (*vals)[j];
    if (abs(v) > amax) { amax = abs(v); maxv = v; }
  }
  let d = maxv / -8.0;
  let id = select(0.0, recip_exact(d), d != 0.0);
  ss(ob, d);
  var q: array<u32, 32>;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let x = (*vals)[j] * id + 8.5;
    q[j] = min(15u, u32(max(x, 0.0)));
  }
  packNibbles(ob, &q);
}
fn writeQ4Affine(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var vmin = (*vals)[0]; var vmax = (*vals)[0];
  for (var j = 1u; j < 32u; j = j + 1u) {
    vmin = min(vmin, (*vals)[j]);
    vmax = max(vmax, (*vals)[j]);
  }
  let d1 = div_exact(vmax - vmin, 15.0, recip_exact(15.0));
  let id = select(0.0, recip_exact(d1), d1 != 0.0);
  ss(ob * 2u, d1);
  ss(ob * 2u + 1u, vmin + 8.0 * d1);
  var q: array<u32, 32>;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let x = ((*vals)[j] - vmin) * id + 0.5;
    q[j] = min(15u, u32(max(x, 0.0)));
  }
  packNibbles(ob, &q);
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let base = bi * 34u;
  let d = f16at(base);
  var vals: array<f32, 32>;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let b = gbyte(base + 2u + j);
    vals[j] = d * f32((i32(b) << 24u) >> 24u);
  }
  writeQ4Sym(ob, &vals);
}`},Q5_K_TO_KQ4:{id:"q5_k_to_kq4req",workgroupSize:64,bakedWgsl:!0,buildWgsl:e=>e.f16Scales?`enable f16;
@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f16>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = f16(v); }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn packNibbles(ob: u32, nib: ptr<function, array<u32, 32>>) {
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let e = w * 4u + k;
      word = word | ((((*nib)[e] | ((*nib)[e + 16u] << 4u)) & 0xFFu) << (8u * k));
    }
    out_bits[ob * 4u + w] = word;
  }
}
fn writeQ4Sym(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var amax = 0.0; var maxv = 0.0;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let v = (*vals)[j];
    if (abs(v) > amax) { amax = abs(v); maxv = v; }
  }
  let d = maxv / -8.0;
  let id = select(0.0, recip_exact(d), d != 0.0);
  ss(ob, d);
  var q: array<u32, 32>;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let x = (*vals)[j] * id + 8.5;
    q[j] = min(15u, u32(max(x, 0.0)));
  }
  packNibbles(ob, &q);
}
fn writeQ4Affine(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var vmin = (*vals)[0]; var vmax = (*vals)[0];
  for (var j = 1u; j < 32u; j = j + 1u) {
    vmin = min(vmin, (*vals)[j]);
    vmax = max(vmax, (*vals)[j]);
  }
  let d1 = div_exact(vmax - vmin, 15.0, recip_exact(15.0));
  let id = select(0.0, recip_exact(d1), d1 != 0.0);
  ss(ob * 2u, d1);
  ss(ob * 2u + 1u, vmin + 8.0 * d1);
  var q: array<u32, 32>;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let x = ((*vals)[j] - vmin) * id + 0.5;
    q[j] = min(15u, u32(max(x, 0.0)));
  }
  packNibbles(ob, &q);
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q5val(qlw: u32, qhw: u32, m: u32, hi: bool, sbk: u32, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let hibit = ((((qhw >> (8u * m)) & 0xFFu) >> sbk) & 1u) * 16u;
  let p = dl * f32(nibble + hibit);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 176u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 48u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(base + 16u, &qh);
  var vals: array<f32, 32>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w];
    vals[w * 4u + 0u] = q5val(qlw, qhw, 0u, hi, sbk, dl, ml);
    vals[w * 4u + 1u] = q5val(qlw, qhw, 1u, hi, sbk, dl, ml);
    vals[w * 4u + 2u] = q5val(qlw, qhw, 2u, hi, sbk, dl, ml);
    vals[w * 4u + 3u] = q5val(qlw, qhw, 3u, hi, sbk, dl, ml);
  }
  writeQ4Affine(ob, &vals);
}`:`@group(0) @binding(0) var<storage, read> src: array<u32>;
@group(0) @binding(1) var<storage, read_write> out_bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> out_scales: array<f32>;
struct Params { numBlocks: u32, blockOffset: u32, aux0: u32, aux1: u32 }
@group(0) @binding(3) var<uniform> params: Params;
fn ss(i: u32, v: f32) { out_scales[i] = v; }
fn sbyte(wordBase: u32, j: u32) -> u32 { return (src[wordBase + (j >> 2u)] >> (8u * (j & 3u))) & 0xFFu; }
fn gbyte(j: u32) -> u32 { return (src[j >> 2u] >> (8u * (j & 3u))) & 0xFFu; }
fn f16at(j: u32) -> f32 { return unpack2x16float(gbyte(j) | (gbyte(j + 1u) << 8u)).x; }
fn word32at(j: u32) -> u32 { return gbyte(j) | (gbyte(j + 1u) << 8u) | (gbyte(j + 2u) << 16u) | (gbyte(j + 3u) << 24u); }
fn load16(B: u32, out: ptr<function, array<u32, 4>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 4u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 4u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn load32(B: u32, out: ptr<function, array<u32, 8>>) {
  let wb = B >> 2u; let sh = (B & 3u) * 8u;
  if (sh == 0u) { for (var w = 0u; w < 8u; w = w + 1u) { (*out)[w] = src[wb + w]; } }
  else { let inv = 32u - sh; var prev = src[wb]; for (var w = 0u; w < 8u; w = w + 1u) { let cur = src[wb + 1u + w]; (*out)[w] = (prev >> sh) | (cur << inv); prev = cur; } }
}
fn b4(w: ptr<function, array<u32, 4>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn b8(w: ptr<function, array<u32, 8>>, k: u32) -> u32 { return ((*w)[k >> 2u] >> ((k & 3u) * 8u)) & 0xFFu; }
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }
fn packNibbles(ob: u32, nib: ptr<function, array<u32, 32>>) {
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var k = 0u; k < 4u; k = k + 1u) {
      let e = w * 4u + k;
      word = word | ((((*nib)[e] | ((*nib)[e + 16u] << 4u)) & 0xFFu) << (8u * k));
    }
    out_bits[ob * 4u + w] = word;
  }
}
fn writeQ4Sym(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var amax = 0.0; var maxv = 0.0;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let v = (*vals)[j];
    if (abs(v) > amax) { amax = abs(v); maxv = v; }
  }
  let d = maxv / -8.0;
  let id = select(0.0, recip_exact(d), d != 0.0);
  ss(ob, d);
  var q: array<u32, 32>;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let x = (*vals)[j] * id + 8.5;
    q[j] = min(15u, u32(max(x, 0.0)));
  }
  packNibbles(ob, &q);
}
fn writeQ4Affine(ob: u32, vals: ptr<function, array<f32, 32>>) {
  var vmin = (*vals)[0]; var vmax = (*vals)[0];
  for (var j = 1u; j < 32u; j = j + 1u) {
    vmin = min(vmin, (*vals)[j]);
    vmax = max(vmax, (*vals)[j]);
  }
  let d1 = div_exact(vmax - vmin, 15.0, recip_exact(15.0));
  let id = select(0.0, recip_exact(d1), d1 != 0.0);
  ss(ob * 2u, d1);
  ss(ob * 2u + 1u, vmin + 8.0 * d1);
  var q: array<u32, 32>;
  for (var j = 0u; j < 32u; j = j + 1u) {
    let x = ((*vals)[j] - vmin) * id + 0.5;
    q[j] = min(15u, u32(max(x, 0.0)));
  }
  packNibbles(ob, &q);
}
fn scByte(base: u32, j: u32) -> u32 { return gbyte(base + 4u + j); }
fn getSM(base: u32, j: u32) -> vec2<u32> {
  if (j < 4u) { return vec2<u32>(scByte(base, j) & 63u, scByte(base, j + 4u) & 63u); }
  let dd = (scByte(base, j + 4u) & 0x0Fu) | ((scByte(base, j - 4u) >> 6u) << 4u);
  let mm = (scByte(base, j + 4u) >> 4u) | ((scByte(base, j) >> 6u) << 4u);
  return vec2<u32>(dd, mm);
}
fn q5val(qlw: u32, qhw: u32, m: u32, hi: bool, sbk: u32, dl: f32, ml: f32) -> f32 {
  let q = (qlw >> (8u * m)) & 0xFFu;
  let nibble = select(q & 0x0Fu, q >> 4u, hi);
  let hibit = ((((qhw >> (8u * m)) & 0xFFu) >> sbk) & 1u) * 16u;
  let p = dl * f32(nibble + hibit);
  return p - ml;
}
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let bi = gid.x + gid.y * nwg.x * 64u;
  if (bi >= params.numBlocks) { return; }
  let ob = params.blockOffset + bi;
  let sb = bi / 8u; let sbk = bi % 8u; let base = sb * 176u;
  let d = f16at(base); let dmin = f16at(base + 2u);
  let sm = getSM(base, sbk);
  let dl = d * f32(sm.x); let ml = dmin * f32(sm.y);
  let qlOff = base + 48u + (sbk >> 1u) * 32u; let hi = (sbk & 1u) == 1u;
  var ql: array<u32, 8>; load32(qlOff, &ql);
  var qh: array<u32, 8>; load32(base + 16u, &qh);
  var vals: array<f32, 32>;
  for (var w = 0u; w < 8u; w = w + 1u) {
    let qlw = ql[w]; let qhw = qh[w];
    vals[w * 4u + 0u] = q5val(qlw, qhw, 0u, hi, sbk, dl, ml);
    vals[w * 4u + 1u] = q5val(qlw, qhw, 1u, hi, sbk, dl, ml);
    vals[w * 4u + 2u] = q5val(qlw, qhw, 2u, hi, sbk, dl, ml);
    vals[w * 4u + 3u] = q5val(qlw, qhw, 3u, hi, sbk, dl, ml);
  }
  writeQ4Affine(ob, &vals);
}`}};var ot={},ft={},Ot={};var dm=`
fn div_exact(x: f32, s: f32, t: f32) -> f32 { let q0 = x * t; let r = fma(-s, q0, x); return fma(r, t, q0); }
fn recip_exact(s: f32) -> f32 { let t0 = 1.0 / s; let t1 = fma(fma(-s, t0, 1.0), t0, t0); return fma(fma(-s, t1, 1.0), t1, t1); }`;var Wl=`${dm}
fn r_half_up(x: f32) -> f32 { return floor(x + 0.5); }`;function Jk(e){return Wi(`${e?`enable f16;
`:""}@group(0) @binding(0) var<storage, read> emb: array<f32>;
@group(0) @binding(1) var<storage, read_write> packed: array<u32>;
@group(0) @binding(2) var<storage, read_write> scales: array<${e?"f16":"f32"}>;
struct P { chunkRows: u32, cols: u32, rowOffset: u32 }
@group(0) @binding(3) var<uniform> params: P;
${Wl}
fn qi8(x: f32) -> i32 { return clamp(i32(r_half_up(x)), -127, 127); }
var<workgroup> wmax: array<f32, 256>;
@compute @workgroup_size(256)
fn main(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_id) lid3: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let localRow = wid.x + wid.y * nwg.x;
  if (localRow >= params.chunkRows) { return; }
  let row = params.rowOffset + localRow;
  let lid = lid3.x; let cols = params.cols; let rowOff = localRow * cols;
  var m = 0.0;
  for (var c = lid; c < cols; c = c + 256u) { m = max(m, abs(emb[rowOff + c])); }
  wmax[lid] = m; workgroupBarrier();
  for (var s = 128u; s > 0u; s = s >> 1u) { if (lid < s) { wmax[lid] = max(wmax[lid], wmax[lid + s]); } workgroupBarrier(); }
  let maxAbs = wmax[0];
  let scale = select(1.0, div_exact(maxAbs, 127.0, recip_exact(127.0)), maxAbs > 0.0);
  let inv = recip_exact(scale);
  if (lid == 0u) { scales[row] = ${e?"f16(scale)":"scale"}; }
  let cols4 = cols >> 2u;
  for (var c4 = lid; c4 < cols4; c4 = c4 + 256u) {
    let base = rowOff + c4 * 4u;
    let q0 = qi8(emb[base] * inv); let q1 = qi8(emb[base + 1u] * inv);
    let q2 = qi8(emb[base + 2u] * inv); let q3 = qi8(emb[base + 3u] * inv);
    packed[row * cols4 + c4] = (u32(q0) & 0xFFu) | ((u32(q1) & 0xFFu) << 8u) | ((u32(q2) & 0xFFu) << 16u) | ((u32(q3) & 0xFFu) << 24u);
  }
}`)}function eA(e){return Wi(`${e?`enable f16;
`:""}@group(0) @binding(0) var<storage, read> emb: array<f32>;
@group(0) @binding(1) var<storage, read_write> bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> scales: array<${e?"f16":"f32"}>;
struct P { chunkBlocks: u32, cols: u32, blkOffset: u32 }
@group(0) @binding(3) var<uniform> params: P;
${Wl}
fn nib(q: i32) -> u32 { return u32(clamp(q, 0, 15)); }
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let localBlk = gid.x + gid.y * nwg.x * 64u;
  if (localBlk >= params.chunkBlocks) { return; }
  let blk = params.blkOffset + localBlk;
  let base = localBlk * 32u;
  var amax = 0.0; var vmax = 0.0;
  for (var j = 0u; j < 32u; j = j + 1u) { let v = emb[base + j]; let a = abs(v); if (a > amax) { amax = a; vmax = v; } }
  let d = vmax * -0.125;
  let id = select(0.0, recip_exact(d), d != 0.0);
  scales[blk] = ${e?"f16(d)":"d"};
  for (var w = 0u; w < 4u; w = w + 1u) {
    var word = 0u;
    for (var bb = 0u; bb < 4u; bb = bb + 1u) {
      let k = w * 4u + bb;
      let xlo = emb[base + k] * id; let xhi = emb[base + k + 16u] * id;
      let qlo = nib(i32(r_half_up(xlo)) + 8);
      let qhi = nib(i32(r_half_up(xhi)) + 8);
      word = word | (((qlo | (qhi << 4u)) & 0xFFu) << (bb * 8u));
    }
    bits[blk * 4u + w] = word;
  }
}`)}function tA(){return Wi(`@group(0) @binding(0) var<storage, read> emb: array<f32>;
@group(0) @binding(1) var<storage, read_write> bits: array<u32>;
@group(0) @binding(2) var<storage, read_write> scales: array<u32>;
struct P { chunkBlocks: u32, cols: u32, blkOffset: u32 }
@group(0) @binding(3) var<uniform> params: P;
${Wl}
fn code2(x: f32) -> u32 { return u32(clamp(i32(r_half_up(x)), 0, 3)); }
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
  let localBlk = gid.x + gid.y * nwg.x * 64u;
  if (localBlk >= params.chunkBlocks) { return; }
  let blk = params.blkOffset + localBlk;
  let base = localBlk * 32u;
  var lo = emb[base]; var hi = emb[base];
  for (var j = 1u; j < 32u; j = j + 1u) { let v = emb[base + j]; lo = min(lo, v); hi = max(hi, v); }
  var minv = lo;
  var step = div_exact(hi - lo, 3.0, recip_exact(3.0));
  var q: array<u32, 32>;
  for (var it = 0u; it < 3u; it = it + 1u) {
    if (!(step > 0.0)) { break; }
    let inv = recip_exact(step);
    for (var j = 0u; j < 32u; j = j + 1u) { let d = emb[base + j] - minv; let x = fma(d, inv, 0.0); q[j] = code2(x); }
    var sq = 0.0; var sqq = 0.0; var sw = 0.0; var sqw = 0.0;
    for (var j = 0u; j < 32u; j = j + 1u) {
      let qi = f32(q[j]); let wv = emb[base + j];
      sq = sq + qi;
      let qq = fma(qi, qi, 0.0); sqq = sqq + qq;
      sw = sw + wv;
      let qw = fma(qi, wv, 0.0); sqw = sqw + qw;
    }
    let a = fma(32.0, sqq, 0.0); let b = fma(sq, sq, 0.0); let det = a - b;
    if (det == 0.0) { break; }
    let rd = recip_exact(det);
    let p0 = fma(sqq, sw, 0.0); let p1 = fma(sq, sqw, 0.0); let numMin = p0 - p1;
    let p2 = fma(32.0, sqw, 0.0); let p3 = fma(sq, sw, 0.0); let numStep = p2 - p3;
    let nextMin = div_exact(numMin, det, rd);
    let nextStep = div_exact(numStep, det, rd);
    if (!(nextStep > 0.0)) { break; }
    minv = nextMin; step = nextStep;
  }
  let packedScale = pack2x16float(vec2<f32>(step, minv));
  let sv = unpack2x16float(packedScale);
  let stepH = sv.x; let minH = sv.y;
  scales[blk] = packedScale;
  let inv = select(0.0, recip_exact(stepH), stepH > 0.0);
  var w0 = 0u; var w1 = 0u;
  for (var j = 0u; j < 16u; j = j + 1u) {
    let d0 = emb[base + j] - minH; let x0 = fma(d0, inv, 0.0);
    let d1 = emb[base + 16u + j] - minH; let x1 = fma(d1, inv, 0.0);
    w0 = w0 | (code2(x0) << (2u * j));
    w1 = w1 | (code2(x1) << (2u * j));
  }
  bits[blk * 2u] = w0;
  bits[blk * 2u + 1u] = w1;
}`)}var rA=32*1024*1024;async function Hi(e,t,r,n,s,a,i){let u=i.embed??!0,o=i.head??"q4_0",l=o==="q4_0",c=o==="scan2";if(!u&&!o)throw new Error("transcodeEmbedRows: targets must select at least one of embed / head");let d=e.device.limits.maxComputeWorkgroupsPerDimension,p=n/4,f=n/32,g=u?e.empty("uint32",[r*p]):null,m=u?e.empty("float32",[r]):null,h=l?e.empty("uint32",[r*f*4]):null,v=l?e.empty(s?"float16":"float32",[r*f]):null,b=c?e.empty("uint32",[r*f*2]):null,_=c?e.empty("uint32",[r*f]):null,C=Math.max(1,Math.min(r,a??(Math.floor(rA/(n*4))||1)));for(let y=0;y<r;y+=C){let E=Math.min(C,r-y),k;try{k=await t(y,E)}catch(z){throw await Kt(e,[g,m,h,v,b,_]),z}let S=k instanceof Float32Array?e.tensorFromTypedArray("float32",[E*n],k):k,A=g&&m?e.createUniformU32([E,n,y,0],"embedQ8-params"):null,w=h&&v?e.createUniformU32([E*f,n,y*f,0],"embedQ4-params"):null,I=b&&_?e.createUniformU32([E*f,n,y*f,0],"embedQ2-params"):null;if(g&&m&&A&&await e.runProgram({name:"com.xenova.transcode.embedQ8Rows",source:Jk(!1),entryPoint:"main",cacheKey:"com.xenova.transcode:embedQ8Rows:f32",bindings:[{tensor:S,type:"read-only-storage",binding:0},{tensor:g,type:"storage",binding:1},{tensor:m,type:"storage",binding:2},{buffer:A,type:"uniform",binding:3}],dispatchWorkgroups:[Math.min(E,d),Math.ceil(E/d),1]}),h&&v&&w){let z=E*f,O=Math.ceil(z/64);await e.runProgram({name:"com.xenova.transcode.embedQ4Rows",source:eA(s),entryPoint:"main",cacheKey:`com.xenova.transcode:embedQ4Rows:${s?"f16":"f32"}`,bindings:[{tensor:S,type:"read-only-storage",binding:0},{tensor:h,type:"storage",binding:1},{tensor:v,type:"storage",binding:2},{buffer:w,type:"uniform",binding:3}],dispatchWorkgroups:[Math.min(O,d),Math.ceil(O/d),1]})}if(b&&_&&I){let z=Math.ceil(E*f/64);await e.runProgram({name:"com.xenova.transcode.embedQ2Rows",source:tA(),entryPoint:"main",cacheKey:"com.xenova.transcode:embedQ2Rows",bindings:[{tensor:S,type:"read-only-storage",binding:0},{tensor:b,type:"storage",binding:1},{tensor:_,type:"storage",binding:2},{buffer:I,type:"uniform",binding:3}],dispatchWorkgroups:[Math.min(z,d),Math.ceil(z/d),1]})}await e.queueIdle(),S.destroy(),A?.destroy?.(),w?.destroy?.(),I?.destroy?.()}return{embedQ8:g,embedQ8Scales:m,lmHeadQ4:h,lmHeadQ4Scales:v,lmHeadQ2:b,lmHeadQ2Scales:_}}function nA(e,t){let r=/^model\.layers\.(\d+)\.(.+)\.weight$/.exec(e);if(!r)return null;let[,n,s]=r,a=t[s];return a?`blk.${n}.${a}.weight`:null}var sA={"self_attn.q_proj":"attn_q","self_attn.k_proj":"attn_k","self_attn.v_proj":"attn_v","self_attn.o_proj":"attn_output","self_attn.q_norm":"attn_q_norm","self_attn.k_norm":"attn_k_norm","mlp.gate_proj":"ffn_gate","mlp.up_proj":"ffn_up","mlp.down_proj":"ffn_down",input_layernorm:"attn_norm",post_attention_layernorm:"ffn_norm"};function zl(e){return e==="model.embed_tokens.weight"?"token_embd.weight":e==="model.norm.weight"?"output_norm.weight":e==="lm_head.weight"?"output.weight":nA(e,sA)??e}function Kl(e,t){return{has:r=>e.has(t(r)),describe(r){let n=e.info(t(r));return{dtype:Xo(n.type),shape:[...n.shape],elementCount:n.elementCount}},async read(r){let n=t(r),s=e.info(n),a=await e.tensorBytes(n),i=Vr(s.type,a);return i?new Uint8Array(i.buffer,i.byteOffset,i.byteLength):a},async close(){}}}function Ul(e,t){return{async streamAll(r,{concurrency:n,chunkMaxBytes:s,names:a=null,signal:i}={}){if(!a)throw new Error("makeGGUFWeightSource.streamAll requires the plan's tensor names");let u=new Map(a.map(o=>[t(o),o]));await e.streamTensors([...u.keys()],async(o,l)=>{let c=Vr(e.info(o).type,l),d=c?new Uint8Array(c.buffer,c.byteOffset,c.byteLength):l;await r({bytes:d,tensors:[{name:u.get(o),offset:0,length:d.byteLength}]})},{concurrency:n,chunkMaxBytes:s,signal:i})},close(){}}}function tt(e){return e.slice(e.lastIndexOf(".")+1)}var Bn=64*1024*1024;function aA(e,t){for(;t!==0;)[e,t]=[t,e%t];return e}function iA(e){for(let t=1;t<=8;++t){let r=t*kn;try{let n=kr(e,r);return{elementsPerSourceBlock:r,bytesPerSourceBlock:n,outputBlocksPerSourceBlock:t}}catch{}}throw new Error(`Cannot determine GGML source-block geometry for type ${e}`)}function fm(e,t,r,n=Bn){if(!Number.isSafeInteger(t)||t<=0)throw new Error(`Invalid block-quant element count ${t}`);if(!Number.isSafeInteger(r)||r<=0)throw new Error(`Invalid block-quant byte length ${r}`);if(!Number.isSafeInteger(n)||n<=0)throw new Error(`Invalid block-quant chunk byte cap ${n}`);let s=Math.min(n,Bn),a=iA(e);if(t%a.elementsPerSourceBlock!==0)throw new Error(`GGML type ${e} element count ${t} is not divisible by source block ${a.elementsPerSourceBlock}`);let i=t/a.elementsPerSourceBlock,u=i*a.bytesPerSourceBlock;if(u!==r)throw new Error(`GGML type ${e} byte length ${r} does not match expected ${u}`);let o=4/aA(a.bytesPerSourceBlock,4),l=Math.floor(s/a.bytesPerSourceBlock),c=l-l%o;if(c<=0&&i>l)throw new Error(`GGML type ${e} source block cannot form a 4-byte-aligned chunk within ${s} bytes`);let d=[],p=0;for(;p<i;){let f=i-p,g=f<=l?f:c,m=p*a.bytesPerSourceBlock,h=m+g*a.bytesPerSourceBlock;d.push({relBegin:m,relEnd:h,outputBlockBegin:p*a.outputBlocksPerSourceBlock,outputBlockCount:g*a.outputBlocksPerSourceBlock,directMappable:m%4===0&&h%4===0}),p+=g}return d}var uA=new Set([T.PRISM_PQ2_0,T.PRISM_PTQ1_0,T.Q1_0,T.Q2_0,T.TQ1_0,T.TQ2_0,T.IQ2_XXS,T.IQ2_XS,T.IQ2_S,T.IQ3_XXS,T.IQ3_S,T.IQ1_S,T.IQ1_M]);function Nl(e){return e===T.Q4_0?4:uA.has(e)?null:mi(e)?8:null}var Ir={q_proj:["q_proj","k_proj","v_proj"],k_proj:["q_proj","k_proj","v_proj"],v_proj:["q_proj","k_proj","v_proj"],gate_proj:["gate_proj","up_proj"],up_proj:["gate_proj","up_proj"],linear_in_proj_qkv:["linear_in_proj_qkv"],linear_in_proj_z:["linear_in_proj_z"],attn_q:["attn_q","attn_k","attn_v"],attn_k:["attn_q","attn_k","attn_v"],attn_v:["attn_q","attn_k","attn_v"],gate_exps:["gate_exps","up_exps"],up_exps:["gate_exps","up_exps"]},oA={...Ir,q_proj:["q_proj"],k_proj:["k_proj"],v_proj:["v_proj"]},Hl={...oA,gate_proj:["gate_proj"],up_proj:["up_proj"]};var lA=new Set([T.Q4_K,T.Q4_1]);function qs(e,t,r,n=Ir){let s=new Set;for(let a of e){let i=tt(a.id);if(!t.has(i))continue;let u=r(a.id);if(u===null)continue;let o=n[i]??[i],l=a.id.slice(0,a.id.length-i.length);o.every(c=>r(l+c)===u)&&s.add(a.id)}return s}function $i(e,t,r,n,s){let a=new Map(e.map(i=>[i.id,i]));return i=>{let u=a.get(i);if(!u||s&&!s.has(tt(i)))return null;let o=r(u.name);return t.has(o)&&n(t.info(o).type)?"1":null}}function pm(e,t,r,n,s){return qs(e,n,$i(e,t,r,a=>lA.has(a),n),s)}var gm={linear_in_proj_qkv:["linear_in_proj_qkv","linear_in_proj_z"],linear_in_proj_z:["linear_in_proj_qkv","linear_in_proj_z"],q_proj:["q_proj","k_proj","v_proj"],k_proj:["q_proj","k_proj","v_proj"],v_proj:["q_proj","k_proj","v_proj"],gate_proj:["gate_proj","up_proj"],up_proj:["gate_proj","up_proj"]};function mm(e,t,r,n,s=Ir){let a=new Map(e.map(o=>[o.id,o])),i=o=>{let l=a.get(o);if(!l||!n.has(tt(o)))return!1;let c=r(l.name);return t.has(c)&&Nl(t.info(c).type)!==null},u=new Set;for(let o of e){let l=tt(o.id),c=s[l];if(!c||!i(o.id))continue;let d=o.id.slice(0,o.id.length-l.length);c.every(p=>i(d+p))||u.add(o.id)}return u}function en(e,t,r,n,s,a){let i=tt(e.id);if(!n.has(i)||s?.has(e.id)||a?.has(e.id))return!1;let u=r(e.name);return t.has(u)&&Nl(t.info(u).type)!==null}function $l(e,t,r,n,s,a){let i=!1,u=!0;for(let o of e)en(o,t,r,n,s,a)&&(i=!0,Nl(t.info(r(o.name)).type)!==4&&(u=!1));return i?u?4:8:null}function Bs(e,t){let r=new Map,n=u=>{let o=r.get(u);if(o===void 0)return r.set(u,u),u;if(o===u)return u;let l=n(o);return r.set(u,l),l},s=(u,o)=>{let l=n(u),c=n(o);l!==c&&r.set(c,l)},a=new Set(e.map(u=>u.id));for(let u of e){let o=tt(u.id),l=u.id.slice(0,u.id.length-o.length);for(let c of t?[Ir,t]:[Ir])for(let d of c[o]??[]){let p=l+d;a.has(p)&&s(u.id,p)}}let i=new Map;for(let u of e){if(u.packedGroup===void 0)continue;let o=i.get(u.packedGroup);o===void 0?i.set(u.packedGroup,u.id):s(o,u.id)}return new Map(e.map(u=>[u.id,n(u.id)]))}function In(e,t){if(e.length===0)return null;if(e.length>1)throw new Error(`${t} split across ${e.length} device-sized buffers, but its consumer binds one pack whole. Pack a smaller group at a time, or teach the consumer to resolve a pack per weight id.`);return e[0]}function cA(e,t){return bm(e,t)}function dA(e,t){return bm(e,t)}function bm(e,t){if(t%256!==0)return null;let r=e===T.Q2_0?Ot.Q2_0:e===T.PRISM_PQ2_0?Ot.PRISM_PQ2_0:e===T.PRISM_PTQ1_0?Ot.PRISM_PTQ1_0:void 0;return r&&!r.per16?r.lutId:null}var hm={q6_k:"q6k",q5_k:"q5k",q3_k:"q3k",q2_k:"q2k",q4_k_native_compact:"q4knc"};function _m(e,t){return e===T.Q4_K?t%256===0?"q4_k_native_compact":"q4_k":e===T.Q4_1?"q4_k":e===T.Q6_K&&t%256===0?"q6_k":e===T.Q5_K&&t%256===0?"q5_k":e===T.Q3_K&&t%256===0?"q3_k":e===T.Q2_K&&t%256===0?"q2_k":null}function fA(e,t){let n=_m(e,t);return n!=="q3_k"&&n!=="q2_k"&&n!=="q6_k"&&n!=="q5_k"?null:n}function pA(e,t){return!1}function gA(e,t,r){return!1}function mA(e,t){return e&&t==="q8_rows"}var km=256;function hA(e,t){return e?Math.ceil(t/km)<=Wt:!1}function Vi(e,t,r,n,s,a){let i={embedFormat:null,headPacked:!1,headFormat:null,headLut2:null,embedLut2:null,headRescoreQ8:!1,headScan2:!1,headAliasesEmbed:!1,headKSuperScan:!1};if(r.prismHadamard){let _=s?"token_embd.weight":"output.weight",C=e.has(_)?e.info(_).type:-1,y=C===T.Q2_0||C===T.PRISM_PQ2_0||C===T.PRISM_PTQ1_0,E=n.find(j=>j.id==="embed_tokens"),k=E?t(E.name):null,S=k&&e.has(k)?e.info(k).type:-1,A=S===T.Q2_0||S===T.PRISM_PQ2_0||S===T.PRISM_PTQ1_0,w=y&&r.hidden_size%256===0,I=w?cA(C,r.hidden_size):null,z=A&&r.hidden_size%256===0,O=z?dA(S,r.hidden_size):null;return{...i,headPacked:w,headLut2:I,embedLut2:O,embedFormat:z?O!==null?"lut2_32":"q4_0":null,headAliasesEmbed:s&&I!==null&&O===I}}let u=r.hidden_size;if(!a||!or(r)||u%32!==0)return i;let o=n.find(_=>_.id==="embed_tokens");if(!o)return i;let l=t(o.name);if(!e.has(l))return i;let c=e.info(l).type,p=_m(c,u)??(Zo(c)?"q8_rows":null),f=s?c:(()=>{let _=n.find(y=>y.id==="lm_head"),C=_?t(_.name):null;return C&&e.has(C)?e.info(C).type:null})(),g=f!==null&&ys(r)&&Zo(f),m=g?fA(f,u)??(pA(s,p)?"q8_rows":null):null,h=s&&m!==null&&m===p,v=g&&m===null&&mA(s,p),b=gA(s,m,r.vocab_size);return{embedFormat:p,headPacked:g,headFormat:m,headLut2:null,embedLut2:null,headRescoreQ8:v,headScan2:hA(v,r.vocab_size),headAliasesEmbed:h,headKSuperScan:b}}function Is(e,t,r,n,s=null){let a=t*(r/256),i=t*(r/32);if(e==="q4_0")return{bits:i*4,scales:i,scalesDtype:"float32"};if(e==="lut2_32"){if(s!==8&&s!==9)throw new Error(`lut2_32 gather needs codebook 8 or 9, got ${s}`);return{bits:i*2,scales:a*(s===9?1:3),scalesDtype:"uint32"}}return e==="q8_rows"?{bits:t*(r/4),scales:t,scalesDtype:"float32"}:e==="q6_k"?{bits:a*52,scales:a,scalesDtype:"float32"}:e==="q5_k"?{bits:a*44,scales:a*2,scalesDtype:"float32"}:e==="q3_k"?{bits:a*28,scales:a,scalesDtype:"float32"}:e==="q2_k"?{bits:a*20,scales:a*2,scalesDtype:"float32"}:e==="q4_k_native_compact"?{bits:a*32,scales:a*5,scalesDtype:"uint32"}:{bits:i*4,scales:i*2,scalesDtype:n?"float16":"float32"}}async function vm(e,t,r,n,s){let{spec:a,u32PerSuper:i,scalesPerSuper:u}=s.kSuperRoutes[r],o=In(await s.packKSuperGGUFWeights({...e,entries:[t]},{ids:new Set([t.id]),spec:a,u32PerSuper:i,scalesPerSuper:u,scalesF16:n}),`the ${r} gather table for ${t.name}`);if(o&&qr(o.offsets,t.id)!==0)throw new Error(`${t.name} must be packed at block 0 of its gather buffer`);return o}async function Am(e,t,r,n){let s=new Set,a={skipDenseIds:s},i=[],u=o=>(i.push(o),o);try{return await vA(a,s,u,e,t,r,n)}catch(o){throw await Kt(e.runtime,[a,i]),o}}async function vA(e,t,r,n,s,a,i){let{runtime:u,entries:o,gguf:l,nameToGGUF:c}=n,d=s.hidden_size,p=s.vocab_size,f=Vi(l,c,s,o,a,u.device.features.has("shader-f16"));if(!f.embedFormat&&!f.headPacked)return null;let g=o.find(S=>S.id==="embed_tokens");if(!g)return null;let m=c(g.name),h=a?g:o.find(S=>S.id==="lm_head"),v=h?c(h.name):null,b=f.headPacked&&v!==null;if(s.prismHadamard){for(let[S,A]of[[g,"embed"],[h,"head"]]){if(!S||(A==="embed"?f.embedFormat===null:!b)||A==="embed"&&f.headAliasesEmbed&&b)continue;let w=l.info(c(S.name)).type,I=A==="head"?f.headLut2:f.embedLut2;if(I!==null){let j=In(r(await i.packLutGGUFWeights({...n,entries:[S]},{lutIds:new Set([S.id]),per16:!1,compact:!0,lut2:!0})),`Prism two-bit ${A}`);if(!j)throw new Error(`Prism ${A} did not produce a two-bit LUT pack`);if(qr(j.offsets,S.id)!==0)throw new Error(`${S.name} must be packed at block 0 of its ${A} buffer`);A==="head"&&(e.lmHeadQ4=j.bits,e.lmHeadQ4Scales=j.scales,e.lmHeadLut=I,t.add("lm_head")),(A==="embed"||f.headAliasesEmbed)&&(e.embedBits=j.bits,e.embedScales=j.scales,e.embedFormat="lut2_32",e.embedLut=I,t.add(g.id));continue}let z=w===T.Q2_0?qe.Q2_0Q4:w===T.PRISM_PQ2_0?qe.PRISM_PQ2_0Q4:qe.PRISM_PTQ1_0Q4,O=In(r(await i.packKSuperGGUFWeights({...n,entries:[S]},{ids:new Set([S.id]),spec:z,u32PerSuper:32,scalesPerSuper:8,scalesF16:A==="head"&&u.device.features.has("shader-f16")})),`Prism exact ${A}`);if(!O)throw new Error(`Prism ${A} did not produce an exact Q4 pack`);A==="embed"?(e.embedBits=O.bits,e.embedScales=O.scales,e.embedFormat="q4_0"):(e.lmHeadQ4=O.bits,e.lmHeadQ4Scales=O.scales),t.add(A==="head"?"lm_head":S.id)}return e}let _=l.info(m).type,C=f.embedFormat,y=C?hm[C]:void 0;if(C&&y){let S=r(await vm(n,g,y,!1,i));if(!S)throw new Error(`${m} resolved to the ${y} gather but produced no pack`);e.embedBits=S.bits,e.embedScales=S.scales,e.embedFormat=C,t.add("embed_tokens")}else if(f.embedFormat==="q4_k"){let S=In(r(await i.packKq4GGUFWeights({...n,entries:[g]},{kq4Ids:new Set([g.id])})),`the kq4 gather table for ${m}`);if(!S)throw new Error(`${m} resolved to the kq4 gather but produced no pack`);if(qr(S.offsets,g.id)!==0)throw new Error(`${m} must be packed at block 0 of its gather buffer`);e.embedBits=S.bits,e.embedScales=S.scales,e.embedFormat="q4_k",t.add("embed_tokens")}else if(f.embedFormat==="q8_rows"){let S=b&&v===m,w=S&&f.headFormat!=="q8_rows"?f.headScan2?"scan2":"q4_0":!1,I=r(await Vl(u,l,m,_,p,d,{embed:!0,head:w}));if(!I.embedQ8||!I.embedQ8Scales)throw new Error(`${m} produced no q8_rows gather pack`);let z=w==="scan2"?I.lmHeadQ2:I.lmHeadQ4,O=w==="scan2"?I.lmHeadQ2Scales:I.lmHeadQ4Scales;e.embedBits=I.embedQ8,e.embedScales=I.embedQ8Scales,e.embedFormat="q8_rows",t.add("embed_tokens"),S&&f.headFormat==="q8_rows"?(e.lmHeadQ8=I.embedQ8,e.lmHeadQ8Scales=I.embedQ8Scales,t.add("lm_head")):w&&z&&O&&(e.lmHeadQ4=z,e.lmHeadQ4Scales=O,w==="scan2"&&(e.lmHeadScanBits=2),t.add("lm_head"),f.headRescoreQ8&&(e.lmHeadQ8=I.embedQ8,e.lmHeadQ8Scales=I.embedQ8Scales))}let E=b&&!e.lmHeadQ4&&!e.lmHeadQ8?f.headFormat:null;if(E==="q8_rows")throw new Error(`${v} resolved to the q8_rows head, which only the tied gather pass can bind`);let k=E;if(k&&f.headAliasesEmbed&&e.embedBits&&e.embedScales)f.headKSuperScan?await wm(u,l,v,k,p,d,e.embedBits,e.embedScales,e):(e.lmHeadQ4=e.embedBits,e.lmHeadQ4Scales=e.embedScales,e.lmHeadFormat=k),t.add("lm_head");else if(k&&h){let S=hm[k];if(!S)throw new Error(`${v} resolved to the ${k} head grid, which has no k-super route`);let A=r(await vm(n,h,S,!1,i));if(!A)throw new Error(`${v} resolved to the ${S} head grid but produced no pack`);f.headKSuperScan?await wm(u,l,v,k,p,d,A.bits,A.scales,e):(e.lmHeadQ4=A.bits,e.lmHeadQ4Scales=A.scales,e.lmHeadFormat=k),t.add("lm_head")}else if(b&&!e.lmHeadQ4&&!e.lmHeadQ8){let S=r(await Vl(u,l,v,l.info(v).type,p,d,{embed:!1,head:"q4_0"}));if(!S.lmHeadQ4||!S.lmHeadQ4Scales)throw new Error(`${v} produced no Q4_0 head pack`);e.lmHeadQ4=S.lmHeadQ4,e.lmHeadQ4Scales=S.lmHeadQ4Scales,t.add("lm_head")}return t.size>0?e:null}function wA(e,t){let n={[T.Q4_0]:{spec:lr.Q4_0,blockElements:32},[T.Q4_1]:{spec:lr.Q4_1,blockElements:32},[T.Q5_0]:{spec:lr.Q5_0,blockElements:32},[T.Q5_1]:{spec:lr.Q5_1,blockElements:32},[T.Q4_K]:{spec:lr.Q4_K,blockElements:256},[T.Q5_K]:{spec:lr.Q5_K,blockElements:256},[T.Q6_K]:{spec:lr.Q6_K,blockElements:256}}[e];return n?.spec&&t%n.blockElements===0?n.spec:void 0}async function wm(e,t,r,n,s,a,i,u,o){let l=await Vl(e,t,r,t.info(r).type,s,a,{embed:!1,head:"scan2"});if(!l.lmHeadQ2||!l.lmHeadQ2Scales)throw new Error(`${r} produced no 2-bit scan grid for its k-super head`);o.lmHeadQ4=l.lmHeadQ2,o.lmHeadQ4Scales=l.lmHeadQ2Scales,o.lmHeadScanBits=2,o.lmHeadQ8=i,o.lmHeadQ8Scales=u,o.lmHeadRescoreFormat=n}async function Vl(e,t,r,n,s,a,i){let u=Pg(t,r,kr(n,a)),o=wA(n,a);if(!o)return Hi(e,async(p,f)=>Yo(n,await u(p,f)),s,a,!0,void 0,i);let l=e.empty("float32",[1],"embedDequant.scales"),c=new Jr,d=new Map;try{return await Hi(e,async(p,f)=>{c.destroyAll();let g=e.empty("float32",[f*a],"embedDequant.chunk");try{await Ui(e,o,await u(p,f),g,l,0,f*a/32,!1,d,c)}catch(m){throw g.destroy(),m}return g},s,a,!0,void 0,i)}finally{c.destroyAll(),l.destroy()}}var Zl={q6k:{wordsPerBlock:6.5,scalesPerBlock:.125,metadataU32:!1},q5k:{wordsPerBlock:5.5,scalesPerBlock:.25,metadataU32:!1},q3k:{wordsPerBlock:3.5,scalesPerBlock:.125,metadataU32:!1},q2k:{wordsPerBlock:2.5,scalesPerBlock:.25,metadataU32:!1},q4knc:{wordsPerBlock:4,scalesPerBlock:.625,metadataU32:!0}};function ym(e,t,r){let n=Number(e.device.limits.maxBufferSize),s=Number(e.device.limits.maxStorageBufferBindingSize);if(t%4!==0||t<=0||Number.isFinite(n)&&t>n||Number.isFinite(s)&&t>s)return null;let a=e.beginMappedTensorUpload("uint32",[t/4],`${r}.direct-source`);if(!a||a.destination.byteLength!==t){try{a?.abort()}catch{}return null}return{bytes:a.destination,commit:()=>a.commit(),rollback:i=>a.rollback(i),abort:()=>a.abort()}}function bA(e,t,r,n){if(e.byteLength%Ht!==0)throw new Error(`Q8_0 byte length ${e.byteLength} is not a multiple of ${Ht}`);let s=e.byteLength/Ht,a=new DataView(e.buffer,e.byteOffset,e.byteLength);for(let i=0;i<s;++i){let u=i*Ht;r[n+i]=Qe(a.getUint16(u,!0));let o=u+2;for(let l=0;l<8;++l){let c=0;for(let d=0;d<4;++d)c|=(e[o+l*4+d]&255)<<d*8;t[(n+i)*8+l]=c>>>0}}}function _A(e){let t=e.length/$r,r=new Uint32Array(t*8),n=new Float32Array(t);for(let s=0;s<t;++s){let a=s*$r,i=0;for(let l=0;l<$r;++l){let c=Math.abs(e[a+l]);c>i&&(i=c)}let u=i/127,o=u>0?1/u:0;n[s]=u;for(let l=0;l<8;++l){let c=0;for(let d=0;d<4;++d){let p=Math.round(e[a+l*4+d]*o);p>127?p=127:p<-127&&(p=-127),c|=(p&255)<<d*8}r[s*8+l]=c>>>0}}return{bits:r,scales:n}}function kA(e){switch(e){case T.Q4_K:return Lt.Q4_K;case T.Q6_K:return Lt.Q6_K;case T.Q5_K:return Lt.Q5_K;case T.Q3_K:return Lt.Q3_K;case T.Q2_K:return Lt.Q2_K;case T.IQ4_XS:return Lt.IQ4_XS;case T.IQ4_NL:return Lt.IQ4_NL;case T.Q5_0:return Lt.Q5_0;case T.Q5_1:return Lt.Q5_1;default:return null}}function AA(e,t){return e===4?qe.Q4_0:t===T.Q8_0?qe.Q8_0:t===T.Q8_1?qe.Q8_1:t===T.Q8_K?qe.Q8_KQ8:kA(t)}var yA=8;async function Rs({runtime:e,entries:t,gguf:r,nameToGGUF:n,signal:s,streamByteBudget:a,shardUnitOf:i},{routeName:u,include:o,guardElements:l=kn,bitsLength:c,scalesLength:d,scalesF16:p,metadataStorage:f="float",specForSlot:g,cpuFallback:m}){let h=[];for(let Q of t){if(!o(Q))continue;let N=n(Q.name),q=r.info(N);if(q.elementCount%l!==0)throw new Error(`${u} tensor ${N} element count ${q.elementCount} not divisible by ${l}`);h.push({id:Q.id,gname:N,blocks:q.elementCount/kn,type:q.type})}if(h.length===0)return[];let v=0,b=h.map(Q=>{let N={gname:Q.gname,baseBlock:v,blocks:Q.blocks,type:Q.type,shardIndex:0};return v+=Q.blocks,N}),_=f==="float"&&p(b),C=f!=="float"?"uint32":_?"float16":"float32",y=xr(e.device.limits),E=ve(C),k=Q=>Math.max(c(Q)*Uint32Array.BYTES_PER_ELEMENT,d(Q)*E),S=Cr(h.map(Q=>{let N=i?.(Q.id);return{id:Q.id,blockCount:Q.blocks,...N===void 0?{}:{unitKey:N}}}),Tr({byteLimit:y,maxBlocks:h.reduce((Q,N)=>Q+N.blocks,0),bytesForBlocks:k}),u),A=S.shardBlockCounts;for(let Q=0;Q<A.length;++Q){let N=c(A[Q])*Uint32Array.BYTES_PER_ELEMENT,q=d(A[Q])*E;if(!(N<=y&&q<=y))throw new Error(`${u} pack ${Q} needs ${N} bytes of bits and ${q} of scales, over this device's ${y}-byte buffer limit. One indivisible co-bound weight group is larger than a single buffer, or the support probe planned a different split than the loader.`)}let w=A.map(()=>Dt()),I=A.map(()=>Dt()),z=A.map(()=>new Set),O=h.map(Q=>{let N=S.placements.get(Q.id);return ut(w[N.shardIndex],Q.id,N.blockOffset),ut(I[N.shardIndex],Q.id,Q.blocks),z[N.shardIndex].add(Q.id),{gname:Q.gname,baseBlock:N.blockOffset,blocks:Q.blocks,type:Q.type,shardIndex:N.shardIndex}}),j=Q=>Q.map((N,q)=>({bits:N.bits,scales:N.scales,offsets:w[q],blockCounts:I[q],packedIds:z[q],layout:O.filter(x=>x.shardIndex===q)})),F=new Map(O.map(Q=>[Q.gname,Q])),K=wl();if(O.every(Q=>g(Q)!==null)){let Q=A.map(D=>({bits:e.empty("uint32",[c(D)]),scales:e.empty(C,[d(D)])})),N=new Map,q=new Set,x=new Map,B=[],U=[],M=0,G=D=>{let se=D?new Set(D):null,V=B.filter(le=>!se||se.has(le.pending));B=B.filter(le=>se&&!se.has(le.pending));let ee=new Set;for(let[le,me]of x)se&&!se.has(le)||(ee.add(me),x.delete(le),q.delete(me));if(!se){for(let le of q)ee.add(le);q.clear()}return{direct:V,pools:ee}},$=D=>{let{direct:se,pools:V}=G(D);for(let ee of V)ee.destroyAll();for(let{staging:ee}of se)ee.destroy()},X=async D=>{try{await e.queueIdle()}catch{}let{direct:se,pools:V}=G(D);for(let{destination:ee,staging:le}of se)try{await ee.rollback(le)}catch{}for(let ee of V)ee.destroyAll()},H=async()=>{if(U.length===0)return;let D=U;U=[],M=0;try{await Ql(e,D),await e.queueIdle()}catch(se){throw await X(D),se}$(D)},W=async(D,se,V,ee,le)=>{let me=g(D),{bits:We,scales:Oe}=Q[D.shardIndex];if(!V){let de=new Jr;q.add(de);let Ae=Ni(e,me,se,We,Oe,ee,le,_,N,de);return x.set(Ae,de),Ae}let te=null;try{if(se.buffer!==V.bytes.buffer||se.byteOffset!==V.bytes.byteOffset||se.byteLength!==V.bytes.byteLength)throw new Error(`Direct GGUF source bytes for ${D.gname} do not alias the destination`);let de=Ki(e,me,N);te=await V.commit();let Ae=zi(e,me,te,We,Oe,ee,le,_,de);return B.push({destination:V,staging:te,pending:Ae}),Ae}catch(de){if(te)try{await V.rollback(te)}catch{}else try{await V.abort()}catch{}throw de}},P=r.streamTensorRange?.bind(r),R=P?O.filter(D=>r.info(D.gname).byteLength>Bn):[],Y=new Set(R.map(D=>D.gname)),ne=R.length===0?O:O.filter(D=>!Y.has(D.gname)),re=r.supportsDirectDestinations===!0?new Map(ne.flatMap(D=>{let se=r.info(D.gname).byteLength;return se%4!==0?[]:[[D.gname,()=>ym(e,se,D.gname)]]})):void 0;try{ne.length>0&&(await r.streamTensors(ne.map(D=>D.gname),async(D,se,V)=>{let ee=F.get(D),le=se.byteLength;U.length>0&&M+le>Bn&&await H();let me=await W(ee,se,V,ee.baseBlock,ee.blocks);U.push(me),M+=le,(U.length>=yA||M>=Bn)&&await H(),await K(le)},{signal:s,directDestinations:re,byteBudget:a}),await H(),await e.queueIdle(),$());for(let D of R){let se=r.info(D.gname);for(let V of fm(D.type,se.elementCount,se.byteLength)){let ee=!1,le=r.supportsDirectDestinations===!0&&V.directMappable?()=>ym(e,V.relEnd-V.relBegin,`${D.gname}[${V.relBegin}:${V.relEnd}]`):void 0;if(await P(D.gname,V.relBegin,V.relEnd,async(me,We)=>{if(ee)throw new Error(`GGUF range stream delivered ${D.gname} chunk more than once`);ee=!0;let Oe=me.byteLength,te=V.relEnd-V.relBegin;if(Oe!==te)throw new Error(`GGUF range stream delivered ${Oe} bytes for ${D.gname}; expected ${te}`);try{let de=await W(D,me,We,D.baseBlock+V.outputBlockBegin,V.outputBlockCount);U.push(de),await H(),await e.queueIdle()}catch(de){for(let Ae of U)Ae.release();throw U=[],M=0,await X(),de}$()},{signal:s,directDestination:le}),!ee)throw new Error(`GGUF range stream did not deliver ${D.gname} chunk`);await K(V.relEnd-V.relBegin)}}}catch(D){for(let se of U)se.release();U=[],M=0,await X();for(let se of Q)se.bits.destroy(),se.scales.destroy();throw D}return $(),j(Q)}if(!m){let Q=O.find(N=>g(N)===null);throw new Error(`${u}: no transcode kernel for ${Q.gname} (type ${Q.type})`)}if(f!=="float")throw new Error(`${u}: uint32 metadata routes require a GPU transcode kernel for every slot`);let Z=A.map(Q=>({bits:new Uint32Array(c(Q)),scales:new Float32Array(d(Q))}));return await r.streamTensors([...F.keys()],async(Q,N)=>{let q=F.get(Q),x=Z[q.shardIndex];m(q,N,x.bits,x.scales),await K(N.byteLength)},{signal:s}),j(Z.map(Q=>({bits:e.tensorFromTypedArray("uint32",[Q.bits.length],Q.bits),scales:_?e.tensorFromTypedArray("float16",[Q.scales.length],Hr(Q.scales)):e.tensorFromTypedArray("float32",[Q.scales.length],Q.scales)})))}async function SA(e,{packSuffixes:t,excludeKq4Ids:r,forceDenseIds:n}){let{runtime:s,entries:a,gguf:i,nameToGGUF:u}=e,o=$l(a,i,u,t,r,n);if(o===null)return[];let l=o;return(await Rs(e,{routeName:"packed",include:d=>en(d,i,u,t,r,n),bitsLength:d=>d*l,scalesLength:d=>d,scalesF16:()=>s.device.features.has("shader-f16"),specForSlot:d=>AA(o,d.type),cpuFallback:(d,p,f,g)=>{if(d.type===T.Q8_0)bA(p,f,g,d.baseBlock);else{let m=Vr(d.type,p);if(!m)throw new Error(`packQ4GGUFWeights: ${d.gname} type ${d.type} is not dequantizable`);let h=_A(m);f.set(h.bits,d.baseBlock*8),g.set(h.scales,d.baseBlock)}}})).map(d=>({...d,packedQuantBits:o}))}async function Sm(e,{kq4Ids:t,scalesF16:r}){return await Rs(e,{routeName:"kq4",include:s=>t.has(s.id),bitsLength:s=>s*4,scalesLength:s=>s*2,scalesF16:()=>(r??!0)&&e.runtime.device.features.has("shader-f16"),specForSlot:s=>s.type===T.Q4_1?qe.Q4_1:s.type===T.Q5_K?jl.Q5_K_TO_KQ4:qe.Q4_K})}var Zi=new Set(["gate_proj","up_proj","down_proj","o_proj","q_proj","k_proj","v_proj","linear_out_proj","conv_out_proj","conv_in_proj","linear_in_proj_qkv","linear_in_proj_z"]),xm=new Set([...Zi,"gate_exps","up_exps","down_exps"]),xA=new Set(["conv_in_proj","conv_out_proj"]),TA=new Set([...xm].filter(e=>!xA.has(e))),Xl={q6k:{format:"q6_k",ggmlType:T.Q6_K,env:"Q6K_PACK",u32PerSuper:52,scalesPerSuper:1,spec:qe.Q6_KN,metadataU32:!1},q5k:{format:"q5_k",ggmlType:T.Q5_K,env:"Q5K_PACK",suffixes:xm,u32PerSuper:44,scalesPerSuper:2,spec:qe.Q5_KN,metadataU32:!1},q3k:{format:"q3_k",ggmlType:T.Q3_K,env:"Q3K_PACK",suffixes:Zi,u32PerSuper:28,scalesPerSuper:1,spec:qe.Q3_KN,metadataU32:!1},q2k:{format:"q2_k",ggmlType:T.Q2_K,env:"Q2K_PACK",suffixes:Zi,u32PerSuper:20,scalesPerSuper:2,spec:qe.Q2_KN,metadataU32:!1},q4knc:{format:"q4_k_native_compact",ggmlType:T.Q4_K,env:"Q4KNC_PACK",suffixes:TA,u32PerSuper:32,scalesPerSuper:5,spec:qe.Q4_KNC,metadataU32:!0}};function CA(e,t,r,n,s,a,i,u,o,l){if(!t.has(e.format))return new Set;let c=e.format==="q3_k"||e.format==="q2_k"?u??i:i,d=c?new Set([...a].filter(g=>c.has(g))):a,p=e.suffixes?o??e.suffixes:void 0,f=p?new Set([...d].filter(g=>p.has(g))):d;return f.size===0?new Set:BA(r,n,s,f,e.ggmlType,l)}function qA(e,t,r,n,s,a,i,u,o){return Object.fromEntries(Object.entries(Xl).map(([l,c])=>[l,CA(c,e,t,r,n,s,a,i,u,o)]))}function BA(e,t,r,n,s,a){return qs(e,n,$i(e,t,r,i=>i===s),a)}async function Tm(e,{ids:t,spec:r,u32PerSuper:n,scalesPerSuper:s,scalesF16:a=!1}){let i=r.metadataStorage??"float";if(i==="uint32"&&a)throw new Error(`k-super transcode ${r.id} cannot combine uint32 metadata with f16 scale storage`);return(await Rs(e,{routeName:"k-super",include:o=>t.has(o.id),guardElements:256,bitsLength:o=>o/8*n,scalesLength:o=>o/8*s,scalesF16:()=>a&&e.runtime.device.features.has("shader-f16"),metadataStorage:i,specForSlot:()=>r})).map(o=>({...o,u32PerSuper:n,scalesPerSuper:s,metadataStorage:i}))}async function Es(e,t,r,n){if(r.size===0)return[];let{spec:s,u32PerSuper:a,scalesPerSuper:i}=Xl[t];return Tm(e,{ids:r,spec:s,u32PerSuper:a,scalesPerSuper:i,scalesF16:n})}function IA(e,t,r,n,s){return qs(e,n,$i(e,t,r,fi,n),s)}async function EA(e,{iqIds:t}){return await Rs(e,{routeName:"IQ",include:n=>t.has(n.id),bitsLength:n=>n*8,scalesLength:n=>n*2,scalesF16:()=>e.runtime.device.features.has("shader-f16"),specForSlot:n=>PA(n.type),cpuFallback:(n,s,a,i)=>{let u=Pp(n.type,s);a.set(u.bits,n.baseBlock*8),i.set(u.scales,n.baseBlock*2)}})}function PA(e){return e===T.Q2_0?qe.Q2_0IQ:e===T.PRISM_PQ2_0?qe.PRISM_PQ2_0IQ:e===T.PRISM_PTQ1_0?qe.PRISM_PTQ1_0IQ:e===T.TQ1_0?qe.TQ1_0IQ:e===T.TQ2_0?qe.TQ2_0IQ:e===T.IQ2_XXS?qe.IQ2_XXS:e===T.IQ2_XS?qe.IQ2_XS:e===T.IQ2_S?qe.IQ2_S:e===T.IQ3_S?qe.IQ3_S:e===T.IQ3_XXS?qe.IQ3_XXS:null}var Xi=new Map([[T.IQ2_XXS,ot.IQ2_XXS],[T.IQ2_XS,ot.IQ2_XS],[T.IQ2_S,ot.IQ2_S],[T.IQ3_XXS,ot.IQ3_XXS],[T.IQ3_S,ot.IQ3_S],[T.IQ4_NL,ot.IQ4_NL],[T.IQ4_XS,ot.IQ4_XS],[T.Q3_K,ot.Q3_K],[T.MXFP4,ot.MXFP4],[T.NVFP4,ot.NVFP4],[T.IQ1_S,ot.IQ1_S],[T.IQ1_M,ot.IQ1_M]].filter(e=>e[1]!==void 0)),Ps=new Map([[T.Q2_0,Ot.Q2_0],[T.PRISM_PQ2_0,Ot.PRISM_PQ2_0],[T.PRISM_PTQ1_0,Ot.PRISM_PTQ1_0],[T.IQ1_S,Ot.IQ1_S],[T.IQ1_M,Ot.IQ1_M]].filter(e=>e[1]!==void 0)),Cm=new Map([[T.IQ2_XXS,ft.IQ2_XXS],[T.IQ2_XS,ft.IQ2_XS],[T.IQ2_S,ft.IQ2_S],[T.IQ3_XXS,ft.IQ3_XXS],[T.IQ3_S,ft.IQ3_S],[T.IQ4_XS,ft.IQ4_XS],[T.Q3_K,ft.Q3_K],[T.MXFP4,ft.MXFP4],[T.NVFP4,ft.NVFP4],[T.IQ1_S,ft.IQ1_S],[T.IQ1_M,ft.IQ1_M]].filter(e=>e[1]!==void 0));var RA=new Set(["conv_in_proj"]),GA=new Set([...Zi,"linear_in_proj_qkv","linear_in_proj_z"].filter(e=>!RA.has(e)));function DA(){return GA}function qm(e){let t=Xi.get(e)??Ps.get(e);return t?{lutId:t.lutId,per16:t.per16}:null}function LA(e,t,r,n,s,a,i){let u=()=>({lut32Ids:new Set,lut16Ids:new Set,lut32cIds:new Set,lut16cIds:new Set,lut2_32Ids:new Set,lut2_16Ids:new Set}),o=new Map(e.map(m=>[m.id,m])),l=(m,h)=>s?.has(tt(m))&&i?.has(tt(m))&&[8,9].includes(Ps.get(h)?.lutId??0)?Ps.get(h):null,c=m=>{let h=o.get(m);if(!h||!n.has(tt(m)))return null;let v=r(h.name);return t.has(v)?Xi.get(t.info(v).type)??l(m,t.info(v).type):null},d=m=>{if(s===void 0)return null;let h=o.get(m);if(!h||!s.has(tt(m)))return null;let v=r(h.name);return t.has(v)?Cm.get(t.info(v).type)??l(m,t.info(v).type):null},p=m=>{if(i===void 0||d(m)===null||!i.has(tt(m)))return null;let h=o.get(m),v=r(h.name);return t.has(v)?Ps.get(t.info(v).type)??null:null},f=qs(e,n,m=>{let h=c(m);return h?`${h.lutId}|${h.per16?16:32}|${d(m)?"c":"x"}|${p(m)?2:4}`:null},a),g=u();for(let m of f){let h=c(m).per16;if(p(m)!==null){(h?g.lut2_16Ids:g.lut2_32Ids).add(m);continue}(d(m)!==null?h?g.lut16cIds:g.lut32cIds:h?g.lut16Ids:g.lut32Ids).add(m)}return g}async function cr(e,{lutIds:t,per16:r,compact:n=!1,lut2:s=!1}){let{runtime:a,entries:i,gguf:u,nameToGGUF:o}=e;if(s&&!n)throw new Error("packLutGGUFWeights: the 2-bit LUT pack has no expanded form");let l=s?Ps:n?Cm:Xi,c=new Set(i.filter(f=>t.has(f.id)&&l.get(u.info(o(f.name)).type)?.lutId===9).map(f=>f.id));if(c.size>0&&c.size<t.size){let f=new Set([...t].filter(m=>!c.has(m))),g=await cr(e,{lutIds:f,per16:r,compact:n,lut2:s});try{let m=await cr(e,{lutIds:c,per16:r,compact:n,lut2:s});return[...g,...m]}catch(m){for(let h of g)h.bits.destroy(),h.scales.destroy();throw m}}let d=c.size>0?1:r?5:3;return(await Rs(e,{routeName:n?"LUT-compact":"LUT",include:f=>{if(!t.has(f.id))return!1;let g=o(f.name),m=u.info(g),h=l.get(m.type);if(!h||h.per16!==r)throw new Error(`packLutGGUFWeights: ${g} type ${m.type} is not a per${r?16:32}${s?" 2-bit":""} LUT route`);return!0},...n?{guardElements:256,metadataStorage:"atomicU32"}:{},bitsLength:f=>f*(s?2:4),scalesLength:f=>n?f/8*d:f*(r?2:1),scalesF16:f=>!n&&a.device.features.has("shader-f16")&&f.every(g=>Xi.get(g.type)?.scaleF16Exact===!0),specForSlot:f=>l.get(f.type).spec})).map(f=>{let g=new Map;for(let m of i)f.packedIds.has(m.id)&&g.set(m.id,l.get(u.info(o(m.name)).type).lutId);return{...f,lutIds:g,lut16:r?1:0,...n?{compact:1}:{},...s?{lut2:1}:{}}})}var _t=[];function Yl(e,t,r,n,s={}){let{nativeFormats:a=new Set,nativeSuffixes:i,routeSuffixes:u,fusedGroups:o=!0,decodeGroups:l,iqRawTrunkSuffixes:c}=s,d=o?l??Ir:{},p=u??DA(),f=s.lutcSuffixes??i,g=a.has("lut32_compact")||a.has("lut16_compact")?f?new Set([...p].filter(G=>f.has(G))):p:void 0,m=new Map(e.map(G=>[G.id,G])),h=qA(a,e,t,r,n,i,s.subFourKSuperSuffixes,u,d),{q6k:v,q5k:b,q3k:_,q2k:C,q4knc:y}=h,E=pm(e,t,r,n,d);for(let G of y)E.delete(G);let k=new Map(_t.map(G=>[G.key,new Set])),S=new Set([...k.values()].flatMap(G=>[...G])),A=s.lut2Suffixes,w=g&&A?new Set([...g].filter(G=>A.has(G))):void 0,{lut32Ids:I,lut16Ids:z,lut32cIds:O,lut16cIds:j,lut2_32Ids:F,lut2_16Ids:K}=LA(e,t,r,n,g,d,w),Z=[I,z,O,j,F,K];for(let G of S)for(let $ of Z)$.delete(G);for(let G of Object.values(h))for(let $ of G)for(let X of Z)X.delete($);let Q=new Set(Z.flatMap(G=>[...G])),N=mm(e,t,r,n,d),q=new Set([...IA(e,t,r,n,d)].filter(G=>!Q.has(G)&&!S.has(G))),x=new Set([...E,...Object.values(h).flatMap(G=>[...G]),...Q,...S]),B=[E,...Z,q,...Object.values(h),...k.values()],U=G=>B.some($=>$.has(G))||en(m.get(G)??{id:G,name:G,length:0},t,r,n,x,N);for(let G of e){let $=tt(G.id),X=s.denseGroups?.[$]??gm[$];if(!X)continue;let H=G.id.slice(0,G.id.length-$.length),W=X.map(P=>H+P).filter(P=>m.has(P));if(!(W.length===0||W.every(P=>U(P))))for(let P of W){for(let R of B)R.delete(P);Q.delete(P),S.delete(P),x.delete(P),N.add(P)}}let M=$l(e,t,r,n,x,N)??4;return OA({kq4:E,...h,lut32:I,lut16:z,lut32c:O,lut16c:j,lut2_32:F,lut2_16:K,iq:q,...Object.fromEntries([...k].map(([G,$])=>[`iqRaw:${G}`,$]))}),{kq4Ids:E,kSuperIds:h,q6kIds:v,q5kIds:b,q3kIds:_,q2kIds:C,q4kncIds:y,lut32Ids:I,lut16Ids:z,lut32cIds:O,lut16cIds:j,lut2_32Ids:F,lut2_16Ids:K,iqIds:q,iqRawIds:k,allIqRawIds:S,lutRoutedIds:Q,forceDenseIds:N,excludeKq4Ids:x,quantBits:M}}function OA(e){let t=new Map;for(let[r,n]of Object.entries(e))for(let s of n){let a=t.get(s);if(a!==void 0)throw new Error(`resolveGGUFPackRoutes: ${s} is claimed by both the ${a} and ${r} packs, so it would be packed twice and cost its bytes twice. Subtract one route from the other.`);t.set(s,r)}}function MA(e,t){let r=e.filter(s=>s.transform&&t.has(s.id));if(r.length===0)return;let n=r.map(s=>s.id).join(", ");throw new Error(`GGUF packing would silently drop WeightEntry.transform for ${r.length} tensor(s): ${n}. Either exclude these ids from packSuffixes so they load dense, or apply the transform inside the packer.`)}async function Bm(e,{packSuffixes:t,...r}){let{entries:n,gguf:s,nameToGGUF:a}=e,{kq4Ids:i,q6kIds:u,q5kIds:o,q3kIds:l,q2kIds:c,q4kncIds:d,lut32Ids:p,lut16Ids:f,lut32cIds:g,lut16cIds:m,lut2_32Ids:h,lut2_16Ids:v,iqIds:b,iqRawIds:_,excludeKq4Ids:C,forceDenseIds:y}=Yl(n,s,a,t,r),E=Bs(n,r.decodeGroups);e={...e,shardUnitOf:w=>E.get(w)};let k=w=>{let I=r.scaleStorage?.[w];return I===void 0?void 0:I==="f16"},S=[],A=w=>(S.push(w),w);try{let w={q4:A(await SA(e,{packSuffixes:t,excludeKq4Ids:C,forceDenseIds:y})),kq4:A(await Sm(e,{kq4Ids:i,scalesF16:k("kq4")})),q6k:A(await Es(e,"q6k",u,k("q6k"))),q5k:A(await Es(e,"q5k",o,k("q5k"))),q3k:A(await Es(e,"q3k",l,k("q3k"))),q2k:A(await Es(e,"q2k",c,k("q2k"))),q4knc:A(await Es(e,"q4knc",d,k("q4knc"))),lut32:A(await cr(e,{lutIds:p,per16:!1})),lut16:A(await cr(e,{lutIds:f,per16:!0})),lut32c:A(await cr(e,{lutIds:g,per16:!1,compact:!0})),lut16c:A(await cr(e,{lutIds:m,per16:!0,compact:!0})),lut2_32:A(await cr(e,{lutIds:h,per16:!1,compact:!0,lut2:!0})),lut2_16:A(await cr(e,{lutIds:v,per16:!0,compact:!0,lut2:!0})),iq:A(await EA(e,{iqIds:b}))};for(let z of _t)w[z.key]=[];let I=new Set(Object.values(w).flatMap(z=>(Array.isArray(z)?z:[]).flatMap(O=>[...O.packedIds])));return MA(n,I),{packs:w,skipDenseIds:I}}catch(w){throw await Kt(e.runtime,S),w}}function Im(e,t,r){return Am(e,t,r,{kSuperRoutes:Xl,packKSuperGGUFWeights:Tm,packKq4GGUFWeights:Sm,packLutGGUFWeights:cr})}var Em=32,FA=["q6k","q5k","q3k","q2k","q4knc"],QA=["lut32","lut16","lut32c","lut16c","lut2_32","lut2_16","lut2_128"],jA=["kq4","iq"],WA=_t.map(e=>e.key),Jl=[...jA,...FA,...QA,...WA],Gs=2048,zA=[16,32,...Oi];function KA(){return{}}function UA(e,t){return{device:e,recordSupportOp:t,empty:(r,n)=>fe(je(r),n),tensorFromTypedArray:(r,n)=>fe(je(r),n),createUniformU32:()=>KA(),host:{writeBuffer:()=>{}},clearBuffers:()=>{}}}function NA(e,t,r,n,s,a,i,u,o,l=()=>null){let c=new Set,d=[];for(let k of e){if(k.id==="lm_head"&&n||!r(k))continue;let S=k.length/ji;if(!Number.isSafeInteger(S)||S<=0)throw new Error(`Q1 weight ${k.id} length ${k.length} is not a positive multiple of ${ji}`);d.push({id:k.id,blockCount:S,packedGroup:k.packedGroup}),c.add(k.id)}let p=Fl(d,i),f=new Map(p.locations),g=Dt(),m=Dt(),h=new Set,v=Object.fromEntries(Jl.map(k=>[k,{offsets:Dt(),blockCounts:Dt(),packedIds:new Set,totalBlocks:0}])),b=0;for(let k of e){if(c.has(k.id)||!t(k))continue;let S=k.length/Em,A=l(k);if(A){let w=v[A];ut(w.offsets,k.id,w.totalBlocks),ut(w.blockCounts,k.id,S),w.packedIds.add(k.id),w.totalBlocks+=S;continue}ut(g,k.id,b),ut(m,k.id,S),h.add(k.id),b+=S}let _=Dt(),C=[],y=null;for(let k of e)if(!c.has(k.id)&&!h.has(k.id)&&!Jl.some(S=>v[S].packedIds.has(k.id))&&!(s&&k.id==="embed_tokens")&&!(a&&!n&&k.id==="lm_head")){if(k.id==="lm_head"&&n){if(c.has("embed_tokens")){let S=f.get("embed_tokens");if(!S)throw new Error("Q1 support layout is missing tied embed_tokens");f.set(k.id,S)}else y=k.id;continue}C.push({id:k.id,length:k.length})}let E=Ii({entries:C,elementBytes:u,limits:o});for(let k of C)ut(_,k.id,E.placements.get(k.id).elementOffset);return y&&ut(_,y,_.top.embed_tokens??0),{offsets:_,q4Offsets:g,q4BlockCounts:m,totalBlocks:b,densePlan:E,q1Locations:f,q1PackedIds:c,kSuper:v,q1ShardBlockCounts:p.shardBlockCounts,maxQ1SourceBlocks:d.reduce((k,S)=>Math.max(k,S.blockCount),0)}}function HA({config:e,entries:t,isPacked:r=()=>!1,isQ1:n=()=>!1,packedQuantBits:s=4,packSlots:a,decodeGroups:i,tieWordEmbeddings:u,canUseFloat16:o=or,canUseQ8LmHead:l=_l,canUseQ4LmHead:c=ys,embedHeadRoutes:d,emitGraph:p,emitPrefillGraph:f,device:g}){let m=ze(g),h=m.features.has("shader-f16"),v=h&&o(e),b=v?"float16":"float32",_=v?"float16":"float32",C=h?"float16":"float32",y=t.some(ie=>n(ie)),E=!y&&v&&c(e),k=!y&&v&&!E&&l(e),S=y?null:d?.embedFormat??null,A=!y&&(d?.headPacked??!1),w=A?d?.headFormat??null:null,I=A?d?.headLut2??null:null,z=S==="lut2_32"?d?.embedLut2??null:null,O=(w!==null||I!==null)&&(d?.headAliasesEmbed??!1),j=ie=>{let ae=a?.(ie)?.buffer;return Jl.find(Se=>Se===ae)??null},{offsets:F,q4Offsets:K,q4BlockCounts:Z,totalBlocks:Q,densePlan:N,q1Locations:q,q1PackedIds:x,kSuper:B,q1ShardBlockCounts:U,maxQ1SourceBlocks:M}=NA(t,r,n,u,S!==null,A,Ml(m.limits),ve(b),m.limits,j),G=Bs(t,i),$=[],X=UA(m,(ie,ae)=>$.push({op:ie,request:ae})),H=N.shardElementCounts.map(ie=>fe(b,[Math.max(1,ie)])),W=Object.fromEntries(N.shardIndexByUnit),P=H[0],R=Q>0?{bits:fe("uint32",[Q*s]),scales:fe(C,[Q]),offsets:K,blockCounts:Z,packedIds:new Set(t.filter(ie=>r(ie)&&!x.has(ie.id)&&!j(ie)).map(ie=>ie.id)),packedQuantBits:s}:null,Y=ie=>{let ae=B[ie],Se=t.find(Re=>ae.packedIds.has(Re.id));if(!Se)return null;let ce=a(Se);return{bits:fe("uint32",[ae.totalBlocks*ce.wordsPerBlock]),scales:fe(ce.metadataU32?"uint32":"float32",[ae.totalBlocks*ce.scalesPerBlock]),offsets:ae.offsets,blockCounts:ae.blockCounts,packedIds:ae.packedIds,u32PerSuper:ce.wordsPerBlock*8,scalesPerSuper:ce.scalesPerBlock*8}},ne=ie=>{let ae=B[ie],Se=t.find(Re=>ae.packedIds.has(Re.id));if(!Se)return null;let ce=a(Se);return{bits:fe("uint32",[ae.totalBlocks*ce.wordsPerBlock]),scales:fe(ce.metadataU32?"uint32":ce.scalesF32?"float32":C,[ae.totalBlocks*ce.scalesPerBlock]),offsets:ae.offsets,blockCounts:ae.blockCounts,packedIds:ae.packedIds}},re=ie=>{let ae=B[ie];if(!ae)return null;let Se=t.find(Qt=>ae.packedIds.has(Qt.id));if(!Se)return null;let ce=a(Se),Re=Math.ceil(ae.totalBlocks*ce.wordsPerBlock),Yt=_t.find(Qt=>Qt.key===ie)?.gridInBits===!0;return{bits:fe("uint32",[Yt?Re+4096:Re]),scales:fe(C,[Math.ceil(ae.totalBlocks*ce.scalesPerBlock)]),grid:fe("uint32",[512]),offsets:ae.offsets,blockCounts:ae.blockCounts,packedIds:ae.packedIds,...Yt?{gridWordBase:Re}:{}}},D=ie=>{let ae=B[ie],Se=t.find(jt=>ae.packedIds.has(jt.id));if(!Se)return null;let ce=a(Se),Re=new Map;for(let jt of t)ae.packedIds.has(jt.id)&&Re.set(jt.id,a(jt)?.lut??0);let Yt=ie==="lut16"||ie==="lut16c"||ie==="lut2_16",Qt=ie==="lut2_32"||ie==="lut2_16"||ie==="lut2_128",At=ie==="lut32c"||ie==="lut16c"||Qt;return{bits:fe("uint32",[Math.ceil(ae.totalBlocks*ce.wordsPerBlock)]),scales:fe(ce.metadataU32?"uint32":"float32",[Math.ceil(ae.totalBlocks*ce.scalesPerBlock)]),offsets:ae.offsets,blockCounts:ae.blockCounts,packedIds:ae.packedIds,lutIds:Re,lut16:Yt?1:0,...At?{compact:1}:{},...Qt?{lut2:1}:{}}},se=D("lut32"),V=D("lut16"),ee=D("lut32c"),le=D("lut16c"),me=D("lut2_32"),We=D("lut2_128"),Oe=D("lut2_16"),te=ne("kq4"),de=ne("iq"),Ae=Y("q6k"),be=Y("q5k"),he=Y("q3k"),$e=Y("q2k"),Be=Y("q4knc"),Ce=U.length>0?{shards:U.map(ie=>({bits:fe("uint32",[ie*4]),scales:fe(C,[ie]),blockCount:ie})),locations:q,colMajorIds:!u&&e.hidden_size%128===0&&x.has("lm_head")?new Set(["lm_head"]):new Set}:null,Ve=w==="q8_rows",pt=A&&!Ve&&(d?.headRescoreQ8??!1),kc=Ve?null:w,Er=w?Is(w,e.vocab_size,e.hidden_size,h):null,vu=pt&&(d?.headScan2??!1),fr=!Ve&&Er!==null&&(d?.headKSuperScan??!1),sn=e.vocab_size*(e.hidden_size/32),v1=I===null?0:I===9?1:3,Ks=Ve?null:I!==null?fe("uint32",[sn*2]):vu||fr?fe("uint32",[sn*2]):Er?fe("uint32",[Er.bits]):E||A?fe("uint32",[sn*4]):null,Us=Ve?null:I!==null?fe("uint32",[Math.ceil(sn/8)*v1]):vu||fr?fe("uint32",[sn]):Er?fe("float32",[Er.scales]):E||A?fe(C,[sn]):null,Ns=Ve||pt?Is("q8_rows",e.vocab_size,e.hidden_size,h):null,Hs=fr?fe("uint32",[Er.bits]):Ns?fe("uint32",[Ns.bits]):k&&!A?fe("uint32",[e.vocab_size*(e.hidden_size/4)]):null,$s=fr?fe("float32",[Er.scales]):Ns?fe("float32",[Ns.scales]):k&&!A?fe("float32",[e.vocab_size]):null,On=S?Is(S,e.vocab_size,e.hidden_size,h,z):null,Vs=On?fe("uint32",[On.bits]):null,Zs=On?fe(On.scalesDtype,[On.scales]):null,Ac={runtime:X,config:e,weights:P,offsets:F,denseShards:H,denseShardIndex:W,packs:{q1:Ce,q4:R?[R]:[],kq4:te?[te]:[],iq:de?[de]:[],q6k:Ae?[Ae]:[],q5k:be?[be]:[],q3k:he?[he]:[],q2k:$e?[$e]:[],q4knc:Be?[Be]:[],lut32:se?[se]:[],lut16:V?[V]:[],lut32c:ee?[ee]:[],lut16c:le?[le]:[],lut2_32:[...me?[me]:[],...We?[We]:[]],lut2_16:Oe?[Oe]:[],...Object.fromEntries(_t.map(({key:ie})=>{let ae=re(ie);return[ie,ae?[ae]:[]]}))},lmHeadQ4:Ks,lmHeadQ4Scales:Us,lmHeadQ8:Hs,lmHeadQ8Scales:$s,embedBits:Vs,embedScales:Zs,embedFormat:S,embedLut:z,lmHeadFormat:fr?null:kc,lmHeadLut:I,lmHeadScanBits:vu||fr?2:null,lmHeadRescoreFormat:fr?kc:null,cacheDtype:_,programCache:new Map,defaultPositionIdsCache:new Map,ropeCache:{cos:fe("float32",[Gs,e.head_dim>>1]),sin:fe("float32",[Gs,e.head_dim>>1])},partialRopeCache:{cos:fe("float32",[Gs,(e.rotary_dim??e.head_dim)>>1]),sin:fe("float32",[Gs,(e.rotary_dim??e.head_dim)>>1])}},yc=Br.allocate(X,e,1,Gs,_),wu=p(Ac,yc);if(qn(wu.graph,X,{weights:wu.weights,states:wu.states}),f)for(let ie of zA){let ae=f(Ac,yc,ie);if(!ae)break;qn(ae.graph,X,{weights:ae.weights,states:ae.states,...ae.inputs?{inputs:ae.inputs}:{}})}let w1=$.map(({op:ie,request:ae})=>({op:ie,request:ae,id:ie.manifest.id})),rt=H.map((ie,ae)=>({name:H.length===1?"weights":`weights.${ae}`,dtype:b,shape:ie.shape,boundWhole:!0}));if(a){let ie=new Map;for(let Se of t){let ce=a(Se);if(!ce)continue;let Re=ie.get(ce.buffer)??{wordsPerBlock:ce.wordsPerBlock,scalesPerBlock:ce.scalesPerBlock,scalesF32:!!ce.scalesF32,metadataU32:!!ce.metadataU32,entries:[]};if(Re.wordsPerBlock!==ce.wordsPerBlock||Re.scalesPerBlock!==ce.scalesPerBlock)throw new Error(`pack buffer ${ce.buffer} is priced two ways: ${Re.wordsPerBlock}/${Re.scalesPerBlock} per block and ${ce.wordsPerBlock}/${ce.scalesPerBlock} at ${Se.id}`);let Yt=G.get(Se.id);Re.entries.push({id:Se.id,blockCount:Se.length/Em,...Yt===void 0?{}:{unitKey:Yt}}),Re.scalesF32=Re.scalesF32||!!ce.scalesF32,Re.metadataU32=Re.metadataU32||!!ce.metadataU32,ie.set(ce.buffer,Re)}let ae=xr(m.limits);for(let[Se,ce]of ie){let Re=ce.metadataU32?"uint32":ce.scalesF32?"float32":C,Yt=ve(Re),Qt=Cr(ce.entries,Tr({byteLimit:ae,maxBlocks:ce.entries.reduce((At,jt)=>At+jt.blockCount,0),bytesForBlocks:At=>Math.max(At*ce.wordsPerBlock*Uint32Array.BYTES_PER_ELEMENT,Math.ceil(At*ce.scalesPerBlock)*Yt)}),Se);for(let At=0;At<Qt.shardBlockCounts.length;++At){let jt=Qt.shardBlockCounts.length===1?"":`.${At}`,Tc=Qt.shardBlockCounts[At];rt.push({name:`${Se}Bits${jt}`,dtype:"uint32",shape:[Tc*ce.wordsPerBlock],boundWhole:!0}),rt.push({name:`${Se}Scales${jt}`,dtype:Re,shape:[Math.ceil(Tc*ce.scalesPerBlock)],boundWhole:!0})}}}else R&&(rt.push({name:"q4Bits",dtype:R.bits.dtype,shape:R.bits.shape}),rt.push({name:"q4Scales",dtype:R.scales.dtype,shape:R.scales.shape}));if(Ce){for(let ae=0;ae<Ce.shards.length;++ae){let Se=Ce.shards[ae];rt.push({name:`q1Bits.${ae}`,dtype:Se.bits.dtype,shape:Se.bits.shape,boundWhole:!0}),rt.push({name:`q1Scales.${ae}`,dtype:Se.scales.dtype,shape:Se.scales.shape,boundWhole:!0})}let ie=Ol(M);rt.push({name:"q1TranscodeSource",dtype:"uint32",shape:[ie/4],boundWhole:!0})}let Sc=O&&!fr;Ks&&!Sc&&rt.push({name:"lmHeadQ4",dtype:Ks.dtype,shape:Ks.shape,boundWhole:!0}),Us&&!Sc&&rt.push({name:"lmHeadQ4Scales",dtype:Us.dtype,shape:Us.shape,boundWhole:!0});let xc=pt||O;return Hs&&!xc&&rt.push({name:"lmHeadQ8",dtype:Hs.dtype,shape:Hs.shape,boundWhole:!0}),$s&&!xc&&rt.push({name:"lmHeadQ8Scales",dtype:$s.dtype,shape:$s.shape,boundWhole:!0}),Vs&&rt.push({name:"embedBits",dtype:Vs.dtype,shape:Vs.shape,boundWhole:!0}),Zs&&rt.push({name:"embedScales",dtype:Zs.dtype,shape:Zs.shape,boundWhole:!0}),{opRequests:w1,tensors:rt.map(ie=>ie.name==="q1TranscodeSource"?ie:{...ie,allocation:!0})}}function ec({config:e,device:t,...r}){return El(HA({config:e,device:t,...r}),t)}var Yi=class{#e=!1;#t;constructor(t){this.#t=t}get config(){return this.#t.config}async checkDeviceSupport(t){return await this.#t.checkSupport(t)}async unsupportedReason(t){return(await this.#t.checkSupport(t)).reason}async load(t,r={}){if(this.#e)throw new Error("prepared model has already been loaded or closed");this.#e=!0;try{let n=this.#t.loadWeights(t,r),s=Promise.resolve(this.#t.checkSupport(t.device)),[a,i]=await Promise.allSettled([s,n]);if(a.status==="fulfilled"&&a.value.reason)throw i.status==="fulfilled"&&i.value?.dispose?.(),new Error(a.value.reason);if(i.status==="rejected")throw i.reason;if(a.status==="rejected")throw i.value?.dispose?.(),a.reason;return i.value}finally{await this.#t.closeSource()}}async close(){this.#e||(this.#e=!0,await this.#t.closeSource())}};function $A(e,t){return t instanceof e.ConfigClass?t:new e.ConfigClass(t)}function Pm(e,t,r,n){let s=$A(e,r),{weights:a,offsets:i}=Eg({runtime:t,entries:e.expectedWeightEntries(s),stateDict:n,tieWordEmbeddings:s.tie_word_embeddings,modelName:e.MODEL_NAME});return new e({runtime:t,config:s,weights:a,offsets:i})}var En=class{#e=new Map;#t=!1;#r;constructor(t=r=>r.dispose()){this.#r=t}getOrBuild(t,r){if(this.#t)return Promise.reject(new Error("SessionPool is closed"));let n=this.#e.get(t);if(n?.state==="dispose-failed")return Promise.reject(new Error("SessionPool entry cannot be reused until its failed disposal is retried"));if(n)return n.promise;let s,a=Promise.resolve().then(()=>{if(s.invalidation)throw s.invalidation;return r()}).then(i=>{if(s.invalidation){let u=s.invalidation;try{this.#r(i)}catch(o){throw new AggregateError([u,o],"Invalidated session build and cleanup both failed")}throw u}return s.state="ready",s.session=i,i}).catch(i=>{throw this.#e.get(t)===s&&this.#e.delete(t),i});return s={state:"building",promise:a},this.#e.set(t,s),a}release(t){let r=this.#e.get(t);if(r){if(r.state==="building"){this.#e.delete(t),r.invalidation=new Error("SessionPool entry was released while its session was building");return}try{this.#r(r.session),this.#e.delete(t)}catch(n){throw r.state="dispose-failed",n}}}disposeAll(){this.#n("SessionPool was disposed while its session was building")}close(){this.#t=!0,this.#n("SessionPool was closed while its session was building")}#n(t){let r=[];for(let[n,s]of this.#e){if(s.state==="building"){this.#e.delete(n),s.invalidation=new Error(t);continue}try{this.#r(s.session),this.#e.delete(n)}catch(a){s.state="dispose-failed",r.push(a)}}ct(r,"Multiple pooled sessions failed to dispose")}};function Rm(e,...t){let r=[];for(let n of t)try{n.release(e)}catch(s){r.push(s)}try{e.dispose()}catch(n){r.push(n)}ct(r,"Generation cache and bound sessions failed to dispose")}function eu(e,t){return{cacheKeysT:e.keyStateForLayer(t),cacheValuesT:e.valueStateForLayer(t)}}function Ls(e,t){return Lm(e,t)}function VA(e,t,r){return Lm(e,r.map(n=>`layers.${t}.${n}`))}function Lm(e,t){let r=e.packs.q1;if(!r)throw new Error("Q1 resource views requested but no q1 pack is loaded");if(t.length===0)throw new Error("Q1 resource views require at least one weight id");let n=Ji(r,t[0]);for(let o of t.slice(1)){let l=Ji(r,o);if(l.groupIndex!==n.groupIndex)throw new Error(`Q1 co-bound weights must declare one packedGroup: ${t.join(", ")}`);if(l.shardIndex!==n.shardIndex)throw new Error(`Q1 co-bound weights resolved to different shards: ${t.join(", ")}`)}let s=r.shards[n.shardIndex];if(!s)throw new Error(`Missing Q1 shard ${n.shardIndex} for ${t[0]}`);let a=JA(r,s,t),i=new Set(t),u=o=>{if(!i.has(o))throw new Error(`Q1 packed weight ${o} is not part of this bound view`);let l=Ji(r,o);if(l.groupIndex!==n.groupIndex||l.shardIndex!==n.shardIndex)throw new Error(`Q1 packed weight ${o} is outside the requested group`);return l.blockOffset-a.baseBlock};return{q1BitsT:Mt(s.bits,a.baseBlock*4,a.blockCount*4),q1ScalesT:Mt(s.scales,a.baseBlock,a.blockCount),offset:u,colMajor:o=>r.colMajorIds.has(o)?1:0}}function tu(e,t){return(r,n,s)=>{let a=VA(t,r,n);return{bitsT:e(`L${r}.q1bits.${s}`,a.q1BitsT),scalesT:e(`L${r}.q1scales.${s}`,a.q1ScalesT),offset:i=>a.offset(i)}}}function ZA(e,t,r){return XA(e,r.map(n=>`layers.${t}.${n}`))}function ru(e,t){return(r,n,s)=>{let a=ZA(t,r,n);return{bitsT:e(`L${r}.q4bits.${s}`,a.q4BitsT),scalesT:e(`L${r}.q4scales.${s}`,a.q4ScalesT),offset:i=>a.offset(i),format:a.format,quantBits:a.quantBits,hasMin:a.hasMin,iq:a.iq,lut:a.lut,lut16:a.lut16,iqRaw:a.iqRaw,gridT:a.gridT?e(`iqgrid${a.iqRaw}`,a.gridT):null,...a.gridWordBase===void 0?{}:{gridWordBase:a.gridWordBase}}}}function nu(e,t){let r=t.denseShards;if(!r||r.length<=1){let n=e("weights",t.weights);return()=>n}return n=>{let s=Bg(t,n);return e(`weights.${s}`,r[s])}}var tc=[{pack:e=>e.packs.q6k,format:"q6_k",align:512},{pack:e=>e.packs.q5k,format:"q5_k",align:256},{pack:e=>e.packs.q3k,format:"q3_k",align:512},{pack:e=>e.packs.q2k,format:"q2_k",align:256},{pack:e=>e.packs.q4knc,format:"q4_k_native_compact",align:512}],Gm=256;function XA(e,t){let r=YA(e,t);for(let[n,s]of[["bits",r.q4BitsT],["scales",r.q4ScalesT]]){let a=s.byteOffset??0;if(a%Gm!==0)throw new Error(`packed ${n} view for ${t.join(", ")} (${r.format}) starts at byte ${a}, which is not a multiple of ${Gm}: this route's block alignment is too coarse for its per-block scale stride.`)}return r}function YA(e,t){let r=_t.map(p=>({route:e.packs[p.key],iqRaw:p.iqRaw,wordsPerBlock:p.wordsPerBlock}));for(let p of r){let f=Ft(p.route,t[0]);if(!f)continue;let{baseBlock:g,blockCount:m}=Ds(t,_=>({offset:kt(f.offsets,_,"offset"),count:kt(f.blockCounts,_,"block count")}),1024),h=g*p.wordsPerBlock,v=f.gridWordBase,b=v===void 0?m*p.wordsPerBlock:f.bits.byteLength/Uint32Array.BYTES_PER_ELEMENT-h;return{q4BitsT:Mt(f.bits,h,b),q4ScalesT:Mt(f.scales,g/8,Math.ceil(m/8)),offset:_=>kt(f.offsets,_,"offset")-g,format:"q4_0",hasMin:0,iq:0,lut:0,lut16:0,quantBits:4,iqRaw:p.iqRaw,gridT:f.grid,...v===void 0?{}:{gridWordBase:v-h}}}let n=Mm(e,t[0]);if(n){let p=n.lutIds.get(t[0])??0;for(let b of t)if(n.lutIds.get(b)!==p)throw new Error(`co-bound LUT weights disagree on lutId: ${t.join(", ")}`);let f=n.compact===1,g=n.lut2===1?2:4,m=n.lut16?2:1,h=p===9?1:n.lut16?5:3,v=f?Ds(t,b=>({offset:kt(n.offsets,b,"offset"),count:kt(n.blockCounts,b,"block count")}),512):Dm(n,t);return{q4BitsT:Mt(n.bits,v.baseBlock*g,v.blockCount*g),q4ScalesT:f?Mt(n.scales,v.baseBlock/8*h,Math.ceil(v.blockCount/8)*h):Mt(n.scales,v.baseBlock*m,v.blockCount*m),offset:b=>kt(n.offsets,b,"offset")-v.baseBlock,format:n.lut2===1?n.lut16?"lut2_16":"lut2_32":f?n.lut16?"lut16_compact":"lut32_compact":n.lut16?"lut16":"lut32",hasMin:0,iq:0,lut:p,lut16:n.lut16,quantBits:4,iqRaw:0,gridT:null}}for(let{pack:p,format:f,align:g}of tc){let m=Ft(p(e),t[0]);if(!m)continue;let{baseBlock:h,blockCount:v}=Ds(t,b=>({offset:kt(m.offsets,b,"offset"),count:kt(m.blockCounts,b,"block count")}),g);return{q4BitsT:Mt(m.bits,h/8*m.u32PerSuper,v/8*m.u32PerSuper),q4ScalesT:Mt(m.scales,h/8*m.scalesPerSuper,v/8*m.scalesPerSuper),offset:b=>kt(m.offsets,b,"offset")-h,format:f,hasMin:0,iq:0,lut:0,lut16:0,quantBits:4,iqRaw:0,gridT:null}}let s=Ft(e.packs.iq,t[0]),a=Ft(e.packs.kq4,t[0]),i=Ft(e.packs.q4,t[0]),u=s?{pack:s,wordsPerBlock:8,scalesPerBlock:2,hasMin:0,iqFlag:1,format:"iq"}:a?{pack:a,wordsPerBlock:4,scalesPerBlock:2,hasMin:1,iqFlag:0,format:"q4_k"}:i?{pack:i,wordsPerBlock:i.packedQuantBits,scalesPerBlock:1,hasMin:0,iqFlag:0,format:i.packedQuantBits===8?"q8_0":"q4_0"}:null;if(!u)throw new Error(`No packed-quant route holds ${t[0]}`);let{pack:o,wordsPerBlock:l,scalesPerBlock:c}=u,d=Dm(o,t);return{q4BitsT:Mt(o.bits,d.baseBlock*l,d.blockCount*l),q4ScalesT:Mt(o.scales,d.baseBlock*c,d.blockCount*c),offset:p=>kt(o.offsets,p,"offset")-d.baseBlock,format:u.format,hasMin:u.hasMin,iq:u.iqFlag,lut:0,lut16:0,quantBits:l,iqRaw:0,gridT:null}}function Ds(e,t,r){let n=Number.POSITIVE_INFINITY,s=0;for(let i of e){let{offset:u,count:o}=t(i);n=Math.min(n,u),s=Math.max(s,u+o)}let a=Math.floor(n/r)*r;return{baseBlock:a,blockCount:s-a}}function Dm(e,t){return Ds(t,r=>({offset:kt(e.offsets,r,"offset"),count:kt(e.blockCounts,r,"block count")}),ve(e.scales.dtype)===2?128:64)}function Ji(e,t){let r=e.locations.get(t);if(!r)throw new Error(`Missing Q1 packed weight location for ${t}`);return r}function JA(e,t,r){return Ds(r,n=>{let s=Ji(e,n);if(s.blockOffset+s.blockCount>t.blockCount)throw new Error(`Q1 packed weight ${n} exceeds shard ${s.shardIndex}`);return{offset:s.blockOffset,count:s.blockCount}},ve(t.scales.dtype)===2?128:64)}function Om(e,t){if(e==="lut32_compact"||e==="lut16_compact")throw new Error(`${t} is outside lutcRouteSuffixes(), so it has no compact-LUT kernel`);return e}function Zt(e,t){if(e==="lut2_32"||e==="lut2_16")throw new Error(`${t} is outside lut2Suffixes(), so it has no ternary 2-bit LUT kernel`);return e}function Pn(e,t){if(e==="q3_k"||e==="q2_k")throw new Error(`${t} has no native ${e} kernel (q3_k/q2_k skip the conv in-proj)`);return e}function su(e,t){let r=s=>{let{q4:a,kq4:i,iq:u,lut32:o,lut16:l,lut32c:c,lut16c:d,lut2_32:p,lut2_16:f}=e.packs,g=m=>(m??[]).findIndex(h=>qr(h.offsets,s)!==void 0);for(let{key:m}of _t){let h=g(e.packs[m]);if(h>=0)return`${m}#${h}`}for(let[m,h]of[["lut32",o],["lut16",l],["lut32c",c],["lut16c",d],["lut2_32",p],["lut2_16",f]]){let v=g(h);if(v>=0)return`${m}#${v}:${h[v].lutIds.get(s)??0}`}for(let{pack:m,format:h}of tc){let v=g(m(e));if(v>=0)return`${h}#${v}`}for(let[m,h]of[["iq",u],["kq4",i],["q4",a]]){let v=g(h);if(v>=0)return`${m}#${v}`}return null},n=r(t[0]);return n!==null&&t.every(s=>r(s)===n)}function He(e,t){return Ft(e.packs.q4,t)!==null||ey(e,t)||ty(e,t)||Mm(e,t)!==null||tc.some(({pack:r})=>Ft(r(e),t)!==null)||_t.some(({key:r})=>Ft(e.packs[r],t)!==null)}function ey(e,t){return Ft(e.packs.kq4,t)!==null}function ty(e,t){return Ft(e.packs.iq,t)!==null}function Mm(e,t){for(let r of[e.packs.lut32,e.packs.lut16,e.packs.lut32c,e.packs.lut16c,e.packs.lut2_32,e.packs.lut2_16]){let n=Ft(r,t);if(n)return n}return null}function Ft(e,t){for(let r of e??[])if(qr(r.offsets,t)!==void 0)return r;return null}function kt(e,t,r){let n=qr(e,t);if(n===void 0||!Number.isInteger(n)||n<0)throw new Error(`Missing packed weight ${r} for ${t}`);return n}function Mt(e,t,r){if(!Number.isInteger(t)||t<0)throw new Error(`tensor view elementOffset must be a non-negative integer, got ${t}`);if(!Number.isInteger(r)||r<=0)throw new Error(`tensor view elementCount must be a positive integer, got ${r}`);let n=ve(e.dtype),s=e.byteOffset??0,a=s+t*n,i=r*n;if(a+i>s+e.byteLength)throw new Error(`tensor view range exceeds tensor byteLength for dtype ${e.dtype}`);return{...e,shape:[r],size:r,byteOffset:a,byteLength:i}}function ry(e,t,r){let n=t/2,s=new Float32Array(e*n),a=new Float32Array(e*n),i=new Float64Array(n);for(let u=0;u<n;++u)i[u]=1/Math.pow(r,u*2/t);for(let u=0;u<e;++u){let o=u*n;for(let l=0;l<n;++l){let c=u*i[l];s[o+l]=Math.cos(c),a[o+l]=Math.sin(c)}}return{cos:s,sin:a}}function Rn(e){let t=e.rotary_dim;return typeof t=="number"&&t>0?t:e.head_dim}function au(e,t){let r=Rn(e.config);return r===e.config.head_dim?ny(e,t):(e.partialRopeCache=Fm(e,r,t,e.partialRopeCache),e.partialRopeCache)}function ny(e,t){return e.ropeCache=Fm(e,e.config.head_dim,t,e.ropeCache),e.ropeCache}function Fm(e,t,r,n){if(!Number.isSafeInteger(r)||r<1)throw new Error(`RoPE cache needs a positive cache length, got ${r}`);if(n&&n.cos.shape[0]>=r)return n;let s=e.config.rope_parameters?.rope_theta;if(typeof s!="number")throw new Error("RoPE cache requires config.rope_parameters.rope_theta");let a=2**Math.ceil(Math.log2(r)),i=t/2,{cos:u,sin:o}=ry(a,t,s);return{cos:e.runtime.tensorFromTypedArray("float32",[a,i],u),sin:e.runtime.tensorFromTypedArray("float32",[a,i],o),...n?{superseded:n}:{}}}var Qm=32,jm=8,sy=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj","input_layernorm","post_attention_layernorm","q_layernorm","k_layernorm"],rE=sy.length;var ay=[16,32,64,128,256],iy=256;function rc(e){let t=e.head_dim;if(t===128)return 64;let r=Os(e);return t<=r||t===r*2?r:ay.find(n=>t<=n)??0}function iu(e){return e.head_dim%4===0&&e.head_dim<=iy&&rc(e)>0}function Os(e){return e.head_dim<=64?64:128}function uu(e,t){return e.layer_types?e.layer_types[t]:"full_attention"}function tn(e){return!!e.attention_qk_norm}function Xt(e){return e.packs.q1!=null}function Ms(e){return e.features.has("shader-f16")&&e.features.has("subgroups")&&e.features.has("chromium-experimental-subgroup-matrix")}function Gn(e){if(!e)return!1;let t=e.features;return t.has("shader-f16")&&t.has("subgroups")&&t.has("chromium-experimental-subgroup-matrix")}function Dn(e,t){return e&&(t===2||t===3)}function Fs(e,t){return e&&t!==1&&t!==7&&t!==8}function nc(e,t=!1,r=!0){if(!e)return new Set;if(!e.features.has("shader-f16"))return new Set;if(!r&&!(e.features.has("subgroups")&&e.features.has("chromium-experimental-subgroup-matrix")))return new Set;let n=Gn(e)?["linear_in_proj_qkv","linear_in_proj_z"]:[],s=Gn(e)&&t?["q_proj","k_proj","v_proj"]:[],a=["down_proj"],i=["o_proj","linear_out_proj"];return new Set(["gate_proj","up_proj",...s,...n,...a,...i])}var uy=new Set(["intermediate","output"]),Ln=class{tensors=[];nodes=[];aliases=[];boundNames=new Map;ownedHandles=new WeakSet;nodesByTensor=new Map;viewChildren=new Map;get nodeCount(){return this.nodes.length}input(t,r,n){return this.declareTensor("input",t,r,n)}weight(t,r,n){return this.declareTensor("weight",t,r,n)}state(t,r,n){return this.declareTensor("state",t,r,n)}stepInput(t,r,n){return this.declareTensor("stepInput",t,r,n)}uniform(t,r){if(r<=0||r%4!==0)throw new Error(`graph uniform "${t}" needs a positive multiple-of-4 byte length (got ${r})`);return this.declareTensor("uniform",t,"uint32",[r/4])}scratch(t,r,n){return this.declareTensor("intermediate",t,r,n)}view(t,r,n,s,a){if(t=this.currentTensor(t),t.logicalDtype!==void 0&&!Lr(t.logicalDtype,n))throw new Error(`graph view of "${t.name}" cannot reinterpret logical dtype ${t.logicalDtype} as ${n} storage`);if(!Number.isInteger(r)||r<0)throw new Error(`graph view of "${t.name}" needs a non-negative integer elementOffset (got ${r})`);if(!s.every(p=>Number.isInteger(p)&&p>=0))throw new Error(`graph view "${a??`${t.name}.view@${r}`}" has a non-integer shape [${s}]`);let i=ve(n),u=r*i,o=we(s)*i,l=we(t.shape)*ve(t.dtype);if(u+o>l)throw new Error(`graph view "${a??`${t.name}.view@${r}`}" exceeds its parent "${t.name}" range (${u}+${o} > ${l} bytes)`);let c=t.logicalDtype,d=t.id;if(t.view){let f=t.view.elementOffset*ve(t.dtype)+u;if(f%i!==0)throw new Error(`graph view "${a??`${t.name}.view@${r}`}" cannot represent byte offset ${f} in ${n} elements`);t=this.currentTensor(this.tensors[t.view.parent]),r=f/i}return this.addView(t,r,n,s,a,c,d)}storageView(t,r={}){if(t=this.currentTensor(t),t.logicalDtype===void 0||t.logicalDtype===t.dtype||!Lr(t.logicalDtype,t.dtype))throw new Error(`graph tensor "${t.name}" has no logical/storage projection to expose`);let n=r.dtype??t.dtype;if(!Lr(t.logicalDtype,n))throw new Error(`storage view of "${t.name}" cannot represent logical dtype ${t.logicalDtype} as ${n} storage`);if(ve(n)!==ve(t.dtype))throw new Error(`storage view of "${t.name}" cannot change its element size (${t.dtype} to ${n})`);let s=r.shape??t.shape;if(we(s)!==we(t.shape))throw new Error(`storage view of "${t.name}" cannot change its element count (${we(t.shape)} to ${we(s)})`);let a=t.id,i=0;if(t.view){let u=t.view.elementOffset*ve(t.dtype),o=ve(n);if(u%o!==0)throw new Error(`storage view of "${t.name}" cannot represent byte offset ${u} in ${n} elements`);t=this.currentTensor(this.tensors[t.view.parent]),i=u/o}return this.addView(t,i,n,s,r.name,void 0,a)}addView(t,r,n,s,a,i,u){if(t.view)throw new Error(`internal error: graph view "${t.name}" was not flattened to its storage parent`);if(t.kind==="uniform")throw new Error(`graph uniform "${t.name}" cannot be viewed`);if(!Number.isInteger(r)||r<0)throw new Error(`graph view of "${t.name}" needs a non-negative integer elementOffset (got ${r})`);let o={id:this.tensors.length,kind:"intermediate",dtype:n,...i===void 0?{}:{logicalDtype:i},shape:[...s],name:a??`${t.name}.view@${r}`,view:{parent:t.id,elementOffset:r}};this.tensors.push(o),this.ownedHandles.add(o);let l=this.viewChildren.get(u);return l?l.push(o.id):this.viewChildren.set(u,[o.id]),o}op(t,r,n={}){let s=this.opFor(t,n.conformance),a={},i={},u={};for(let[p,f]of Object.entries(r)){if(f===void 0)continue;if(!oy(f))throw new Error(`${t}: binding "${p}" is not a graph tensor \u2014 scalars belong in options.args`);let g=this.currentTensor(f);u[p]=g,a[p]=g,i[p]=g.id}let o=s.inferOutputs(a,{args:n.args,attrs:n.attrs}),l={},c={};for(let[p,f]of Object.entries(o)){let g=u[p],m=je(f.dtype);if(g!==void 0){if(!da(g.shape,f.shape))throw new Error(`${t}: output "${p}" bound to tensor "${g.name}" of shape [${g.shape}] but the manifest infers [${f.shape}]`);l[p]=this.withLogicalDtype(g,f.logicalDtype),c[p]=g.id;continue}let h=this.withLogicalDtype(this.allocate("intermediate",`${t.split(".").pop()}.${p}#${this.nodes.length}`,m,f.shape),f.logicalDtype);l[p]=h,c[p]=h.id}let d=this.nodes.length;this.nodes.push({op:t,...n.conformance?{conformance:n.conformance}:{},inputs:i,outputs:c,...n.args?{args:{...n.args}}:{},...n.attrs?{attrs:{...n.attrs}}:{},...n.tunables?{tunables:{...n.tunables}}:{}});for(let p of new Set([...Object.values(i),...Object.values(c)])){let f=this.nodesByTensor.get(p);f||(f=new Set,this.nodesByTensor.set(p,f)),f.add(d)}return l}alias(t,r){if(t=this.currentTensor(t),r=this.currentTensor(r),t.id===r.id)throw new Error(`alias: a tensor cannot alias itself ("${t.name}")`);for(let[a,i]of[["from",t],["to",r]])if(!uy.has(i.kind))throw new Error(`alias: ${a} tensor "${i.name}" is ${i.kind}; only intermediate/output tensors can share a buffer`);if(t.dtype!==r.dtype)throw new Error(`alias("${t.name}","${r.name}"): dtype mismatch (${t.dtype} vs ${r.dtype})`);let n=we(t.shape)*ve(t.dtype),s=we(r.shape)*ve(r.dtype);if(n!==s)throw new Error(`alias("${t.name}","${r.name}"): byte-length mismatch (${n} vs ${s}); a shared buffer needs an exact-size fit`);this.aliases.push({from:t.id,to:r.id})}output(t,r){let n=this.currentTensor(t),s={...n,kind:"output",name:r??n.name};return this.tensors[n.id]=s,this.ownedHandles.add(s),s}finish(t){return Object.freeze({name:t.name,params:Object.freeze({...t.params??{}}),tensors:Object.freeze([...this.tensors]),nodes:Object.freeze([...this.nodes]),...this.aliases.length>0?{aliases:Object.freeze([...this.aliases])}:{}})}opFor(t,r){return Xr(t,{conformance:r})}declareTensor(t,r,n,s){let a=this.boundNames.get(r);if(a)throw new Error(`graph tensor name "${r}" already declared (${a.kind})`);let i=this.allocate(t,r,n,s);return this.boundNames.set(r,i),i}allocate(t,r,n,s){if(!s.every(i=>Number.isInteger(i)&&i>=0))throw new Error(`graph tensor "${r}" has a non-integer shape [${s}]`);let a={id:this.tensors.length,kind:t,dtype:n,shape:[...s],name:r};return this.tensors.push(a),this.ownedHandles.add(a),a}withLogicalDtype(t,r){if(t=this.currentTensor(t),t.logicalDtype===r)return t;let{logicalDtype:n,...s}=t,a={...s,...r===void 0?{}:{logicalDtype:r}},i=new Map([[t.id,a]]),u=[{parent:t,nextLogicalDtype:r}];for(;u.length>0;){let{parent:o,nextLogicalDtype:l}=u.pop();for(let c of this.viewChildren.get(o.id)??[]){let d=this.tensors[c];if(d.logicalDtype!==o.logicalDtype)continue;if(l!==void 0&&!Lr(l,d.dtype))throw new Error(`graph view "${d.name}" cannot represent logical dtype ${l} as ${d.dtype} storage`);let{logicalDtype:p,...f}=d,g={...f,...l===void 0?{}:{logicalDtype:l}};i.set(d.id,g),u.push({parent:d,nextLogicalDtype:l})}}this.revalidateNodesReferencing([...i.keys()],i);for(let[o,l]of i)this.tensors[o]=l,this.ownedHandles.add(l);return a}revalidateNodesReferencing(t,r){let n=new Set;for(let s of t)for(let a of this.nodesByTensor.get(s)??[])n.add(a);for(let s of n){let a=this.nodes[s],i={};for(let[u,o]of Object.entries({...a.inputs,...a.outputs}))i[u]=r.get(o)??this.tensors[o];try{this.opFor(a.op,a.conformance).inferOutputs(i,{args:a.args,attrs:a.attrs})}catch(u){let o=u instanceof Error?u.message:String(u);throw new Error(`graph node ${s} (${a.op}) became invalid after a logical dtype refinement: ${o}`,{cause:u})}}}currentTensor(t){if(!this.ownedHandles.has(t))throw new Error(`graph tensor "${t.name}" (id ${t.id}) does not belong to this builder`);let r=this.tensors[t.id];if(r===void 0)throw new Error(`graph tensor "${t.name}" (id ${t.id}) does not belong to this builder`);return r}};function ou(e){let t={},r={},n=new Map;return{w:(i,u)=>{let o=n.get(i);if(o)return o;t[i]=u;let l=e.weight(i,u.dtype,u.shape);return n.set(i,l),l},state:(i,u)=>(r[i]=u,e.state(i,u.dtype,u.shape)),boundWeights:t,states:r}}function oy(e){return e!==null&&typeof e=="object"&&typeof e.id=="number"&&typeof e.dtype=="string"&&Array.isArray(e.shape)}var ly={q_proj:"attn_q",k_proj:"attn_k",v_proj:"attn_v",o_proj:"attn_output",gate_proj:"ffn_gate",up_proj:"ffn_up",down_proj:"ffn_down",linear_in_proj_qkv:"attn_qkv",linear_in_proj_z:"attn_gate",linear_out_proj:"ssm_out"};function sc(e,t,r){let n=e.config.prismHadamard;if(!n)return null;let s=/^layers\.(\d+)\.(.+)$/.exec(t),a=s?`blk.${s[1]}.${ly[s[2]]}.weight`:t==="embed_tokens"?"token_embd.weight":t==="lm_head"?"output.weight":t,i=n.tiedOutput&&t==="lm_head"&&!r;return!(r?n.inverseWeights:n.weights).includes(a)&&!i?null:{spec:n,grouped:(n.gdnVGrouped&&s?.[2]==="linear_out_proj")===!0}}function Wm(e,t,r,n){e.hadamardSigns??=new Map;let s=e.hadamardSigns.get(n);return s||(s=e.runtime.tensorFromTypedArray("float32",[n],Float32Array.from(r[n])),e.hadamardSigns.set(n,s)),t(`prism.signs.${n}`,s)}function zm(e,t){return sc(e,t,!1)!==null}function cy(e,t,r){let n=new Map;return(s,a,i=!1)=>{let u=sc(t,a,i);if(!u)return s;let{spec:o,grouped:l}=u,c=s.shape.at(-1),d=`${s.id}/${i}/${l}`,p=n.get(d);if(p)return p;let f=`${s.name}.hadamard${i?".inverse":""}${l?".grouped":""}`;if(l){let m=t.config.linear_value_head_dim,h=t.config.linear_num_key_heads,v=t.config.linear_num_value_heads/h;if(v>1){let b=s.shape.reduce((y,E)=>y*E,1)/c,_=e.view(s,0,s.dtype,[b,v,h,m],`${f}.tiled`),C=e.op("ai.onnx.Transpose",{x:_,y:e.scratch(`${f}.permuted`,s.dtype,[b,h,v,m])},{attrs:{perm:[0,2,1,3]}}).y;s=e.view(C,0,s.dtype,s.shape,`${f}.input`)}}let g=e.op("com.xenova.BlockHadamard",{xT:s,signsT:Wm(t,r,o.signs,c),yT:e.scratch(f,s.dtype,s.shape)},{args:{width:c,blockSize:o.blockSize,inverse:i}}).yT;return n.set(d,g),g}}function Km(e,t,r){let n=cy(e,t,r);return(s,a,i,u,o)=>{let l=sc(t,a,!1),c=s.shape.at(-1),d=s.shape.reduce((p,f)=>p*f,1)/c;if(!l||l.grouped){let p=e.op("co.huggingface.RMSNorm",{xT:s,wT:i.weights,yT:o??e.scratch(u,"float32",s.shape)},{args:{rows:d,dim:c,eps:i.eps,weightBaseOffset:i.offset}}).yT;return n(p,a)}return e.op("com.xenova.BlockHadamard",{xT:s,signsT:Wm(t,r,l.spec.signs,c),normWeightsT:i.weights,yT:e.scratch(`${u}.hadamard`,"float32",s.shape),...o?{normedT:o}:{}},{args:{width:c,blockSize:l.spec.blockSize,inverse:!1,rmsEps:i.eps,normOffset:i.offset}}).yT}}function Um(e,t){let r=e;for(let n of t)r=dy(r,n);return r}function dy(e,t){let r=new Map(e.tensors.map(f=>[f.id,f.id])),n=f=>{let g=f;for(;r.has(g)&&r.get(g)!==g;)g=r.get(g);for(;r.has(f)&&r.get(f)!==g;){let m=r.get(f);r.set(f,g),f=m}return g},s=(f,g)=>{r.set(n(f),n(g))};for(let f of e.tensors)f.view&&s(f.id,f.view.parent);for(let f of e.aliases??[])s(f.from,f.to);let a=new Set(e.tensors.filter(f=>f.kind!=="intermediate").map(f=>n(f.id))),i=new Map;e.nodes.forEach((f,g)=>{for(let m of Object.values(f.outputs)){let h=n(m),v=i.get(h)??new Set;v.add(g),i.set(h,v)}});let u=new Map,o=f=>{let g=n(f);u.set(g,(u.get(g)??0)+1)};for(let f of e.nodes){let g=new Set(Object.values(f.outputs));for(let m of Object.values(f.inputs))g.has(m)||o(m)}for(let f of e.tensors)f.kind==="output"&&o(f.id);let l=new Map;e.nodes.forEach((f,g)=>{if(f.op!==t.consumer)return;let m=f.inputs[t.consumerInput];m!==void 0&&l.set(m,l.has(m)?-1:g)});let c=new Map,d=new Set;if(e.nodes.forEach((f,g)=>{if(f.op!==t.producer)return;let m=f.outputs[t.producerOutput];if(m===void 0||(u.get(n(m))??0)!==1||a.has(n(m))||i.get(n(m))?.size!==1)return;let h=l.get(m);if(h===void 0||h<=g||Object.values(f.outputs).some(y=>y!==m&&(a.has(n(y))||(u.get(n(y))??0)>0)))return;let v=new Set(Object.values(f.inputs).map(n));for(let y=g+1;y<h;++y)if(Object.values(e.nodes[y].outputs).some(E=>v.has(n(E))))return;let b=e.nodes[h];if(Object.values(b.outputs).some(y=>v.has(n(y))))return;let _=t.fuse(f,b,e);if(_===null)return;let C=new Set(Object.values(_.outputs));Object.values(b.outputs).some(y=>!C.has(y))||(c.set(h,_),d.add(g))}),c.size===0)return e;let p=[];return e.nodes.forEach((f,g)=>{d.has(g)||p.push(c.get(g)??f)}),Object.freeze({...e,nodes:Object.freeze(p)})}var ac=class{steps=[];prefillSteps=null;chainedSteps=null;get stepCount(){return this.steps.length}get prefillStepCount(){return(this.prefillSteps??this.steps).length}get chainedStepCount(){return(this.chainedSteps??this.steps).length}async step(t,r){return this.writeStepInputs(t,r),this.collector.enqueue(this.steps),(await this.runtime.readTensor(this.idsTensor))[0]}submitStep(t,r){return this.writeStepInputs(t,r),this.collector.enqueue(t===null?this.chainedSteps??this.steps:this.steps),this.runtime.readTensor(this.idsTensor).then(n=>n[0])}enqueueStep(t,r,{full:n=!1}={}){this.writeStepInputs(t,r),this.collector.enqueue(n?this.steps:this.prefillSteps??this.steps)}async replayKnownInputs(t,r){for(let n=0;n<t.length;++n)this.enqueueStep(t[n],r+n);await this.runtime.queueIdle()}finish(){}measureGpuMs(){return this.runtime.measurePreparedSequenceGpuMs?this.runtime.measurePreparedSequenceGpuMs(this.chainedSteps??this.steps):Promise.resolve(null)}},ic=class extends ac{#e=null;get compiled(){if(!this.#e)throw new Error("Graph decode session is not built");return this.#e}set compiled(t){this.#e=t}detachCompiled(t){this.#e===t&&(this.#e=null)}get collector(){return this.compiled.collector}get idsTensor(){return this.compiled.tensor("input_ids")}dispose(){this.#e?.dispose(),this.#e=null}};function Nm(e,t){if(t===void 0)return null;let r=e.nodes.length-t;if(!Number.isInteger(t)||t<=0||r<=0)throw new Error(`graph "${e.name}": next-token tail must be a positive proper suffix (got ${t} of ${e.nodes.length} nodes)`);return r}var rn="sampling",Hm=32,fy=Object.freeze({top_k:0,temperature:0,top_p:1,min_p:0,seed:0,position:0}),lu=class{#e=null;#t=!1;set(t){this.#e=t}write(t,r,n){let s=this.#e;s===null&&!this.#t||(n(r()(s===null?fy:{...s,position:t})),this.#t=s!==null)}};function $m(e){return e.tensors.some(t=>t.kind==="uniform"&&t.name===rn)}function Vm(e){return new Error(`graph "${e.name}" picks its next token inside a fused lm_head; sampling needs a head that writes the logits, so this model decodes greedily only`)}function Zm(e,t){try{t.dispose()}catch(r){throw new AggregateError([e,r],"Graph session build and cleanup both failed")}throw e}var cu=class extends ic{model;cache;emission;#e;#t=new Map;#r=new lu;#n=!1;constructor(t,r){super(),this.model=t,this.cache=r}stepUniformPack(t){let r=this.#t.get(t);return r||(r=Cs(this.#e,t,this.compiled.nodeVariants),this.#t.set(t,r)),r}get runtime(){return this.model.runtime}async prepareBuild(){}onCompiled(){}async build(t,{fuse:r=!1}={}){await this.prepareBuild();let{emission:n,graph:s}=this.buildGraph(t,r),a=n.nextTokenTailNodeCount,i=n.gpuChainedPrefixNodeCount,u=Nm(s,a);if(i!==void 0&&(!Number.isInteger(i)||i<=0||i>=(u??s.nodes.length)))throw new Error(`graph "${s.name}": GPU-chained prefix must be a positive proper prefix of the reusable body (got ${i} of ${u??s.nodes.length} body nodes)`);let o=qn(s,this.runtime,{weights:n.weights,states:n.states,...n.inputs?{inputs:n.inputs}:{}});this.emission=n,this.#e=s,this.#t.clear(),this.#n=$m(s),this.compiled=o;try{if(u===null)this.prefillSteps=null,this.steps=await o.buildSteps(),this.chainedSteps=i===void 0?null:await o.buildSteps({startNode:i});else{let l=i===void 0?[]:await o.buildSteps({endNode:i}),c=await o.buildSteps({...i!==void 0?{startNode:i}:{},endNode:u}),d=await o.buildSteps({startNode:u});if(c.length===0||d.length===0||i!==void 0&&l.length===0)throw new Error(`graph "${s.name}": next-token tail split produced an empty prepared sequence`);this.prefillSteps=[...l,...c],this.steps=[...this.prefillSteps,...d],this.chainedSteps=i===void 0?null:[...c,...d]}this.onCompiled()}catch(l){this.steps=[],this.prefillSteps=null,this.chainedSteps=null,this.#t.clear(),this.detachCompiled(o),Zm(l,o)}}setSampling(t){if(t!==null&&!this.#n)throw Vm(this.#e);this.#r.set(t)}writeStepInputs(t,r){t!==null&&this.runtime.host.writeBuffer(this.compiled.tensor("input_ids").buffer,0,new Uint32Array([t])),this.writeStepUniforms(r),this.#n&&this.#r.write(r,()=>this.stepUniformPack(rn),n=>this.runtime.host.writeBuffer(this.compiled.uniformBuffer(rn),0,n))}},du=class{model;cache;blockLen;emission;#e=null;steps=[];bodySteps=null;#t=new Map;#r=new lu;#n=!1;constructor(t,r,n){this.model=t,this.cache=r,this.blockLen=n}get compiled(){if(!this.#e)throw new Error("Graph prefill session is not built");return this.#e}stepUniformPack(t){let r=this.#t.get(t);return r||(r=Cs(this.emission.graph,t,this.compiled.nodeVariants),this.#t.set(t,r)),r}get runtime(){return this.model.runtime}async prepareBuild(){}onCompiled(){}async build(){await this.prepareBuild();let t=this.buildEmission(),r=Nm(t.graph,t.nextTokenTailNodeCount),n=qn(t.graph,this.runtime,{weights:t.weights,states:t.states,...t.inputs?{inputs:t.inputs}:{}});this.emission=t,this.#t.clear(),this.#n=$m(t.graph),this.#e=n;try{if(r===null)this.bodySteps=null,this.steps=await n.buildSteps();else{let s=await n.buildSteps({endNode:r}),a=await n.buildSteps({startNode:r});if(s.length===0||a.length===0)throw new Error(`graph "${t.graph.name}": next-token tail split produced an empty prepared sequence`);this.bodySteps=s,this.steps=[...s,...a]}this.onCompiled()}catch(s){this.steps=[],this.bodySteps=null,this.#t.clear(),this.#e===n&&(this.#e=null),Zm(s,n)}}writeInputs(t,r){if(t.length===0||t.length>this.blockLen)throw new Error(`prefill block ${this.blockLen} cannot run ${t.length} tokens`);let n=new Uint32Array(this.blockLen);n.set(t),this.runtime.host.writeBuffer(this.compiled.tensor("input_ids").buffer,0,n),this.writeRunInputs(t.length,r),this.#n&&this.#r.write(r+t.length-1,()=>this.stepUniformPack(rn),s=>this.runtime.host.writeBuffer(this.compiled.uniformBuffer(rn),0,s))}setSampling(t){if(t!==null&&!this.#n)throw Vm(this.emission.graph);this.#r.set(t)}enqueueBody(t,r=0){if(!this.bodySteps)throw new Error(`prefill graph "${this.emission.graph.name}" has no removable next-token tail`);this.writeInputs(t,r),this.compiled.collector.enqueue(this.bodySteps)}async run(t,r=0){return this.writeInputs(t,r),this.compiled.collector.enqueue(this.steps),(await this.runtime.readTensor(this.compiled.tensor("next_token")))[0]}dispose(){this.#e?.dispose(),this.#e=null}};var py=256,gy=4,my=8;function oc(e){let t=e===2,[r,n]=t?[2,16]:[1,8],s=void 0,a=s?Number(s):NaN;return Number.isInteger(a)&&a>=r&&a<=n?a:t?my:gy}var hy=256;function lc(e,t){let r=Math.ceil(e/hy);return r>0&&r<=t?r:null}function vy(e){let t=e.config;if(dt(t))throw new Error("emitLlamaDecode: linear-attention layers decode through the Qwen3.5 decode graph");if(Xt(e)&&Gt(t))throw new Error("emitLlamaDecode: Q1 weights are not supported with convolution layers");if(Xt(e)&&!tn(t))throw new Error("emitLlamaDecode: Q1 weights need qk-norm (the packed path applies it)")}var cc=e=>Number(Math.pow(e,-.5).toPrecision(9));function dc(e){return zm(e,"lm_head")}function pu(e,t){let r=e.nodeCount;if(dc(t.model)){let c=`${t.nameSuffix??""}prism.head`,d=Km(e,t.model,t.w)(t.hidden,"lm_head",{weights:t.weights,eps:t.rmsEps,offset:t.finalNormOffset},`${c}.normed`),p=e.scratch(`${c}.logits`,"float32",[1,t.vocabSize]),f=e.view(d,0,"float32",[1,t.hiddenSize],`${c}.input`),g={M:1,inFeatures:t.hiddenSize,outFeatures:t.vocabSize,outStride:t.vocabSize,dstColStart:0};if(t.model.lmHeadQ4&&t.model.lmHeadQ4Scales){let m=t.model.lmHeadLut!==null&&t.model.lmHeadLut!==void 0?{format:"lut2_32",lut:t.model.lmHeadLut}:{format:"q4_0"};e.op("com.xenova.LlamaPrefillMatmul",{aT:f,bitsT:t.w(`${c}.bits`,t.model.lmHeadQ4),scalesT:t.w(`${c}.scales`,t.model.lmHeadQ4Scales),yT:p},{args:{...g,blockOffset:0,...m}})}else e.op("com.xenova.LlamaPrefillMatmul",{aT:f,weightsT:t.weights,yT:p},{args:{...g,weightOffset:t.lmHeadOffset}});return e.op("com.xenova.SampleNextToken",{logitsT:e.view(p,0,"float32",[t.vocabSize],`${c}.scores`),outT:t.ids,paramsT:e.uniform(`${t.nameSuffix??""}${rn}`,Hm)},{args:{maxTopK:jn,outputOffset:t.outputOffset??0}}),e.nodeCount-r}let n=!!(t.model.lmHeadQ4&&t.model.lmHeadQ4Scales),s=!!(t.model.lmHeadQ8&&t.model.lmHeadQ8Scales),a=!t.q1&&n&&s?lc(t.vocabSize,t.model.runtime.device.limits.maxComputeWorkgroupsPerDimension):null;if(a===null&&t.model.lmHeadScanBits===2)throw new Error(`lm_head is packed as a 2-bit scan grid, but the two-stage head cannot serve a vocabulary of ${t.vocabSize} on this device`);if(a!==null)return wy(e,t,a),e.nodeCount-r;let i=t.q1||n?512:py,u=t.nameSuffix??"",o=e.scratch(`${u}bestValues`,"float32",[i]),l=e.scratch(`${u}bestIds`,"uint32",[i]);if(t.q1){let c=e.scratch(`${u}final.normed`,t.normedDtype,[t.hiddenSize]);e.op("co.huggingface.RMSNorm",{xT:t.hidden,wT:t.weights,yT:c},{args:{rows:1,dim:t.hiddenSize,eps:t.rmsEps,weightBaseOffset:t.finalNormOffset}});let d=Ls(t.model,["lm_head"]);e.op("com.xenova.LlamaDecodeLmHeadArgmax",{normedT:c,bitsT:t.w("lmhead.q1bits",d.q1BitsT),scalesT:t.w("lmhead.q1scales",d.q1ScalesT),bestValuesT:o,bestIdsT:l},{args:{hiddenSize:t.hiddenSize,vocabSize:t.vocabSize,lmHeadOffset:d.offset("lm_head"),numBlocks:i,colMajor:d.colMajor("lm_head"),format:"q1_0"}})}else n?e.op("com.xenova.LlamaDecodeLmHeadArgmax",{hiddenT:t.hidden,bitsT:t.w("lmhead.q4bits",t.model.lmHeadQ4),scalesT:t.w("lmhead.q4scales",t.model.lmHeadQ4Scales),weightsT:t.weights,bestValuesT:o,bestIdsT:l},{args:{hiddenSize:t.hiddenSize,vocabSize:t.vocabSize,lmHeadOffset:0,finalNormOffset:t.finalNormOffset,numBlocks:i,...t.model.lmHeadFormat?{format:t.model.lmHeadFormat}:{}}}):s?e.op("com.xenova.LlamaDecodeLmHeadArgmax",{hiddenT:t.hidden,weightsT:t.weights,bitsT:t.w("lmhead.q8bits",t.model.lmHeadQ8),scalesT:t.w("lmhead.q8scales",t.model.lmHeadQ8Scales),bestValuesT:o,bestIdsT:l},{args:{vocabSize:t.vocabSize,hiddenSize:t.hiddenSize,numBlocks:i,rmsEps:t.rmsEps,finalNormOffset:t.finalNormOffset,format:"q8_rows"}}):e.op("com.xenova.LlamaDecodeLmHeadArgmax",{hiddenT:t.hidden,weightsT:t.weights,bestValuesT:o,bestIdsT:l},{args:{hiddenSize:t.hiddenSize,vocabSize:t.vocabSize,rmsEps:t.rmsEps,finalNormOffset:t.finalNormOffset,lmHeadOffset:t.lmHeadOffset,numBlocks:i}});return e.op("com.xenova.CandidateArgMax",{valuesT:o,indicesT:l,outT:t.ids},{args:{outputOffset:t.outputOffset??0,count:i}}),e.nodeCount-r}function wy(e,t,r){let n=oc(t.model.lmHeadScanBits),s=r*n,a=t.nameSuffix??"",i=e.scratch(`${a}head.scanValues`,"float32",[s]),u=e.scratch(`${a}head.scanIds`,"uint32",[s]);e.op("com.xenova.LlamaDecodeLmHeadArgmax",{hiddenT:t.hidden,bitsT:t.w("lmhead.q4bits",t.model.lmHeadQ4),scalesT:t.w("lmhead.q4scales",t.model.lmHeadQ4Scales),weightsT:t.weights,bestValuesT:i,bestIdsT:u},{args:{hiddenSize:t.hiddenSize,vocabSize:t.vocabSize,lmHeadOffset:0,finalNormOffset:t.finalNormOffset,numBlocks:r,topK:n,...t.model.lmHeadScanBits===2?{quantBits:2}:{},...t.model.lmHeadFormat?{format:t.model.lmHeadFormat}:{}}});let o=e.scratch(`${a}head.rescored`,"float32",[s]);e.op("com.xenova.LlamaDecodeLmHeadRescore",{aT:t.hidden,normT:t.weights,bitsT:t.w("lmhead.q8bits",t.model.lmHeadQ8),scalesT:t.w("lmhead.q8scales",t.model.lmHeadQ8Scales),candIdsT:u,outValuesT:o},{args:{inFeatures:t.hiddenSize,numCands:s,zeroPoint:0,mask:255,format:t.model.lmHeadRescoreFormat??"q8_rows",fuseNorm:1,normOffset:t.finalNormOffset}}),e.op("com.xenova.CandidateArgMax",{valuesT:o,indicesT:u,outT:t.ids},{args:{outputOffset:t.outputOffset??0,count:s}})}function fc(e,t,r){vy(e);let n=e.config,s=tn(n),a=Xt(e),i=n.hidden_size,u=n.num_hidden_layers,o=n.num_attention_heads,l=n.num_key_value_heads,c=n.head_dim,d=l*c,p=Gt(n)?Fe(n.conv_L_cache,"config.conv_L_cache"):0,f=n.intermediate_size,g=n.vocab_size,m=Number(n.rms_norm_eps),h=t.maxLength,v=cc(c),b=rc(n),_=Rn(n),C=au(e,h),y=e.offsets.top,E=e.runtime.device.features.has("shader-f16"),k=E&&(e.weights.dtype==="float16"||a)?"float16":"float32",S=a&&E?"float16":"float32",A=new Ln,{w,state:I,boundWeights:z,states:O}=ou(A),j=nu(w,e),F=j("top"),K=w("rope.cos",C.cos),Z=w("rope.sin",C.sin),Q=A.uniform("params",16),N=Gn(e.runtime.device),q=ru(w,e),x=tu(w,e),B=(P,R,Y,ne)=>{let re=`layers.${R}.${Y}`;if(a){let D=x(R,[Y],`o.${Y}`);return A.op("com.xenova.LlamaDecodeResidualProjection",{hiddenT:P,bitsT:D.bitsT,scalesT:D.scalesT,attnT:ne},{args:{hiddenSize:i,oOffset:D.offset(re),format:"q1_0"}}).hiddenT}if(He(e,re)){let D=q(R,[Y],`o.${Y}`);return A.op("com.xenova.LlamaDecodeResidualProjection",{hiddenT:P,bitsT:D.bitsT,scalesT:D.scalesT,attnT:ne,...D.gridT&&!Fs(N,D.iqRaw)?{gridT:D.gridT}:{}},{args:{hiddenSize:i,oOffset:D.offset(re),format:Zt(D.format,"output projection"),lut:D.lut,iqRaw:D.iqRaw}}).hiddenT}return A.op("com.xenova.LlamaDecodeResidualProjection",{hiddenT:P,weightsT:j(R),attnT:ne},{args:{hiddenSize:i,oOffset:e.offsets.layers[R][Y]}}).hiddenT},U=Si(n),M=(P,R)=>{if(!xn(U,R))throw new Error(`moeMlpInto: layer ${R} is not a MoE layer`);let Y=e.offsets.layers[R],ne=A.scratch(`L${R}.moeProbs`,"float32",[U.numExperts]),re=A.scratch(`L${R}.moeRouteIdx`,"uint32",[U.topK]),D=A.scratch(`L${R}.moeRouteW`,"float32",[U.topK]);A.op("com.xenova.LlamaDecodeMoeRoute",{hiddenT:P,weightsT:j(R),probsT:ne},{args:{hiddenSize:i,numExperts:U.numExperts,rmsEps:m,normOffset:Y.post_attention_layernorm,routerOffset:Y.moe_router}});let se={hasBias:U.useExpertBias?1:0,biasOffset:U.useExpertBias?Y.moe_bias:0,normTopk:U.normTopkProb?1:0,routedScale:U.routedScale},V=A.scratch(`L${R}.moeInter`,k,[U.topK*U.moeIntermediateSize]);if(He(e,`layers.${R}.gate_exps`)){let ee=q(R,["gate_exps","up_exps"],"moegateup");A.op("com.xenova.LlamaPrefillMoeGateUp",{hiddenT:P,bitsT:ee.bitsT,scalesT:ee.scalesT,probsT:ne,decodeRouteIdxT:re,routeWT:D,intermediateT:V,weightsT:j(R),...ee.gridT&&!Dn(N,ee.iqRaw)?{gridT:ee.gridT}:{}},{args:{hiddenSize:i,moeIntermediateSize:U.moeIntermediateSize,numExperts:U.numExperts,topK:U.topK,rmsEps:m,normOffset:Y.post_attention_layernorm,...se,gateOffset:ee.offset(`layers.${R}.gate_exps`),upOffset:ee.offset(`layers.${R}.up_exps`),format:Zt(Pn(ee.format,"MoE expert gate/up"),"MoE expert gate/up"),lut:ee.lut,iqRaw:ee.iqRaw}})}else A.op("com.xenova.LlamaPrefillMoeGateUp",{hiddenT:P,weightsT:j(R),probsT:ne,decodeRouteIdxT:re,routeWT:D,intermediateT:V},{args:{hiddenSize:i,moeIntermediateSize:U.moeIntermediateSize,numExperts:U.numExperts,topK:U.topK,rmsEps:m,normOffset:Y.post_attention_layernorm,...se,gateOffset:Y.gate_exps,upOffset:Y.up_exps}});if(He(e,`layers.${R}.down_exps`)){let ee=q(R,["down_exps"],"moedown");return A.op("com.xenova.LlamaPrefillMoeDown",{hiddenT:P,bitsT:ee.bitsT,scalesT:ee.scalesT,intermediateT:V,routeIdxT:re,routeWT:D,...ee.gridT&&!Dn(N,ee.iqRaw)?{gridT:ee.gridT}:{}},{args:{hiddenSize:i,moeIntermediateSize:U.moeIntermediateSize,numExperts:U.numExperts,topK:U.topK,downOffset:ee.offset(`layers.${R}.down_exps`),format:Zt(Pn(ee.format,"MoE expert down"),"MoE expert down"),lut:ee.lut,iqRaw:ee.iqRaw}}).hiddenT}return A.op("com.xenova.LlamaPrefillMoeDown",{hiddenT:P,weightsT:j(R),intermediateT:V,routeIdxT:re,routeWT:D},{args:{hiddenSize:i,moeIntermediateSize:U.moeIntermediateSize,numExperts:U.numExperts,topK:U.topK,downOffset:Y.down_exps}}).hiddenT},G=(P,R)=>{if(xn(U,R))return M(P,R);let Y=e.offsets.layers[R],ne=A.scratch(`L${R}.intermediate`,k,[f]);if(a){let re=x(R,["gate_proj","up_proj"],"gateup");A.op("com.xenova.LlamaDecodeGateUp",{hiddenT:P,bitsT:re.bitsT,scalesT:re.scalesT,intermediateT:ne,weightsT:j(R)},{args:{hiddenSize:i,intermediateSize:f,rmsEps:m,normOffset:Y.post_attention_layernorm,gateOffset:re.offset(`layers.${R}.gate_proj`),upOffset:re.offset(`layers.${R}.up_proj`),format:"q1_0"}});let D=x(R,["down_proj"],"down");return A.op("com.xenova.LlamaDecodeResidualProjection",{hiddenT:P,bitsT:D.bitsT,scalesT:D.scalesT,intermediateT:ne},{args:{hiddenSize:i,intermediateSize:f,downOffset:D.offset(`layers.${R}.down_proj`),format:"q1_0"}}).hiddenT}if(He(e,`layers.${R}.gate_proj`)){let re=`layers.${R}.gate_proj`,D=`layers.${R}.up_proj`,se=su(e,[re,D]),V=q(R,se?["gate_proj","up_proj"]:["gate_proj"],"gateup"),ee=se?null:q(R,["up_proj"],"gateupU"),le=V.gridT??ee?.gridT??null,me=ee&&V.iqRaw&&ee.iqRaw&&ee.iqRaw!==V.iqRaw?ee.gridT:null,We=V.gridT?V.iqRaw:ee?.iqRaw??0,Oe=ee?.iqRaw??0,te={...le&&!Fs(N,We)?{gridT:le}:{},...me&&!Fs(N,Oe)?{upGridT:me}:{}};if(ee){let de=A.scratch(`L${R}.postNormed`,"float32",[i]);A.op("co.huggingface.RMSNorm",{xT:P,wT:j(R),yT:de},{args:{rows:1,dim:i,eps:m,weightBaseOffset:Y.post_attention_layernorm}}),A.op("com.xenova.LlamaDecodeGateUp",{normedT:de,bitsT:V.bitsT,scalesT:V.scalesT,upBitsT:ee.bitsT,upScalesT:ee.scalesT,intermediateT:ne,...te},{args:{hiddenSize:i,intermediateSize:f,gateOffset:V.offset(re),upOffset:ee.offset(D),format:V.format,lut:V.lut,iqRaw:V.iqRaw,upFormat:ee.format,upLut:ee.lut,upIqRaw:ee.iqRaw}})}else A.op("com.xenova.LlamaDecodeGateUp",{hiddenT:P,bitsT:V.bitsT,scalesT:V.scalesT,intermediateT:ne,weightsT:j(R),...te},{args:{hiddenSize:i,intermediateSize:f,rmsEps:m,normOffset:Y.post_attention_layernorm,gateOffset:V.offset(re),upOffset:V.offset(D),format:Zt(V.format,"fused gate|up"),lut:V.lut,iqRaw:V.iqRaw}})}else A.op("com.xenova.LlamaDecodeGateUp",{hiddenT:P,weightsT:j(R),intermediateT:ne},{args:{hiddenSize:i,intermediateSize:f,rmsEps:m,normOffset:Y.post_attention_layernorm,gateOffset:Y.gate_proj,upOffset:Y.up_proj}});if(He(e,`layers.${R}.down_proj`)){let re=q(R,["down_proj"],"down");return A.op("com.xenova.LlamaDecodeResidualProjection",{hiddenT:P,bitsT:re.bitsT,scalesT:re.scalesT,intermediateT:ne,...re.gridT&&!Fs(N,re.iqRaw)?{gridT:re.gridT}:{}},{args:{hiddenSize:i,intermediateSize:f,downOffset:re.offset(`layers.${R}.down_proj`),format:Zt(re.format,"down projection"),lut:re.lut,iqRaw:re.iqRaw}}).hiddenT}return A.op("com.xenova.LlamaDecodeResidualProjection",{hiddenT:P,weightsT:j(R),intermediateT:ne},{args:{hiddenSize:i,intermediateSize:f,downOffset:Y.down_proj}}).hiddenT},$=A.stepInput("input_ids","uint32",[1]),X=A.scratch("embed.hidden","float32",[i]),H;if(a){let P=Ls(e,["embed_tokens"]);H=A.op("com.xenova.LlamaEmbed",{inputT:$,bitsT:w("embed.q1bits",P.q1BitsT),scalesT:w("embed.q1scales",P.q1ScalesT),hiddenT:X,paramsT:Q},{args:{hiddenSize:i,vocabSize:g,embedOffset:P.offset("embed_tokens"),tokenOffset:0,format:"q1_0"}}).hiddenT}else e.embedBits&&e.embedScales?H=A.op("com.xenova.LlamaEmbed",{inputT:$,bitsT:w("embed.bits",e.embedBits),scalesT:w("embed.scales",e.embedScales),hiddenT:X,paramsT:Q},{args:{hiddenSize:i,vocabSize:g,tokenOffset:0,format:e.embedFormat??"q8_rows",...e.embedLut!==null&&e.embedLut!==void 0?{lut:e.embedLut}:{}}}).hiddenT:H=A.op("com.xenova.LlamaEmbed",{inputT:$,weightsT:F,hiddenT:X,paramsT:Q},{args:{hiddenSize:i,vocabSize:g,embedOffset:y.embed_tokens,tokenOffset:0}}).hiddenT;for(let P=0;P<u;++P){let R=e.offsets.layers[P];if(uu(n,P)==="conv"){if(!t.convStates)throw new Error("emitLlamaDecode: conv layer requires cache.convStates");let Y=I(`conv.state.${P}`,t.convStateForLayer(P)),ne=A.scratch(`L${P}.convY`,"float32",[i]);if(He(e,`layers.${P}.conv_in_proj`)){let re=A.scratch(`L${P}.convProj`,"float32",[3*i]),D=q(P,["conv_in_proj"],"convin");A.op("com.xenova.LlamaDecodeConvInProj",{hiddenT:H,bitsT:D.bitsT,scalesT:D.scalesT,weightsT:j(P),convProjT:re},{args:{hiddenSize:i,rmsEps:m,normOffset:R.input_layernorm,convInOffset:D.offset(`layers.${P}.conv_in_proj`),format:Zt(Om(D.format,"conv in-proj"),"conv in-proj"),lut:D.lut}}),A.op("com.xenova.LlamaDecodeShortConv",{bcxT:re,weightsT:j(P),convStatesT:Y,convYT:ne},{args:{layer:0,hiddenSize:i,convLCache:p,convWeightOffset:R.conv_weight}})}else{let re=A.scratch(`L${P}.normed`,"float32",[i]);A.op("co.huggingface.RMSNorm",{xT:H,wT:j(P),yT:re},{args:{rows:1,dim:i,eps:m,weightBaseOffset:R.input_layernorm}}),A.op("com.xenova.LlamaDecodeShortConv",{normedT:re,weightsT:j(P),convStatesT:Y,convYT:ne},{args:{layer:0,hiddenSize:i,convLCache:p,convInOffset:R.conv_in_proj,convWeightOffset:R.conv_weight}})}H=B(H,P,"conv_out_proj",ne)}else{let Y=eu(t,P),ne=I(`cache.key.${P}`,Y.cacheKeysT),re=I(`cache.value.${P}`,Y.cacheValuesT),D;if(a){let le=A.scratch(`L${P}.qkv`,"float32",[i+2*d]),me=x(P,["q_proj","k_proj","v_proj"],"qkv");A.op("com.xenova.LlamaDecodeQkvProj",{hiddenT:H,bitsT:me.bitsT,scalesT:me.scalesT,qkvT:le,normWeightsT:j(P)},{args:{hiddenSize:i,numKvHeads:l,headDim:c,rmsEps:m,normOffset:R.input_layernorm,qOffset:me.offset(`layers.${P}.q_proj`),kOffset:me.offset(`layers.${P}.k_proj`),vOffset:me.offset(`layers.${P}.v_proj`),format:"q1_0"}}),A.op("com.xenova.LlamaDecodeRopeCacheKv",{stageQkvT:le,weightsT:j(P),cacheKeysT:ne,cacheValuesT:re,cosT:K,sinT:Z,paramsT:Q},{args:{layer:0,cacheLen:h,pastLen:r,hiddenSize:i,numHeads:o,numKvHeads:l,headDim:c,..._<c?{rotaryDim:_}:{},rmsEps:m,qNormOffset:R.q_layernorm,kNormOffset:R.k_layernorm}}),D=le}else if(s){let le=A.scratch(`L${P}.q`,"float32",[i]),me=A.scratch(`L${P}.k`,"float32",[d]),We=A.scratch(`L${P}.v`,"float32",[d]);if(He(e,`layers.${P}.q_proj`)){let Oe=q(P,["q_proj","k_proj","v_proj"],"qkv");A.op("com.xenova.LlamaDecodeQkvProj",{hiddenT:H,bitsT:Oe.bitsT,scalesT:Oe.scalesT,normWeightsT:j(P),qT:le,kT:me,vT:We},{args:{hiddenSize:i,numKvHeads:l,headDim:c,rmsEps:m,normOffset:R.input_layernorm,qOffset:Oe.offset(`layers.${P}.q_proj`),kOffset:Oe.offset(`layers.${P}.k_proj`),vOffset:Oe.offset(`layers.${P}.v_proj`),format:Oe.format,lut:Oe.lut}})}else{let Oe=A.scratch(`L${P}.normed`,"float32",[i]);A.op("co.huggingface.RMSNorm",{xT:H,wT:j(P),yT:Oe},{args:{rows:1,dim:i,eps:m,weightBaseOffset:R.input_layernorm}}),A.op("com.xenova.LlamaDecodeQkvProj",{normedT:Oe,weightsT:j(P),qT:le,kT:me,vT:We},{args:{hiddenSize:i,numKvHeads:l,headDim:c,qOffset:R.q_proj,kOffset:R.k_proj,vOffset:R.v_proj}})}A.op("com.xenova.LlamaDecodeRopeCacheKv",{stageQT:le,stageKT:me,stageVT:We,weightsT:j(P),cacheKeysT:ne,cacheValuesT:re,cosT:K,sinT:Z,paramsT:Q},{args:{layer:0,cacheLen:h,pastLen:r,numHeads:o,numKvHeads:l,headDim:c,..._<c?{rotaryDim:_}:{},rmsEps:m,qNormOffset:R.q_layernorm,kNormOffset:R.k_layernorm}}),D=le}else{let le=A.scratch(`L${P}.q`,"float32",[i]),me=He(e,`layers.${P}.q_proj`)?q(P,["q_proj"],"q"):null;A.op("com.xenova.LlamaDecodeQkvProj",{hiddenT:H,weightsT:j(P),qT:le,...me?{bitsT:me.bitsT,scalesT:me.scalesT}:{},cacheKeysT:ne,cacheValuesT:re,cosT:K,sinT:Z,paramsT:Q},{args:{layer:0,cacheLen:h,pastLen:r,hiddenSize:i,rmsEps:m,normOffset:R.input_layernorm,numHeads:o,numKvHeads:l,headDim:c,..._<c?{rotaryDim:_}:{},qOffset:me?me.offset(`layers.${P}.q_proj`):R.q_proj,...me?{format:me.format}:{},kOffset:R.k_proj,vOffset:R.v_proj}}),D=le}let se=iu(n),V=A.scratch(`L${P}.attn`,"float32",[o*c]),ee=A.op("com.xenova.CachedDecodeAttention",{qT:D,cacheKeysT:ne,cacheValuesT:re,attnT:V,paramsT:Q},{args:{layer:0,cacheLen:h,pastLen:r,numHeads:o,numKvHeads:l,headDim:c,scale:v,workgroupSize:se?b:Os(n),...se?{tileSize:16,numKvChunks:Qm,minKeysPerChunk:jm}:{}}}).attnT;H=B(H,P,"o_proj",ee)}H=G(H,P)}let W=pu(A,{w,model:e,hidden:H,weights:F,ids:$,hiddenSize:i,vocabSize:g,rmsEps:m,finalNormOffset:y.model_norm,lmHeadOffset:y.lm_head,q1:a,normedDtype:S});return{graph:A.finish({name:"llama-decode",params:{positionOffset:r}}),weights:z,states:O,paramsName:"params",nextTokenTailNodeCount:W}}var by={id:"rms-qkv",producer:"co.huggingface.RMSNorm",producerOutput:"yT",consumer:"com.xenova.LlamaDecodeQkvProj",consumerInput:"normedT",fuse:(e,t,r)=>{let n=e.args??{},s=t.args??{};return t.inputs.qGateT!==void 0||t.outputs.qGateT!==void 0||r.tensors.find(a=>a.id===e.inputs.xT)?.dtype!=="float32"||r.tensors.find(a=>a.id===e.outputs.yT)?.dtype!=="float32"||Object.keys(t.inputs).some(a=>a!=="normedT"&&a!=="weightsT"&&t.outputs[a]===void 0)||n.rows!==1||typeof n.weightBaseOffset!="number"||typeof n.dim!="number"||typeof n.eps!="number"||(n.weightOffset??0)!==0||n.castBeforeScale||(n.exact??0)!==0||e.inputs.wT===void 0||t.inputs.weightsT!==e.inputs.wT||s.hiddenSize!==n.dim?null:{op:"com.xenova.LlamaDecodeQkvProj",inputs:{hiddenT:e.inputs.xT,weightsT:e.inputs.wT},outputs:{...t.outputs},args:{hiddenSize:n.dim,numKvHeads:s.numKvHeads,headDim:s.headDim,rmsEps:n.eps,normOffset:n.weightBaseOffset,qOffset:s.qOffset,kOffset:s.kOffset,vOffset:s.vOffset}}}},_y=[by],uc=class extends cu{writeStepUniforms(t){let r=this.stepUniformPack(this.emission.paramsName)({past_len:t,cache_len:this.cache.maxLength,token_offset:0});this.runtime.host.writeBuffer(this.compiled.uniformBuffer(this.emission.paramsName),0,r)}},fu=class extends uc{buildGraph(t,r){let n=fc(this.model,this.cache,t);return{emission:n,graph:r?Um(n.graph,_y):n.graph}}};function Qs(e){let t=e.config;return!(Xt(e)||dt(t))}var ky=480,Ay=96,yy=8,Sy=8;function Xm(e,t,r){if(e<=8||e>16)return 1;let n=Math.ceil(t/128),s=Math.min(yy,Math.max(1,Math.round(ky/n)));return Math.min(s,Math.max(1,Math.floor(r/32/Sy)))}function dr(e,t,r,n,s,a,i,u,o,l){let c=t&&r==="float16"&&o%64===0&&u%32===0?Xm(n,o,u):1,d=c>1?{partialsT:e.scratch(`${l.name}.c0.partials`,"float32",[c*n*o])}:{};return e.op("com.xenova.LlamaPrefillMatmul",{aT:s,weightsT:a,yT:l,...d},{args:{weightOffset:i,M:n,inFeatures:u,outFeatures:o,...c>1?{kSplits:c}:{}}}).yT}var xy=64,Ty=new Set(["q4_0","q4_1","q4_k","q4_k_native","q5_0","q8_0","q8_0_pair","q2_k","q3_k","q6_k","lut16","lut32","iq","q5_k","q4_k_native_compact","lut16_compact","lut32_compact","lut2_16","lut2_32"]);function Cy(e,t,r=xy){return t>=r?e.stepInput("real_rows","uint32",[1]):void 0}function qy({packedQ1:e,activationDtype:t,rows:r,inFeatures:n,intermediateSize:s}){return e&&t==="float16"&&n%128===0&&s%64===0&&s>=n&&(r>=64||r===16&&2*s>=3*n)}function By({g:e,model:t,denseW:r,q4w:n,q1w:s,q1RealLenT:a,realRowsT:i,nativeLut2PolicyRows:u,T:o,H:l,inter:c,eps:d,actGemm:p,fusedQ4:f,q1Activation:g,transformInput:m}){let h=t.runtime.device.features.has("shader-f16")&&!0,v=t.runtime.device.adapterInfo,b=v.subgroupMinSize===32&&v.subgroupMaxSize===32||t.runtime.device.features.has("subgroup-size-control")&&v.subgroupMinSize!==void 0&&v.subgroupMinSize<=32&&v.subgroupMaxSize!==void 0&&v.subgroupMaxSize>=32,_=o>=512&&b&&(v.subgroupMatrixConfigs===void 0||v.subgroupMatrixConfigs.some(M=>M.componentType==="f16"&&M.M===8&&M.N===8&&M.K===8)),C=Gn(t.runtime.device),y=Xt(t),E=(M,G,$,X,H,W,P)=>e.op("com.xenova.LlamaPrefillDequant",{bitsT:$.bitsT,scalesT:$.scalesT,denseT:M},{args:{denseElems:G,blockOffset:X,outRows:H,inFeatures:W,dstElemOffset:P,format:"q1_0"}}).denseT,k=(M,G,$,X,H,W,P,R)=>e.op("com.xenova.LlamaPrefillMatmul",{aT:M,bitsT:G.bitsT,scalesT:G.scalesT,yT:W,...a&&X%64===0?{realLenT:a}:{}},{args:{M:o,inFeatures:H,outFeatures:X,blockOffset:$,outStride:P,dstColStart:R,format:"q1_0"}}).yT,S=(M,G)=>M%4===0&&G%128===0&&o<=65535,A=M=>M.format==="lut2_32"&&(M.lut===8||M.lut===9),w=M=>!f&&p==="float32"&&A(M)&&(!h||o>16&&o<=64),I=t.runtime.device,z=v.subgroupMinSize===16&&v.subgroupMaxSize===32&&t.runtime.device.features.has("subgroups")&&I.wgslLanguageFeatures?.has("packed_4x8_integer_dot_product")===!0,O=o>16&&!0&&(z||!1),j=M=>!f&&t.runtime.device.features.has("shader-f16")&&!0&&!w(M)&&!M.iqRaw&&Ty.has(M.format),F=(M,G,$,X,H,W,P,R)=>{let ne=(f||G.format==="q4_k_native_compact"&&Math.ceil(X/32)<=Ay)&&p==="float16"&&X%64===0&&H%32===0?Xm(o,X,H):1,re=ne>1?{partialsT:e.scratch(`${W.name}.c${R}.partials`,"float32",[ne*o*X])}:{},D=O&&A(G);return e.op("com.xenova.LlamaPrefillMatmul",{aT:M,bitsT:G.bitsT,scalesT:G.scalesT,yT:W,...re,...i&&(D||(f?G.format!=="q4_k_native_compact"||_:j(G)))?{realLenT:i}:{},...G.gridT&&!C?{gridT:G.gridT}:{}},{args:{M:o,inFeatures:H,outFeatures:X,blockOffset:$,outStride:P,dstColStart:R,format:G.format,lut:G.lut,iqRaw:G.iqRaw,...w(G)&&!D?{exact:1}:{},...D?{activationQuant:"int8"}:{},...ne>1?{kSplits:ne}:{}},...f&&A(G)&&u!==void 0?{tunables:{LUT2_PREFILL_POLICY_ROWS:u}}:{}}).yT},K=(M,G,$)=>(f||p==="float16"||A(M)&&$%256===0)&&(M.format!=="q5_k"||$%256===0)&&G%64===0&&$%32===0,Z=(M,G,$,X,H,W,P)=>e.op("com.xenova.LlamaPrefillDequant",{bitsT:$.bitsT,scalesT:$.scalesT,denseT:M},{args:{denseElems:G,blockOffset:X,outRows:H,inFeatures:W,dstElemOffset:P,format:$.format,lut:$.lut}}).denseT,Q=(M,G,$,X,H,W,P,R)=>{if(G=m?.(G,`layers.${M}.${$}`)??G,y){let Y=s(M,[$],$),ne=Y.offset(`layers.${M}.${$}`);if(S(H,W))return k(G,Y,ne,H,W,P,H,0);let re=H*W,D=E(e.scratch(`${R}.w`,p,[re]),re,Y,ne,H,W,0);return dr(e,f,p,o,G,D,0,W,H,P)}if(He(t,`layers.${M}.${$}`)){let Y=n(M,[$],$);if(K(Y,H,W))return F(G,Y,Y.offset(`layers.${M}.${$}`),H,W,P,H,0);let ne=H*W,re=Z(e.scratch(`${R}.w`,p,[ne]),ne,Y,Y.offset(`layers.${M}.${$}`),H,W,0);return dr(e,f,p,o,G,re,0,W,H,P)}return dr(e,f,p,o,G,r(M),X,W,H,P)},N=(M,G,$,X,H,W,P)=>Q(M,G,$,X,H,W,e.scratch(P,p,[o,H]),P),q=(M,G,$,X)=>{G=m?.(G,`layers.${M}.gate_proj`)??G;let H=e.scratch(X,p,[o,2*c]);if(y){let W=s(M,["gate_proj","up_proj"],"gateup");if(S(c,l)){let Y=k(G,W,W.offset(`layers.${M}.gate_proj`),c,l,H,2*c,0);return k(G,W,W.offset(`layers.${M}.up_proj`),c,l,Y,2*c,c)}let P=2*c*l,R=e.scratch(`${X}.w`,p,[P]);return R=E(R,P,W,W.offset(`layers.${M}.gate_proj`),c,l,0),R=E(R,P,W,W.offset(`layers.${M}.up_proj`),c,l,c*l),dr(e,f,p,o,G,R,0,l,2*c,H)}if(He(t,`layers.${M}.gate_proj`)){let W=`layers.${M}.gate_proj`,P=`layers.${M}.up_proj`,R=su(t,[W,P]),Y=n(M,R?["gate_proj","up_proj"]:["gate_proj"],"gateup"),ne=R?Y:n(M,["up_proj"],"gateupU");if(K(Y,c,l)&&K(ne,c,l)){let se=F(G,Y,Y.offset(W),c,l,H,2*c,0);return F(G,ne,ne.offset(P),c,l,se,2*c,c)}let re=2*c*l,D=e.scratch(`${X}.w`,p,[re]);return D=Z(D,re,Y,Y.offset(W),c,l,0),D=Z(D,re,ne,ne.offset(P),c,l,c*l),dr(e,f,p,o,G,D,0,l,2*c,H)}return dr(e,f,p,o,G,r(M),$,l,2*c,H)},x=Si(t.config),B=(M,G,$)=>{if(!xn(x,M))throw new Error(`moeMlpInto: layer ${M} is not a MoE layer`);let X=e.op("co.huggingface.RMSNorm",{xT:$,wT:r(M),yT:e.scratch(`L${M}.ffnorm`,p,[o,l])},{args:{rows:o,dim:l,eps:d,weightBaseOffset:G.post_attention_layernorm}}).yT,H=e.scratch(`L${M}.moeRouteIdx`,"uint32",[o*x.topK]),W=e.scratch(`L${M}.moeRouteW`,"float32",[o*x.topK]);e.op("com.xenova.LlamaPrefillMoeRoute",{normedT:X,weightsT:r(M),routeIdxT:H,routeWT:W},{args:{hiddenSize:l,seqLen:o,numExperts:x.numExperts,topK:x.topK,routerOffset:G.moe_router,hasBias:x.useExpertBias?1:0,biasOffset:x.useExpertBias?G.moe_bias:0,normTopk:x.normTopkProb?1:0,routedScale:x.routedScale}});let P=x.moeIntermediateSize,R=e.scratch(`L${M}.moeInter`,p,[o,x.topK*P]),Y=He(t,`layers.${M}.gate_exps`),ne=He(t,`layers.${M}.down_exps`),re=o*x.topK,D=null;if(Y||ne){let V=Math.ceil(re/16)+x.numExperts;D={list:e.scratch(`L${M}.moeSlotList`,"uint32",[re]),tiles:e.scratch(`L${M}.moeTiles`,"uint32",[V*3]),tileCount:e.scratch(`L${M}.moeTileCount`,"uint32",[1])},e.op("com.xenova.LlamaPrefillMoeGroup",{routeIdxT:H,listT:D.list,tilesT:D.tiles,tileCountT:D.tileCount},{args:{seqLen:o,numExperts:x.numExperts,topK:x.topK}})}if(Y&&D){let V=n(M,["gate_exps","up_exps"],"moegateup");e.op("com.xenova.LlamaPrefillMoeGateUpGrouped",{normedT:X,bitsT:V.bitsT,scalesT:V.scalesT,listT:D.list,tilesT:D.tiles,tileCountT:D.tileCount,intermediateT:R,...V.gridT&&!Dn(C,V.iqRaw)?{gridT:V.gridT}:{}},{args:{hiddenSize:l,moeIntermediateSize:P,seqLen:o,numExperts:x.numExperts,topK:x.topK,gateOffset:V.offset(`layers.${M}.gate_exps`),upOffset:V.offset(`layers.${M}.up_exps`),format:Zt(Pn(V.format,"MoE expert gate/up"),"MoE expert gate/up"),lut:V.lut,iqRaw:V.iqRaw}})}else e.op("com.xenova.LlamaPrefillMoeGateUp",{normedT:X,weightsT:r(M),routeIdxT:H,intermediateT:R},{args:{hiddenSize:l,moeIntermediateSize:P,seqLen:o,numExperts:x.numExperts,topK:x.topK,gateOffset:G.gate_exps,upOffset:G.up_exps}});let se=e.scratch(`L${M}.moedown`,p,[o,l]);if(ne&&D){let V=n(M,["down_exps"],"moedown");e.op("com.xenova.LlamaPrefillMoeDownGrouped",{intermediateT:R,bitsT:V.bitsT,scalesT:V.scalesT,listT:D.list,tilesT:D.tiles,tileCountT:D.tileCount,routeWT:W,yT:se,...V.gridT&&!Dn(C,V.iqRaw)?{gridT:V.gridT}:{}},{args:{hiddenSize:l,moeIntermediateSize:P,seqLen:o,numExperts:x.numExperts,topK:x.topK,downOffset:V.offset(`layers.${M}.down_exps`),format:Zt(Pn(V.format,"MoE expert down"),"MoE expert down"),lut:V.lut,iqRaw:V.iqRaw}})}else e.op("com.xenova.LlamaPrefillMoeDown",{intermediateT:R,weightsT:r(M),routeIdxT:H,routeWT:W,yT:se},{args:{hiddenSize:l,moeIntermediateSize:P,seqLen:o,numExperts:x.numExperts,topK:x.topK,downOffset:G.down_exps}});e.op("com.xenova.Axpy",{xT:$,yT:se},{args:{count:o*l,alpha:1}})};return{dequant:Z,fused:F,canFuse:K,projInto:Q,proj:N,gateUp:q,mlpInto:(M,G,$)=>{if(xn(x,M)){B(M,G,$);return}let X=e.op("co.huggingface.RMSNorm",{xT:$,wT:r(M),yT:e.scratch(`L${M}.ffnorm`,p,[o,l])},{args:{rows:o,dim:l,eps:d,weightBaseOffset:G.post_attention_layernorm}}).yT,H=e.scratch(`L${M}.mlpact`,p,[o,c]),W;if(qy({packedQ1:y,activationDtype:p,rows:o,inFeatures:l,intermediateSize:c})){let R=s(M,["gate_proj","up_proj"],"gateup");W=e.op("com.xenova.LlamaPrefillGateUp",{aT:X,bitsT:R.bitsT,scalesT:R.scalesT,yT:H,...a?{realLenT:a}:{}},{args:{M:o,inFeatures:l,intermediateSize:c,gateOffset:R.offset(`layers.${M}.gate_proj`),upOffset:R.offset(`layers.${M}.up_proj`),format:"q1_0",...g?{activation:g}:{}}}).yT}else{let R=q(M,X,G.gate_proj,`L${M}.gateup`);W=e.op("co.huggingface.GatedMlpActivation",{xT:R,yT:H},{args:{rows:o,mlpInner:c,...g?{activation:g}:{}}}).yT}let P=N(M,W,"down_proj",G.down_proj,l,c,`L${M}.down`);e.op("com.xenova.Axpy",{xT:$,yT:P},{args:{count:o*l,alpha:1}})}}}function Iy(e){let{g:t,w:r,weights:n,model:s,ids:a,act:i,H:u,vocab:o,T:l,embedOffset:c}=e;if(Xt(s)){let d=Ls(s,["embed_tokens"]);return t.op("com.xenova.LlamaEmbed",{inputT:a,bitsT:r("embed.q1bits",d.q1BitsT),scalesT:r("embed.q1scales",d.q1ScalesT),hiddenT:t.scratch("embed.hidden",i,[l,u])},{args:{hiddenSize:u,vocabSize:o,seqLen:l,embedOffset:d.offset("embed_tokens"),format:"q1_0"}}).hiddenT}return s.embedBits&&s.embedScales?t.op("com.xenova.LlamaEmbed",{inputT:a,bitsT:r("embed.bits",s.embedBits),scalesT:r("embed.scales",s.embedScales),hiddenT:t.scratch("embed.hidden",i,[l,u])},{args:{hiddenSize:u,vocabSize:o,seqLen:l,format:s.embedFormat??"q8_rows",...s.embedLut!==null&&s.embedLut!==void 0?{lut:s.embedLut}:{}}}).hiddenT:t.op("com.xenova.LlamaEmbed",{inputT:a,weightsT:n,hiddenT:t.scratch("embed.hidden",i,[l,u])},{args:{embedOffset:c,hiddenSize:u,vocabSize:o,seqLen:l}}).hiddenT}function Ey(e){let{g:t,w:r,weights:n,model:s,hidden:a,act:i,H:u,vocab:o,T:l,eps:c,finalNormOffset:d,lmHeadOffset:p}=e,f=t.scratch("last_row",i,[u]),g=t.uniform("last_row_uni",32);t.op("com.xenova.StridedCopy",{srcT:a,dstT:f,p:g},{args:{rows:1,srcStride:u,srcStart:(l-1)*u,dstStride:u,dstStart:0,copyCols:u}});let m=t.scratch("next_token","uint32",[1]);if(e.nextTokenPick)return t.op("com.xenova.StridedCopy",{srcT:e.nextTokenPick.tokens,dstT:m,p:e.nextTokenPick.uni},{args:{rows:1,srcStride:1,srcStart:l-1,dstStride:1,dstStart:0,copyCols:1}}),t.output(m,"next_token"),g.name;let h=Xt(s),v=s.runtime.device.features.has("shader-f16");return pu(t,{w:r,model:s,hidden:f,weights:n,ids:m,hiddenSize:u,vocabSize:o,rmsEps:c,finalNormOffset:d,lmHeadOffset:p,q1:h,normedDtype:h&&v?"float16":"float32"}),t.output(m,"next_token"),g.name}var Ym=512;function Py(e,t,r,n,s){return!t&&!!(e.lmHeadQ4&&e.lmHeadQ4Scales)&&e.lmHeadScanBits!==2&&r>=2&&r<=16&&n%256===0&&s<=256*Ym}function gc(e,t,r,n){if(!Qs(e))throw new Error("emitLlamaPrefill: only full-causal attention models (plain or dense qk-norm; dense or Q8/Q4/Q4_K/IQ packed o_proj+MLP) are supported");let s=e.config,a=tn(s),i=Gt(s),u=i?Fe(s.conv_L_cache,"config.conv_L_cache"):0,o=r,l=s.hidden_size,c=i?Li(l,u):0,d=s.num_hidden_layers,p=s.num_attention_heads,f=s.num_key_value_heads,g=s.head_dim,m=p*g,h=f*g,v=m+2*h,b=s.intermediate_size,_=s.vocab_size,C=Number(s.rms_norm_eps),y=t.maxLength,E=cc(g),k=64,S="float32",A=e.runtime.device.features.has("shader-f16"),w=A&&e.weights.dtype==="float16"?"float16":"float32",I=Ms(e.runtime.device),z=au(e,y),O=Rn(e.config),j=e.offsets.top,F=new Ln,{w:K,state:Z,boundWeights:Q,states:N}=ou(F),q=nu(K,e),x=q("top"),B=K("rope.cos",z.cos),U=K("rope.sin",z.sin),M=F.uniform("params",16),G=ru(K,e),$=tu(K,e),X=Cy(F,o),{dequant:H,fused:W,canFuse:P,proj:R,mlpInto:Y}=By({g:F,model:e,denseW:q,q4w:G,q1w:$,realRowsT:X,T:o,H:l,inter:b,eps:C,actGemm:w,fusedQ4:I}),ne=(te,de,Ae)=>{let be=F.scratch(`L${te}.qkv`,w,[o,v]);if(He(e,`layers.${te}.q_proj`)){if(!He(e,`layers.${te}.k_proj`)&&!He(e,`layers.${te}.v_proj`)){let Ce=G(te,["q_proj"],"q");if(Ce.format!=="q4_k_native_compact"||de.v_proj!==de.k_proj+h*l)throw new Error("Mixed QKV prefill requires native Q4_K query and contiguous dense K/V");if((o<=16||o>=32)&&w==="float16"&&I)return F.op("com.xenova.LlamaPrefillMatmul",{aT:Ae,bitsT:Ce.bitsT,scalesT:Ce.scalesT,weightsT:q(te),yT:be},{args:{M:o,inFeatures:l,outFeatures:v,outStride:v,blockOffset:Ce.offset(`layers.${te}.q_proj`),weightOffset:de.k_proj,denseTailStart:m,dstColStart:0,format:Ce.format}}).yT;let Ve=v*l,pt=F.scratch(`L${te}.qkvDense`,w,[Ve]);return pt=H(pt,Ve,Ce,Ce.offset(`layers.${te}.q_proj`),m,l,0),pt=F.op("com.xenova.StridedCopy",{srcT:q(te),dstT:pt},{args:{rows:1,srcStride:2*h*l,srcStart:de.k_proj,dstStride:Ve,dstStart:m*l,copyCols:2*h*l}}).dstT,dr(F,I,w,o,Ae,pt,0,l,v,be)}let he=G(te,["q_proj","k_proj","v_proj"],"qkv");if(P(he,m,l)&&h%64===0){let Ce=W(Ae,he,he.offset(`layers.${te}.q_proj`),m,l,be,v,0);return Ce=W(Ae,he,he.offset(`layers.${te}.k_proj`),h,l,Ce,v,m),W(Ae,he,he.offset(`layers.${te}.v_proj`),h,l,Ce,v,m+h)}let $e=v*l,Be=F.scratch(`L${te}.qkvDense`,w,[$e]);return Be=H(Be,$e,he,he.offset(`layers.${te}.q_proj`),m,l,0),Be=H(Be,$e,he,he.offset(`layers.${te}.k_proj`),h,l,m*l),Be=H(Be,$e,he,he.offset(`layers.${te}.v_proj`),h,l,(m+h)*l),dr(F,I,w,o,Ae,Be,0,l,v,be)}return dr(F,I,w,o,Ae,q(te),de.q_proj,l,v,be)},re=F.stepInput("input_ids","uint32",[o]),D=Iy({g:F,w:K,weights:x,model:e,ids:re,act:S,H:l,vocab:_,T:o,embedOffset:j.embed_tokens}),se=n?.tapLayers.length??0,V=w,ee=n?F.scratch("dspark.features",V,[o,se*l]):null,le=n?.convSnapshots?Z("dspark.conv_snapshots",n.convSnapshots):null;for(let te=0;te<d;++te){let de=e.offsets.layers[te];if(ee&&n){let Ae=n.tapLayers.indexOf(te);Ae>=0&&F.op("com.xenova.StridedCopy",{srcT:D,dstT:ee},{args:{rows:o,srcStride:l,srcStart:0,dstStride:se*l,dstStart:Ae*l,copyCols:l}})}if(i&&uu(s,te)==="conv"){if(!t.convStates)throw new Error("emitLlamaPrefill: conv layer requires cache.convStates");let Ae=Z(`conv.state.${te}`,t.convStateForLayer(te)),be=F.op("co.huggingface.RMSNorm",{xT:D,wT:q(te),yT:F.scratch(`L${te}.convnorm`,w,[o,l])},{args:{rows:o,dim:l,eps:C,weightBaseOffset:de.input_layernorm}}).yT,he=R(te,be,"conv_in_proj",de.conv_in_proj,3*l,l,`L${te}.bcx`),$e=F.op("com.xenova.LlamaPrefillConv",{bcxT:he,weightsT:q(te),convStatesT:Ae,convYT:F.scratch(`L${te}.convY`,w,[o,l]),params:M,...le?{snapshotsT:le}:{}},{args:{hiddenSize:l,convLCache:u,convWeightOffset:de.conv_weight,seqLen:o,...le?{snapshotOffset:te*c,snapshotStride:t.convStates.shape[0]}:{}}}).convYT,Be=R(te,$e,"conv_out_proj",de.conv_out_proj,l,l,`L${te}.convout`);F.op("com.xenova.Axpy",{xT:D,yT:Be},{args:{count:o*l,alpha:1}})}else{let Ae=eu(t,te),be=Z(`cache.key.${te}`,Ae.cacheKeysT),he=Z(`cache.value.${te}`,Ae.cacheValuesT),$e=F.op("co.huggingface.RMSNorm",{xT:D,wT:q(te),yT:F.scratch(`L${te}.normed`,w,[o,l])},{args:{rows:o,dim:l,eps:C,weightBaseOffset:de.input_layernorm}}).yT,Be=ne(te,de,$e);F.op("com.xenova.LlamaPrefillRopeCacheKv",{qkvT:Be,cacheKeysT:be,cacheValuesT:he,cosT:B,sinT:U,params:M,...a?{weightsT:q(te)}:{}},{args:{layer:0,cacheLen:y,pastLen:0,seqLen:o,numHeads:p,numKvHeads:f,headDim:g,...O<g?{rotaryDim:O}:{},...a?{qNormOffset:de.q_layernorm,kNormOffset:de.k_layernorm,rmsEps:C}:{}}});let Ce=F.op("com.xenova.CachedDecodeAttention",{qkvT:Be,cacheKeysT:be,cacheValuesT:he,attnT:F.scratch(`L${te}.attn`,w,[o,m]),params:M},{args:{layer:0,cacheLen:y,pastLen:0,seqLen:o,numHeads:p,numKvHeads:f,headDim:g,scale:E,workgroupSize:k}}).attnT,Ve=R(te,Ce,"o_proj",de.o_proj,l,m,`L${te}.oproj`);F.op("com.xenova.Axpy",{xT:D,yT:Ve},{args:{count:o*l,alpha:1}})}Y(te,de,D)}n&&ee&&n.inject(F,{featuresT:ee,cosT:B,sinT:U,paramsT:M,rows:o,actGemm:w,w:K,state:Z});let me=F.nodeCount,We;if(n?.allRowsHead){let te=Xt(e),de=F.scratch("verify_tokens","uint32",[o]),Ae=!te&&e.lmHeadQ4&&e.lmHeadQ4Scales&&e.lmHeadQ8&&e.lmHeadQ8Scales?lc(_,e.runtime.device.limits.maxComputeWorkgroupsPerDimension):null;if(Ae!==null&&o>=2&&o<=16){let be=oc(e.lmHeadScanBits),he=Ae*be,$e=F.scratch("V.head.scanValues","float32",[o*he]),Be=F.scratch("V.head.scanIds","uint32",[o*he]);F.op("com.xenova.LlamaDecodeLmHeadArgmax",{hiddenT:D,bitsT:K("lmhead.q4bits",e.lmHeadQ4),scalesT:K("lmhead.q4scales",e.lmHeadQ4Scales),weightsT:x,bestValuesT:$e,bestIdsT:Be},{args:{hiddenSize:l,vocabSize:_,lmHeadOffset:0,finalNormOffset:j.model_norm,numBlocks:Ae,topK:be,mRows:o,...e.lmHeadScanBits===2?{quantBits:2}:{},...e.lmHeadFormat?{format:e.lmHeadFormat}:{}}});for(let Ce=0;Ce<o;++Ce){let Ve=F.scratch(`V.r${Ce}.candIds`,"uint32",[he]);F.op("com.xenova.StridedCopy",{srcT:Be,dstT:Ve},{args:{rows:1,srcStride:he,srcStart:Ce*he,dstStride:he,dstStart:0,copyCols:he}});let pt=F.scratch(`V.r${Ce}.rescored`,"float32",[he]);F.op("com.xenova.LlamaDecodeLmHeadRescore",{aT:F.view(D,Ce*l,S,[l],`V.row${Ce}`),normT:x,bitsT:K("lmhead.q8bits",e.lmHeadQ8),scalesT:K("lmhead.q8scales",e.lmHeadQ8Scales),candIdsT:Ve,outValuesT:pt},{args:{inFeatures:l,numCands:he,zeroPoint:0,mask:255,format:e.lmHeadRescoreFormat??"q8_rows",fuseNorm:1,normOffset:j.model_norm}}),F.op("com.xenova.CandidateArgMax",{valuesT:pt,indicesT:Ve,outT:de},{args:{outputOffset:Ce,count:he}})}}else if(Py(e,te,o,l,_)){let be=Ym,he=F.scratch("V.head.bestValues","float32",[o*be]),$e=F.scratch("V.head.bestIds","uint32",[o*be]);F.op("com.xenova.LlamaDecodeLmHeadArgmax",{hiddenT:D,bitsT:K("lmhead.q4bits",e.lmHeadQ4),scalesT:K("lmhead.q4scales",e.lmHeadQ4Scales),weightsT:x,bestValuesT:he,bestIdsT:$e},{args:{hiddenSize:l,vocabSize:_,lmHeadOffset:0,finalNormOffset:j.model_norm,numBlocks:be,mRows:o,...e.lmHeadFormat?{format:e.lmHeadFormat}:{}}});for(let Be=0;Be<o;++Be)F.op("com.xenova.CandidateArgMax",{valuesT:F.view(he,Be*be,"float32",[be],`V.r${Be}.bestValues`),indicesT:F.view($e,Be*be,"uint32",[be],`V.r${Be}.bestIds`),outT:de},{args:{outputOffset:Be,count:be}})}else for(let be=0;be<o;++be)pu(F,{w:K,model:e,hidden:F.view(D,be*l,S,[l],`V.row${be}`),weights:x,ids:de,hiddenSize:l,vocabSize:_,rmsEps:C,finalNormOffset:j.model_norm,lmHeadOffset:j.lm_head,q1:te,normedDtype:te&&A?"float16":"float32",nameSuffix:`V.r${be}.`,outputOffset:be});F.output(de,"verify_tokens"),We={tokens:de,uni:F.uniform("next_token_pick_uni",32)}}let Oe=Ey({g:F,w:K,weights:x,model:e,hidden:D,act:S,H:l,vocab:_,T:o,eps:C,finalNormOffset:j.model_norm,lmHeadOffset:j.lm_head,...We?{nextTokenPick:We}:{}});return{graph:F.finish({name:"llama-prefill",params:{T:o}}),weights:Q,states:N,paramsName:"params",lastRowUniform:Oe,...We?{nextTokenPickUniform:We.uni.name}:{},...X?{realRowsInput:X.name}:{},nextTokenTailNodeCount:F.nodeCount-me}}var pc=class extends du{writeRunInputs(t,r){let n=this.runtime.host,s=this.model.config.hidden_size,a=this.stepUniformPack(this.emission.paramsName);n.writeBuffer(this.compiled.uniformBuffer(this.emission.paramsName),0,a({past_len:r,cache_len:this.cache.maxLength,seq_len:this.blockLen,...a.fields.has("real_len")?{real_len:t}:{}})),n.writeBuffer(this.compiled.uniformBuffer(this.emission.lastRowUniform),0,this.stepUniformPack(this.emission.lastRowUniform)({rows:1,copyCols:s,srcStride:s,srcStart:(t-1)*s,dstStride:s,dstStart:0})),this.emission.nextTokenPickUniform&&n.writeBuffer(this.compiled.uniformBuffer(this.emission.nextTokenPickUniform),0,this.stepUniformPack(this.emission.nextTokenPickUniform)({rows:1,copyCols:1,srcStride:1,srcStart:t-1,dstStride:1,dstStart:0})),this.emission.realRowsInput&&n.writeBuffer(this.compiled.tensor(this.emission.realRowsInput).buffer,0,new Uint32Array([t]))}},gu=class extends pc{buildEmission(){return gc(this.model,this.cache,this.blockLen)}};var Ry=(e,t,r)=>Qs(e)?gc(e,t,r):null,Jm=["q6_k","q5_k","q3_k","q2_k","q4_k_native_compact","lut32_compact","lut16_compact"];function e1(e){let{entriesFor:t,nameToGGUF:r,packSuffixes:n}=e,s=new Set(e.nativeFormats??[]),a=e.nativeSuffixes?new Set(e.nativeSuffixes):void 0,i=e.lutcSuffixes?new Set(e.lutcSuffixes):void 0,u=d=>{let p=typeof e.lut2Suffixes=="function"?e.lut2Suffixes(d):e.lut2Suffixes;return p?new Set(p):void 0},o=e.subFourKSuperSuffixes?new Set(e.subFourKSuperSuffixes):void 0;function l(d,p,f){let g=t(d),m=Cn(g,K=>f.has(r(K))),h=K=>{let Z=r(K.name);return K.id==="lm_head"&&!f.has(Z)?f.has("token_embd.weight")&&f.info("token_embd.weight").type===T.Q1_0:f.has(Z)&&f.info(Z).type===T.Q1_0},{kq4Ids:v,kSuperIds:b,lut32Ids:_,lut16Ids:C,lut32cIds:y,lut16cIds:E,lut2_32Ids:k,lut2_16Ids:S,iqIds:A,iqRawIds:w,excludeKq4Ids:I,forceDenseIds:z,quantBits:O}=Yl(g,f,r,n,{nativeFormats:s,nativeSuffixes:a,lutcSuffixes:i,lut2Suffixes:u(d),subFourKSuperSuffixes:o,decodeGroups:e.decodeGroups,denseGroups:e.denseGroups,iqRawTrunkSuffixes:nc(ze(p),e.gatedQkv,or(d))}),j=K=>{let Z=r(K.name);return f.has(Z)?qm(f.info(Z).type)?.lutId??0:0},F=K=>{for(let Z of _t)if(w.get(Z.key).has(K.id))return{buffer:Z.key,wordsPerBlock:Z.wordsPerBlock,scalesPerBlock:.125};if(v.has(K.id))return{buffer:"kq4",wordsPerBlock:4,scalesPerBlock:2};for(let[Z,Q]of Object.entries(b)){if(!Q.has(K.id))continue;let N=Zl[Z];return{buffer:Z,wordsPerBlock:N.wordsPerBlock,scalesPerBlock:N.scalesPerBlock,scalesF32:!0,metadataU32:N.metadataU32}}if(_.has(K.id))return{buffer:"lut32",wordsPerBlock:4,scalesPerBlock:1,scalesF32:!0,lut:j(K)};if(C.has(K.id))return{buffer:"lut16",wordsPerBlock:4,scalesPerBlock:2,scalesF32:!0,lut:j(K)};if(y.has(K.id))return{buffer:"lut32c",wordsPerBlock:4,scalesPerBlock:.375,metadataU32:!0,lut:j(K)};if(E.has(K.id))return{buffer:"lut16c",wordsPerBlock:4,scalesPerBlock:.625,metadataU32:!0,lut:j(K)};if(k.has(K.id)){let Z=j(K);return{buffer:Z===9?"lut2_128":"lut2_32",wordsPerBlock:2,scalesPerBlock:Z===9?.125:.375,metadataU32:!0,lut:Z}}return S.has(K.id)?{buffer:"lut2_16",wordsPerBlock:2,scalesPerBlock:.625,metadataU32:!0,lut:j(K)}:A.has(K.id)?{buffer:"iq",wordsPerBlock:8,scalesPerBlock:2}:en(K,f,r,n,I,z)?{buffer:"q4",wordsPerBlock:O,scalesPerBlock:1}:null};return ec({config:d,device:p,entries:g,emitGraph:e.emitGraph,emitPrefillGraph:e.emitPrefillGraph??Ry,isPacked:K=>F(K)!==null,isQ1:h,packedQuantBits:O,packSlots:F,decodeGroups:e.decodeGroups,tieWordEmbeddings:m,embedHeadRoutes:Vi(f,r,d,g,m,ze(p).features.has("shader-f16"))})}async function c(d,p,f,{onProgress:g,signal:m}){let h=t(f);p.planReads?.(h.map(A=>r(A.name)));let v=h.map(A=>({...A,name:r(A.name)})),b=Cn(h,A=>p.has(r(A))),_=Kl(p,r),C=Ul(p,r),{packs:y,skipDenseIds:E}=await Bm({runtime:d,entries:h,gguf:p,nameToGGUF:r,signal:m},{packSuffixes:n,nativeFormats:s,nativeSuffixes:a,lutcSuffixes:i,lut2Suffixes:u(f),subFourKSuperSuffixes:o,decodeGroups:e.decodeGroups,denseGroups:e.denseGroups,iqRawTrunkSuffixes:nc(d.device,e.gatedQkv,or(f))}),k=null,S=null;try{let A=Cn(h,w=>p.has(r(w)));k=await Im({runtime:d,entries:h,gguf:p,nameToGGUF:r,signal:m},f,A);for(let w of k?.skipDenseIds??[])E.add(w);return S=await bl({runtime:d,config:f,entries:h,reader:_,source:C,modelName:e.modelName,onProgress:g,signal:m,skipDenseIds:E,...k?.lmHeadQ4||k?.lmHeadQ8?{canUseQ4LmHead:()=>!1,canUseQ8LmHead:()=>!1}:{}}),e.construct({...S,runtime:d,config:f,packs:y,...k?.embedBits&&k.embedScales?{embedBits:k.embedBits,embedScales:k.embedScales,embedFormat:k.embedFormat,embedLut:k.embedLut??null}:{},...k?.lmHeadQ4&&k.lmHeadQ4Scales?{lmHeadQ4:k.lmHeadQ4,lmHeadQ4Scales:k.lmHeadQ4Scales,lmHeadFormat:k.lmHeadFormat??null,lmHeadLut:k.lmHeadLut??null,lmHeadScanBits:k.lmHeadScanBits??null,lmHeadRescoreFormat:k.lmHeadRescoreFormat??null}:{},...k?.lmHeadQ8&&k.lmHeadQ8Scales?{lmHeadQ8:k.lmHeadQ8,lmHeadQ8Scales:k.lmHeadQ8Scales}:{}})}catch(A){throw await Kt(d,[y,k,S]),A}}return{configFromGGUF:e.configFromGGUF,checkSupport:l,loadWeights:c}}function t1(e,t,r,n){let s=t.configFromGGUF(e);return new Yi({config:s,checkSupport:a=>t.checkSupport(s,a,e),loadWeights:(a,{signal:i})=>t.loadWeights(a,e,s,{onProgress:r,signal:i}),closeSource:n})}async function Gy(e,t,r){let n=e instanceof yr?e:await yr.open(e,t);try{return t1(n,r,t.onProgress??null,()=>n.close())}catch(s){throw await n.close(),s}}function Dy(e,t,r){return t1(e,r,t.onProgress??null,async()=>{})}async function Ly(e){let{readFile:t}=await Promise.resolve().then(()=>(ul(),il));return new $t(await t(e))}function r1(e){let t=(n,s={})=>Gy(n,s,e(s)),r=(n,s)=>Dy(s.gguf,{onProgress:s.onProgress},e(s)).load(n);return{prepare:t,async load(n,s,a={}){return(await t(s,a)).load(n,{signal:a.signal})},async loadGGUFFile(n,s,a={}){return r(n,{...a,gguf:await Ly(s)})},loadGGUF:r}}function mu(e,t){return{cache:Br.allocate(e.runtime,e.config,1,t,e.cacheDtype)}}function js(e){return dt(e)||e.hidden_size>128||e.num_hidden_layers>4||e.vocab_size>4096}var Oy=8192;function n1(e){return Math.max(1,Math.min(Oy,Ws(e)))}function Ws(e){let{config:t,runtime:r,cacheDtype:n}=e;if(!js(t))return t.max_position_embeddings;let s=r.device.limits;if(!s)return t.max_position_embeddings;let a=n==="float16"?2:4,i=t.num_key_value_heads*t.head_dim*a,u=Zr(t).length,o=Fe(s.maxComputeWorkgroupStorageSize,"limits.maxComputeWorkgroupStorageSize"),l=Os(t)*4,c=u===0||dt(t)||iu(t)?1/0:Math.floor((o-l)/4),d=typeof s.maxStorageBufferBindingSize=="number"&&s.maxStorageBufferBindingSize>0?s.maxStorageBufferBindingSize:1/0,p=Number.isFinite(d)&&u>0?Math.max(1,Math.floor(d/i)):1/0,f=typeof s.maxBufferSize=="number"&&s.maxBufferSize>0?s.maxBufferSize:1/0,g=Number.isFinite(f)&&u>0?Math.max(1,Math.floor(f/(u*i))):1/0,m=u>0?Sl(1,t.num_key_value_heads,t.head_dim,a):1,h=Math.floor(Math.min(c,p,g)/m)*m;return Math.max(1,Math.min(t.max_position_embeddings,h))}var My=()=>{throw new Error("This feature is not included in this bundle.")},s1=My;function mc(e,t){if(typeof e!="number"||!Number.isInteger(e)||e<0)throw new Error(`${t} requires a non-negative integer max_new_tokens`)}function a1(e,t){if(e)throw new Error(`${t} V1 only supports greedy decoding`)}function i1(e,t,r,n){if(!(e instanceof Br))throw new Error("past_key_values must be a LlamaDynamicCache");if(e.runtime!==t)throw new Error("past_key_values does not belong to this runtime");if(e.batchSize!==n)throw new Error(`past_key_values batch size ${e.batchSize} does not match input batch size ${n}`);let s=Zr(r);if(e.kvLayers.length!==s.length||e.kvLayers.some((i,u)=>i!==s[u]))throw new Error(`past_key_values KV layers [${e.kvLayers}] do not match expected [${s}]`);let a=[s.length,n,e.maxLength,r.num_key_value_heads,r.head_dim];if(zs(e.keyStates,a,"past_key_values.keyStates"),zs(e.valueStates,a,"past_key_values.valueStates"),Gt(r)&&zs(e.convStates,[xl(r,n)],"past_key_values.convStates"),dt(r)){if(!Array.isArray(e.linearConvStates)||!Array.isArray(e.linearRecurrentStates))throw new Error("past_key_values linear attention states are required");let i=yi(r);for(let u=0;u<r.num_hidden_layers;++u)i[u]==="linear_attention"&&(zs(e.linearConvStates[u],[n,Fe(r.linear_conv_dim,"config.linear_conv_dim"),Fe(r.linear_conv_kernel_dim,"config.linear_conv_kernel_dim")-1],`past_key_values.linearConvStates[${u}]`),zs(e.linearRecurrentStates[u],[n,Fe(r.linear_num_value_heads,"config.linear_num_value_heads"),Fe(r.linear_key_head_dim,"config.linear_key_head_dim"),Fe(r.linear_value_head_dim,"config.linear_value_head_dim")],`past_key_values.linearRecurrentStates[${u}]`))}}function zs(e,t,r){if(!st(e)||e.dtype!=="float32"&&e.dtype!=="float16")throw new Error(`${r} must be a float32 or float16 WebGpuTensor`);if(e.shape.length!==t.length||e.shape.some((n,s)=>n!==t[s]))throw new Error(`${r} shape [${e.shape}] does not match expected [${t}]`)}function hc(e,t,r){let n=0,s=e[t];for(let a=1;a<r;++a){let i=e[t+a];i>s&&(s=i,n=a)}return n}function wc(e,t){if(e instanceof Uint32Array)return{shape:[1,e.length],data:e};if(!Array.isArray(e))throw new Error(`${t} must be a uint32 tensor, Uint32Array, or array`);let r=vc(e,t);if(r.length===1&&r.unshift(1),r.length!==2)throw new Error(`${t} must be rank 1 or rank 2`);let n=new Uint32Array(we(r)),s=0,a=i=>{if(Array.isArray(i)){for(let u of i)a(u);return}if(typeof i!="number"||!Number.isInteger(i)||i<0)throw new Error("token ids must be non-negative integers");n[s++]=i};return a(e),{shape:r,data:n}}function vc(e,t){if(!Array.isArray(e))return[];let r=e.length;if(r===0)throw new Error(`${t} arrays must not be empty`);let n=Array.isArray(e[0])?vc(e[0],t):[];for(let s=1;s<e.length;++s){let a=Array.isArray(e[s])?vc(e[s],t):[];if(a.length!==n.length||a.some((i,u)=>i!==n[u]))throw new Error(`${t} must be a rectangular array`)}return[r,...n]}var Fy=4,bc=r1(()=>f1),nn=class{runtime;config;weights;offsets;denseShards;denseShardIndex;lmHeadQ8;lmHeadQ8Scales;lmHeadQ4;lmHeadQ4Scales;lmHeadFormat;lmHeadLut;lmHeadScanBits;lmHeadRescoreFormat;embedBits;embedScales;embedFormat;embedLut;packs;cacheDtype;programCache;defaultPositionIdsCache;llamaForwardLayerOffsetsT;ropeCache;partialRopeCache;hadamardSigns;#e=new En;#t=new En(t=>t.disposeAll());#r;constructor({runtime:t,config:r,weights:n,offsets:s,denseShards:a,denseShardIndex:i,lmHeadQ8:u=null,lmHeadQ8Scales:o=null,lmHeadQ4:l=null,lmHeadQ4Scales:c=null,lmHeadFormat:d=null,lmHeadLut:p=null,lmHeadScanBits:f=null,lmHeadRescoreFormat:g=null,embedBits:m=null,embedScales:h=null,embedFormat:v=null,embedLut:b=null,cacheDtype:_="float32",packs:C={}}){this.runtime=t,this.config=ks(r),this.weights=n,this.offsets=s,this.denseShards=a??[n],this.denseShardIndex=i??{},this.lmHeadQ8=u,this.lmHeadQ8Scales=o,this.lmHeadQ4=l,this.lmHeadQ4Scales=c,this.lmHeadFormat=d,this.lmHeadLut=p,this.lmHeadScanBits=f,this.lmHeadRescoreFormat=g,this.embedBits=m,this.embedScales=h,this.embedFormat=v,this.embedLut=b,this.packs=C,this.cacheDtype=_,this.programCache=new Map,this.defaultPositionIdsCache=new Map,this.llamaForwardLayerOffsetsT=null,this.ropeCache=null}static ConfigClass=ur;static MODEL_NAME="Llama";static expectedWeightEntries=xi;static DECODE_GRAPH_EMIT=(t,r)=>fc(t,r,0);static loadStateDict(t,r,n){return Pm(this,t,r,n)}static prepareGGUF=bc.prepare;static loadGGUFFile=bc.loadGGUFFile;static loadGGUF=bc.loadGGUF;canSample(){return js(this.config)&&dc(this)}dispose(){this.#e.disposeAll(),this.#t.disposeAll(),Pe(this.denseShards),Pe(this.lmHeadQ8),Pe(this.lmHeadQ8Scales),Pe(this.lmHeadQ4),Pe(this.lmHeadQ4Scales),Pe(this.embedBits),Pe(this.embedScales),Pe(this.packs),Pe(this.ropeCache),Pe(this.partialRopeCache),Pe(this.hadamardSigns),this.hadamardSigns=void 0,Pe(this.defaultPositionIdsCache),Pe(this.llamaForwardLayerOffsetsT),this.programCache.clear(),this.ropeCache=null,this.partialRopeCache=null,this.llamaForwardLayerOffsetsT=null,this.lmHeadQ8=null,this.lmHeadQ8Scales=null,this.lmHeadQ4=null,this.lmHeadFormat=null,this.lmHeadLut=null,this.lmHeadScanBits=null,this.lmHeadRescoreFormat=null,this.lmHeadQ4Scales=null,this.embedBits=null,this.embedScales=null,this.embedFormat=null,this.embedLut=null,this.packs={}}async forward(t={}){return s1(this,t)}async generate(t){let{input_ids:r,max_new_tokens:n,do_sample:s=!1,force_token_path:a=!1}=t;mc(n,"generate"),a1(s,"LlamaForCausalLM.generate");let{shape:i,data:u}=wc(r,"input_ids");if(i.length!==2)throw new Error("generate expects input_ids shape [batch, sequence]");let[o,l]=i,c=l+n;if(c>this.config.max_position_embeddings)throw new Error(`generated length ${c} exceeds max_position_embeddings ${this.config.max_position_embeddings}`);let d=new Uint32Array(o*c);for(let g=0;g<o;++g)for(let m=0;m<l;++m)d[g*c+m]=u[g*l+m];if(n===0)return this.runtime.tensorFromTypedArray("uint32",[o,c],d);if(a||js(this.config))return this.#s({data:u,batchSize:o,promptLength:l,totalLength:c,maxNewTokens:n});let p=null,f=this.runtime.tensorFromTypedArray("uint32",[o,l],u);try{for(let g=0;g<n;++g){let m=f;if(m===null)throw new Error("generate lost its next input tensor");let h=await this.forward({input_ids:m,past_key_values:p,use_cache:!0,logits_to_keep:1});p=h.past_key_values;let v=new Uint32Array(o);try{let b=await this.runtime.readTensor(h.logits),_=h.logits.shape[1];for(let C=0;C<o;++C){let y=(C*_+_-1)*this.config.vocab_size,E=hc(Kn(b),y,this.config.vocab_size);v[C]=E,d[C*c+l+g]=E}}finally{h.logits.destroy()}m.destroy(),f=null,g+1<n&&(f=this.runtime.tensorFromTypedArray("uint32",[o,1],v))}return this.runtime.tensorFromTypedArray("uint32",[o,c],d)}finally{f?.destroy(),p?.dispose()}}async*streamTokenIds(t){yield*this.#n(t,{fromCache:!1})}createGenerationState(t={}){let r=t.batchSize??t.batch_size??1,n=Ws(this),s=Fc(t,n1(this));if(r!==1)throw new Error("createGenerationState V1 supports batch size 1");if(!Number.isInteger(s)||s<1||s>n)throw new Error(`maxLength must be an integer in [1, ${n}]`);return mu(this,s)}disposeGenerationState(t){let r=this.specDecodeRunner();Rm(t.cache,this.#e,this.#t,...r?[r]:[])}async*streamTokenIdsFromCache(t){yield*this.#n(t,{fromCache:!0})}async*#n(t,{fromCache:r}){let n=r?"streamTokenIdsFromCache":"streamTokenIds",{input_ids:s,max_new_tokens:a,generation_state:i=null,past_key_values:u=null,force_token_path:o=!1}=t,l=r?i?.cache??u:null;mc(a,n);let{eosTokenId:c,stopOnEos:d}=ea(t,{eosTokenId:this.config.eos_token_id}),p=un(t);if(p&&!this.canSample())throw new Error(`LlamaForCausalLM.${n}: this model decodes greedily only`);if(r&&l==null)throw new Error("streamTokenIdsFromCache requires generation_state, cache, or past_key_values");let{shape:f,data:g}=wc(s,"input_ids");if(f.length!==2)throw new Error(`${n} expects input_ids shape [batch, sequence]`);let[m,h]=f;if(m!==1)throw new Error(`${n} V1 supports batch size 1`);if(r){if(h<1)throw new Error("streamTokenIdsFromCache requires at least one input token");if(i1(l,this.runtime,this.config,m),l.get_seq_length()+h+a>l.maxLength)throw new Error(`cache length ${l.get_seq_length()} + input length ${h} + max_new_tokens ${a} exceeds maxLength ${l.maxLength}`)}else{let b=h+a;if(b>this.config.max_position_embeddings)throw new Error(`generated length ${b} exceeds max_position_embeddings ${this.config.max_position_embeddings}`)}if(a===0)return;if(!o&&!js(this.config)){if(p)throw new Error(`LlamaForCausalLM.${n}: this model's forward path decodes greedily only`);yield*this.#i({tokenIds:g,inputLength:h,cache:l,maxNewTokens:a,eosTokenId:c,stopOnEos:d,signal:t.signal??null});return}let v=!r;if(r||(l=mu(this,h+a).cache),!l)throw new Error(`${n} requires a generation cache`);try{yield*this.#o({tokenIds:g,inputLength:h,cache:l,eosTokenId:c,generationArgs:t,sampling:p})}finally{v&&this.disposeGenerationState({cache:l})}}async#s({data:t,batchSize:r,promptLength:n,totalLength:s,maxNewTokens:a}){if(r!==1)throw new Error("large-model generation V1 supports batch size 1");let i=new Uint32Array(s);if(i.set(t),a===0)return this.runtime.tensorFromTypedArray("uint32",[1,s],i);let u=mu(this,s),o=0;try{for await(let l of this.#o({tokenIds:i,inputLength:n,cache:u.cache,generationArgs:{max_new_tokens:a,stop_on_eos:!1}}))i[n+o]=l,o+=1}finally{this.disposeGenerationState(u)}return this.runtime.tensorFromTypedArray("uint32",[1,s],i)}async*#i({tokenIds:t,inputLength:r,cache:n=null,maxNewTokens:s,eosTokenId:a,stopOnEos:i,signal:u=null}){let o=n===null,l=this.runtime.tensorFromTypedArray("uint32",[1,r],t);try{for(let c=0;c<s;++c){let d=l;if(d===null)throw new Error("streamForwardGreedy lost its next input tensor");let p=await this.forward({input_ids:d,past_key_values:n,use_cache:!0,logits_to_keep:1});n=p.past_key_values;let f;try{let g=await this.runtime.readTensor(p.logits),m=p.logits.shape[1];f=hc(Kn(g),(m-1)*this.config.vocab_size,this.config.vocab_size)}finally{p.logits.destroy()}if(d.destroy(),l=null,i&&ta(f,a))return;u?.throwIfAborted(),yield f,c+1<s&&(l=this.runtime.tensorFromTypedArray("uint32",[1,1],new Uint32Array([f])))}}finally{l?.destroy(),o&&n?.dispose()}}async*#o({tokenIds:t,inputLength:r,cache:n,eosTokenId:s,generationArgs:a,sampling:i=null}){let u=this.specDecodeRunner();if(u){if(i)throw new Error("speculative decoding verifies greedy drafts; it cannot sample");for await(let l of u.generate(t.subarray(0,r),n,a,c=>this.#a(c),s??null))a.signal?.throwIfAborted(),yield l;return}let o=this.#p(n,{eosTokenId:s,pipelineDepth:ss(a,Fy,Nr),sampling:i});yield*Af(o,t.subarray(0,r),a)}specDecodeRunner(){return null}makeGraphDecodeSession(t){return new fu(this,t)}#a(t){return this.#e.getOrBuild(t,async()=>{let r=this.makeGraphDecodeSession(t);return await r.build(t.get_seq_length(),{fuse:!0}),r})}prefillSupported(){return Qs(this)}makeGraphPrefillSession(t,r){return new gu(this,t,r)}#u(){return this.#r??=this.prefillSupported()}async#l(t,r){return(await this.#t.getOrBuild(t,()=>new En)).getOrBuild(r,async()=>{let s=this.makeGraphPrefillSession(t,r);return await s.build(),s})}async warmup(t,r){if(t.maxLength<3)return;if(await t.mutableStateCheckpoint?.(),await t.alternateMutableStateCheckpoint?.(),!this.#u()){await Co(this,t,[1]);return}let n=r?.promptTokens!==void 0?this.#c(r.promptTokens,t):Cl((a,i,u)=>this.planPrefillChunks(a,i,u),t.maxLength),s=n.find(a=>a+2<=t.maxLength);if(s!==void 0&&!this.specDecodeRunner()&&await this.#a(t),await Co(this,t,[s??1]),!(this.specDecodeRunner()||r?.deferRemainingPrefill))for(let a of n)a!==s&&await this.#l(t,a)}async warmRemainingPrefill(t){if(!this.#u()||this.specDecodeRunner())return;let r=Cl((n,s,a)=>this.planPrefillChunks(n,s,a),t.maxLength);for(let n of r)await this.#l(t,n)}#c(t,r){return Tl(this.planPrefillChunks(t,0,r.maxLength))}async#f(t,r,n,s=null){let a=await this.#a(t);a.setSampling(s);let i=r.length-1,u=!1;for(let o=0;o<i;++o)a.enqueueStep(r[o],n+o,{full:u});return a.submitStep(r[i],n+i)}prefillMinTokens(){return $g/2}planPrefillChunks(t,r,n){return t>=16&&this.config.model_type==="llama"&&this.weights.dtype==="float16"&&Ms(this.runtime.device)&&!tn(this.config)&&!Gt(this.config)&&!dt(this.config)&&this.config.hidden_size%256===0&&this.config.intermediate_size%256===0?Yg({tokenCount:t,position:r,maxLength:n,shortBlock:16,normalMinTokens:this.prefillMinTokens(),bucketMinTokens:23,bucketMaxTokens:32,bucketBlock:32,fitCache:!0}):Xg({tokenCount:t,position:r,maxLength:n,minGraphTokens:this.prefillMinTokens()})}async#d(t,r,n,s,a=null){if(!this.#u())return this.#f(t,r,n,a);let i=this.planPrefillChunks(r.length,n,t.maxLength),u=s?.signal??null,o=s?.onProgress??null,l=u!==null||o!==null,c=null,d=0,p=n,f=0;for(let g=0;g<i.length;++g){g>0&&u?.throwIfAborted();let m=i[g],h=r.subarray(d,d+m.tokenCount),v=g===i.length-1;if(m.kind==="graph"){let b=await this.#l(t,m.blockLen);v?(b.setSampling(v?a:null),f=await b.run(h,p)):b.enqueueBody(h,p)}else f=await this.#f(t,h,p,v?a:null);if(d+=m.tokenCount,p+=m.tokenCount,l&&!v){let b=this.runtime.queueIdle(),_=d;b.then(()=>o?.(_),()=>{}),c&&await c,c=b}}return f}#p(t,{eosTokenId:r,pipelineDepth:n,sampling:s}){return{cache:t,eosTokenId:r,pipelineDepth:n,sampling:s,prefill:(a,i,u)=>this.#d(t,a,i,u,s),beginDecode:async()=>{let a=await this.#a(t);return a.setSampling(s),a}}}},u1=new Set(["o_proj","gate_proj","up_proj","down_proj"]);function Qy(e){let t=e.metadata["llama.vocab_size"];return typeof t=="number"&&t>0?t:Ep(e)}function hu(e){let t=e.metadata,r=t["general.architecture"];if(r!=="llama")throw new Error(`Expected llama GGUF architecture, got ${JSON.stringify(r)}`);let n=ar(t,"llama.attention.head_count","llama"),s=ar(t,"llama.embedding_length","llama"),a=ar(t,"llama.block_count","llama");return new ur({vocab_size:Qy(e),hidden_size:s,intermediate_size:ar(t,"llama.feed_forward_length","llama"),num_hidden_layers:a,num_attention_heads:n,num_key_value_heads:ar(t,"llama.attention.head_count_kv","llama",n),head_dim:ar(t,"llama.attention.key_length","llama",Math.floor(s/n)),max_position_embeddings:ar(t,"llama.context_length","llama"),rms_norm_eps:pi(t["llama.attention.layer_norm_rms_epsilon"]),rope_parameters:{rope_theta:ar(t,"llama.rope.freq_base","llama",1e4),rope_type:"default"},hidden_act:"silu",use_cache:!0,attention_bias:!1,attention_dropout:0,mlp_bias:!1,eos_token_id:gi(t),bos_token_id:pi(t["tokenizer.ggml.bos_token_id"]),pad_token_id:pi(t["tokenizer.ggml.padding_token_id"])??null,tie_word_embeddings:!e.has("output.weight")})}function d1(e){return e1({configFromGGUF:hu,entriesFor:t=>e?xi(t):pg(xi(t),{...t,rotary_dim:t.head_dim}),nameToGGUF:zl,packSuffixes:e?new Set([...u1,"q_proj"]):u1,denseGroups:e?{q_proj:[],k_proj:[],v_proj:[]}:void 0,decodeGroups:Hl,nativeFormats:Jm,emitGraph:nn.DECODE_GRAPH_EMIT,modelName:nn.MODEL_NAME,construct:t=>new nn(t)})}function o1(e,t,r){let n=ze(t),s=n.adapterInfo,a=s.subgroupMinSize===32&&s.subgroupMaxSize===32||n.features.has("subgroup-size-control")&&s.subgroupMinSize!==void 0&&s.subgroupMinSize<=32&&s.subgroupMaxSize!==void 0&&s.subgroupMaxSize>=32;if(!Ms(n)||!a||n.limits.maxStorageBuffersPerShaderStage<9||e.hidden_size%256!==0||e.head_dim%4!==0||e.num_attention_heads*e.head_dim!==e.hidden_size||e.num_key_value_heads*e.head_dim%64!==0||s.subgroupMatrixConfigs!==void 0&&!s.subgroupMatrixConfigs.some(i=>i.componentType==="f16"&&i.M===8&&i.N===8&&i.K===8))return!1;for(let i=0;i<e.num_hidden_layers;++i)for(let[u,o]of[["q",e.num_attention_heads],["k",e.num_key_value_heads]]){let l=`blk.${i}.attn_${u}.weight`;if(!r.has(l))return!1;let c=r.info(l);if(c.shape.length!==2||c.shape[0]!==o*e.head_dim||c.shape[1]!==e.hidden_size||u==="q"&&c.type!==T.Q4_K)return!1}return!0}var l1=d1(!1),c1=d1(!0),f1={configFromGGUF:hu,checkSupport(e,t,r){return(o1(e,t,r)?c1:l1).checkSupport(e,t,r)},loadWeights(e,t,r,n){if(!o1(r,e.device,t))return l1.loadWeights(e,t,r,n);let s=mg(t,{...r,rotary_dim:r.head_dim});return c1.loadWeights(e,s,r,n)}};function jy(e,t,r){return f1.checkSupport(e,t,r)}function p1(e,t,r){let n=jy(e,t,r);return n instanceof Promise?n.then(s=>s.reason):n.reason}var Wy=8192,zy=1024;function g1({defaultModelId:e,defaultFile:t,defaultContextLength:r=Wy,defaultMaxNewTokens:n=zy}){return{defaultModelId:e,defaultFile:()=>t,unsupportedReason:(s,a)=>p1(hu(s),a,s),loadModel:(s,a,i)=>nn.loadGGUF(s,{gguf:a,onProgress:i}),createGenerationState:(s,a)=>{let i=Ws(s);return s.createGenerationState({maxLength:Math.max(1,Math.min(a??Math.min(i,r),i))})},defaultMaxNewTokens:n}}var h1="openbmb/MiniCPM5-2B-GGUF",Ky="MiniCPM5-2B-Q4_K_M.gguf",m1=g1({defaultModelId:h1,defaultFile:Ky}),_c=class e extends Ja{static DEFAULT_MODEL_ID=h1;constructor(t){super(t)}static checkAvailability(t=null,r={}){return lg(m1,t,r)}static async from_pretrained(t=null,r={}){let n=await dg(m1,s=>new Ya(s),t,r);return new e(n)}},HP=_c;export{Rr as BusyError,ra as ContextOverflowError,Ky as DEFAULT_FILE,h1 as DEFAULT_MODEL_ID,Gr as DeviceLostError,_c as MiniCPM5ForCausalLM,HP as default};