computer-vision-course/pr_428/en/_app/immutable/nodes/29.ecb0fb72.js

import{s as Ws,o as Os,n as Ce}from"../chunks/scheduler.35aab934.js";import{S as Rs,i as Us,e as l,s as n,c as $,q as r,H as f,h as Ns,a as p,d as e,b as i,f as S,g as k,j as c,r as h,u as d,k as We,l as m,m as s,n as M,t as T,o as H,p as C}from"../chunks/index.de17b971.js";import{T as He}from"../chunks/Tip.514d77c4.js";import{C as Bs}from"../chunks/CopyLLMTxtMenu.a0605f42.js";import{H as F,E as Ds}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.9a22621c.js";function Gs(j){let o,u=`Long convolutions are similar to standard convolutions except the kernel is the size of the input.
It is equivalent to having a global receptive field instead of a local one.
Having an implicitly parametrized convolution means that the convolution filters values are not directly learned. Instead, learning a function that can recover thoses values is preferred.`;return{c(){o=l("p"),o.textContent=u},l(v){o=p(v,"P",{"data-svelte-h":!0}),c(o)!=="svelte-obk4bf"&&(o.textContent=u)},m(v,w){s(v,o,w)},p:Ce,d(v){v&&e(o)}}}function Vs(j){let o,u=`Gating mechanisms control the path through which information flows in the network. They help to define how long an information should be remembered. Usally they consist in elementwise multiplications.
An interresting blog article about gating can be found <a href="https://medium.com/autonomous-agents/a-math-deep-dive-on-gating-in-neural-architectures-b49775810dde">here</a>.`;return{c(){o=l("p"),o.innerHTML=u},l(v){o=p(v,"P",{"data-svelte-h":!0}),c(o)!=="svelte-1jr8qmm"&&(o.innerHTML=u)},m(v,w){s(v,o,w)},p:Ce,d(v){v&&e(o)}}}function Zs(j){let o,u,v="H3 Paper",w;return{c(){o=r("It's important to note that the mapping function can be conceptualized within various abstract models, such as Neural Field or State Space Models (S4) as discussed in "),u=l("a"),u.textContent=v,w=r("."),this.h()},l(g){o=h(g,"It's important to note that the mapping function can be conceptualized within various abstract models, such as Neural Field or State Space Models (S4) as discussed in "),u=p(g,"A",{href:!0,"data-svelte-h":!0}),c(u)!=="svelte-y21tg9"&&(u.textContent=v),w=h(g,"."),this.h()},h(){We(u,"href","https://arxiv.org/abs/2212.14052")},m(g,L){s(g,o,L),s(g,u,L),s(g,w,L)},p:Ce,d(g){g&&(e(o),e(u),e(w))}}}function Js(j){let o,u,v='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>y</mi><mo>=</mo><mi>f</mi><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo><mo>=</mo><mi>a</mi><mo>×</mo><mi>x</mi><mo>+</mo><mi>b</mi></mrow><annotation encoding="application/x-tex">y=f(x)= a \\times x + b</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.03588em;">y</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.10764em;">f</span><span class="mopen">(</span><span class="mord mathnormal">x</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6667em;vertical-align:-0.0833em;"></span><span class="mord mathnormal">a</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.6667em;vertical-align:-0.0833em;"></span><span class="mord mathnormal">x</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">b</span></span></span></span>',w;return{c(){o=r("One way to have an intuition about implicit parametrization is to think about an afine function"),u=new f(!1),w=r(" we want to learn. Instead of learning every single point positions it is more efficient to learn a and b and compute the points when needed."),this.h()},l(g){o=h(g,"One way to have an intuition about implicit parametrization is to think about an afine function"),u=d(g,!1),w=h(g," we want to learn. Instead of learning every single point positions it is more efficient to learn a and b and compute the points when needed."),this.h()},h(){u.a=w},m(g,L){s(g,o,L),u.m(v,g,L),s(g,w,L)},p:Ce,d(g){g&&(e(o),u.d(),e(w))}}}function Ks(j){let o,u,v,w,g,L,G,Et,V,At,Z,Wt,J,Oe="While Transformer is a well established and very capable architecture, the quadratic computational cost is an expensive price to pay, especially in inference.",Ot,K,Re=`Hyena is a new type of operator that serves as a substitute for the attention mechanism.
Developed by Hazy Research, it features a subquadratic computational efficiency, constructed by interleaving implicitly parametrized long convolutions and data-controlled gating.`,Rt,E,Ut,A,Nt,Q,Ue=`<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/outlook_hyena_images/transformer2hyena.png" alt="transformer2hyena.png"/>
The Hyena operator consists of recursively computing convolutions and multiplicative element-wise gating operations with one projection at a time, until all projections are exhausted. This approach builds on top of the <a href="https://arxiv.org/abs/2212.14052" rel="nofollow">Hungry Hungry Hippo (H3)</a> mechanism, also developed by the same researchers. The H3 mechanism is characterized by its data-controlled, parametric decomposition, acting as a surrogate attention mechanism.`,Bt,X,Ne=`Another way of understanding Hyena is to consider it as a generalization of the H3 layer for an arbitrary number of projections, where the Hyena layer extends recursively H3 with a different choice of parametrization for the long convolution.
<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/outlook_hyena_images/hyena_recurence.png" alt="hyena_recurence.png"/>`,Dt,Y,Gt,tt,Be="The attention mechanism is characterized by two fundamental properties:",Vt,W,It,De="It possesses a global contextual awareness, enabling it to assess interactions between pairs of visual tokens within a sequence.",Le,P,ze,Zt,ds='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>q</mi></mrow><annotation encoding="application/x-tex">q</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span></span></span></span>',Jt,Kt,vs='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>k</mi></mrow><annotation encoding="application/x-tex">k</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span></span></span></span>',Qt,Xt,ys='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>v</mi></mrow><annotation encoding="application/x-tex">v</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.4306em;"></span><span class="mord mathnormal" style="margin-right:0.03588em;">v</span></span></span></span>',Yt,ta,at,Ge='<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/outlook_hyena_images/self-attention-schema.png" alt="Alt text"/>',aa,x,qe,ea,ws='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>q</mi></mrow><annotation encoding="application/x-tex">q</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span></span></span></span>',sa,na,xs='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>k</mi></mrow><annotation encoding="application/x-tex">k</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span></span></span></span>',ia,la,bs='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>v</mi></mrow><annotation encoding="application/x-tex">v</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.4306em;"></span><span class="mord mathnormal" style="margin-right:0.03588em;">v</span></span></span></span>',pa,ma,_s='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>W</mi><mi>q</mi></msub></mrow><annotation encoding="application/x-tex">W_q</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.9694em;vertical-align:-0.2861em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">W</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">q</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span></span></span></span>',oa,ra,$s='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>W</mi><mi>k</mi></msub></mrow><annotation encoding="application/x-tex">W_k</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">W</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.03148em;">k</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',ha,ca,ks='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>W</mi><mi>v</mi></msub></mrow><annotation encoding="application/x-tex">W_v</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">W</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">v</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',ga,ua,et,Ve=`For a given visual token, we can compute an attention score using thoses projections. The attention score determines how much focus to give on other parts of the input image.<br/>
For a nice detailled explainer of Attention you can refer on this  <a href="https://jalammar.github.io/illustrated-transformer/" rel="nofollow">illustrated blog article</a>.`,fa,st,Ze="In an attempt to replicate these characteristics, the Hyena operator incorporates two key elements:",da,nt,Je="<li>It employs long convolution to provide a sense of global context, akin to the first property of the attention mechanism.</li> <li>For data dependency, Hyena uses element-wise gating. This is essentially an element-wise multiplication of input projections, mirroring the data-dependent nature of traditional attention.</li>",va,O,Pe,ya,Ms='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>O</mi><mo stretchy="false">(</mo><mi>L</mi><mo>×</mo><msub><mrow><mi>log</mi><mo>⁡</mo></mrow><mn>2</mn></msub><mi>L</mi></mrow><annotation encoding="application/x-tex">O(L \\times \\log_2 L</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.02778em;">O</span><span class="mopen">(</span><span class="mord mathnormal">L</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.9386em;vertical-align:-0.2441em;"></span><span class="mop"><span class="mop">lo<span style="margin-right:0.01389em;">g</span></span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.207em;"><span style="top:-2.4559em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">2</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2441em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathnormal">L</span></span></span></span>',wa,xa,it,ba,lt,Ke=`Let’s delve into the second-order recursion of the Hyena operator, which simplifies its representation for illustrative purposes.
<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/outlook_hyena_images/hyena-order2-schema.png" alt="hyena_mechanism.png"/>`,_a,z,Ie,$a,Ts='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>q</mi></mrow><annotation encoding="application/x-tex">q</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.03588em;">q</span></span></span></span>',ka,Ma,Hs='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>k</mi></mrow><annotation encoding="application/x-tex">k</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.03148em;">k</span></span></span></span>',Ta,Ha,Cs='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>v</mi></mrow><annotation encoding="application/x-tex">v</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.4306em;"></span><span class="mord mathnormal" style="margin-right:0.03588em;">v</span></span></span></span>',Ca,La,q,Se,za,Ls='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>T</mi><mi>q</mi></msub></mrow><annotation encoding="application/x-tex">T_q</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.9694em;vertical-align:-0.2861em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">T</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">q</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.2861em;"><span></span></span></span></span></span></span></span></span></span>',qa,Pa,zs='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>T</mi><mi>k</mi></msub></mrow><annotation encoding="application/x-tex">T_k</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">T</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.03148em;">k</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Ia,Sa,qs='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>T</mi><mi>v</mi></msub></mrow><annotation encoding="application/x-tex">T_v</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">T</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.1514em;"><span style="top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.03588em;">v</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',ja,Fa,y,je,Ea,Ps='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>u</mi><mo>∈</mo><msup><mi mathvariant="double-struck">R</mi><mi>L</mi></msup></mrow><annotation encoding="application/x-tex">u \\in \\mathbb{R}^{L}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.5782em;vertical-align:-0.0391em;"></span><span class="mord mathnormal">u</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">∈</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.8413em;"></span><span class="mord"><span class="mord mathbb">R</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.8413em;"><span style="top:-3.063em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight">L</span></span></span></span></span></span></span></span></span></span></span></span>',Aa,Wa,Is='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>L</mi></mrow><annotation encoding="application/x-tex">L</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord mathnormal">L</span></span></span></span>',Oa,St,Qe="implicit convolutions",Fe,Ra,Ss='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>h</mi></mrow><annotation encoding="application/x-tex">h</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">h</span></span></span></span>',Ua,Na,js='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>γ</mi><mi>θ</mi></msub></mrow><annotation encoding="application/x-tex">\\gamma_{\\theta}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.05556em;">γ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0556em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Ba,Da,Fs='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>γ</mi><mi>θ</mi></msub></mrow><annotation encoding="application/x-tex">\\gamma_{\\theta}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.05556em;">γ</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.0556em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mathnormal mtight" style="margin-right:0.02778em;">θ</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Ga,Va,Es='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>T</mi><mi>h</mi></msub></mrow><annotation encoding="application/x-tex">T_h</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.13889em;">T</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3361em;"><span style="top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">h</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Za,Ja,pt,Xe="This implies that instead of learning the values of the convolution filter directly, we learn a mapping from a temporal positional encoding to the values, which is more computationally efficient, especially for long sequences.",Ka,R,Qa,mt,Xa,ot,Ye='A linear convolution can be formulated as a matrix multiplication in which one of the inputs is reshaped into a <a href="https://en.wikipedia.org/wiki/Toeplitz_matrix" rel="nofollow">Toeplitz matrix</a>.',Ya,rt,ts=`This transformation leads to greater parameter efficiency.
Instead of directly learning fixed kernel weight values, a parametrized function is employed.
This function intelligently deduces the values of the kernel weights and their dimensions during the network’s forward pass, optimizing resource use.`,te,U,ae,ht,as=`In practice, convolutions are accelerated to a subquadratic time complexity by the Cooley-Tukey fast Fourier transform (FFT) algorithm.
Some work has been conducted to speed up this computation like FastFFTConv based on Monarch decomposition.`,ee,ct,se,gt,es=`<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/outlook_hyena_images/nd_hyena.png" alt="nd_hyena.png"/>
In essence, Hyena can be performed in two steps:`,ne,N,jt,ss="Compute a set of N+1 linear projections similarly of attention (it can be more than 3 projections).",Ee,ut,Ae,ie,As='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>H</mi><mo stretchy="false">(</mo><mi>u</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">H(u)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.08125em;">H</span><span class="mopen">(</span><span class="mord mathnormal">u</span><span class="mclose">)</span></span></span></span>',le,pe,ft,me,dt,ns="The H3 mechanism proposition went close to the perplexity of multi-headed attention mechanisms, but there was still a narrow gap in terms of perplexity that had to be bridged.",oe,vt,is=`A variety of attention replacements have been proposed over the last few years, and evaluating the quality of a new architecture during the exploratory phase remains challenging.
Creating a versatile layer that can effectively process N-Dimensional data within deep neural networks while maintaining good expressiveness is a significant area of ongoing research.`,re,yt,ls=`Empirically, Hyena operators are able to significantly shrink the quality gap with attention at scale, reaching similar perplexity and downstream performance with a smaller computational budget and without hybridization of attention.
It has already achieved a state-of-the-art status for <a href="https://arxiv.org/abs/2306.15794" rel="nofollow">DNA sequence modeling</a> and shows great promise in the field of large language models with Stripped-Hyena-7B.`,he,wt,ps="Similarly to Attention, Hyena can be used in computer vision tasks. In image classification, Hyena is able to match attention in accuracy when training on ImageNet-1k from scratch.",ce,xt,ms=`<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/outlook_hyena_images/hyena_vision_benchmarks.png" alt="hyena_vision_benchmarks.png"/>
Hyena has been applied to N-Dimensional data with the Hyena N-D layer and can be used as direct drop-in replacement within the ViT, Swin, DeiT backbones.`,ge,bt,os=`<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/outlook_hyena_images/vit_vs_hyenavit.png" alt="vit_vs_hyenavit.png"/>
here is a noticeable enhancement in GPU memory efficiency with the increase in the number of image patches.`,ue,_t,rs=`Hyena Hierarchy facilitates the development of larger, more efficient convolution models for long sequences.
The potential for Hyena type models for computer vision would be a more efficient GPU memory consumption of patches, that would allow:`,fe,$t,hs="<li>The processing of larger, higher-resolution images</li> <li>The use of smaller patches, allowing a fine-graine feature representation</li>",de,kt,cs="These qualities would be particularly beneficial in areas such as Medical Imaging and Remote Sensing.",ve,Mt,ye,Tt,gs="Building new layers from simple design principles is an emerging research field that is progressing very quickly.",we,Ht,us=`The H3 mechanism serves as the foundation for many State Space Model based (SSM) architectures, typically featuring a structure that alternates between a block inspired by linear attention and a multi-layer perceptron (MLP) block.
Hyena, as an enhancement of this approach, has paved the way for even more efficient architectures such as Mamba and its derivatives for vision (Vision Mamba, VMamba etc…).`,xe,Ct,be,Lt,fs='<li>Hyena offical repo: <a href="https://github.com/HazyResearch/safari" rel="nofollow">Convolutions for Sequence Modeling</a></li> <li>On the landscape of subquadratic models: <a href="https://hazyresearch.stanford.edu/blog/2023-06-08-hyena-safari" rel="nofollow">The Safari of Deep Signal Processing: Hyena and Beyond · Hazy Research (stanford.edu)</a></li> <li>On speeding up the FFT algorithm: <a href="https://hazyresearch.stanford.edu/blog/2023-11-13-flashfftconv" rel="nofollow">FlashFFTConv: Efficient Convolutions for Long Sequences with Tensor Cores · Hazy Research (stanford.edu)</a></li> <li>On the subquadratic model landscape: <a href="https://hazyresearch.stanford.edu/blog/2023-12-11-zoology1-analysis" rel="nofollow">Zoology (Blogpost 1): Measuring and Improving Recall in Efficient Language Models · Hazy Research (stanford.edu)</a></li> <li>Hyena applied to computer vision: <a href="https://arxiv.org/abs/2309.13600" rel="nofollow">[2309.13600] Multi-Dimensional Hyena for Spatial Inductive Bias (arxiv.org)</a></li> <li>An improved approach: <a href="https://arxiv.org/abs/2401.09417" rel="nofollow">[2401.09417] Vision Mamba: Efficient Visual Representation Learning with Bidirectional State Space Model (arxiv.org)</a></li>',_e,zt,$e,Ft,ke;return g=new Bs({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),G=new F({props:{title:"Hyena",local:"hyena",headingTag:"h1"}}),V=new F({props:{title:"Overview",local:"overview",headingTag:"h2"}}),Z=new F({props:{title:"What is Hyena",local:"what-is-hyena",headingTag:"h3"}}),E=new He({props:{$$slots:{default:[Gs]},$$scope:{ctx:j}}}),A=new He({props:{$$slots:{default:[Vs]},$$scope:{ctx:j}}}),Y=new F({props:{title:"From Attention to Hyena operator",local:"from-attention-to-hyena-operator",headingTag:"h3"}}),it=new F({props:{title:"Hyena operator",local:"hyena-operator",headingTag:"h3"}}),R=new He({props:{$$slots:{default:[Zs]},$$scope:{ctx:j}}}),mt=new F({props:{title:"Implicit convolutions",local:"implicit-convolutions",headingTag:"h3"}}),U=new He({props:{$$slots:{default:[Js]},$$scope:{ctx:j}}}),ct=new F({props:{title:"Wrapping Up Everything",local:"wrapping-up-everything",headingTag:"h3"}}),ft=new F({props:{title:"Why Hyena Matters",local:"why-hyena-matters",headingTag:"h2"}}),Mt=new F({props:{title:"Towards Transformers Alternatives",local:"towards-transformers-alternatives",headingTag:"h2"}}),Ct=new F({props:{title:"Further Reading",local:"further-reading",headingTag:"h2"}}),zt=new Ds({props:{source:"https://github.com/huggingface/computer-vision-course/blob/main/chapters/en/unit13/hyena.mdx"}}),{c(){o=l("meta"),u=n(),v=l("p"),w=n(),$(g.$$.fragment),L=n(),$(G.$$.fragment),Et=n(),$(V.$$.fragment),At=n(),$(Z.$$.fragment),Wt=n(),J=l("p"),J.textContent=Oe,Ot=n(),K=l("p"),K.textContent=Re,Rt=n(),$(E.$$.fragment),Ut=n(),$(A.$$.fragment),Nt=n(),Q=l("p"),Q.innerHTML=Ue,Bt=n(),X=l("p"),X.innerHTML=Ne,Dt=n(),$(Y.$$.fragment),Gt=n(),tt=l("p"),tt.textContent=Be,Vt=n(),W=l("ol"),It=l("li"),It.textContent=De,Le=n(),P=l("li"),ze=r("It is data-dependent, meaning the operation of the attention equation varies based on the input data itself, specifically the input projections "),Zt=new f(!1),Jt=r(","),Kt=new f(!1),Qt=r(","),Xt=new f(!1),Yt=r("."),ta=n(),at=l("p"),at.innerHTML=Ge,aa=n(),x=l("p"),qe=r("The attention mechanism is defined by three projections: query"),ea=new f(!1),sa=r(", key"),na=new f(!1),ia=r(", value"),la=new f(!1),pa=r(", that are generated by mutiliplying the input visual token by three matrices"),ma=new f(!1),oa=r(","),ra=new f(!1),ha=r(" and"),ca=new f(!1),ga=r(" that are learned during training."),ua=n(),et=l("p"),et.innerHTML=Ve,fa=n(),st=l("p"),st.textContent=Ze,da=n(),nt=l("ol"),nt.innerHTML=Je,va=n(),O=l("p"),Pe=r("In the realm of computational efficiency, the Hyena operator attains an evaluation time complexity of"),ya=new f(!1),wa=r("), indicating a noteworthy enhancement in processing speed."),xa=n(),$(it.$$.fragment),ba=n(),lt=l("p"),lt.innerHTML=Ke,_a=n(),z=l("p"),Ie=r("In this order, we compute 3 projections analogous to"),$a=new f(!1),ka=r(","),Ma=new f(!1),Ta=r(" and"),Ha=new f(!1),Ca=r(" attention vectors from the Attention mechanism."),La=n(),q=l("p"),Se=r("However, unlike the attention mechanism, which typically uses a single dense layer for projecting the input sequence into representations, Hyena incorporates both a dense layer and standard convolutions that are performed on each channels (refered as"),za=new f(!1),qa=r(","),Pa=new f(!1),Ia=r(" and"),Sa=new f(!1),ja=r(" on the schema, but it is an explicit convolution in practice). The softmax function is also discared."),Fa=n(),y=l("p"),je=r("The core idea is to repeatedly apply linear operators that are fast to evaluate to an input sequence"),Ea=new f(!1),Aa=r("  with"),Wa=new f(!1),Oa=r(` the length of the sequence.
Because global convolutions have a large number of parameters, they are expensive to train. A notable design choice is the use of `),St=l("strong"),St.textContent=Qe,Fe=r(`.
Unlike standard convolutional layers, the convolution filter`),Ra=new f(!1),Ua=r(" is learned implicitly with a small neural network"),Na=new f(!1),Ba=r(` (also called the Hyena Filter).
This network takes the positional index and potentially positional encodings as inputs. From the outputs of`),Da=new f(!1),Ga=r(" one can construct a Toeplitz matrix"),Va=new f(!1),Za=r("."),Ja=n(),pt=l("p"),pt.textContent=Xe,Ka=n(),$(R.$$.fragment),Qa=n(),$(mt.$$.fragment),Xa=n(),ot=l("p"),ot.innerHTML=Ye,Ya=n(),rt=l("p"),rt.textContent=ts,te=n(),$(U.$$.fragment),ae=n(),ht=l("p"),ht.textContent=as,ee=n(),$(ct.$$.fragment),se=n(),gt=l("p"),gt.innerHTML=es,ne=n(),N=l("ol"),jt=l("li"),jt.textContent=ss,Ee=n(),ut=l("li"),Ae=r("Mixing up the projections: The matrix"),ie=new f(!1),le=r(" is defined by a combination of matrix multiplications."),pe=n(),$(ft.$$.fragment),me=n(),dt=l("p"),dt.textContent=ns,oe=n(),vt=l("p"),vt.textContent=is,re=n(),yt=l("p"),yt.innerHTML=ls,he=n(),wt=l("p"),wt.textContent=ps,ce=n(),xt=l("p"),xt.innerHTML=ms,ge=n(),bt=l("p"),bt.innerHTML=os,ue=n(),_t=l("p"),_t.textContent=rs,fe=n(),$t=l("ul"),$t.innerHTML=hs,de=n(),kt=l("p"),kt.textContent=cs,ve=n(),$(Mt.$$.fragment),ye=n(),Tt=l("p"),Tt.textContent=gs,we=n(),Ht=l("p"),Ht.textContent=us,xe=n(),$(Ct.$$.fragment),be=n(),Lt=l("ul"),Lt.innerHTML=fs,_e=n(),$(zt.$$.fragment),$e=n(),Ft=l("p"),this.h()},l(t){const a=Ns("svelte-u9bgzb",document.head);o=p(a,"META",{name:!0,content:!0}),a.forEach(e),u=i(t),v=p(t,"P",{}),S(v).forEach(e),w=i(t),k(g.$$.fragment,t),L=i(t),k(G.$$.fragment,t),Et=i(t),k(V.$$.fragment,t),At=i(t),k(Z.$$.fragment,t),Wt=i(t),J=p(t,"P",{"data-svelte-h":!0}),c(J)!=="svelte-ht7sw"&&(J.textContent=Oe),Ot=i(t),K=p(t,"P",{"data-svelte-h":!0}),c(K)!=="svelte-1ts1bdi"&&(K.textContent=Re),Rt=i(t),k(E.$$.fragment,t),Ut=i(t),k(A.$$.fragment,t),Nt=i(t),Q=p(t,"P",{"data-svelte-h":!0}),c(Q)!=="svelte-cumhsh"&&(Q.innerHTML=Ue),Bt=i(t),X=p(t,"P",{"data-svelte-h":!0}),c(X)!=="svelte-1m7gu65"&&(X.innerHTML=Ne),Dt=i(t),k(Y.$$.fragment,t),Gt=i(t),tt=p(t,"P",{"data-svelte-h":!0}),c(tt)!=="svelte-1nqx0mq"&&(tt.textContent=Be),Vt=i(t),W=p(t,"OL",{});var qt=S(W);It=p(qt,"LI",{"data-svelte-h":!0}),c(It)!=="svelte-1tafi0t"&&(It.textContent=De),Le=i(qt),P=p(qt,"LI",{});var I=S(P);ze=h(I,"It is data-dependent, meaning the operation of the attention equation varies based on the input data itself, specifically the input projections "),Zt=d(I,!1),Jt=h(I,","),Kt=d(I,!1),Qt=h(I,","),Xt=d(I,!1),Yt=h(I,"."),I.forEach(e),qt.forEach(e),ta=i(t),at=p(t,"P",{"data-svelte-h":!0}),c(at)!=="svelte-10s9qu"&&(at.innerHTML=Ge),aa=i(t),x=p(t,"P",{});var b=S(x);qe=h(b,"The attention mechanism is defined by three projections: query"),ea=d(b,!1),sa=h(b,", key"),na=d(b,!1),ia=h(b,", value"),la=d(b,!1),pa=h(b,", that are generated by mutiliplying the input visual token by three matrices"),ma=d(b,!1),oa=h(b,","),ra=d(b,!1),ha=h(b," and"),ca=d(b,!1),ga=h(b," that are learned during training."),b.forEach(e),ua=i(t),et=p(t,"P",{"data-svelte-h":!0}),c(et)!=="svelte-upxya2"&&(et.innerHTML=Ve),fa=i(t),st=p(t,"P",{"data-svelte-h":!0}),c(st)!=="svelte-ip342p"&&(st.textContent=Ze),da=i(t),nt=p(t,"OL",{"data-svelte-h":!0}),c(nt)!=="svelte-x7l1j4"&&(nt.innerHTML=Je),va=i(t),O=p(t,"P",{});var Pt=S(O);Pe=h(Pt,"In the realm of computational efficiency, the Hyena operator attains an evaluation time complexity of"),ya=d(Pt,!1),wa=h(Pt,"), indicating a noteworthy enhancement in processing speed."),Pt.forEach(e),xa=i(t),k(it.$$.fragment,t),ba=i(t),lt=p(t,"P",{"data-svelte-h":!0}),c(lt)!=="svelte-o1kkd8"&&(lt.innerHTML=Ke),_a=i(t),z=p(t,"P",{});var B=S(z);Ie=h(B,"In this order, we compute 3 projections analogous to"),$a=d(B,!1),ka=h(B,","),Ma=d(B,!1),Ta=h(B," and"),Ha=d(B,!1),Ca=h(B," attention vectors from the Attention mechanism."),B.forEach(e),La=i(t),q=p(t,"P",{});var D=S(q);Se=h(D,"However, unlike the attention mechanism, which typically uses a single dense layer for projecting the input sequence into representations, Hyena incorporates both a dense layer and standard convolutions that are performed on each channels (refered as"),za=d(D,!1),qa=h(D,","),Pa=d(D,!1),Ia=h(D," and"),Sa=d(D,!1),ja=h(D," on the schema, but it is an explicit convolution in practice). The softmax function is also discared."),D.forEach(e),Fa=i(t),y=p(t,"P",{});var _=S(y);je=h(_,"The core idea is to repeatedly apply linear operators that are fast to evaluate to an input sequence"),Ea=d(_,!1),Aa=h(_,"  with"),Wa=d(_,!1),Oa=h(_,` the length of the sequence.
Because global convolutions have a large number of parameters, they are expensive to train. A notable design choice is the use of `),St=p(_,"STRONG",{"data-svelte-h":!0}),c(St)!=="svelte-9gehus"&&(St.textContent=Qe),Fe=h(_,`.
Unlike standard convolutional layers, the convolution filter`),Ra=d(_,!1),Ua=h(_," is learned implicitly with a small neural network"),Na=d(_,!1),Ba=h(_,` (also called the Hyena Filter).
This network takes the positional index and potentially positional encodings as inputs. From the outputs of`),Da=d(_,!1),Ga=h(_," one can construct a Toeplitz matrix"),Va=d(_,!1),Za=h(_,"."),_.forEach(e),Ja=i(t),pt=p(t,"P",{"data-svelte-h":!0}),c(pt)!=="svelte-1vjdzhl"&&(pt.textContent=Xe),Ka=i(t),k(R.$$.fragment,t),Qa=i(t),k(mt.$$.fragment,t),Xa=i(t),ot=p(t,"P",{"data-svelte-h":!0}),c(ot)!=="svelte-r38dwj"&&(ot.innerHTML=Ye),Ya=i(t),rt=p(t,"P",{"data-svelte-h":!0}),c(rt)!=="svelte-kz4ziy"&&(rt.textContent=ts),te=i(t),k(U.$$.fragment,t),ae=i(t),ht=p(t,"P",{"data-svelte-h":!0}),c(ht)!=="svelte-iqose5"&&(ht.textContent=as),ee=i(t),k(ct.$$.fragment,t),se=i(t),gt=p(t,"P",{"data-svelte-h":!0}),c(gt)!=="svelte-1tokxbd"&&(gt.innerHTML=es),ne=i(t),N=p(t,"OL",{});var Me=S(N);jt=p(Me,"LI",{"data-svelte-h":!0}),c(jt)!=="svelte-bncpwv"&&(jt.textContent=ss),Ee=i(Me),ut=p(Me,"LI",{});var Te=S(ut);Ae=h(Te,"Mixing up the projections: The matrix"),ie=d(Te,!1),le=h(Te," is defined by a combination of matrix multiplications."),Te.forEach(e),Me.forEach(e),pe=i(t),k(ft.$$.fragment,t),me=i(t),dt=p(t,"P",{"data-svelte-h":!0}),c(dt)!=="svelte-otyit9"&&(dt.textContent=ns),oe=i(t),vt=p(t,"P",{"data-svelte-h":!0}),c(vt)!=="svelte-si830m"&&(vt.textContent=is),re=i(t),yt=p(t,"P",{"data-svelte-h":!0}),c(yt)!=="svelte-17z8vyk"&&(yt.innerHTML=ls),he=i(t),wt=p(t,"P",{"data-svelte-h":!0}),c(wt)!=="svelte-as36q9"&&(wt.textContent=ps),ce=i(t),xt=p(t,"P",{"data-svelte-h":!0}),c(xt)!=="svelte-9amuno"&&(xt.innerHTML=ms),ge=i(t),bt=p(t,"P",{"data-svelte-h":!0}),c(bt)!=="svelte-9eeax6"&&(bt.innerHTML=os),ue=i(t),_t=p(t,"P",{"data-svelte-h":!0}),c(_t)!=="svelte-qoance"&&(_t.textContent=rs),fe=i(t),$t=p(t,"UL",{"data-svelte-h":!0}),c($t)!=="svelte-1x78zkx"&&($t.innerHTML=hs),de=i(t),kt=p(t,"P",{"data-svelte-h":!0}),c(kt)!=="svelte-1kv3mdj"&&(kt.textContent=cs),ve=i(t),k(Mt.$$.fragment,t),ye=i(t),Tt=p(t,"P",{"data-svelte-h":!0}),c(Tt)!=="svelte-10si01d"&&(Tt.textContent=gs),we=i(t),Ht=p(t,"P",{"data-svelte-h":!0}),c(Ht)!=="svelte-10mzyb8"&&(Ht.textContent=us),xe=i(t),k(Ct.$$.fragment,t),be=i(t),Lt=p(t,"UL",{"data-svelte-h":!0}),c(Lt)!=="svelte-bxo5fl"&&(Lt.innerHTML=fs),_e=i(t),k(zt.$$.fragment,t),$e=i(t),Ft=p(t,"P",{}),S(Ft).forEach(e),this.h()},h(){We(o,"name","hf:doc:metadata"),We(o,"content",Qs),Zt.a=Jt,Kt.a=Qt,Xt.a=Yt,ea.a=sa,na.a=ia,la.a=pa,ma.a=oa,ra.a=ha,ca.a=ga,ya.a=wa,$a.a=ka,Ma.a=Ta,Ha.a=Ca,za.a=qa,Pa.a=Ia,Sa.a=ja,Ea.a=Aa,Wa.a=Oa,Ra.a=Ua,Na.a=Ba,Da.a=Ga,Va.a=Za,ie.a=le},m(t,a){m(document.head,o),s(t,u,a),s(t,v,a),s(t,w,a),M(g,t,a),s(t,L,a),M(G,t,a),s(t,Et,a),M(V,t,a),s(t,At,a),M(Z,t,a),s(t,Wt,a),s(t,J,a),s(t,Ot,a),s(t,K,a),s(t,Rt,a),M(E,t,a),s(t,Ut,a),M(A,t,a),s(t,Nt,a),s(t,Q,a),s(t,Bt,a),s(t,X,a),s(t,Dt,a),M(Y,t,a),s(t,Gt,a),s(t,tt,a),s(t,Vt,a),s(t,W,a),m(W,It),m(W,Le),m(W,P),m(P,ze),Zt.m(ds,P),m(P,Jt),Kt.m(vs,P),m(P,Qt),Xt.m(ys,P),m(P,Yt),s(t,ta,a),s(t,at,a),s(t,aa,a),s(t,x,a),m(x,qe),ea.m(ws,x),m(x,sa),na.m(xs,x),m(x,ia),la.m(bs,x),m(x,pa),ma.m(_s,x),m(x,oa),ra.m($s,x),m(x,ha),ca.m(ks,x),m(x,ga),s(t,ua,a),s(t,et,a),s(t,fa,a),s(t,st,a),s(t,da,a),s(t,nt,a),s(t,va,a),s(t,O,a),m(O,Pe),ya.m(Ms,O),m(O,wa),s(t,xa,a),M(it,t,a),s(t,ba,a),s(t,lt,a),s(t,_a,a),s(t,z,a),m(z,Ie),$a.m(Ts,z),m(z,ka),Ma.m(Hs,z),m(z,Ta),Ha.m(Cs,z),m(z,Ca),s(t,La,a),s(t,q,a),m(q,Se),za.m(Ls,q),m(q,qa),Pa.m(zs,q),m(q,Ia),Sa.m(qs,q),m(q,ja),s(t,Fa,a),s(t,y,a),m(y,je),Ea.m(Ps,y),m(y,Aa),Wa.m(Is,y),m(y,Oa),m(y,St),m(y,Fe),Ra.m(Ss,y),m(y,Ua),Na.m(js,y),m(y,Ba),Da.m(Fs,y),m(y,Ga),Va.m(Es,y),m(y,Za),s(t,Ja,a),s(t,pt,a),s(t,Ka,a),M(R,t,a),s(t,Qa,a),M(mt,t,a),s(t,Xa,a),s(t,ot,a),s(t,Ya,a),s(t,rt,a),s(t,te,a),M(U,t,a),s(t,ae,a),s(t,ht,a),s(t,ee,a),M(ct,t,a),s(t,se,a),s(t,gt,a),s(t,ne,a),s(t,N,a),m(N,jt),m(N,Ee),m(N,ut),m(ut,Ae),ie.m(As,ut),m(ut,le),s(t,pe,a),M(ft,t,a),s(t,me,a),s(t,dt,a),s(t,oe,a),s(t,vt,a),s(t,re,a),s(t,yt,a),s(t,he,a),s(t,wt,a),s(t,ce,a),s(t,xt,a),s(t,ge,a),s(t,bt,a),s(t,ue,a),s(t,_t,a),s(t,fe,a),s(t,$t,a),s(t,de,a),s(t,kt,a),s(t,ve,a),M(Mt,t,a),s(t,ye,a),s(t,Tt,a),s(t,we,a),s(t,Ht,a),s(t,xe,a),M(Ct,t,a),s(t,be,a),s(t,Lt,a),s(t,_e,a),M(zt,t,a),s(t,$e,a),s(t,Ft,a),ke=!0},p(t,[a]){const qt={};a&2&&(qt.$$scope={dirty:a,ctx:t}),E.$set(qt);const I={};a&2&&(I.$$scope={dirty:a,ctx:t}),A.$set(I);const b={};a&2&&(b.$$scope={dirty:a,ctx:t}),R.$set(b);const Pt={};a&2&&(Pt.$$scope={dirty:a,ctx:t}),U.$set(Pt)},i(t){ke||(T(g.$$.fragment,t),T(G.$$.fragment,t),T(V.$$.fragment,t),T(Z.$$.fragment,t),T(E.$$.fragment,t),T(A.$$.fragment,t),T(Y.$$.fragment,t),T(it.$$.fragment,t),T(R.$$.fragment,t),T(mt.$$.fragment,t),T(U.$$.fragment,t),T(ct.$$.fragment,t),T(ft.$$.fragment,t),T(Mt.$$.fragment,t),T(Ct.$$.fragment,t),T(zt.$$.fragment,t),ke=!0)},o(t){H(g.$$.fragment,t),H(G.$$.fragment,t),H(V.$$.fragment,t),H(Z.$$.fragment,t),H(E.$$.fragment,t),H(A.$$.fragment,t),H(Y.$$.fragment,t),H(it.$$.fragment,t),H(R.$$.fragment,t),H(mt.$$.fragment,t),H(U.$$.fragment,t),H(ct.$$.fragment,t),H(ft.$$.fragment,t),H(Mt.$$.fragment,t),H(Ct.$$.fragment,t),H(zt.$$.fragment,t),ke=!1},d(t){t&&(e(u),e(v),e(w),e(L),e(Et),e(At),e(Wt),e(J),e(Ot),e(K),e(Rt),e(Ut),e(Nt),e(Q),e(Bt),e(X),e(Dt),e(Gt),e(tt),e(Vt),e(W),e(ta),e(at),e(aa),e(x),e(ua),e(et),e(fa),e(st),e(da),e(nt),e(va),e(O),e(xa),e(ba),e(lt),e(_a),e(z),e(La),e(q),e(Fa),e(y),e(Ja),e(pt),e(Ka),e(Qa),e(Xa),e(ot),e(Ya),e(rt),e(te),e(ae),e(ht),e(ee),e(se),e(gt),e(ne),e(N),e(pe),e(me),e(dt),e(oe),e(vt),e(re),e(yt),e(he),e(wt),e(ce),e(xt),e(ge),e(bt),e(ue),e(_t),e(fe),e($t),e(de),e(kt),e(ve),e(ye),e(Tt),e(we),e(Ht),e(xe),e(be),e(Lt),e(_e),e($e),e(Ft)),e(o),C(g,t),C(G,t),C(V,t),C(Z,t),C(E,t),C(A,t),C(Y,t),C(it,t),C(R,t),C(mt,t),C(U,t),C(ct,t),C(ft,t),C(Mt,t),C(Ct,t),C(zt,t)}}}const Qs='{"title":"Hyena","local":"hyena","sections":[{"title":"Overview","local":"overview","sections":[{"title":"What is Hyena","local":"what-is-hyena","sections":[],"depth":3},{"title":"From Attention to Hyena operator","local":"from-attention-to-hyena-operator","sections":[],"depth":3},{"title":"Hyena operator","local":"hyena-operator","sections":[],"depth":3},{"title":"Implicit convolutions","local":"implicit-convolutions","sections":[],"depth":3},{"title":"Wrapping Up Everything","local":"wrapping-up-everything","sections":[],"depth":3}],"depth":2},{"title":"Why Hyena Matters","local":"why-hyena-matters","sections":[],"depth":2},{"title":"Towards Transformers Alternatives","local":"towards-transformers-alternatives","sections":[],"depth":2},{"title":"Further Reading","local":"further-reading","sections":[],"depth":2}],"depth":1}';function Xs(j){return Os(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class nn extends Rs{constructor(o){super(),Us(this,o,Xs,Ks,Ws,{})}}export{nn as component};