import{s as ja,n as Ta,o as qa}from"../chunks/scheduler.9bc65507.js";import{S as Na,i as Ia,g as n,s as o,u as Z,m as g,H as v,A as Ua,h as l,f as a,c as h,j as c,v as ss,y as d,n as u,B as f,k as r,o as t,a as m,w as as,d as ts,t as es,x as ns}from"../chunks/index.78c82d43.js";import{I as ls}from"../chunks/IconCopyLink.bedaa44d.js";function Ka(sa){let w,is,D,ms,x,M,F,A,Js,O,aa="Attention mechanisms",ps,P,ta=`Most transformer models use full attention in the sense that the attention matrix is square. It can be a big
computational bottleneck when you have long texts. Longformer and reformer are models that try to be more efficient and
use a sparse version of the attention matrix to speed up training.`,rs,y,$,J,S,Vs,Q,ea="LSH attention",os,H,na=`<a href="#reformer">Reformer</a> uses LSH attention. In the softmax(QK^t), only the biggest elements (in the softmax
dimension) of the matrix QK^t are going to give useful contributions. So for each query q in Q, we can consider only
the keys k in K that are close to q. A hash function is used to determine if q and k are close. The attention mask is
modified to mask the current token (except at the first position), because it will give a query and a key equal (so
very similar to each other). Since the hash can be a bit random, several hash functions are used in practice
(determined by a n_rounds parameter) and then are averaged together.`,hs,_,E,V,j,Ws,R,la="Local attention",cs,T,ia=`<a href="#longformer">Longformer</a> uses local attention: often, the local context (e.g., what are the two tokens to the
left and right?) is enough to take action for a given token. Also, by stacking attention layers that have a small
window, the last layer will have a receptive field of more than just the tokens in the window, allowing them to build a
representation of the whole sentence.`,gs,q,ma=`Some preselected input tokens are also given global attention: for those few tokens, the attention matrix can access
all tokens and this process is symmetric: all other tokens have access to those specific tokens (on top of the ones in
their local window). This is shown in Figure 2d of the paper, see below for a sample attention mask:`,ds,z,pa='<img scale="50 %" align="center" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/local_attention_mask.png"/>',us,N,ra=`Using those attention matrices with less parameters then allows the model to have inputs having a bigger sequence
length.`,vs,b,C,W,I,Xs,G,oa="Other tricks",fs,k,L,X,U,Ys,B,ha="Axial positional encodings",ws,i,K,ca="Reformer",Zs,xs,ka='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi></mrow><annotation encoding="application/x-tex">l</annotation></semantics></math></span><span class="katex-html hidden" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span></span></span></span>',ys,_s,Ma='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>d</mi></mrow><annotation encoding="application/x-tex">d</annotation></semantics></math></span><span class="katex-html hidden" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">d</span></span></span></span>',bs,ks,$a='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi></mrow><annotation encoding="application/x-tex">l</annotation></semantics></math></span><span class="katex-html hidden" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span></span></span></span>',Ms,$s,Ea='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>d</mi></mrow><annotation encoding="application/x-tex">d</annotation></semantics></math></span><span class="katex-html hidden" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">d</span></span></span></span>',Es,zs,za='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>l</mi><mn>1</mn></msub><mo>×</mo><msub><mi>d</mi><mn>1</mn></msub></mrow><annotation encoding="application/x-tex">l_{1} \\times d_{1}</annotation></semantics></math></span><span class="katex-html hidden" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Cs,Ls,Ca='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>l</mi><mn>2</mn></msub><mo>×</mo><msub><mi>d</mi><mn>2</mn></msub></mrow><annotation encoding="application/x-tex">l_{2} \\times d_{2}</annotation></semantics></math></span><span class="katex-html hidden" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',As,Ps,La='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>l</mi><mn>1</mn></msub><mo>×</mo><msub><mi>l</mi><mn>2</mn></msub><mo>=</mo><mi>l</mi></mrow><annotation encoding="application/x-tex">l_{1} \\times l_{2} = l</annotation></semantics></math></span><span class="katex-html hidden" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">×</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span></span></span></span>',Ss,Hs,Aa='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>d</mi><mn>1</mn></msub><mo>+</mo><msub><mi>d</mi><mn>2</mn></msub><mo>=</mo><mi>d</mi></mrow><annotation encoding="application/x-tex">d_{1} + d_{2} = d</annotation></semantics></math></span><span class="katex-html hidden" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">d</span></span></span></span>',js,Ts,Pa='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>j</mi></mrow><annotation encoding="application/x-tex">j</annotation></semantics></math></span><span class="katex-html hidden" aria-hidden="true"><span class="base"><span class="strut" style="height:0.854em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.05724em;">j</span></span></span></span>',qs,Ns,Sa='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>j</mi><mi mathvariant="normal">%</mi><mi>l</mi><mn>1</mn></mrow><annotation encoding="application/x-tex">j \\% l1</annotation></semantics></math></span><span class="katex-html hidden" aria-hidden="true"><span class="base"><span class="strut" style="height:0.9444em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.05724em;">j</span><span class="mord">%</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord">1</span></span></span></span>',Is,Us,Ha='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>j</mi><mi mathvariant="normal">/</mi><mi mathvariant="normal">/</mi><mi>l</mi><mn>1</mn></mrow><annotation encoding="application/x-tex">j // l1</annotation></semantics></math></span><span class="katex-html hidden" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.05724em;">j</span><span class="mord">//</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord">1</span></span></span></span>',Ks,Os,Y,Qs;return A=new ls({}),S=new ls({}),j=new ls({}),I=new ls({}),U=new ls({}),{c(){w=n("meta"),is=o(),D=n("p"),ms=o(),x=n("h1"),M=n("a"),F=n("span"),Z(A.$$.fragment),Js=o(),O=n("span"),O.textContent=aa,ps=o(),P=n("p"),P.textContent=ta,rs=o(),y=n("h2"),$=n("a"),J=n("span"),Z(S.$$.fragment),Vs=o(),Q=n("span"),Q.textContent=ea,os=o(),H=n("p"),H.innerHTML=na,hs=o(),_=n("h2"),E=n("a"),V=n("span"),Z(j.$$.fragment),Ws=o(),R=n("span"),R.textContent=la,cs=o(),T=n("p"),T.innerHTML=ia,gs=o(),q=n("p"),q.textContent=ma,ds=o(),z=n("div"),z.innerHTML=pa,us=o(),N=n("p"),N.textContent=ra,vs=o(),b=n("h2"),C=n("a"),W=n("span"),Z(I.$$.fragment),Xs=o(),G=n("span"),G.textContent=oa,fs=o(),k=n("h3"),L=n("a"),X=n("span"),Z(U.$$.fragment),Ys=o(),B=n("span"),B.textContent=ha,ws=o(),i=n("p"),K=n("a"),K.textContent=ca,Zs=g(` uses axial positional encodings: in traditional transformer models, the positional encoding
E is a matrix of size `),xs=new v(!1),ys=g(" by "),_s=new v(!1),bs=g(", "),ks=new v(!1),Ms=g(" being the sequence length and "),$s=new v(!1),Es=g(` the dimension of the
hidden state. If you have very long texts, this matrix can be huge and take way too much space on the GPU. To alleviate
that, axial positional encodings consist of factorizing that big matrix E in two smaller matrices E1 and E2, with
dimensions `),zs=new v(!1),Cs=g(" and "),Ls=new v(!1),As=g(", such that "),Ps=new v(!1),Ss=g(` and
`),Hs=new v(!1),js=g(` (with the product for the lengths, this ends up being way smaller). The embedding for time
step `),Ts=new v(!1),qs=g(" in E is obtained by concatenating the embeddings for timestep "),Ns=new v(!1),Is=g(" in E1 and "),Us=new v(!1),Ks=g(`
in E2.`),Os=o(),Y=n("p"),this.h()},l(s){const e=Ua("svelte-1phssyn",document.head);w=l(e,"META",{name:!0,content:!0}),e.forEach(a),is=h(s),D=l(s,"P",{}),c(D).forEach(a),ms=h(s),x=l(s,"H1",{class:!0});var Rs=c(x);M=l(Rs,"A",{id:!0,class:!0,href:!0});var ga=c(M);F=l(ga,"SPAN",{});var da=c(F);ss(A.$$.fragment,da),da.forEach(a),ga.forEach(a),Js=h(Rs),O=l(Rs,"SPAN",{"data-svelte-h":!0}),d(O)!=="svelte-16ux99"&&(O.textContent=aa),Rs.forEach(a),ps=h(s),P=l(s,"P",{"data-svelte-h":!0}),d(P)!=="svelte-1badtw"&&(P.textContent=ta),rs=h(s),y=l(s,"H2",{class:!0});var Gs=c(y);$=l(Gs,"A",{id:!0,class:!0,href:!0});var ua=c($);J=l(ua,"SPAN",{});var va=c(J);ss(S.$$.fragment,va),va.forEach(a),ua.forEach(a),Vs=h(Gs),Q=l(Gs,"SPAN",{"data-svelte-h":!0}),d(Q)!=="svelte-znoqzy"&&(Q.textContent=ea),Gs.forEach(a),os=h(s),H=l(s,"P",{"data-svelte-h":!0}),d(H)!=="svelte-m13qzv"&&(H.innerHTML=na),hs=h(s),_=l(s,"H2",{class:!0});var Bs=c(_);E=l(Bs,"A",{id:!0,class:!0,href:!0});var fa=c(E);V=l(fa,"SPAN",{});var wa=c(V);ss(j.$$.fragment,wa),wa.forEach(a),fa.forEach(a),Ws=h(Bs),R=l(Bs,"SPAN",{"data-svelte-h":!0}),d(R)!=="svelte-u4flcm"&&(R.textContent=la),Bs.forEach(a),cs=h(s),T=l(s,"P",{"data-svelte-h":!0}),d(T)!=="svelte-60jd3u"&&(T.innerHTML=ia),gs=h(s),q=l(s,"P",{"data-svelte-h":!0}),d(q)!=="svelte-1x90fhu"&&(q.textContent=ma),ds=h(s),z=l(s,"DIV",{class:!0,"data-svelte-h":!0}),d(z)!=="svelte-d4kpls"&&(z.innerHTML=pa),us=h(s),N=l(s,"P",{"data-svelte-h":!0}),d(N)!=="svelte-11i8g0o"&&(N.textContent=ra),vs=h(s),b=l(s,"H2",{class:!0});var Ds=c(b);C=l(Ds,"A",{id:!0,class:!0,href:!0});var xa=c(C);W=l(xa,"SPAN",{});var ya=c(W);ss(I.$$.fragment,ya),ya.forEach(a),xa.forEach(a),Xs=h(Ds),G=l(Ds,"SPAN",{"data-svelte-h":!0}),d(G)!=="svelte-lco77z"&&(G.textContent=oa),Ds.forEach(a),fs=h(s),k=l(s,"H3",{class:!0});var Fs=c(k);L=l(Fs,"A",{id:!0,class:!0,href:!0});var _a=c(L);X=l(_a,"SPAN",{});var ba=c(X);ss(U.$$.fragment,ba),ba.forEach(a),_a.forEach(a),Ys=h(Fs),B=l(Fs,"SPAN",{"data-svelte-h":!0}),d(B)!=="svelte-xczdv8"&&(B.textContent=ha),Fs.forEach(a),ws=h(s),i=l(s,"P",{});var p=c(i);K=l(p,"A",{href:!0,"data-svelte-h":!0}),d(K)!=="svelte-s9wjf3"&&(K.textContent=ca),Zs=u(p,` uses axial positional encodings: in traditional transformer models, the positional encoding
E is a matrix of size `),xs=f(p,!1),ys=u(p," by "),_s=f(p,!1),bs=u(p,", "),ks=f(p,!1),Ms=u(p," being the sequence length and "),$s=f(p,!1),Es=u(p,` the dimension of the
hidden state. If you have very long texts, this matrix can be huge and take way too much space on the GPU. To alleviate
that, axial positional encodings consist of factorizing that big matrix E in two smaller matrices E1 and E2, with
dimensions `),zs=f(p,!1),Cs=u(p," and "),Ls=f(p,!1),As=u(p,", such that "),Ps=f(p,!1),Ss=u(p,` and
`),Hs=f(p,!1),js=u(p,` (with the product for the lengths, this ends up being way smaller). The embedding for time
step `),Ts=f(p,!1),qs=u(p," in E is obtained by concatenating the embeddings for timestep "),Ns=f(p,!1),Is=u(p," in E1 and "),Us=f(p,!1),Ks=u(p,`
in E2.`),p.forEach(a),Os=h(s),Y=l(s,"P",{}),c(Y).forEach(a),this.h()},h(){r(w,"name","hf:doc:metadata"),r(w,"content",JSON.stringify(Oa)),r(M,"id","attention-mechanisms"),r(M,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(M,"href","#attention-mechanisms"),r(x,"class","relative group"),r($,"id","lsh-attention"),r($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r($,"href","#lsh-attention"),r(y,"class","relative group"),r(E,"id","local-attention"),r(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(E,"href","#local-attention"),r(_,"class","relative group"),r(z,"class","flex justify-center"),r(C,"id","other-tricks"),r(C,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(C,"href","#other-tricks"),r(b,"class","relative group"),r(L,"id","axial-positional-encodings"),r(L,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(L,"href","#axial-positional-encodings"),r(k,"class","relative group"),r(K,"href","#reformer"),xs.a=ys,_s.a=bs,ks.a=Ms,$s.a=Es,zs.a=Cs,Ls.a=As,Ps.a=Ss,Hs.a=js,Ts.a=qs,Ns.a=Is,Us.a=Ks},m(s,e){t(document.head,w),m(s,is,e),m(s,D,e),m(s,ms,e),m(s,x,e),t(x,M),t(M,F),as(A,F,null),t(x,Js),t(x,O),m(s,ps,e),m(s,P,e),m(s,rs,e),m(s,y,e),t(y,$),t($,J),as(S,J,null),t(y,Vs),t(y,Q),m(s,os,e),m(s,H,e),m(s,hs,e),m(s,_,e),t(_,E),t(E,V),as(j,V,null),t(_,Ws),t(_,R),m(s,cs,e),m(s,T,e),m(s,gs,e),m(s,q,e),m(s,ds,e),m(s,z,e),m(s,us,e),m(s,N,e),m(s,vs,e),m(s,b,e),t(b,C),t(C,W),as(I,W,null),t(b,Xs),t(b,G),m(s,fs,e),m(s,k,e),t(k,L),t(L,X),as(U,X,null),t(k,Ys),t(k,B),m(s,ws,e),m(s,i,e),t(i,K),t(i,Zs),xs.m(ka,i),t(i,ys),_s.m(Ma,i),t(i,bs),ks.m($a,i),t(i,Ms),$s.m(Ea,i),t(i,Es),zs.m(za,i),t(i,Cs),Ls.m(Ca,i),t(i,As),Ps.m(La,i),t(i,Ss),Hs.m(Aa,i),t(i,js),Ts.m(Pa,i),t(i,qs),Ns.m(Sa,i),t(i,Is),Us.m(Ha,i),t(i,Ks),m(s,Os,e),m(s,Y,e),Qs=!0},p:Ta,i(s){Qs||(ts(A.$$.fragment,s),ts(S.$$.fragment,s),ts(j.$$.fragment,s),ts(I.$$.fragment,s),ts(U.$$.fragment,s),Qs=!0)},o(s){es(A.$$.fragment,s),es(S.$$.fragment,s),es(j.$$.fragment,s),es(I.$$.fragment,s),es(U.$$.fragment,s),Qs=!1},d(s){s&&(a(is),a(D),a(ms),a(x),a(ps),a(P),a(rs),a(y),a(os),a(H),a(hs),a(_),a(cs),a(T),a(gs),a(q),a(ds),a(z),a(us),a(N),a(vs),a(b),a(fs),a(k),a(ws),a(i),a(Os),a(Y)),a(w),ns(A),ns(S),ns(j),ns(I),ns(U)}}}const Oa={local:"attention-mechanisms",sections:[{local:"lsh-attention",title:"LSH attention"},{local:"local-attention",title:"Local attention"},{local:"other-tricks",sections:[{local:"axial-positional-encodings",title:"Axial positional encodings"}],title:"Other tricks"}],title:"Attention mechanisms"};function Qa(sa){return qa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Da extends Na{constructor(w){super(),Ia(this,w,Qa,Ka,ja,{})}}export{Da as component};
