import{s as ht,o as ft,n as pt}from"../chunks/scheduler.9bc65507.js";import{S as ct,i as dt,g as o,s as h,u as G,m as qe,A as mt,h as r,f as t,c as f,j as l,v as O,y as d,n as ze,k as a,o as s,a as i,w as R,d as W,t as D,x as K}from"../chunks/index.78c82d43.js";import{T as ut}from"../chunks/Tip.87d55b76.js";import{I as ne}from"../chunks/IconCopyLink.bedaa44d.js";function vt(he){let c,I="for PyTorch >= 1.14.0. JIT-mode could benefit any models for prediction and evaluaion since dict input is supported in jit.trace",v,m,u=`for PyTorch < 1.14.0. JIT-mode could benefit models whose forward parameter order matches the tuple input order in jit.trace, like question-answering model
In the case where the forward parameter order does not match the tuple input order in jit.trace, like text-classification models, jit.trace will fail and we are capturing this with the exception here to make it fallback. Logging is used to notify users.`;return{c(){c=o("p"),c.textContent=I,v=h(),m=o("p"),m.textContent=u},l(p){c=r(p,"P",{"data-svelte-h":!0}),d(c)!=="svelte-1eeqt0g"&&(c.textContent=I),v=f(p),m=r(p,"P",{"data-svelte-h":!0}),d(m)!=="svelte-122cw99"&&(m.textContent=u)},m(p,g){i(p,c,g),i(p,v,g),i(p,m,g)},p:pt,d(p){p&&(t(c),t(v),t(m))}}}function gt(he){let c,I,v,m,u,p,g,j,je,Q,Ne="Efficient Inference on CPU",fe,A,Je="This guide focuses on inferencing large models efficiently on CPU.",pe,_,$,oe,L,Ae,V,Ue="<code>BetterTransformer</code> for faster inference",ce,S,Xe='We have recently integrated <code>BetterTransformer</code> for faster inference on CPU for text, image and audio models. Check the documentation about this integration <a href="https://huggingface.co/docs/optimum/bettertransformer/overview" rel="nofollow">here</a> for more details.',de,w,P,re,H,Le,Y,Be="PyTorch JIT-mode (TorchScript)",me,M,Fe='For a gentle introduction to TorchScript, see the Introduction to <a href="https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html#tracing-modules" rel="nofollow">PyTorch TorchScript tutorial</a>.',ue,y,b,ie,q,Se,Z,Ge="IPEX Graph Optimization with JIT-mode",ve,z,Oe='Check more detailed information for <a href="https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/features/graph_optimization.html" rel="nofollow">IPEX Graph Optimization</a>.',ge,T,C,ae,N,He,ee,Re="IPEX installation:",_e,J,We='IPEX release is following PyTorch, check the approaches for <a href="https://intel.github.io/intel-extension-for-pytorch/" rel="nofollow">IPEX installation</a>.',we,x,k,se,U,Me,te,De="Usage of JIT-mode",ye,E,Te,X,Ke='Take an example of the use cases on <a href="https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering" rel="nofollow">Transformers question-answering</a>',xe,B,Qe=`<li><p>Inference using jit mode on CPU:</p> <pre>python run_qa.py \\
--model_name_or_path csarron/bert-base-uncased-squad-v1 \\
--dataset_name squad \\
--do_eval \\
--max_seq_length 384 \\
--doc_stride 128 \\
--output_dir /tmp/ \\
--no_cuda \\
<b>--jit_mode_eval </b></pre></li> <li><p>Inference with IPEX using jit mode on CPU:</p> <pre>python run_qa.py \\
--model_name_or_path csarron/bert-base-uncased-squad-v1 \\
--dataset_name squad \\
--do_eval \\
--max_seq_length 384 \\
--doc_stride 128 \\
--output_dir /tmp/ \\
--no_cuda \\
<b>--use_ipex \\</b>
<b>--jit_mode_eval</b></pre></li>`,$e,le,Pe;return j=new ne({}),L=new ne({}),H=new ne({}),q=new ne({}),N=new ne({}),U=new ne({}),E=new ut({props:{warning:!0,$$slots:{default:[vt]},$$scope:{ctx:he}}}),{c(){c=o("meta"),I=h(),v=o("p"),m=h(),u=o("h1"),p=o("a"),g=o("span"),G(j.$$.fragment),je=h(),Q=o("span"),Q.textContent=Ne,fe=h(),A=o("p"),A.textContent=Je,pe=h(),_=o("h2"),$=o("a"),oe=o("span"),G(L.$$.fragment),Ae=h(),V=o("span"),V.innerHTML=Ue,ce=h(),S=o("p"),S.innerHTML=Xe,de=h(),w=o("h2"),P=o("a"),re=o("span"),G(H.$$.fragment),Le=h(),Y=o("span"),Y.textContent=Be,me=qe(`


TorchScript is a way to create serializable and optimizable models from PyTorch code. Any TorchScript program can be saved from a Python process and loaded in a process where there is no Python dependency.
Comparing to default eager mode, jit mode in PyTorch normally yields better performance for model inference from optimization methodologies like operator fusion.
`),M=o("p"),M.innerHTML=Fe,ue=h(),y=o("h3"),b=o("a"),ie=o("span"),G(q.$$.fragment),Se=h(),Z=o("span"),Z.textContent=Ge,ve=qe(`


Intel® Extension for PyTorch provides further optimizations in jit mode for Transformers series models. It is highly recommended for users to take advantage of Intel® Extension for PyTorch with jit mode. Some frequently used operator patterns from Transformers models are already supported in Intel® Extension for PyTorch with jit mode fusions. Those fusion patterns like Multi-head-attention fusion, Concat Linear, Linear+Add, Linear+Gelu, Add+LayerNorm fusion and etc. are enabled and perform well. The benefit of the fusion is delivered to users in a transparent fashion. According to the analysis, ~70% of most popular NLP tasks in question-answering, text-classification, and token-classification can get performance benefits with these fusion patterns for both Float32 precision and BFloat16 Mixed precision.
`),z=o("p"),z.innerHTML=Oe,ge=h(),T=o("h4"),C=o("a"),ae=o("span"),G(N.$$.fragment),He=h(),ee=o("span"),ee.textContent=Re,_e=h(),J=o("p"),J.innerHTML=We,we=h(),x=o("h3"),k=o("a"),se=o("span"),G(U.$$.fragment),Me=h(),te=o("span"),te.textContent=De,ye=qe(`


To enable JIT-mode in Trainer for evaluaion or prediction, users should add \`jit_mode_eval\` in Trainer command arguments.
`),G(E.$$.fragment),Te=h(),X=o("p"),X.innerHTML=Ke,xe=h(),B=o("ul"),B.innerHTML=Qe,$e=h(),le=o("p"),this.h()},l(e){const n=mt("svelte-1phssyn",document.head);c=r(n,"META",{name:!0,content:!0}),n.forEach(t),I=f(e),v=r(e,"P",{}),l(v).forEach(t),m=f(e),u=r(e,"H1",{class:!0});var F=l(u);p=r(F,"A",{id:!0,class:!0,href:!0});var Ve=l(p);g=r(Ve,"SPAN",{});var Ye=l(g);O(j.$$.fragment,Ye),Ye.forEach(t),Ve.forEach(t),je=f(F),Q=r(F,"SPAN",{"data-svelte-h":!0}),d(Q)!=="svelte-i16ncg"&&(Q.textContent=Ne),F.forEach(t),fe=f(e),A=r(e,"P",{"data-svelte-h":!0}),d(A)!=="svelte-remud"&&(A.textContent=Je),pe=f(e),_=r(e,"H2",{class:!0});var be=l(_);$=r(be,"A",{id:!0,class:!0,href:!0});var Ze=l($);oe=r(Ze,"SPAN",{});var et=l(oe);O(L.$$.fragment,et),et.forEach(t),Ze.forEach(t),Ae=f(be),V=r(be,"SPAN",{"data-svelte-h":!0}),d(V)!=="svelte-1e0dhew"&&(V.innerHTML=Ue),be.forEach(t),ce=f(e),S=r(e,"P",{"data-svelte-h":!0}),d(S)!=="svelte-1tf28r"&&(S.innerHTML=Xe),de=f(e),w=r(e,"H2",{class:!0});var Ce=l(w);P=r(Ce,"A",{id:!0,class:!0,href:!0});var tt=l(P);re=r(tt,"SPAN",{});var nt=l(re);O(H.$$.fragment,nt),nt.forEach(t),tt.forEach(t),Le=f(Ce),Y=r(Ce,"SPAN",{"data-svelte-h":!0}),d(Y)!=="svelte-ps12u5"&&(Y.textContent=Be),Ce.forEach(t),me=ze(e,`


TorchScript is a way to create serializable and optimizable models from PyTorch code. Any TorchScript program can be saved from a Python process and loaded in a process where there is no Python dependency.
Comparing to default eager mode, jit mode in PyTorch normally yields better performance for model inference from optimization methodologies like operator fusion.
`),M=r(e,"P",{"data-svelte-h":!0}),d(M)!=="svelte-p4cknk"&&(M.innerHTML=Fe),ue=f(e),y=r(e,"H3",{class:!0});var ke=l(y);b=r(ke,"A",{id:!0,class:!0,href:!0});var ot=l(b);ie=r(ot,"SPAN",{});var rt=l(ie);O(q.$$.fragment,rt),rt.forEach(t),ot.forEach(t),Se=f(ke),Z=r(ke,"SPAN",{"data-svelte-h":!0}),d(Z)!=="svelte-14dibu3"&&(Z.textContent=Ge),ke.forEach(t),ve=ze(e,`


Intel® Extension for PyTorch provides further optimizations in jit mode for Transformers series models. It is highly recommended for users to take advantage of Intel® Extension for PyTorch with jit mode. Some frequently used operator patterns from Transformers models are already supported in Intel® Extension for PyTorch with jit mode fusions. Those fusion patterns like Multi-head-attention fusion, Concat Linear, Linear+Add, Linear+Gelu, Add+LayerNorm fusion and etc. are enabled and perform well. The benefit of the fusion is delivered to users in a transparent fashion. According to the analysis, ~70% of most popular NLP tasks in question-answering, text-classification, and token-classification can get performance benefits with these fusion patterns for both Float32 precision and BFloat16 Mixed precision.
`),z=r(e,"P",{"data-svelte-h":!0}),d(z)!=="svelte-1mws6j8"&&(z.innerHTML=Oe),ge=f(e),T=r(e,"H4",{class:!0});var Ee=l(T);C=r(Ee,"A",{id:!0,class:!0,href:!0});var it=l(C);ae=r(it,"SPAN",{});var at=l(ae);O(N.$$.fragment,at),at.forEach(t),it.forEach(t),He=f(Ee),ee=r(Ee,"SPAN",{"data-svelte-h":!0}),d(ee)!=="svelte-utfx4z"&&(ee.textContent=Re),Ee.forEach(t),_e=f(e),J=r(e,"P",{"data-svelte-h":!0}),d(J)!=="svelte-hnoa5g"&&(J.innerHTML=We),we=f(e),x=r(e,"H3",{class:!0});var Ie=l(x);k=r(Ie,"A",{id:!0,class:!0,href:!0});var st=l(k);se=r(st,"SPAN",{});var lt=l(se);O(U.$$.fragment,lt),lt.forEach(t),st.forEach(t),Me=f(Ie),te=r(Ie,"SPAN",{"data-svelte-h":!0}),d(te)!=="svelte-2gz65i"&&(te.textContent=De),Ie.forEach(t),ye=ze(e,`


To enable JIT-mode in Trainer for evaluaion or prediction, users should add \`jit_mode_eval\` in Trainer command arguments.
`),O(E.$$.fragment,e),Te=f(e),X=r(e,"P",{"data-svelte-h":!0}),d(X)!=="svelte-1t7mun5"&&(X.innerHTML=Ke),xe=f(e),B=r(e,"UL",{"data-svelte-h":!0}),d(B)!=="svelte-143w2h7"&&(B.innerHTML=Qe),$e=f(e),le=r(e,"P",{}),l(le).forEach(t),this.h()},h(){a(c,"name","hf:doc:metadata"),a(c,"content",JSON.stringify(_t)),a(p,"id","efficient-inference-on-cpu"),a(p,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),a(p,"href","#efficient-inference-on-cpu"),a(u,"class","relative group"),a($,"id","bettertransformer-for-faster-inference"),a($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),a($,"href","#bettertransformer-for-faster-inference"),a(_,"class","relative group"),a(P,"id","pytorch-jitmode-torchscript"),a(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),a(P,"href","#pytorch-jitmode-torchscript"),a(w,"class","relative group"),a(b,"id","ipex-graph-optimization-with-jitmode"),a(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),a(b,"href","#ipex-graph-optimization-with-jitmode"),a(y,"class","relative group"),a(C,"id","ipex-installation"),a(C,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),a(C,"href","#ipex-installation"),a(T,"class","relative group"),a(k,"id","usage-of-jitmode"),a(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),a(k,"href","#usage-of-jitmode"),a(x,"class","relative group")},m(e,n){s(document.head,c),i(e,I,n),i(e,v,n),i(e,m,n),i(e,u,n),s(u,p),s(p,g),R(j,g,null),s(u,je),s(u,Q),i(e,fe,n),i(e,A,n),i(e,pe,n),i(e,_,n),s(_,$),s($,oe),R(L,oe,null),s(_,Ae),s(_,V),i(e,ce,n),i(e,S,n),i(e,de,n),i(e,w,n),s(w,P),s(P,re),R(H,re,null),s(w,Le),s(w,Y),i(e,me,n),i(e,M,n),i(e,ue,n),i(e,y,n),s(y,b),s(b,ie),R(q,ie,null),s(y,Se),s(y,Z),i(e,ve,n),i(e,z,n),i(e,ge,n),i(e,T,n),s(T,C),s(C,ae),R(N,ae,null),s(T,He),s(T,ee),i(e,_e,n),i(e,J,n),i(e,we,n),i(e,x,n),s(x,k),s(k,se),R(U,se,null),s(x,Me),s(x,te),i(e,ye,n),R(E,e,n),i(e,Te,n),i(e,X,n),i(e,xe,n),i(e,B,n),i(e,$e,n),i(e,le,n),Pe=!0},p(e,[n]){const F={};n&2&&(F.$$scope={dirty:n,ctx:e}),E.$set(F)},i(e){Pe||(W(j.$$.fragment,e),W(L.$$.fragment,e),W(H.$$.fragment,e),W(q.$$.fragment,e),W(N.$$.fragment,e),W(U.$$.fragment,e),W(E.$$.fragment,e),Pe=!0)},o(e){D(j.$$.fragment,e),D(L.$$.fragment,e),D(H.$$.fragment,e),D(q.$$.fragment,e),D(N.$$.fragment,e),D(U.$$.fragment,e),D(E.$$.fragment,e),Pe=!1},d(e){e&&(t(I),t(v),t(m),t(u),t(fe),t(A),t(pe),t(_),t(ce),t(S),t(de),t(w),t(me),t(M),t(ue),t(y),t(ve),t(z),t(ge),t(T),t(_e),t(J),t(we),t(x),t(ye),t(Te),t(X),t(xe),t(B),t($e),t(le)),t(c),K(j),K(L),K(H),K(q),K(N),K(U),K(E,e)}}}const _t={local:"efficient-inference-on-cpu",sections:[{local:"bettertransformer-for-faster-inference",title:"`BetterTransformer` for faster inference"},{local:"pytorch-jitmode-torchscript",sections:[{local:"ipex-graph-optimization-with-jitmode",sections:[{local:"ipex-installation",title:"IPEX installation:"}],title:"IPEX Graph Optimization with JIT-mode"},{local:"usage-of-jitmode",title:"Usage of JIT-mode"}],title:"PyTorch JIT-mode (TorchScript)"}],title:"Efficient Inference on CPU"};function wt(he){return ft(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pt extends ct{constructor(c){super(),dt(this,c,wt,gt,ht,{})}}export{Pt as component};
