import{s as is,o as os,n as rs}from"../chunks/scheduler.56730f09.js";import{S as ps,i as ms,g as s,s as r,u as c,A as fs,h as a,f as t,c as i,j as f,v as h,y as m,k as o,o as p,a as n,w as d,d as v,t as b,x as y}from"../chunks/index.ba464511.js";import{T as as}from"../chunks/Tip.70442610.js";import{I as g}from"../chunks/IconCopyLink.89bde88e.js";import{C as Z}from"../chunks/CodeBlock.2302fd0d.js";function us(Qe){let u,$="이 기능은 다중 GPU 설정에서도 사용할 수 있습니다.";return{c(){u=s("p"),u.textContent=$},l(M){u=a(M,"P",{"data-svelte-h":!0}),m(u)!=="svelte-1wakscf"&&(u.textContent=$)},m(M,E){n(M,u,E)},p:rs,d(M){M&&t(u)}}}function cs(Qe){let u,$="이 기능은 다중 GPU 설정에서도 사용할 수 있습니다.";return{c(){u=s("p"),u.textContent=$},l(M){u=a(M,"P",{"data-svelte-h":!0}),m(u)!=="svelte-1wakscf"&&(u.textContent=$)},m(M,E){n(M,u,E)},p:rs,d(M){M&&t(u)}}}function hs(Qe){let u,$,M,E,w,F,ot,z,Zl,ze,Bl="단일 GPU에서 효율적인 추론",wt,K,Yl='이 가이드 외에도, <a href="perf_train_gpu_one">단일 GPU에서의 훈련 가이드</a>와 <a href="perf_infer_cpu">CPU에서의 추론 가이드</a>에서도 관련 정보를 찾을 수 있습니다.',_t,_,N,pt,O,El,Ke,Sl="Better Transformer: PyTorch 네이티브 Transformer 패스트패스",Tt,D,Ql='PyTorch 네이티브 <a href="https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/" rel="nofollow"><code>nn.MultiHeadAttention</code></a> 어텐션 패스트패스인 BetterTransformer는 <a href="https://huggingface.co/docs/optimum/bettertransformer/overview" rel="nofollow">🤗 Optimum 라이브러리</a>의 통합을 통해 Transformers와 함께 사용할 수 있습니다.',xt,ee,zl='PyTorch의 어텐션 패스트패스는 커널 퓨전과 <a href="https://pytorch.org/docs/stable/nested.html" rel="nofollow">중첩된 텐서</a>의 사용을 통해 추론 속도를 높일 수 있습니다. 자세한 벤치마크는 <a href="https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2" rel="nofollow">이 블로그 글</a>에서 확인할 수 있습니다.',Ct,te,Kl='<a href="https://github.com/huggingface/optimum" rel="nofollow"><code>optimum</code></a> 패키지를 설치한 후에는 추론 중 Better Transformer를 사용할 수 있도록 <code>to_bettertransformer()</code>를 호출하여 관련 내부 모듈을 대체합니다:',Ut,le,Jt,ne,Ol="<code>reverse_bettertransformer()</code> 메소드는 정규화된 transformers 모델링을 사용하기 위해 모델을 저장하기 전 원래의 모델링으로 돌아갈 수 있도록 해줍니다:",kt,se,Gt,ae,Dl='PyTorch 2.0부터는 어텐션 패스트패스가 인코더와 디코더 모두에서 지원됩니다. 지원되는 아키텍처 목록은 <a href="https://huggingface.co/docs/optimum/bettertransformer/overview#supported-models" rel="nofollow">여기</a>에서 확인할 수 있습니다.',Pt,T,H,mt,re,Fl,Oe,en="FP4 혼합 정밀도 추론을 위한 <code>bitsandbytes</code> 통합",Wt,ie,tn="<code>bitsandbytes</code>를 설치하면 GPU에서 손쉽게 모델을 압축할 수 있습니다. FP4 양자화를 사용하면 원래의 전체 정밀도 버전과 비교하여 모델 크기를 최대 8배 줄일 수 있습니다. 아래에서 시작하는 방법을 확인하세요.",jt,V,Zt,x,R,ft,oe,Nl,De,ln="요구 사항",Et,pe,nn=`<li><p>최신 <code>bitsandbytes</code> 라이브러리
<code>pip install bitsandbytes&gt;=0.39.0</code></p></li> <li><p>최신 <code>accelerate</code>를 소스에서 설치
<code>pip install git+https://github.com/huggingface/accelerate.git</code></p></li> <li><p>최신 <code>transformers</code>를 소스에서 설치
<code>pip install git+https://github.com/huggingface/transformers.git</code></p></li>`,Ft,C,I,ut,me,Hl,et,sn="FP4 모델 실행 - 단일 GPU 설정 - 빠른 시작",Nt,fe,an="다음 코드를 실행하여 단일 GPU에서 빠르게 FP4 모델을 실행할 수 있습니다.",Ht,ue,Vt,ce,rn="<code>device_map</code>은 선택 사항입니다. 그러나 <code>device_map = &#39;auto&#39;</code>로 설정하는 것이 사용 가능한 리소스를 효율적으로 디스패치하기 때문에 추론에 있어 권장됩니다.",Rt,U,L,ct,he,Vl,tt,on="FP4 모델 실행 - 다중 GPU 설정",It,de,pn="다중 GPU에서 혼합 4비트 모델을 가져오는 방법은 단일 GPU 설정과 동일합니다(동일한 명령어 사용):",Lt,ve,Xt,be,mn="하지만 <code>accelerate</code>를 사용하여 각 GPU에 할당할 GPU RAM을 제어할 수 있습니다. 다음과 같이 <code>max_memory</code> 인수를 사용하세요:",At,ye,qt,Me,fn="이 예에서는 첫 번째 GPU가 600MB의 메모리를 사용하고 두 번째 GPU가 1GB를 사용합니다.",Bt,J,X,ht,ge,Rl,lt,un="고급 사용법",Yt,$e,cn='이 방법의 더 고급 사용법에 대해서는 <a href="main_classes/quantization">양자화</a> 문서 페이지를 참조하세요.',St,k,A,dt,we,Il,nt,hn="Int8 혼합 정밀도 행렬 분해를 위한 <code>bitsandbytes</code> 통합",Qt,q,zt,_e,dn=`<a href="https://arxiv.org/abs/2208.07339" rel="nofollow"><code>LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale</code></a> 논문에서 우리는 몇 줄의 코드로 Hub의 모든 모델에 대한 Hugging Face 통합을 지원합니다.
이 방법은 <code>float16</code> 및 <code>bfloat16</code> 가중치에 대해 <code>nn.Linear</code> 크기를 2배로 줄이고, <code>float32</code> 가중치에 대해 4배로 줄입니다. 이는 절반 정밀도에서 이상치를 처리함으로써 품질에 거의 영향을 미치지 않습니다.`,Kt,Te,vn='<img src="https://cdn-uploads.huggingface.co/production/uploads/1659861207959-62441d1d9fdefb55a0b7d12c.png" alt="HFxbitsandbytes.png"/>',Ot,xe,bn=`Int8 혼합 정밀도 행렬 분해는 행렬 곱셈을 두 개의 스트림으로 분리합니다: (1) fp16로 곱해지는 체계적인 특이값 이상치 스트림 행렬(0.01%) 및 (2) int8 행렬 곱셈의 일반적인 스트림(99.9%). 이 방법을 사용하면 매우 큰 모델에 대해 예측 저하 없이 int8 추론이 가능합니다.
이 방법에 대한 자세한 내용은 <a href="https://arxiv.org/abs/2208.07339" rel="nofollow">논문</a>이나 <a href="https://huggingface.co/blog/hf-bitsandbytes-integration" rel="nofollow">통합에 관한 블로그 글</a>에서 확인할 수 있습니다.`,Dt,Ce,yn='<img src="https://cdn-uploads.huggingface.co/production/uploads/1660567469965-62441d1d9fdefb55a0b7d12c.gif" alt="MixedInt8.gif"/>',el,Ue,Mn=`커널은 GPU 전용으로 컴파일되어 있기 때문에 혼합 8비트 모델을 실행하려면 GPU가 필요합니다. 이 기능을 사용하기 전에 모델의 1/4(또는 모델 가중치가 절반 정밀도인 경우 절반)을 저장할 충분한 GPU 메모리가 있는지 확인하세요.
이 모듈을 사용하는 데 도움이 되는 몇 가지 참고 사항이 아래에 나와 있습니다. 또는 <a href="#colab-demos">Google colab</a>에서 데모를 따라할 수도 있습니다.`,tl,G,B,vt,Je,Ll,st,gn="요구 사항",ll,ke,$n=`<li><code>bitsandbytes&lt;0.37.0</code>을 사용하는 경우, 8비트 텐서 코어(Turing, Ampere 또는 이후 아키텍처 - 예: T4, RTX20s RTX30s, A40-A100)를 지원하는 NVIDIA GPU에서 실행하는지 확인하세요. <code>bitsandbytes&gt;=0.37.0</code>을 사용하는 경우, 모든 GPU가 지원됩니다.</li> <li>올바른 버전의 <code>bitsandbytes</code>를 다음 명령으로 설치하세요:
<code>pip install bitsandbytes&gt;=0.31.5</code></li> <li><code>accelerate</code>를 설치하세요
<code>pip install accelerate&gt;=0.12.0</code></li>`,nl,P,Y,bt,Ge,Xl,at,wn="혼합 Int8 모델 실행 - 단일 GPU 설정",sl,Pe,_n="필요한 라이브러리를 설치한 후 혼합 8비트 모델을 가져오는 방법은 다음과 같습니다:",al,We,rl,je,Tn="텍스트 생성의 경우:",il,Ze,xn="<li><code>pipeline()</code> 함수 대신 모델의 <code>generate()</code> 메소드를 사용하는 것을 권장합니다. <code>pipeline()</code> 함수로는 추론이 가능하지만, 혼합 8비트 모델에 최적화되지 않았기 때문에 <code>generate()</code> 메소드를 사용하는 것보다 느릴 수 있습니다. 또한, nucleus 샘플링과 같은 일부 샘플링 전략은 혼합 8비트 모델에 대해 <code>pipeline()</code> 함수에서 지원되지 않습니다.</li> <li>입력을 모델과 동일한 GPU에 배치하는 것이 좋습니다.</li>",ol,Ee,Cn="다음은 간단한 예입니다:",pl,Fe,ml,W,S,yt,Ne,Al,rt,Un="혼합 Int8 모델 실행 - 다중 GPU 설정",fl,He,Jn="다중 GPU에서 혼합 8비트 모델을 로드하는 방법은 단일 GPU 설정과 동일합니다(동일한 명령어 사용):",ul,Ve,cl,Re,kn="하지만 <code>accelerate</code>를 사용하여 각 GPU에 할당할 GPU RAM을 제어할 수 있습니다. 다음과 같이 <code>max_memory</code> 인수를 사용하세요:",hl,Ie,dl,Le,Gn="이 예시에서는 첫 번째 GPU가 1GB의 메모리를 사용하고 두 번째 GPU가 2GB를 사용합니다.",vl,j,Q,Mt,Xe,ql,it,Pn="Colab 데모",bl,Ae,Wn=`이 방법을 사용하면 이전에 Google Colab에서 추론할 수 없었던 모델에 대해 추론할 수 있습니다.
Google Colab에서 8비트 양자화를 사용하여 T5-11b(42GB in fp32)를 실행하는 데모를 확인하세요:`,yl,qe,jn='<a href="https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab: T5-11b demo"/></a>',Ml,Be,Zn="또는 BLOOM-3B에 대한 데모를 확인하세요:",gl,Ye,En='<a href="https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab: BLOOM-3b demo"/></a>',$l,gt,wl;return z=new g({}),O=new g({}),le=new Z({props:{code:"bW9kZWwlMjAlM0QlMjBtb2RlbC50b19iZXR0ZXJ0cmFuc2Zvcm1lcigp",highlighted:"model = model.to_bettertransformer()",wrap:!1}}),se=new Z({props:{code:"bW9kZWwlMjAlM0QlMjBtb2RlbC5yZXZlcnNlX2JldHRlcnRyYW5zZm9ybWVyKCklMEFtb2RlbC5zYXZlX3ByZXRyYWluZWQoJTIyc2F2ZWRfbW9kZWwlMjIp",highlighted:`model = model.reverse_bettertransformer()
model.save_pretrained(<span class="hljs-string">&quot;saved_model&quot;</span>)`,wrap:!1}}),re=new g({}),V=new as({props:{$$slots:{default:[us]},$$scope:{ctx:Qe}}}),oe=new g({}),me=new g({}),ue=new Z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`,wrap:!1}}),he=new g({}),ve=new Z({props:{code:"bW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUp",highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`,wrap:!1}}),ye=new Z({props:{code:"bWF4X21lbW9yeV9tYXBwaW5nJTIwJTNEJTIwJTdCMCUzQSUyMCUyMjYwME1CJTIyJTJDJTIwMSUzQSUyMCUyMjFHQiUyMiU3RCUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tM2IlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMjBtYXhfbWVtb3J5JTNEbWF4X21lbW9yeV9tYXBwaW5nJTBBKQ==",highlighted:`max_memory_mapping = {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;600MB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;1GB&quot;</span>}
model_name = <span class="hljs-string">&quot;bigscience/bloom-3b&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>, max_memory=max_memory_mapping
)`,wrap:!1}}),ge=new g({}),we=new g({}),q=new as({props:{$$slots:{default:[cs]},$$scope:{ctx:Qe}}}),Je=new g({}),Ge=new g({}),We=new Z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`,wrap:!1}}),Fe=new Z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMmI1JTIyJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSklMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBcHJvbXB0JTIwJTNEJTIwJTIySGVsbG8lMkMlMjBteSUyMGxsYW1hJTIwaXMlMjBjdXRlJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKSUwQW91dHB1dHMlMjAlM0QlMjB0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKGdlbmVyYXRlZF9pZHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_name)
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)

prompt = <span class="hljs-string">&quot;Hello, my llama is cute&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
generated_ids = model.generate(**inputs)
outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)`,wrap:!1}}),Ne=new g({}),Ve=new Z({props:{code:"bW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUp",highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`,wrap:!1}}),Ie=new Z({props:{code:"bWF4X21lbW9yeV9tYXBwaW5nJTIwJTNEJTIwJTdCMCUzQSUyMCUyMjFHQiUyMiUyQyUyMDElM0ElMjAlMjIyR0IlMjIlN0QlMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTNiJTIyJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjBtb2RlbF9uYW1lJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBsb2FkX2luXzhiaXQlM0RUcnVlJTJDJTIwbWF4X21lbW9yeSUzRG1heF9tZW1vcnlfbWFwcGluZyUwQSk=",highlighted:`max_memory_mapping = {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;1GB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;2GB&quot;</span>}
model_name = <span class="hljs-string">&quot;bigscience/bloom-3b&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>, max_memory=max_memory_mapping
)`,wrap:!1}}),Xe=new g({}),{c(){u=s("meta"),$=r(),M=s("p"),E=r(),w=s("h1"),F=s("a"),ot=s("span"),c(z.$$.fragment),Zl=r(),ze=s("span"),ze.textContent=Bl,wt=r(),K=s("p"),K.innerHTML=Yl,_t=r(),_=s("h2"),N=s("a"),pt=s("span"),c(O.$$.fragment),El=r(),Ke=s("span"),Ke.textContent=Sl,Tt=r(),D=s("p"),D.innerHTML=Ql,xt=r(),ee=s("p"),ee.innerHTML=zl,Ct=r(),te=s("p"),te.innerHTML=Kl,Ut=r(),c(le.$$.fragment),Jt=r(),ne=s("p"),ne.innerHTML=Ol,kt=r(),c(se.$$.fragment),Gt=r(),ae=s("p"),ae.innerHTML=Dl,Pt=r(),T=s("h2"),H=s("a"),mt=s("span"),c(re.$$.fragment),Fl=r(),Oe=s("span"),Oe.innerHTML=en,Wt=r(),ie=s("p"),ie.innerHTML=tn,jt=r(),c(V.$$.fragment),Zt=r(),x=s("h3"),R=s("a"),ft=s("span"),c(oe.$$.fragment),Nl=r(),De=s("span"),De.textContent=ln,Et=r(),pe=s("ul"),pe.innerHTML=nn,Ft=r(),C=s("h3"),I=s("a"),ut=s("span"),c(me.$$.fragment),Hl=r(),et=s("span"),et.textContent=sn,Nt=r(),fe=s("p"),fe.textContent=an,Ht=r(),c(ue.$$.fragment),Vt=r(),ce=s("p"),ce.innerHTML=rn,Rt=r(),U=s("h3"),L=s("a"),ct=s("span"),c(he.$$.fragment),Vl=r(),tt=s("span"),tt.textContent=on,It=r(),de=s("p"),de.textContent=pn,Lt=r(),c(ve.$$.fragment),Xt=r(),be=s("p"),be.innerHTML=mn,At=r(),c(ye.$$.fragment),qt=r(),Me=s("p"),Me.textContent=fn,Bt=r(),J=s("h3"),X=s("a"),ht=s("span"),c(ge.$$.fragment),Rl=r(),lt=s("span"),lt.textContent=un,Yt=r(),$e=s("p"),$e.innerHTML=cn,St=r(),k=s("h2"),A=s("a"),dt=s("span"),c(we.$$.fragment),Il=r(),nt=s("span"),nt.innerHTML=hn,Qt=r(),c(q.$$.fragment),zt=r(),_e=s("p"),_e.innerHTML=dn,Kt=r(),Te=s("p"),Te.innerHTML=vn,Ot=r(),xe=s("p"),xe.innerHTML=bn,Dt=r(),Ce=s("p"),Ce.innerHTML=yn,el=r(),Ue=s("p"),Ue.innerHTML=Mn,tl=r(),G=s("h3"),B=s("a"),vt=s("span"),c(Je.$$.fragment),Ll=r(),st=s("span"),st.textContent=gn,ll=r(),ke=s("ul"),ke.innerHTML=$n,nl=r(),P=s("h3"),Y=s("a"),bt=s("span"),c(Ge.$$.fragment),Xl=r(),at=s("span"),at.textContent=wn,sl=r(),Pe=s("p"),Pe.textContent=_n,al=r(),c(We.$$.fragment),rl=r(),je=s("p"),je.textContent=Tn,il=r(),Ze=s("ul"),Ze.innerHTML=xn,ol=r(),Ee=s("p"),Ee.textContent=Cn,pl=r(),c(Fe.$$.fragment),ml=r(),W=s("h3"),S=s("a"),yt=s("span"),c(Ne.$$.fragment),Al=r(),rt=s("span"),rt.textContent=Un,fl=r(),He=s("p"),He.textContent=Jn,ul=r(),c(Ve.$$.fragment),cl=r(),Re=s("p"),Re.innerHTML=kn,hl=r(),c(Ie.$$.fragment),dl=r(),Le=s("p"),Le.textContent=Gn,vl=r(),j=s("h3"),Q=s("a"),Mt=s("span"),c(Xe.$$.fragment),ql=r(),it=s("span"),it.textContent=Pn,bl=r(),Ae=s("p"),Ae.textContent=Wn,yl=r(),qe=s("p"),qe.innerHTML=jn,Ml=r(),Be=s("p"),Be.textContent=Zn,gl=r(),Ye=s("p"),Ye.innerHTML=En,$l=r(),gt=s("p"),this.h()},l(e){const l=fs("svelte-1phssyn",document.head);u=a(l,"META",{name:!0,content:!0}),l.forEach(t),$=i(e),M=a(e,"P",{}),f(M).forEach(t),E=i(e),w=a(e,"H1",{class:!0});var Se=f(w);F=a(Se,"A",{id:!0,class:!0,href:!0});var $t=f(F);ot=a($t,"SPAN",{});var Fn=f(ot);h(z.$$.fragment,Fn),Fn.forEach(t),$t.forEach(t),Zl=i(Se),ze=a(Se,"SPAN",{"data-svelte-h":!0}),m(ze)!=="svelte-ulweto"&&(ze.textContent=Bl),Se.forEach(t),wt=i(e),K=a(e,"P",{"data-svelte-h":!0}),m(K)!=="svelte-1m7ldpj"&&(K.innerHTML=Yl),_t=i(e),_=a(e,"H2",{class:!0});var _l=f(_);N=a(_l,"A",{id:!0,class:!0,href:!0});var Nn=f(N);pt=a(Nn,"SPAN",{});var Hn=f(pt);h(O.$$.fragment,Hn),Hn.forEach(t),Nn.forEach(t),El=i(_l),Ke=a(_l,"SPAN",{"data-svelte-h":!0}),m(Ke)!=="svelte-472wrc"&&(Ke.textContent=Sl),_l.forEach(t),Tt=i(e),D=a(e,"P",{"data-svelte-h":!0}),m(D)!=="svelte-ee52o1"&&(D.innerHTML=Ql),xt=i(e),ee=a(e,"P",{"data-svelte-h":!0}),m(ee)!=="svelte-8k8zjr"&&(ee.innerHTML=zl),Ct=i(e),te=a(e,"P",{"data-svelte-h":!0}),m(te)!=="svelte-1rnbe8f"&&(te.innerHTML=Kl),Ut=i(e),h(le.$$.fragment,e),Jt=i(e),ne=a(e,"P",{"data-svelte-h":!0}),m(ne)!=="svelte-1u5t675"&&(ne.innerHTML=Ol),kt=i(e),h(se.$$.fragment,e),Gt=i(e),ae=a(e,"P",{"data-svelte-h":!0}),m(ae)!=="svelte-17375w3"&&(ae.innerHTML=Dl),Pt=i(e),T=a(e,"H2",{class:!0});var Tl=f(T);H=a(Tl,"A",{id:!0,class:!0,href:!0});var Vn=f(H);mt=a(Vn,"SPAN",{});var Rn=f(mt);h(re.$$.fragment,Rn),Rn.forEach(t),Vn.forEach(t),Fl=i(Tl),Oe=a(Tl,"SPAN",{"data-svelte-h":!0}),m(Oe)!=="svelte-ahmz2u"&&(Oe.innerHTML=en),Tl.forEach(t),Wt=i(e),ie=a(e,"P",{"data-svelte-h":!0}),m(ie)!=="svelte-sxo64f"&&(ie.innerHTML=tn),jt=i(e),h(V.$$.fragment,e),Zt=i(e),x=a(e,"H3",{class:!0});var xl=f(x);R=a(xl,"A",{id:!0,class:!0,href:!0});var In=f(R);ft=a(In,"SPAN",{});var Ln=f(ft);h(oe.$$.fragment,Ln),Ln.forEach(t),In.forEach(t),Nl=i(xl),De=a(xl,"SPAN",{"data-svelte-h":!0}),m(De)!=="svelte-ifxbuu"&&(De.textContent=ln),xl.forEach(t),Et=i(e),pe=a(e,"UL",{"data-svelte-h":!0}),m(pe)!=="svelte-dwg1v5"&&(pe.innerHTML=nn),Ft=i(e),C=a(e,"H3",{class:!0});var Cl=f(C);I=a(Cl,"A",{id:!0,class:!0,href:!0});var Xn=f(I);ut=a(Xn,"SPAN",{});var An=f(ut);h(me.$$.fragment,An),An.forEach(t),Xn.forEach(t),Hl=i(Cl),et=a(Cl,"SPAN",{"data-svelte-h":!0}),m(et)!=="svelte-aatcb2"&&(et.textContent=sn),Cl.forEach(t),Nt=i(e),fe=a(e,"P",{"data-svelte-h":!0}),m(fe)!=="svelte-60dj5"&&(fe.textContent=an),Ht=i(e),h(ue.$$.fragment,e),Vt=i(e),ce=a(e,"P",{"data-svelte-h":!0}),m(ce)!=="svelte-1xe6z7w"&&(ce.innerHTML=rn),Rt=i(e),U=a(e,"H3",{class:!0});var Ul=f(U);L=a(Ul,"A",{id:!0,class:!0,href:!0});var qn=f(L);ct=a(qn,"SPAN",{});var Bn=f(ct);h(he.$$.fragment,Bn),Bn.forEach(t),qn.forEach(t),Vl=i(Ul),tt=a(Ul,"SPAN",{"data-svelte-h":!0}),m(tt)!=="svelte-1pv275r"&&(tt.textContent=on),Ul.forEach(t),It=i(e),de=a(e,"P",{"data-svelte-h":!0}),m(de)!=="svelte-11nsoog"&&(de.textContent=pn),Lt=i(e),h(ve.$$.fragment,e),Xt=i(e),be=a(e,"P",{"data-svelte-h":!0}),m(be)!=="svelte-7qj9fn"&&(be.innerHTML=mn),At=i(e),h(ye.$$.fragment,e),qt=i(e),Me=a(e,"P",{"data-svelte-h":!0}),m(Me)!=="svelte-1wq62m5"&&(Me.textContent=fn),Bt=i(e),J=a(e,"H3",{class:!0});var Jl=f(J);X=a(Jl,"A",{id:!0,class:!0,href:!0});var Yn=f(X);ht=a(Yn,"SPAN",{});var Sn=f(ht);h(ge.$$.fragment,Sn),Sn.forEach(t),Yn.forEach(t),Rl=i(Jl),lt=a(Jl,"SPAN",{"data-svelte-h":!0}),m(lt)!=="svelte-m53e38"&&(lt.textContent=un),Jl.forEach(t),Yt=i(e),$e=a(e,"P",{"data-svelte-h":!0}),m($e)!=="svelte-19t36l7"&&($e.innerHTML=cn),St=i(e),k=a(e,"H2",{class:!0});var kl=f(k);A=a(kl,"A",{id:!0,class:!0,href:!0});var Qn=f(A);dt=a(Qn,"SPAN",{});var zn=f(dt);h(we.$$.fragment,zn),zn.forEach(t),Qn.forEach(t),Il=i(kl),nt=a(kl,"SPAN",{"data-svelte-h":!0}),m(nt)!=="svelte-1we91mu"&&(nt.innerHTML=hn),kl.forEach(t),Qt=i(e),h(q.$$.fragment,e),zt=i(e),_e=a(e,"P",{"data-svelte-h":!0}),m(_e)!=="svelte-1ynbgg5"&&(_e.innerHTML=dn),Kt=i(e),Te=a(e,"P",{"data-svelte-h":!0}),m(Te)!=="svelte-1tsrdqi"&&(Te.innerHTML=vn),Ot=i(e),xe=a(e,"P",{"data-svelte-h":!0}),m(xe)!=="svelte-sz8751"&&(xe.innerHTML=bn),Dt=i(e),Ce=a(e,"P",{"data-svelte-h":!0}),m(Ce)!=="svelte-y2mgdg"&&(Ce.innerHTML=yn),el=i(e),Ue=a(e,"P",{"data-svelte-h":!0}),m(Ue)!=="svelte-ewmhxc"&&(Ue.innerHTML=Mn),tl=i(e),G=a(e,"H3",{class:!0});var Gl=f(G);B=a(Gl,"A",{id:!0,class:!0,href:!0});var Kn=f(B);vt=a(Kn,"SPAN",{});var On=f(vt);h(Je.$$.fragment,On),On.forEach(t),Kn.forEach(t),Ll=i(Gl),st=a(Gl,"SPAN",{"data-svelte-h":!0}),m(st)!=="svelte-ifxbuu"&&(st.textContent=gn),Gl.forEach(t),ll=i(e),ke=a(e,"UL",{"data-svelte-h":!0}),m(ke)!=="svelte-r9nrya"&&(ke.innerHTML=$n),nl=i(e),P=a(e,"H3",{class:!0});var Pl=f(P);Y=a(Pl,"A",{id:!0,class:!0,href:!0});var Dn=f(Y);bt=a(Dn,"SPAN",{});var es=f(bt);h(Ge.$$.fragment,es),es.forEach(t),Dn.forEach(t),Xl=i(Pl),at=a(Pl,"SPAN",{"data-svelte-h":!0}),m(at)!=="svelte-dis7hm"&&(at.textContent=wn),Pl.forEach(t),sl=i(e),Pe=a(e,"P",{"data-svelte-h":!0}),m(Pe)!=="svelte-1trjkap"&&(Pe.textContent=_n),al=i(e),h(We.$$.fragment,e),rl=i(e),je=a(e,"P",{"data-svelte-h":!0}),m(je)!=="svelte-bekc1k"&&(je.textContent=Tn),il=i(e),Ze=a(e,"UL",{"data-svelte-h":!0}),m(Ze)!=="svelte-daelpd"&&(Ze.innerHTML=xn),ol=i(e),Ee=a(e,"P",{"data-svelte-h":!0}),m(Ee)!=="svelte-1md3fzz"&&(Ee.textContent=Cn),pl=i(e),h(Fe.$$.fragment,e),ml=i(e),W=a(e,"H3",{class:!0});var Wl=f(W);S=a(Wl,"A",{id:!0,class:!0,href:!0});var ts=f(S);yt=a(ts,"SPAN",{});var ls=f(yt);h(Ne.$$.fragment,ls),ls.forEach(t),ts.forEach(t),Al=i(Wl),rt=a(Wl,"SPAN",{"data-svelte-h":!0}),m(rt)!=="svelte-1kvl5l7"&&(rt.textContent=Un),Wl.forEach(t),fl=i(e),He=a(e,"P",{"data-svelte-h":!0}),m(He)!=="svelte-bnkuc8"&&(He.textContent=Jn),ul=i(e),h(Ve.$$.fragment,e),cl=i(e),Re=a(e,"P",{"data-svelte-h":!0}),m(Re)!=="svelte-7qj9fn"&&(Re.innerHTML=kn),hl=i(e),h(Ie.$$.fragment,e),dl=i(e),Le=a(e,"P",{"data-svelte-h":!0}),m(Le)!=="svelte-1bub4cv"&&(Le.textContent=Gn),vl=i(e),j=a(e,"H3",{class:!0});var jl=f(j);Q=a(jl,"A",{id:!0,class:!0,href:!0});var ns=f(Q);Mt=a(ns,"SPAN",{});var ss=f(Mt);h(Xe.$$.fragment,ss),ss.forEach(t),ns.forEach(t),ql=i(jl),it=a(jl,"SPAN",{"data-svelte-h":!0}),m(it)!=="svelte-g9oysy"&&(it.textContent=Pn),jl.forEach(t),bl=i(e),Ae=a(e,"P",{"data-svelte-h":!0}),m(Ae)!=="svelte-1at89fq"&&(Ae.textContent=Wn),yl=i(e),qe=a(e,"P",{"data-svelte-h":!0}),m(qe)!=="svelte-1yb5ek4"&&(qe.innerHTML=jn),Ml=i(e),Be=a(e,"P",{"data-svelte-h":!0}),m(Be)!=="svelte-u2br1q"&&(Be.textContent=Zn),gl=i(e),Ye=a(e,"P",{"data-svelte-h":!0}),m(Ye)!=="svelte-6z7881"&&(Ye.innerHTML=En),$l=i(e),gt=a(e,"P",{}),f(gt).forEach(t),this.h()},h(){o(u,"name","hf:doc:metadata"),o(u,"content",JSON.stringify(ds)),o(F,"id","efficient-inference-on-a-single-gpu"),o(F,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(F,"href","#efficient-inference-on-a-single-gpu"),o(w,"class","relative group"),o(N,"id","better-transformer-pytorchnative-transformer-fastpath"),o(N,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(N,"href","#better-transformer-pytorchnative-transformer-fastpath"),o(_,"class","relative group"),o(H,"id","bitsandbytes-integration-for-fp4-mixedprecision-inference"),o(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(H,"href","#bitsandbytes-integration-for-fp4-mixedprecision-inference"),o(T,"class","relative group"),o(R,"id","requirements-for-fp4-mixedprecision-inference"),o(R,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(R,"href","#requirements-for-fp4-mixedprecision-inference"),o(x,"class","relative group"),o(I,"id","running-fp4-models-single-gpu-setup-quickstart"),o(I,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(I,"href","#running-fp4-models-single-gpu-setup-quickstart"),o(C,"class","relative group"),o(L,"id","running-fp4-models-multi-gpu-setup"),o(L,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(L,"href","#running-fp4-models-multi-gpu-setup"),o(U,"class","relative group"),o(X,"id","advanced-usage"),o(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(X,"href","#advanced-usage"),o(J,"class","relative group"),o(A,"id","bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),o(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(A,"href","#bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),o(k,"class","relative group"),o(B,"id","requirements-for-int8-mixedprecision-matrix-decomposition"),o(B,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(B,"href","#requirements-for-int8-mixedprecision-matrix-decomposition"),o(G,"class","relative group"),o(Y,"id","running-mixedint8-models-single-gpu-setup"),o(Y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(Y,"href","#running-mixedint8-models-single-gpu-setup"),o(P,"class","relative group"),o(S,"id","running-mixedint8-models-multi-gpu-setup"),o(S,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(S,"href","#running-mixedint8-models-multi-gpu-setup"),o(W,"class","relative group"),o(Q,"id","colab-demos"),o(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(Q,"href","#colab-demos"),o(j,"class","relative group")},m(e,l){p(document.head,u),n(e,$,l),n(e,M,l),n(e,E,l),n(e,w,l),p(w,F),p(F,ot),d(z,ot,null),p(w,Zl),p(w,ze),n(e,wt,l),n(e,K,l),n(e,_t,l),n(e,_,l),p(_,N),p(N,pt),d(O,pt,null),p(_,El),p(_,Ke),n(e,Tt,l),n(e,D,l),n(e,xt,l),n(e,ee,l),n(e,Ct,l),n(e,te,l),n(e,Ut,l),d(le,e,l),n(e,Jt,l),n(e,ne,l),n(e,kt,l),d(se,e,l),n(e,Gt,l),n(e,ae,l),n(e,Pt,l),n(e,T,l),p(T,H),p(H,mt),d(re,mt,null),p(T,Fl),p(T,Oe),n(e,Wt,l),n(e,ie,l),n(e,jt,l),d(V,e,l),n(e,Zt,l),n(e,x,l),p(x,R),p(R,ft),d(oe,ft,null),p(x,Nl),p(x,De),n(e,Et,l),n(e,pe,l),n(e,Ft,l),n(e,C,l),p(C,I),p(I,ut),d(me,ut,null),p(C,Hl),p(C,et),n(e,Nt,l),n(e,fe,l),n(e,Ht,l),d(ue,e,l),n(e,Vt,l),n(e,ce,l),n(e,Rt,l),n(e,U,l),p(U,L),p(L,ct),d(he,ct,null),p(U,Vl),p(U,tt),n(e,It,l),n(e,de,l),n(e,Lt,l),d(ve,e,l),n(e,Xt,l),n(e,be,l),n(e,At,l),d(ye,e,l),n(e,qt,l),n(e,Me,l),n(e,Bt,l),n(e,J,l),p(J,X),p(X,ht),d(ge,ht,null),p(J,Rl),p(J,lt),n(e,Yt,l),n(e,$e,l),n(e,St,l),n(e,k,l),p(k,A),p(A,dt),d(we,dt,null),p(k,Il),p(k,nt),n(e,Qt,l),d(q,e,l),n(e,zt,l),n(e,_e,l),n(e,Kt,l),n(e,Te,l),n(e,Ot,l),n(e,xe,l),n(e,Dt,l),n(e,Ce,l),n(e,el,l),n(e,Ue,l),n(e,tl,l),n(e,G,l),p(G,B),p(B,vt),d(Je,vt,null),p(G,Ll),p(G,st),n(e,ll,l),n(e,ke,l),n(e,nl,l),n(e,P,l),p(P,Y),p(Y,bt),d(Ge,bt,null),p(P,Xl),p(P,at),n(e,sl,l),n(e,Pe,l),n(e,al,l),d(We,e,l),n(e,rl,l),n(e,je,l),n(e,il,l),n(e,Ze,l),n(e,ol,l),n(e,Ee,l),n(e,pl,l),d(Fe,e,l),n(e,ml,l),n(e,W,l),p(W,S),p(S,yt),d(Ne,yt,null),p(W,Al),p(W,rt),n(e,fl,l),n(e,He,l),n(e,ul,l),d(Ve,e,l),n(e,cl,l),n(e,Re,l),n(e,hl,l),d(Ie,e,l),n(e,dl,l),n(e,Le,l),n(e,vl,l),n(e,j,l),p(j,Q),p(Q,Mt),d(Xe,Mt,null),p(j,ql),p(j,it),n(e,bl,l),n(e,Ae,l),n(e,yl,l),n(e,qe,l),n(e,Ml,l),n(e,Be,l),n(e,gl,l),n(e,Ye,l),n(e,$l,l),n(e,gt,l),wl=!0},p(e,[l]){const Se={};l&2&&(Se.$$scope={dirty:l,ctx:e}),V.$set(Se);const $t={};l&2&&($t.$$scope={dirty:l,ctx:e}),q.$set($t)},i(e){wl||(v(z.$$.fragment,e),v(O.$$.fragment,e),v(le.$$.fragment,e),v(se.$$.fragment,e),v(re.$$.fragment,e),v(V.$$.fragment,e),v(oe.$$.fragment,e),v(me.$$.fragment,e),v(ue.$$.fragment,e),v(he.$$.fragment,e),v(ve.$$.fragment,e),v(ye.$$.fragment,e),v(ge.$$.fragment,e),v(we.$$.fragment,e),v(q.$$.fragment,e),v(Je.$$.fragment,e),v(Ge.$$.fragment,e),v(We.$$.fragment,e),v(Fe.$$.fragment,e),v(Ne.$$.fragment,e),v(Ve.$$.fragment,e),v(Ie.$$.fragment,e),v(Xe.$$.fragment,e),wl=!0)},o(e){b(z.$$.fragment,e),b(O.$$.fragment,e),b(le.$$.fragment,e),b(se.$$.fragment,e),b(re.$$.fragment,e),b(V.$$.fragment,e),b(oe.$$.fragment,e),b(me.$$.fragment,e),b(ue.$$.fragment,e),b(he.$$.fragment,e),b(ve.$$.fragment,e),b(ye.$$.fragment,e),b(ge.$$.fragment,e),b(we.$$.fragment,e),b(q.$$.fragment,e),b(Je.$$.fragment,e),b(Ge.$$.fragment,e),b(We.$$.fragment,e),b(Fe.$$.fragment,e),b(Ne.$$.fragment,e),b(Ve.$$.fragment,e),b(Ie.$$.fragment,e),b(Xe.$$.fragment,e),wl=!1},d(e){e&&(t($),t(M),t(E),t(w),t(wt),t(K),t(_t),t(_),t(Tt),t(D),t(xt),t(ee),t(Ct),t(te),t(Ut),t(Jt),t(ne),t(kt),t(Gt),t(ae),t(Pt),t(T),t(Wt),t(ie),t(jt),t(Zt),t(x),t(Et),t(pe),t(Ft),t(C),t(Nt),t(fe),t(Ht),t(Vt),t(ce),t(Rt),t(U),t(It),t(de),t(Lt),t(Xt),t(be),t(At),t(qt),t(Me),t(Bt),t(J),t(Yt),t($e),t(St),t(k),t(Qt),t(zt),t(_e),t(Kt),t(Te),t(Ot),t(xe),t(Dt),t(Ce),t(el),t(Ue),t(tl),t(G),t(ll),t(ke),t(nl),t(P),t(sl),t(Pe),t(al),t(rl),t(je),t(il),t(Ze),t(ol),t(Ee),t(pl),t(ml),t(W),t(fl),t(He),t(ul),t(cl),t(Re),t(hl),t(dl),t(Le),t(vl),t(j),t(bl),t(Ae),t(yl),t(qe),t(Ml),t(Be),t(gl),t(Ye),t($l),t(gt)),t(u),y(z),y(O),y(le,e),y(se,e),y(re),y(V,e),y(oe),y(me),y(ue,e),y(he),y(ve,e),y(ye,e),y(ge),y(we),y(q,e),y(Je),y(Ge),y(We,e),y(Fe,e),y(Ne),y(Ve,e),y(Ie,e),y(Xe)}}}const ds={local:"efficient-inference-on-a-single-gpu",sections:[{local:"better-transformer-pytorchnative-transformer-fastpath",title:"Better Transformer: PyTorch 네이티브 Transformer 패스트패스 "},{local:"bitsandbytes-integration-for-fp4-mixedprecision-inference",sections:[{local:"requirements-for-fp4-mixedprecision-inference",title:"요구 사항 "},{local:"running-fp4-models-single-gpu-setup-quickstart",title:"FP4 모델 실행 - 단일 GPU 설정 - 빠른 시작 "},{local:"running-fp4-models-multi-gpu-setup",title:"FP4 모델 실행 - 다중 GPU 설정 "},{local:"advanced-usage",title:"고급 사용법 "}],title:"FP4 혼합 정밀도 추론을 위한 `bitsandbytes` 통합 "},{local:"bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition",sections:[{local:"requirements-for-int8-mixedprecision-matrix-decomposition",title:"요구 사항 "},{local:"running-mixedint8-models-single-gpu-setup",title:"혼합 Int8 모델 실행 - 단일 GPU 설정 "},{local:"running-mixedint8-models-multi-gpu-setup",title:"혼합 Int8 모델 실행 - 다중 GPU 설정 "},{local:"colab-demos",title:"Colab 데모 "}],title:"Int8 혼합 정밀도 행렬 분해를 위한 `bitsandbytes` 통합 "}],title:"단일 GPU에서 효율적인 추론 "};function vs(Qe){return os(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ws extends ps{constructor(u){super(),ms(this,u,vs,hs,is,{})}}export{ws as component};
