import{s as de,n as oe,o as ae}from"../chunks/scheduler.9bc65507.js";import{S as ie,i as ce,g as o,s as i,u as re,A as le,h as a,f as t,c,j as P,v as se,y as r,k,o as H,a as d,w as he,d as ue,t as pe,x as me}from"../chunks/index.78c82d43.js";import{I as ge}from"../chunks/IconCopyLink.bedaa44d.js";function fe(F){let l,M,w,q,s,h,L,u,B,y,J="Padding and truncation",E,p,O="Batched inputs are often different lengths, so they can’t be converted to fixed-size tensors. Padding and truncation are strategies for dealing with this problem, to create rectangular tensors from batches of varying lengths. Padding adds a special <strong>padding token</strong> to ensure shorter sequences will have the same length as either the longest sequence in a batch or the maximum length accepted by the model. Truncation works in the other direction by truncating long sequences.",A,m,D="In most cases, padding your batch to the length of the longest sequence and truncating to the maximum length a model can accept works pretty well. However, the API supports more strategies if you need them. The three arguments you need to are: <code>padding</code>, <code>truncation</code> and <code>max_length</code>.",C,g,K="The <code>padding</code> argument controls padding. It can be a boolean or a string:",S,f,Q=`<li><code>True</code> or <code>&#39;longest&#39;</code>: pad to the longest sequence in the batch (no padding is applied if you only provide
a single sequence).</li> <li><code>&#39;max_length&#39;</code>: pad to a length specified by the <code>max_length</code> argument or the maximum length accepted
by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). Padding will still be applied if you only provide a single sequence.</li> <li><code>False</code> or <code>&#39;do_not_pad&#39;</code>: no padding is applied. This is the default behavior.</li>`,I,_,V="The <code>truncation</code> argument controls truncation. It can be a boolean or a string:",N,x,W=`<li><code>True</code> or <code>&#39;longest_first&#39;</code>: truncate to a maximum length specified by the <code>max_length</code> argument or
the maximum length accepted by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). This will
truncate token by token, removing a token from the longest sequence in the pair until the proper length is
reached.</li> <li><code>&#39;only_second&#39;</code>: truncate to a maximum length specified by the <code>max_length</code> argument or the maximum
length accepted by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). This will only truncate
the second sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.</li> <li><code>&#39;only_first&#39;</code>: truncate to a maximum length specified by the <code>max_length</code> argument or the maximum
length accepted by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). This will only truncate
the first sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.</li> <li><code>False</code> or <code>&#39;do_not_truncate&#39;</code>: no truncation is applied. This is the default behavior.</li>`,$,b,X="The <code>max_length</code> argument controls the length of the padding and truncation. It can be an integer or <code>None</code>, in which case it will default to the maximum length the model can accept. If the model has no specific maximum input length, truncation or padding to <code>max_length</code> is deactivated.",R,T,Z=`The following table summarizes the recommended way to setup padding and truncation. If you use pairs of input sequences in any of the following examples, you can replace <code>truncation=True</code> by a <code>STRATEGY</code> selected in
<code>[&#39;only_first&#39;, &#39;only_second&#39;, &#39;longest_first&#39;]</code>, i.e. <code>truncation=&#39;only_second&#39;</code> or <code>truncation=&#39;longest_first&#39;</code> to control how both sequences in the pair are truncated as detailed before.`,G,v,ee="<thead><tr><th>Truncation</th> <th>Padding</th> <th>Instruction</th></tr></thead> <tbody><tr><td>no truncation</td> <td>no padding</td> <td><code>tokenizer(batch_sentences)</code></td></tr> <tr><td></td> <td>padding to max sequence in batch</td> <td><code>tokenizer(batch_sentences, padding=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=&#39;longest&#39;)</code></td></tr> <tr><td></td> <td>padding to max model input length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;)</code></td></tr> <tr><td></td> <td>padding to specific length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, max_length=42)</code></td></tr> <tr><td></td> <td>padding to a multiple of a value</td> <td>`tokenizer(batch_sentences, padding=True, pad_to_multiple_of=8)</td></tr> <tr><td>truncation to max model input length</td> <td>no padding</td> <td><code>tokenizer(batch_sentences, truncation=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, truncation=STRATEGY)</code></td></tr> <tr><td></td> <td>padding to max sequence in batch</td> <td><code>tokenizer(batch_sentences, padding=True, truncation=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=True, truncation=STRATEGY)</code></td></tr> <tr><td></td> <td>padding to max model input length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=STRATEGY)</code></td></tr> <tr><td></td> <td>padding to specific length</td> <td>Not possible</td></tr> <tr><td>truncation to specific length</td> <td>no padding</td> <td><code>tokenizer(batch_sentences, truncation=True, max_length=42)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)</code></td></tr> <tr><td></td> <td>padding to max sequence in batch</td> <td><code>tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)</code></td></tr> <tr><td></td> <td>padding to max model input length</td> <td>Not possible</td></tr> <tr><td></td> <td>padding to specific length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=True, max_length=42)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=STRATEGY, max_length=42)</code></td></tr></tbody>",Y,z,U;return u=new ge({}),{c(){l=o("meta"),M=i(),w=o("p"),q=i(),s=o("h1"),h=o("a"),L=o("span"),re(u.$$.fragment),B=i(),y=o("span"),y.textContent=J,E=i(),p=o("p"),p.innerHTML=O,A=i(),m=o("p"),m.innerHTML=D,C=i(),g=o("p"),g.innerHTML=K,S=i(),f=o("ul"),f.innerHTML=Q,I=i(),_=o("p"),_.innerHTML=V,N=i(),x=o("ul"),x.innerHTML=W,$=i(),b=o("p"),b.innerHTML=X,R=i(),T=o("p"),T.innerHTML=Z,G=i(),v=o("table"),v.innerHTML=ee,Y=i(),z=o("p"),this.h()},l(e){const n=le("svelte-1phssyn",document.head);l=a(n,"META",{name:!0,content:!0}),n.forEach(t),M=c(e),w=a(e,"P",{}),P(w).forEach(t),q=c(e),s=a(e,"H1",{class:!0});var j=P(s);h=a(j,"A",{id:!0,class:!0,href:!0});var te=P(h);L=a(te,"SPAN",{});var ne=P(L);se(u.$$.fragment,ne),ne.forEach(t),te.forEach(t),B=c(j),y=a(j,"SPAN",{"data-svelte-h":!0}),r(y)!=="svelte-1pb7o4g"&&(y.textContent=J),j.forEach(t),E=c(e),p=a(e,"P",{"data-svelte-h":!0}),r(p)!=="svelte-18l3nu8"&&(p.innerHTML=O),A=c(e),m=a(e,"P",{"data-svelte-h":!0}),r(m)!=="svelte-4dmk3p"&&(m.innerHTML=D),C=c(e),g=a(e,"P",{"data-svelte-h":!0}),r(g)!=="svelte-1xxctal"&&(g.innerHTML=K),S=c(e),f=a(e,"UL",{"data-svelte-h":!0}),r(f)!=="svelte-1pqvfv8"&&(f.innerHTML=Q),I=c(e),_=a(e,"P",{"data-svelte-h":!0}),r(_)!=="svelte-14kdvi5"&&(_.innerHTML=V),N=c(e),x=a(e,"UL",{"data-svelte-h":!0}),r(x)!=="svelte-1eic0m4"&&(x.innerHTML=W),$=c(e),b=a(e,"P",{"data-svelte-h":!0}),r(b)!=="svelte-n95phd"&&(b.innerHTML=X),R=c(e),T=a(e,"P",{"data-svelte-h":!0}),r(T)!=="svelte-1rrzmkp"&&(T.innerHTML=Z),G=c(e),v=a(e,"TABLE",{"data-svelte-h":!0}),r(v)!=="svelte-1btfijn"&&(v.innerHTML=ee),Y=c(e),z=a(e,"P",{}),P(z).forEach(t),this.h()},h(){k(l,"name","hf:doc:metadata"),k(l,"content",JSON.stringify(_e)),k(h,"id","padding-and-truncation"),k(h,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(h,"href","#padding-and-truncation"),k(s,"class","relative group")},m(e,n){H(document.head,l),d(e,M,n),d(e,w,n),d(e,q,n),d(e,s,n),H(s,h),H(h,L),he(u,L,null),H(s,B),H(s,y),d(e,E,n),d(e,p,n),d(e,A,n),d(e,m,n),d(e,C,n),d(e,g,n),d(e,S,n),d(e,f,n),d(e,I,n),d(e,_,n),d(e,N,n),d(e,x,n),d(e,$,n),d(e,b,n),d(e,R,n),d(e,T,n),d(e,G,n),d(e,v,n),d(e,Y,n),d(e,z,n),U=!0},p:oe,i(e){U||(ue(u.$$.fragment,e),U=!0)},o(e){pe(u.$$.fragment,e),U=!1},d(e){e&&(t(M),t(w),t(q),t(s),t(E),t(p),t(A),t(m),t(C),t(g),t(S),t(f),t(I),t(_),t(N),t(x),t($),t(b),t(R),t(T),t(G),t(v),t(Y),t(z)),t(l),me(u)}}}const _e={local:"padding-and-truncation",title:"Padding and truncation"};function xe(F){return ae(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ye extends ie{constructor(l){super(),ce(this,l,xe,fe,de,{})}}export{ye as component};
