import{s as Ee,n as Qe,o as He}from"../chunks/scheduler.9bc65507.js";import{S as qe,i as Fe,g as n,s as r,u,A as Se,h as a,f as e,c as p,j as h,v as b,y as c,k as i,o as l,a as o,w as y,d as g,t as j,x as M}from"../chunks/index.78c82d43.js";import{D as Ms}from"../chunks/Docstring.f3d04bd2.js";import{C as ce}from"../chunks/CodeBlock.48618ec2.js";import{I as As}from"../chunks/IconCopyLink.bedaa44d.js";function Le(me){let T,fs,ds,Ts,k,U,us,z,Ys,es,he="ByT5",ks,_,v,bs,Z,Ds,ts,de="Overview",_s,W,ue=`The ByT5 model was presented in <a href="https://arxiv.org/abs/2105.13626" rel="nofollow">ByT5: Towards a token-free future with pre-trained byte-to-byte models</a> by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir
Kale, Adam Roberts, Colin Raffel.`,ws,R,be="The abstract from the paper is the following:",Js,V,ye=`<em>Most widely-used pre-trained language models operate on sequences of tokens corresponding to word or subword units.
Encoding text as a sequence of tokens requires a tokenizer, which is typically created as an independent artifact from
the model. Token-free models that instead operate directly on raw text (bytes or characters) have many benefits: they
can process text in any language out of the box, they are more robust to noise, and they minimize technical debt by
removing complex and error-prone text preprocessing pipelines. Since byte or character sequences are longer than token
sequences, past work on token-free models has often introduced new model architectures designed to amortize the cost of
operating directly on raw text. In this paper, we show that a standard Transformer architecture can be used with
minimal modifications to process byte sequences. We carefully characterize the trade-offs in terms of parameter count,
training FLOPs, and inference speed, and show that byte-level models are competitive with their token-level
counterparts. We also demonstrate that byte-level models are significantly more robust to noise and perform better on
tasks that are sensitive to spelling and pronunciation. As part of our contribution, we release a new set of
pre-trained byte-level Transformer models based on the T5 architecture, as well as all code and data used in our
experiments.</em>`,Us,N,ge=`This model was contributed by <a href="https://huggingface.co/patrickvonplaten" rel="nofollow">patrickvonplaten</a>. The original code can be
found <a href="https://github.com/google-research/byt5" rel="nofollow">here</a>.`,vs,$,je=`ByT5’s architecture is based on the T5v1.1 model, so one can refer to <a href="t5v1.1">T5v1.1’s documentation page</a>. They
only differ in how inputs should be prepared for the model, see the code examples below.`,Bs,X,Me=`Since ByT5 was pre-trained unsupervisedly, there’s no real advantage to using a task prefix during single-task
fine-tuning. If you are doing multi-task fine-tuning, you should use a prefix.`,xs,w,B,ys,E,Ps,ns,fe="Example",Is,Q,Te="ByT5 works on raw UTF-8 bytes, so it can be used without a tokenizer:",Cs,H,Gs,q,ke="For batched inference and training it is however recommended to make use of the tokenizer:",zs,F,Zs,S,_e=`Similar to <a href="t5">T5</a>, ByT5 was trained on the span-mask denoising task. However,
since the model works directly on characters, the pretraining task is a bit
different. Let’s corrupt some characters of the
input sentence <code>&quot;The dog chases a ball in the park.&quot;</code> and ask ByT5 to predict them
for us.`,Ws,L,Rs,J,x,gs,A,Ks,as,we="ByT5Tokenizer",Vs,m,Y,Os,ls,Je="Construct a ByT5 tokenizer. ByT5 simply uses raw bytes utf-8 encoding.",se,os,Ue=`This tokenizer inherits from <a href="/docs/transformers/v4.33.3/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`,ee,f,D,te,rs,ve=`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A sequence has the following format:`,ne,ps,Be="<li>single sequence: <code>X &lt;/s&gt;</code></li> <li>pair of sequences: <code>A &lt;/s&gt; B &lt;/s&gt;</code></li>",ae,I,P,le,is,xe="Converts a sequence of tokens (string) in a single string.",oe,C,K,re,cs,Ie=`Create a mask from the two sequences passed to be used in a sequence-pair classification task. ByT5 does not
make use of token type ids, therefore a list of zeros is returned.`,pe,G,O,ie,ms,Ce=`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> method.`,Ns,ss,Ge='See <a href="/docs/transformers/v4.33.3/en/model_doc/byt5#transformers.ByT5Tokenizer">ByT5Tokenizer</a> for all details.',$s,js,Xs;return z=new As({}),Z=new As({}),E=new As({}),H=new ce({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFQ1Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTBBaW1wb3J0JTIwdG9yY2glMEElMEFtb2RlbCUyMCUzRCUyMFQ1Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUlMkZieXQ1LXNtYWxsJTIyKSUwQSUwQW51bV9zcGVjaWFsX3Rva2VucyUyMCUzRCUyMDMlMEElMjMlMjBNb2RlbCUyMGhhcyUyMDMlMjBzcGVjaWFsJTIwdG9rZW5zJTIwd2hpY2glMjB0YWtlJTIwdXAlMjB0aGUlMjBpbnB1dCUyMGlkcyUyMDAlMkMxJTJDMiUyMG9mJTIwQnlUNS4lMEElMjMlMjAlM0QlM0UlMjBOZWVkJTIwdG8lMjBzaGlmdCUyMHV0Zi04JTIwY2hhcmFjdGVyJTIwZW5jb2RpbmdzJTIwYnklMjAzJTIwYmVmb3JlJTIwcGFzc2luZyUyMGlkcyUyMHRvJTIwbW9kZWwuJTBBJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9yY2gudGVuc29yKCU1Qmxpc3QoJTIyTGlmZSUyMGlzJTIwbGlrZSUyMGElMjBib3glMjBvZiUyMGNob2NvbGF0ZXMuJTIyLmVuY29kZSglMjJ1dGYtOCUyMikpJTVEKSUyMCUyQiUyMG51bV9zcGVjaWFsX3Rva2VucyUwQSUwQWxhYmVscyUyMCUzRCUyMHRvcmNoLnRlbnNvciglNUJsaXN0KCUyMkxhJTIwdmllJTIwZXN0JTIwY29tbWUlMjB1bmUlMjBibyVDMyVBRXRlJTIwZGUlMjBjaG9jb2xhdC4lMjIuZW5jb2RlKCUyMnV0Zi04JTIyKSklNUQpJTIwJTJCJTIwbnVtX3NwZWNpYWxfdG9rZW5zJTBBJTBBbG9zcyUyMCUzRCUyMG1vZGVsKGlucHV0X2lkcyUyQyUyMGxhYmVscyUzRGxhYmVscykubG9zcyUwQWxvc3MuaXRlbSgp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5ForConditionalGeneration
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>model = T5ForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/byt5-small&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>num_special_tokens = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Model has 3 special tokens which take up the input ids 0,1,2 of ByT5.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># =&gt; Need to shift utf-8 character encodings by 3 before passing ids to model.</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = torch.tensor([<span class="hljs-built_in">list</span>(<span class="hljs-string">&quot;Life is like a box of chocolates.&quot;</span>.encode(<span class="hljs-string">&quot;utf-8&quot;</span>))]) + num_special_tokens

<span class="hljs-meta">&gt;&gt;&gt; </span>labels = torch.tensor([<span class="hljs-built_in">list</span>(<span class="hljs-string">&quot;La vie est comme une boîte de chocolat.&quot;</span>.encode(<span class="hljs-string">&quot;utf-8&quot;</span>))]) + num_special_tokens

<span class="hljs-meta">&gt;&gt;&gt; </span>loss = model(input_ids, labels=labels).loss
<span class="hljs-meta">&gt;&gt;&gt; </span>loss.item()
<span class="hljs-number">2.66</span>`}}),F=new ce({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFQ1Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwVDVGb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmJ5dDUtc21hbGwlMjIpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGYnl0NS1zbWFsbCUyMiklMEElMEFtb2RlbF9pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTBBJTIwJTIwJTIwJTIwJTVCJTIyTGlmZSUyMGlzJTIwbGlrZSUyMGElMjBib3glMjBvZiUyMGNob2NvbGF0ZXMuJTIyJTJDJTIwJTIyVG9kYXklMjBpcyUyME1vbmRheS4lMjIlNUQlMkMlMjBwYWRkaW5nJTNEJTIybG9uZ2VzdCUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIlMEEpJTBBbGFiZWxzX2RpY3QlMjAlM0QlMjB0b2tlbml6ZXIoJTBBJTIwJTIwJTIwJTIwJTVCJTIyTGElMjB2aWUlMjBlc3QlMjBjb21tZSUyMHVuZSUyMGJvJUMzJUFFdGUlMjBkZSUyMGNob2NvbGF0LiUyMiUyQyUyMCUyMkF1am91cmQnaHVpJTIwYydlc3QlMjBsdW5kaS4lMjIlNUQlMkMlMjBwYWRkaW5nJTNEJTIybG9uZ2VzdCUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIlMEEpJTBBbGFiZWxzJTIwJTNEJTIwbGFiZWxzX2RpY3QuaW5wdXRfaWRzJTBBJTBBbG9zcyUyMCUzRCUyMG1vZGVsKCoqbW9kZWxfaW5wdXRzJTJDJTIwbGFiZWxzJTNEbGFiZWxzKS5sb3NzJTBBbG9zcy5pdGVtKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5ForConditionalGeneration, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = T5ForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/byt5-small&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;google/byt5-small&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model_inputs = tokenizer(
<span class="hljs-meta">... </span>    [<span class="hljs-string">&quot;Life is like a box of chocolates.&quot;</span>, <span class="hljs-string">&quot;Today is Monday.&quot;</span>], padding=<span class="hljs-string">&quot;longest&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>labels_dict = tokenizer(
<span class="hljs-meta">... </span>    [<span class="hljs-string">&quot;La vie est comme une boîte de chocolat.&quot;</span>, <span class="hljs-string">&quot;Aujourd&#x27;hui c&#x27;est lundi.&quot;</span>], padding=<span class="hljs-string">&quot;longest&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>labels = labels_dict.input_ids

<span class="hljs-meta">&gt;&gt;&gt; </span>loss = model(**model_inputs, labels=labels).loss
<span class="hljs-meta">&gt;&gt;&gt; </span>loss.item()
<span class="hljs-number">17.9</span>`}}),L=new ce({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JTZXEyU2VxTE0lMEFpbXBvcnQlMjB0b3JjaCUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmJ5dDUtYmFzZSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTS5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGYnl0NS1iYXNlJTIyKSUwQSUwQWlucHV0X2lkc19wcm9tcHQlMjAlM0QlMjAlMjJUaGUlMjBkb2clMjBjaGFzZXMlMjBhJTIwYmFsbCUyMGluJTIwdGhlJTIwcGFyay4lMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfaWRzX3Byb21wdCkuaW5wdXRfaWRzJTBBJTBBJTIzJTIwTm90ZSUyMHRoYXQlMjB3ZSUyMGNhbm5vdCUyMGFkZCUyMCUyMiU3QmV4dHJhX2lkXy4uLiU3RCUyMiUyMHRvJTIwdGhlJTIwc3RyaW5nJTIwZGlyZWN0bHklMEElMjMlMjBhcyUyMHRoZSUyMEJ5dGUlMjB0b2tlbml6ZXIlMjB3b3VsZCUyMGluY29ycmVjdGx5JTIwbWVyZ2UlMjB0aGUlMjB0b2tlbnMlMEElMjMlMjBGb3IlMjBCeVQ1JTJDJTIwd2UlMjBuZWVkJTIwdG8lMjB3b3JrJTIwZGlyZWN0bHklMjBvbiUyMHRoZSUyMGNoYXJhY3RlciUyMGxldmVsJTBBJTIzJTIwQ29udHJhcnklMjB0byUyMFQ1JTJDJTIwQnlUNSUyMGRvZXMlMjBub3QlMjB1c2UlMjBzZW50aW5lbCUyMHRva2VucyUyMGZvciUyMG1hc2tpbmclMkMlMjBidXQlMjBpbnN0ZWFkJTBBJTIzJTIwdXNlcyUyMGZpbmFsJTIwdXRmJTIwY2hhcmFjdGVyJTIwaWRzLiUwQSUyMyUyMFVURi04JTIwaXMlMjByZXByZXNlbnRlZCUyMGJ5JTIwOCUyMGJpdHMlMjBhbmQlMjBCeVQ1JTIwaGFzJTIwMyUyMHNwZWNpYWwlMjB0b2tlbnMuJTBBJTIzJTIwJTNEJTNFJTIwVGhlcmUlMjBhcmUlMjAyKio4JTJCMiUyMCUzRCUyMDI1OSUyMGlucHV0JTIwaWRzJTIwYW5kJTIwbWFzayUyMHRva2VucyUyMGNvdW50JTIwZG93biUyMGZyb20lMjBpbmRleCUyMDI1OC4lMEElMjMlMjAlM0QlM0UlMjBtYXNrJTIwdG8lMjAlMjJUaGUlMjBkb2clMjAlNUIyNTglNURhJTIwYmFsbCUyMCU1QjI1NyU1RHBhcmsuJTIyJTBBJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9yY2gudGVuc29yKCU1QmlucHV0X2lkcyU1QiUzQTglNUQlMjAlMkIlMjAlNUIyNTglNUQlMjAlMkIlMjBpbnB1dF9pZHMlNUIxNCUzQTIxJTVEJTIwJTJCJTIwJTVCMjU3JTVEJTIwJTJCJTIwaW5wdXRfaWRzJTVCMjglM0ElNUQlNUQpJTBBaW5wdXRfaWRzJTBBJTBBJTIzJTIwQnlUNSUyMHByb2R1Y2VzJTIwb25seSUyMG9uZSUyMGNoYXIlMjBhdCUyMGElMjB0aW1lJTIwc28lMjB3ZSUyMG5lZWQlMjB0byUyMHByb2R1Y2UlMjBtYW55JTIwbW9yZSUyMG91dHB1dCUyMGNoYXJhY3RlcnMlMjBoZXJlJTIwLSUzRSUyMHNldCUyMCU2MG1heF9sZW5ndGglM0QxMDAlNjAuJTBBb3V0cHV0X2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKGlucHV0X2lkcyUyQyUyMG1heF9sZW5ndGglM0QxMDApJTVCMCU1RC50b2xpc3QoKSUwQW91dHB1dF9pZHMlMEElMEElMjMlMjAlNUUtJTIwTm90ZSUyMGhvdyUyMDI1OCUyMGRlc2NlbmRzJTIwdG8lMjAyNTclMkMlMjAyNTYlMkMlMjAyNTUlMEElMEElMjMlMjBOb3clMjB3ZSUyMG5lZWQlMjB0byUyMHNwbGl0JTIwb24lMjB0aGUlMjBzZW50aW5lbCUyMHRva2VucyUyQyUyMGxldCdzJTIwd3JpdGUlMjBhJTIwc2hvcnQlMjBsb29wJTIwZm9yJTIwdGhpcyUwQW91dHB1dF9pZHNfbGlzdCUyMCUzRCUyMCU1QiU1RCUwQXN0YXJ0X3Rva2VuJTIwJTNEJTIwMCUwQXNlbnRpbmVsX3Rva2VuJTIwJTNEJTIwMjU4JTBBd2hpbGUlMjBzZW50aW5lbF90b2tlbiUyMGluJTIwb3V0cHV0X2lkcyUzQSUwQSUyMCUyMCUyMCUyMHNwbGl0X2lkeCUyMCUzRCUyMG91dHB1dF9pZHMuaW5kZXgoc2VudGluZWxfdG9rZW4pJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2lkc19saXN0LmFwcGVuZChvdXRwdXRfaWRzJTVCc3RhcnRfdG9rZW4lM0FzcGxpdF9pZHglNUQpJTBBJTIwJTIwJTIwJTIwc3RhcnRfdG9rZW4lMjAlM0QlMjBzcGxpdF9pZHglMEElMjAlMjAlMjAlMjBzZW50aW5lbF90b2tlbiUyMC0lM0QlMjAxJTBBJTBBb3V0cHV0X2lkc19saXN0LmFwcGVuZChvdXRwdXRfaWRzJTVCc3RhcnRfdG9rZW4lM0ElNUQpJTBBb3V0cHV0X3N0cmluZyUyMCUzRCUyMHRva2VuaXplci5iYXRjaF9kZWNvZGUob3V0cHV0X2lkc19saXN0KSUwQW91dHB1dF9zdHJpbmc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSeq2SeqLM
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;google/byt5-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;google/byt5-base&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids_prompt = <span class="hljs-string">&quot;The dog chases a ball in the park.&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(input_ids_prompt).input_ids

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Note that we cannot add &quot;{extra_id_...}&quot; to the string directly</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># as the Byte tokenizer would incorrectly merge the tokens</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># For ByT5, we need to work directly on the character level</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Contrary to T5, ByT5 does not use sentinel tokens for masking, but instead</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># uses final utf character ids.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># UTF-8 is represented by 8 bits and ByT5 has 3 special tokens.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># =&gt; There are 2**8+2 = 259 input ids and mask tokens count down from index 258.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># =&gt; mask to &quot;The dog [258]a ball [257]park.&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = torch.tensor([input_ids[:<span class="hljs-number">8</span>] + [<span class="hljs-number">258</span>] + input_ids[<span class="hljs-number">14</span>:<span class="hljs-number">21</span>] + [<span class="hljs-number">257</span>] + input_ids[<span class="hljs-number">28</span>:]])
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids
tensor([[ <span class="hljs-number">87</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">103</span>, <span class="hljs-number">114</span>, <span class="hljs-number">106</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">258</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">100</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">101</span>, <span class="hljs-number">100</span>, <span class="hljs-number">111</span>, <span class="hljs-number">111</span>, <span class="hljs-number">257</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">115</span>, <span class="hljs-number">100</span>, <span class="hljs-number">117</span>, <span class="hljs-number">110</span>,  <span class="hljs-number">49</span>,   <span class="hljs-number">1</span>]])

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># ByT5 produces only one char at a time so we need to produce many more output characters here -&gt; set \`max_length=100\`.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>output_ids = model.generate(input_ids, max_length=<span class="hljs-number">100</span>)[<span class="hljs-number">0</span>].tolist()
<span class="hljs-meta">&gt;&gt;&gt; </span>output_ids
[<span class="hljs-number">0</span>, <span class="hljs-number">258</span>, <span class="hljs-number">108</span>, <span class="hljs-number">118</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">119</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">114</span>, <span class="hljs-number">113</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">122</span>, <span class="hljs-number">107</span>, <span class="hljs-number">114</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">103</span>, <span class="hljs-number">114</span>, <span class="hljs-number">104</span>, <span class="hljs-number">118</span>, <span class="hljs-number">257</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">108</span>, <span class="hljs-number">113</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">119</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">103</span>, <span class="hljs-number">108</span>, <span class="hljs-number">118</span>, <span class="hljs-number">102</span>, <span class="hljs-number">114</span>, <span class="hljs-number">256</span>, <span class="hljs-number">108</span>, <span class="hljs-number">113</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">119</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>, <span class="hljs-number">35</span>, <span class="hljs-number">115</span>, <span class="hljs-number">100</span>, <span class="hljs-number">117</span>, <span class="hljs-number">110</span>,  <span class="hljs-number">49</span>,  <span class="hljs-number">35</span>,  <span class="hljs-number">87</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">103</span>, <span class="hljs-number">114</span>, <span class="hljs-number">106</span>, <span class="hljs-number">35</span>, <span class="hljs-number">108</span>, <span class="hljs-number">118</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">119</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">114</span>, <span class="hljs-number">113</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">122</span>, <span class="hljs-number">107</span>, <span class="hljs-number">114</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">103</span>, <span class="hljs-number">114</span>, <span class="hljs-number">104</span>, <span class="hljs-number">118</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">100</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">101</span>, <span class="hljs-number">100</span>, <span class="hljs-number">111</span>, <span class="hljs-number">111</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">108</span>, <span class="hljs-number">113</span>, <span class="hljs-number">255</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">108</span>, <span class="hljs-number">113</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">119</span>, <span class="hljs-number">107</span>, <span class="hljs-number">104</span>,  <span class="hljs-number">35</span>, <span class="hljs-number">115</span>, <span class="hljs-number">100</span>, <span class="hljs-number">117</span>, <span class="hljs-number">110</span>,  <span class="hljs-number">49</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># ^- Note how 258 descends to 257, 256, 255</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Now we need to split on the sentinel tokens, let&#x27;s write a short loop for this</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>output_ids_list = []
<span class="hljs-meta">&gt;&gt;&gt; </span>start_token = <span class="hljs-number">0</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>sentinel_token = <span class="hljs-number">258</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">while</span> sentinel_token <span class="hljs-keyword">in</span> output_ids:
<span class="hljs-meta">... </span>    split_idx = output_ids.index(sentinel_token)
<span class="hljs-meta">... </span>    output_ids_list.append(output_ids[start_token:split_idx])
<span class="hljs-meta">... </span>    start_token = split_idx
<span class="hljs-meta">... </span>    sentinel_token -= <span class="hljs-number">1</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>output_ids_list.append(output_ids[start_token:])
<span class="hljs-meta">&gt;&gt;&gt; </span>output_string = tokenizer.batch_decode(output_ids_list)
<span class="hljs-meta">&gt;&gt;&gt; </span>output_string
[<span class="hljs-string">&#x27;&lt;pad&gt;&#x27;</span>, <span class="hljs-string">&#x27;is the one who does&#x27;</span>, <span class="hljs-string">&#x27; in the disco&#x27;</span>, <span class="hljs-string">&#x27;in the park. The dog is the one who does a ball in&#x27;</span>, <span class="hljs-string">&#x27; in the park.&#x27;</span>]`}}),A=new As({}),Y=new Ms({props:{name:"class transformers.ByT5Tokenizer",anchor:"transformers.ByT5Tokenizer",parameters:[{name:"eos_token",val:" = '</s>'"},{name:"unk_token",val:" = '<unk>'"},{name:"pad_token",val:" = '<pad>'"},{name:"extra_ids",val:" = 125"},{name:"additional_special_tokens",val:" = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.ByT5Tokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the end of sequence.
The token used is the <code>sep_token</code>.</p>

					</div>`,name:"eos_token"},{anchor:"transformers.ByT5Tokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.ByT5Tokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;pad&gt;&quot;</code>) &#x2014;
The token used for padding, for example when batching sequences of different lengths.`,name:"pad_token"},{anchor:"transformers.ByT5Tokenizer.extra_ids",description:`<strong>extra_ids</strong> (<code>int</code>, <em>optional</em>, defaults to 100) &#x2014;
Add a number of extra ids added to the end of the vocabulary for use as sentinels. These tokens are
accessible as &#x201C;<extra<em>id{%d}&gt;&#x201D; where &#x201D;{%d}&#x201D; is a number between 0 and extra_ids-1. Extra tokens are
indexed from the end of the vocabulary up to beginning (&#x201C;<extra_id_0>&#x201D; is the last token in the vocabulary
like in ByT5 preprocessing see
<a href="https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117" rel="nofollow">here</a>).</extra_id_0></extra<em>`,name:"extra_ids"},{anchor:"transformers.ByT5Tokenizer.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
Additional special tokens used by the tokenizer.`,name:"additional_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/byt5/tokenization_byt5.py#L28"}}),D=new Ms({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.ByT5Tokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.ByT5Tokenizer.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.ByT5Tokenizer.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/byt5/tokenization_byt5.py#L179",returnDescription:`
<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),P=new Ms({props:{name:"convert_tokens_to_string",anchor:"transformers.ByT5Tokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/byt5/tokenization_byt5.py#L230"}}),K=new Ms({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.ByT5Tokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.ByT5Tokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.ByT5Tokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/byt5/tokenization_byt5.py#L157",returnDescription:`
<p>List of zeros.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),O=new Ms({props:{name:"get_special_tokens_mask",anchor:"transformers.ByT5Tokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.ByT5Tokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.ByT5Tokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.ByT5Tokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/byt5/tokenization_byt5.py#L118",returnDescription:`
<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),{c(){T=n("meta"),fs=r(),ds=n("p"),Ts=r(),k=n("h1"),U=n("a"),us=n("span"),u(z.$$.fragment),Ys=r(),es=n("span"),es.textContent=he,ks=r(),_=n("h2"),v=n("a"),bs=n("span"),u(Z.$$.fragment),Ds=r(),ts=n("span"),ts.textContent=de,_s=r(),W=n("p"),W.innerHTML=ue,ws=r(),R=n("p"),R.textContent=be,Js=r(),V=n("p"),V.innerHTML=ye,Us=r(),N=n("p"),N.innerHTML=ge,vs=r(),$=n("p"),$.innerHTML=je,Bs=r(),X=n("p"),X.textContent=Me,xs=r(),w=n("h3"),B=n("a"),ys=n("span"),u(E.$$.fragment),Ps=r(),ns=n("span"),ns.textContent=fe,Is=r(),Q=n("p"),Q.textContent=Te,Cs=r(),u(H.$$.fragment),Gs=r(),q=n("p"),q.textContent=ke,zs=r(),u(F.$$.fragment),Zs=r(),S=n("p"),S.innerHTML=_e,Ws=r(),u(L.$$.fragment),Rs=r(),J=n("h2"),x=n("a"),gs=n("span"),u(A.$$.fragment),Ks=r(),as=n("span"),as.textContent=we,Vs=r(),m=n("div"),u(Y.$$.fragment),Os=r(),ls=n("p"),ls.textContent=Je,se=r(),os=n("p"),os.innerHTML=Ue,ee=r(),f=n("div"),u(D.$$.fragment),te=r(),rs=n("p"),rs.textContent=ve,ne=r(),ps=n("ul"),ps.innerHTML=Be,ae=r(),I=n("div"),u(P.$$.fragment),le=r(),is=n("p"),is.textContent=xe,oe=r(),C=n("div"),u(K.$$.fragment),re=r(),cs=n("p"),cs.textContent=Ie,pe=r(),G=n("div"),u(O.$$.fragment),ie=r(),ms=n("p"),ms.innerHTML=Ce,Ns=r(),ss=n("p"),ss.innerHTML=Ge,$s=r(),js=n("p"),this.h()},l(s){const t=Se("svelte-1phssyn",document.head);T=a(t,"META",{name:!0,content:!0}),t.forEach(e),fs=p(s),ds=a(s,"P",{}),h(ds).forEach(e),Ts=p(s),k=a(s,"H1",{class:!0});var Es=h(k);U=a(Es,"A",{id:!0,class:!0,href:!0});var ze=h(U);us=a(ze,"SPAN",{});var Ze=h(us);b(z.$$.fragment,Ze),Ze.forEach(e),ze.forEach(e),Ys=p(Es),es=a(Es,"SPAN",{"data-svelte-h":!0}),c(es)!=="svelte-1f3vqc7"&&(es.textContent=he),Es.forEach(e),ks=p(s),_=a(s,"H2",{class:!0});var Qs=h(_);v=a(Qs,"A",{id:!0,class:!0,href:!0});var We=h(v);bs=a(We,"SPAN",{});var Re=h(bs);b(Z.$$.fragment,Re),Re.forEach(e),We.forEach(e),Ds=p(Qs),ts=a(Qs,"SPAN",{"data-svelte-h":!0}),c(ts)!=="svelte-1jsw1pg"&&(ts.textContent=de),Qs.forEach(e),_s=p(s),W=a(s,"P",{"data-svelte-h":!0}),c(W)!=="svelte-1vxwc5j"&&(W.innerHTML=ue),ws=p(s),R=a(s,"P",{"data-svelte-h":!0}),c(R)!=="svelte-vfdo9a"&&(R.textContent=be),Js=p(s),V=a(s,"P",{"data-svelte-h":!0}),c(V)!=="svelte-1jd8hlb"&&(V.innerHTML=ye),Us=p(s),N=a(s,"P",{"data-svelte-h":!0}),c(N)!=="svelte-19tarla"&&(N.innerHTML=ge),vs=p(s),$=a(s,"P",{"data-svelte-h":!0}),c($)!=="svelte-hkjszk"&&($.innerHTML=je),Bs=p(s),X=a(s,"P",{"data-svelte-h":!0}),c(X)!=="svelte-x9ykzc"&&(X.textContent=Me),xs=p(s),w=a(s,"H3",{class:!0});var Hs=h(w);B=a(Hs,"A",{id:!0,class:!0,href:!0});var Ve=h(B);ys=a(Ve,"SPAN",{});var Ne=h(ys);b(E.$$.fragment,Ne),Ne.forEach(e),Ve.forEach(e),Ps=p(Hs),ns=a(Hs,"SPAN",{"data-svelte-h":!0}),c(ns)!=="svelte-1o0sfn1"&&(ns.textContent=fe),Hs.forEach(e),Is=p(s),Q=a(s,"P",{"data-svelte-h":!0}),c(Q)!=="svelte-11rzxcx"&&(Q.textContent=Te),Cs=p(s),b(H.$$.fragment,s),Gs=p(s),q=a(s,"P",{"data-svelte-h":!0}),c(q)!=="svelte-1q3gwa3"&&(q.textContent=ke),zs=p(s),b(F.$$.fragment,s),Zs=p(s),S=a(s,"P",{"data-svelte-h":!0}),c(S)!=="svelte-yy2lov"&&(S.innerHTML=_e),Ws=p(s),b(L.$$.fragment,s),Rs=p(s),J=a(s,"H2",{class:!0});var qs=h(J);x=a(qs,"A",{id:!0,class:!0,href:!0});var $e=h(x);gs=a($e,"SPAN",{});var Xe=h(gs);b(A.$$.fragment,Xe),Xe.forEach(e),$e.forEach(e),Ks=p(qs),as=a(qs,"SPAN",{"data-svelte-h":!0}),c(as)!=="svelte-1eoeabo"&&(as.textContent=we),qs.forEach(e),Vs=p(s),m=a(s,"DIV",{class:!0});var d=h(m);b(Y.$$.fragment,d),Os=p(d),ls=a(d,"P",{"data-svelte-h":!0}),c(ls)!=="svelte-ab2b05"&&(ls.textContent=Je),se=p(d),os=a(d,"P",{"data-svelte-h":!0}),c(os)!=="svelte-14xlfoy"&&(os.innerHTML=Ue),ee=p(d),f=a(d,"DIV",{class:!0});var hs=h(f);b(D.$$.fragment,hs),te=p(hs),rs=a(hs,"P",{"data-svelte-h":!0}),c(rs)!=="svelte-1wjq39d"&&(rs.textContent=ve),ne=p(hs),ps=a(hs,"UL",{"data-svelte-h":!0}),c(ps)!=="svelte-8gh3n2"&&(ps.innerHTML=Be),hs.forEach(e),ae=p(d),I=a(d,"DIV",{class:!0});var Fs=h(I);b(P.$$.fragment,Fs),le=p(Fs),is=a(Fs,"P",{"data-svelte-h":!0}),c(is)!=="svelte-b3k2yi"&&(is.textContent=xe),Fs.forEach(e),oe=p(d),C=a(d,"DIV",{class:!0});var Ss=h(C);b(K.$$.fragment,Ss),re=p(Ss),cs=a(Ss,"P",{"data-svelte-h":!0}),c(cs)!=="svelte-on9fcl"&&(cs.textContent=Ie),Ss.forEach(e),pe=p(d),G=a(d,"DIV",{class:!0});var Ls=h(G);b(O.$$.fragment,Ls),ie=p(Ls),ms=a(Ls,"P",{"data-svelte-h":!0}),c(ms)!=="svelte-1f4f5kp"&&(ms.innerHTML=Ce),Ls.forEach(e),d.forEach(e),Ns=p(s),ss=a(s,"P",{"data-svelte-h":!0}),c(ss)!=="svelte-13fjr7e"&&(ss.innerHTML=Ge),$s=p(s),js=a(s,"P",{}),h(js).forEach(e),this.h()},h(){i(T,"name","hf:doc:metadata"),i(T,"content",JSON.stringify(Ae)),i(U,"id","byt5"),i(U,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(U,"href","#byt5"),i(k,"class","relative group"),i(v,"id","overview"),i(v,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(v,"href","#overview"),i(_,"class","relative group"),i(B,"id","example"),i(B,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(B,"href","#example"),i(w,"class","relative group"),i(x,"id","transformers.ByT5Tokenizer"),i(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(x,"href","#transformers.ByT5Tokenizer"),i(J,"class","relative group"),i(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(m,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(s,t){l(document.head,T),o(s,fs,t),o(s,ds,t),o(s,Ts,t),o(s,k,t),l(k,U),l(U,us),y(z,us,null),l(k,Ys),l(k,es),o(s,ks,t),o(s,_,t),l(_,v),l(v,bs),y(Z,bs,null),l(_,Ds),l(_,ts),o(s,_s,t),o(s,W,t),o(s,ws,t),o(s,R,t),o(s,Js,t),o(s,V,t),o(s,Us,t),o(s,N,t),o(s,vs,t),o(s,$,t),o(s,Bs,t),o(s,X,t),o(s,xs,t),o(s,w,t),l(w,B),l(B,ys),y(E,ys,null),l(w,Ps),l(w,ns),o(s,Is,t),o(s,Q,t),o(s,Cs,t),y(H,s,t),o(s,Gs,t),o(s,q,t),o(s,zs,t),y(F,s,t),o(s,Zs,t),o(s,S,t),o(s,Ws,t),y(L,s,t),o(s,Rs,t),o(s,J,t),l(J,x),l(x,gs),y(A,gs,null),l(J,Ks),l(J,as),o(s,Vs,t),o(s,m,t),y(Y,m,null),l(m,Os),l(m,ls),l(m,se),l(m,os),l(m,ee),l(m,f),y(D,f,null),l(f,te),l(f,rs),l(f,ne),l(f,ps),l(m,ae),l(m,I),y(P,I,null),l(I,le),l(I,is),l(m,oe),l(m,C),y(K,C,null),l(C,re),l(C,cs),l(m,pe),l(m,G),y(O,G,null),l(G,ie),l(G,ms),o(s,Ns,t),o(s,ss,t),o(s,$s,t),o(s,js,t),Xs=!0},p:Qe,i(s){Xs||(g(z.$$.fragment,s),g(Z.$$.fragment,s),g(E.$$.fragment,s),g(H.$$.fragment,s),g(F.$$.fragment,s),g(L.$$.fragment,s),g(A.$$.fragment,s),g(Y.$$.fragment,s),g(D.$$.fragment,s),g(P.$$.fragment,s),g(K.$$.fragment,s),g(O.$$.fragment,s),Xs=!0)},o(s){j(z.$$.fragment,s),j(Z.$$.fragment,s),j(E.$$.fragment,s),j(H.$$.fragment,s),j(F.$$.fragment,s),j(L.$$.fragment,s),j(A.$$.fragment,s),j(Y.$$.fragment,s),j(D.$$.fragment,s),j(P.$$.fragment,s),j(K.$$.fragment,s),j(O.$$.fragment,s),Xs=!1},d(s){s&&(e(fs),e(ds),e(Ts),e(k),e(ks),e(_),e(_s),e(W),e(ws),e(R),e(Js),e(V),e(Us),e(N),e(vs),e($),e(Bs),e(X),e(xs),e(w),e(Is),e(Q),e(Cs),e(Gs),e(q),e(zs),e(Zs),e(S),e(Ws),e(Rs),e(J),e(Vs),e(m),e(Ns),e(ss),e($s),e(js)),e(T),M(z),M(Z),M(E),M(H,s),M(F,s),M(L,s),M(A),M(Y),M(D),M(P),M(K),M(O)}}}const Ae={local:"byt5",sections:[{local:"overview",sections:[{local:"example",title:"Example"}],title:"Overview"},{local:"transformers.ByT5Tokenizer",title:"ByT5Tokenizer"}],title:"ByT5"};function Ye(me){return He(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class et extends qe{constructor(T){super(),Fe(this,T,Ye,Le,Ee,{})}}export{et as component};
