import{S as kn,i as vn,s as Tn,e as o,k as d,w as E,t as s,M as Sn,c as r,d as n,m,a,x as y,h as i,b as p,G as t,g as c,y as x,q as A,o as G,B as M,v as bn,L as Pn}from"../../chunks/vendor-hf-doc-builder.js";import{D as _n}from"../../chunks/Docstring-hf-doc-builder.js";import{C as wn}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as Re}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as $n}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function zn(Pe){let u,I,k,g,v;return g=new wn({props:{code:`from transformers import GPTSw3Tokenizer
tokenizer = GPTSw3Tokenizer.from_pretrained("AI-Sweden/gpt-sw3-126m")
tokenizer("Svenska \xE4r kul!")['input_ids']`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python"><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTSw3Tokenizer</span>
<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">tokenizer = GPTSw3Tokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-126m&quot;</span>)</span>
<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">tokenizer(<span class="hljs-string">&quot;Svenska \xE4r kul!&quot;</span>)[<span class="hljs-string">&#x27;input_ids&#x27;</span>]</span>
[1814, 377, 3617, 63504]`}}),{c(){u=o("p"),I=s("Example usage:"),k=d(),E(g.$$.fragment)},l(h){u=r(h,"P",{});var T=a(u);I=i(T,"Example usage:"),T.forEach(n),k=m(h),y(g.$$.fragment,h)},m(h,T){c(h,u,T),t(u,I),c(h,k,T),x(g,h,T),v=!0},p:Pn,i(h){v||(A(g.$$.fragment,h),v=!0)},o(h){G(g.$$.fragment,h),v=!1},d(h){h&&n(u),h&&n(k),M(g,h)}}}function En(Pe){let u,I,k,g,v,h,T,le,Ue,$e,P,j,pe,F,Ve,ce,Je,ze,L,Ke,B,Qe,Xe,Ee,Z,Ye,ye,ee,Ze,xe,C,et,O,tt,nt,Ae,f,ot,W,rt,at,de,st,it,me,lt,pt,he,ct,dt,fe,mt,ht,ue,ft,ut,ge,gt,Ge,te,_t,Me,H,Ie,$,N,_e,R,wt,we,kt,je,S,ke,ne,vt,Tt,ve,oe,St,bt,Te,re,Pt,Le,z,q,Se,U,$t,be,zt,Ce,_,V,Et,J,yt,K,xt,At,Gt,Q,Mt,ae,It,jt,Lt,D,Ct,se,X,Ne;return h=new Re({}),F=new Re({}),H=new wn({props:{code:`from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("AI-Sweden/gpt-sw3-356m")
model = AutoModelForCausalLM.from_pretrained("AI-Sweden/gpt-sw3-356m")

input_ids = tokenizer("Tr\xE4d \xE4r fina f\xF6r att", return_tensors="pt")["input_ids"]

generated_token_ids = model.generate(inputs=input_ids, max_new_tokens=10, do_sample=True)[0]

print(tokenizer.decode(generated_token_ids))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-356m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-356m&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(<span class="hljs-string">&quot;Tr\xE4d \xE4r fina f\xF6r att&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_token_ids = model.generate(inputs=input_ids, max_new_tokens=<span class="hljs-number">10</span>, do_sample=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(generated_token_ids))
Tr\xE4d \xE4r fina f\xF6r att de \xE4r f\xE4rgstarka. Men ibland \xE4r det fint`}}),R=new Re({}),U=new Re({}),V=new _n({props:{name:"class transformers.GPTSw3Tokenizer",anchor:"transformers.GPTSw3Tokenizer",parameters:[{name:"vocab_file",val:""},{name:"do_lower_case",val:" = False"},{name:"remove_space",val:" = False"},{name:"keep_accents",val:" = False"},{name:"pad_token",val:" = None"},{name:"unk_token",val:" = None"},{name:"eos_token",val:" = None"},{name:"bos_token",val:" = None"},{name:"sp_model_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTSw3Tokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
<a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a <em>.spm</em> extension) that
contains the vocabulary necessary to instantiate a tokenizer.`,name:"vocab_file"},{anchor:"transformers.GPTSw3Tokenizer.do_lower_case",description:`<strong>do_lower_case</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to lowercase the input when tokenizing.`,name:"do_lower_case"},{anchor:"transformers.GPTSw3Tokenizer.remove_space",description:`<strong>remove_space</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to strip the text when tokenizing (removing excess spaces before and after the string).`,name:"remove_space"},{anchor:"transformers.GPTSw3Tokenizer.keep_accents",description:`<strong>keep_accents</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to keep accents when tokenizing.`,name:"keep_accents"},{anchor:"transformers.GPTSw3Tokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The beginning of sequence token that can be used for downstream task, was not seen during pretraining. If
not provided, will default to &#x2019;<s>&#x2019; or &#x2019;&lt;|endoftext|&gt;&#x2019;, depending on model size.</s>`,name:"bos_token"},{anchor:"transformers.GPTSw3Tokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The end of sequence token seen during pretraining. If not provided, will default to &#x2019;&lt;|endoftext|&gt;&#x2019;`,name:"eos_token"},{anchor:"transformers.GPTSw3Tokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead. If not provided, will default to &#x2019;<unk>&#x2018;.</unk>`,name:"unk_token"},{anchor:"transformers.GPTSw3Tokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The token used for padding, for example when batching sequences of different lengths. If not provided, will
default to &#x2019;<pad>&#x2019; or &#x2019;<unk>&#x2019; depending on model size.</unk></pad>`,name:"pad_token"},{anchor:"transformers.GPTSw3Tokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Will be passed to the <code>SentencePieceProcessor.__init__()</code> method. The <a href="https://github.com/google/sentencepiece/tree/master/python" rel="nofollow">Python wrapper for
SentencePiece</a> can be used, among other things,
to set:</p>
<ul>
<li>
<p><code>enable_sampling</code>: Enable subword regularization.</p>
</li>
<li>
<p><code>nbest_size</code>: Sampling parameters for unigram. Invalid for BPE-Dropout.</p>
<ul>
<li><code>nbest_size = {0,1}</code>: No sampling is performed.</li>
<li><code>nbest_size &gt; 1</code>: samples from the nbest_size results.</li>
<li><code>nbest_size &lt; 0</code>: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
using forward-filtering-and-backward-sampling algorithm.</li>
</ul>
</li>
<li>
<p><code>alpha</code>: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
BPE-dropout.</p>
</li>
</ul>`,name:"sp_model_kwargs"},{anchor:"transformers.GPTSw3Tokenizer.sp_model",description:`<strong>sp_model</strong> (<code>SentencePieceProcessor</code>) &#x2014;
The <em>SentencePiece</em> processor that is used for every conversion (string, tokens and IDs).`,name:"sp_model"},{anchor:"transformers.GPTSw3Tokenizer.whitespaces",description:`<strong>whitespaces</strong> (<code>set</code>) &#x2014;
The whitespaces that are replaced in the whitespace normalization in preprocessing.`,name:"whitespaces"},{anchor:"transformers.GPTSw3Tokenizer.non_printing_characters_re",description:`<strong>non_printing_characters_re</strong> (<code>Pattern</code>) &#x2014;
The compiled regular expression to remove non-printing characters in preprocessing.`,name:"non_printing_characters_re"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L42"}}),D=new $n({props:{anchor:"transformers.GPTSw3Tokenizer.example",$$slots:{default:[zn]},$$scope:{ctx:Pe}}}),X=new _n({props:{name:"save_vocabulary",anchor:"transformers.GPTSw3Tokenizer.save_vocabulary",parameters:[{name:"save_directory",val:": str"},{name:"filename_prefix",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/transformers/blob/v4.27.1/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L251"}}),{c(){u=o("meta"),I=d(),k=o("h1"),g=o("a"),v=o("span"),E(h.$$.fragment),T=d(),le=o("span"),Ue=s("GPT-Sw3"),$e=d(),P=o("h2"),j=o("a"),pe=o("span"),E(F.$$.fragment),Ve=d(),ce=o("span"),Je=s("Overview"),ze=d(),L=o("p"),Ke=s(`The GPT-Sw3 model was first proposed in
`),B=o("a"),Qe=s("Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish"),Xe=s(`
by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey \xD6hman,
Fredrik Carlsson, Magnus Sahlgren.`),Ee=d(),Z=o("p"),Ye=s("Since that first paper the authors have extended their work and trained new models on their new 1.2TB corpora named The Nordic Pile."),ye=d(),ee=o("p"),Ze=s(`GPT-Sw3 is a collection of large decoder-only pretrained transformer language models that were developed by AI Sweden
in collaboration with RISE and the WASP WARA for Media and Language. GPT-Sw3 has been trained on a dataset containing
320B tokens in Swedish, Norwegian, Danish, Icelandic, English, and programming code. The model was pretrained using a
causal language modeling (CLM) objective utilizing the NeMo Megatron GPT implementation.`),xe=d(),C=o("p"),et=s("This model was contributed by "),O=o("a"),tt=s("AI Sweden"),nt=s("."),Ae=d(),f=o("p"),ot=s("The implementation uses the "),W=o("a"),rt=s("GPT2Model"),at=s(` coupled
with our `),de=o("code"),st=s("GPTSw3Tokenizer"),it=s(". This means that "),me=o("code"),lt=s("AutoTokenizer"),pt=s(" and "),he=o("code"),ct=s("AutoModelForCausalLM"),dt=s(` map to our tokenizer
implementation and the corresponding GPT2 model implementation respectively.
`),fe=o("em"),mt=s("Note that sentencepiece is required to use our tokenizer and can be installed with:"),ht=d(),ue=o("code"),ft=s("pip install transformers[sentencepiece]"),ut=s(" or "),ge=o("code"),gt=s("pip install sentencepiece"),Ge=d(),te=o("p"),_t=s("Example usage:"),Me=d(),E(H.$$.fragment),Ie=d(),$=o("h2"),N=o("a"),_e=o("span"),E(R.$$.fragment),wt=d(),we=o("span"),kt=s("Documentation resources"),je=d(),S=o("ul"),ke=o("li"),ne=o("a"),vt=s("Text classification task guide"),Tt=d(),ve=o("li"),oe=o("a"),St=s("Token classification task guide"),bt=d(),Te=o("li"),re=o("a"),Pt=s("Causal language modeling task guide"),Le=d(),z=o("h2"),q=o("a"),Se=o("span"),E(U.$$.fragment),$t=d(),be=o("span"),zt=s("GPTSw3Tokenizer"),Ce=d(),_=o("div"),E(V.$$.fragment),Et=d(),J=o("p"),yt=s("Construct an GPTSw3 tokenizer. Based on "),K=o("a"),xt=s("SentencePiece"),At=s("."),Gt=d(),Q=o("p"),Mt=s("This tokenizer inherits from "),ae=o("a"),It=s("PreTrainedTokenizer"),jt=s(` which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`),Lt=d(),E(D.$$.fragment),Ct=d(),se=o("div"),E(X.$$.fragment),this.h()},l(e){const l=Sn('[data-svelte="svelte-1phssyn"]',document.head);u=r(l,"META",{name:!0,content:!0}),l.forEach(n),I=m(e),k=r(e,"H1",{class:!0});var Y=a(k);g=r(Y,"A",{id:!0,class:!0,href:!0});var Nt=a(g);v=r(Nt,"SPAN",{});var qt=a(v);y(h.$$.fragment,qt),qt.forEach(n),Nt.forEach(n),T=m(Y),le=r(Y,"SPAN",{});var Dt=a(le);Ue=i(Dt,"GPT-Sw3"),Dt.forEach(n),Y.forEach(n),$e=m(e),P=r(e,"H2",{class:!0});var qe=a(P);j=r(qe,"A",{id:!0,class:!0,href:!0});var Ft=a(j);pe=r(Ft,"SPAN",{});var Bt=a(pe);y(F.$$.fragment,Bt),Bt.forEach(n),Ft.forEach(n),Ve=m(qe),ce=r(qe,"SPAN",{});var Ot=a(ce);Je=i(Ot,"Overview"),Ot.forEach(n),qe.forEach(n),ze=m(e),L=r(e,"P",{});var De=a(L);Ke=i(De,`The GPT-Sw3 model was first proposed in
`),B=r(De,"A",{href:!0,rel:!0});var Wt=a(B);Qe=i(Wt,"Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish"),Wt.forEach(n),Xe=i(De,`
by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey \xD6hman,
Fredrik Carlsson, Magnus Sahlgren.`),De.forEach(n),Ee=m(e),Z=r(e,"P",{});var Ht=a(Z);Ye=i(Ht,"Since that first paper the authors have extended their work and trained new models on their new 1.2TB corpora named The Nordic Pile."),Ht.forEach(n),ye=m(e),ee=r(e,"P",{});var Rt=a(ee);Ze=i(Rt,`GPT-Sw3 is a collection of large decoder-only pretrained transformer language models that were developed by AI Sweden
in collaboration with RISE and the WASP WARA for Media and Language. GPT-Sw3 has been trained on a dataset containing
320B tokens in Swedish, Norwegian, Danish, Icelandic, English, and programming code. The model was pretrained using a
causal language modeling (CLM) objective utilizing the NeMo Megatron GPT implementation.`),Rt.forEach(n),xe=m(e),C=r(e,"P",{});var Fe=a(C);et=i(Fe,"This model was contributed by "),O=r(Fe,"A",{href:!0,rel:!0});var Ut=a(O);tt=i(Ut,"AI Sweden"),Ut.forEach(n),nt=i(Fe,"."),Fe.forEach(n),Ae=m(e),f=r(e,"P",{});var w=a(f);ot=i(w,"The implementation uses the "),W=r(w,"A",{href:!0,rel:!0});var Vt=a(W);rt=i(Vt,"GPT2Model"),Vt.forEach(n),at=i(w,` coupled
with our `),de=r(w,"CODE",{});var Jt=a(de);st=i(Jt,"GPTSw3Tokenizer"),Jt.forEach(n),it=i(w,". This means that "),me=r(w,"CODE",{});var Kt=a(me);lt=i(Kt,"AutoTokenizer"),Kt.forEach(n),pt=i(w," and "),he=r(w,"CODE",{});var Qt=a(he);ct=i(Qt,"AutoModelForCausalLM"),Qt.forEach(n),dt=i(w,` map to our tokenizer
implementation and the corresponding GPT2 model implementation respectively.
`),fe=r(w,"EM",{});var Xt=a(fe);mt=i(Xt,"Note that sentencepiece is required to use our tokenizer and can be installed with:"),Xt.forEach(n),ht=m(w),ue=r(w,"CODE",{});var Yt=a(ue);ft=i(Yt,"pip install transformers[sentencepiece]"),Yt.forEach(n),ut=i(w," or "),ge=r(w,"CODE",{});var Zt=a(ge);gt=i(Zt,"pip install sentencepiece"),Zt.forEach(n),w.forEach(n),Ge=m(e),te=r(e,"P",{});var en=a(te);_t=i(en,"Example usage:"),en.forEach(n),Me=m(e),y(H.$$.fragment,e),Ie=m(e),$=r(e,"H2",{class:!0});var Be=a($);N=r(Be,"A",{id:!0,class:!0,href:!0});var tn=a(N);_e=r(tn,"SPAN",{});var nn=a(_e);y(R.$$.fragment,nn),nn.forEach(n),tn.forEach(n),wt=m(Be),we=r(Be,"SPAN",{});var on=a(we);kt=i(on,"Documentation resources"),on.forEach(n),Be.forEach(n),je=m(e),S=r(e,"UL",{});var ie=a(S);ke=r(ie,"LI",{});var rn=a(ke);ne=r(rn,"A",{href:!0});var an=a(ne);vt=i(an,"Text classification task guide"),an.forEach(n),rn.forEach(n),Tt=m(ie),ve=r(ie,"LI",{});var sn=a(ve);oe=r(sn,"A",{href:!0});var ln=a(oe);St=i(ln,"Token classification task guide"),ln.forEach(n),sn.forEach(n),bt=m(ie),Te=r(ie,"LI",{});var pn=a(Te);re=r(pn,"A",{href:!0});var cn=a(re);Pt=i(cn,"Causal language modeling task guide"),cn.forEach(n),pn.forEach(n),ie.forEach(n),Le=m(e),z=r(e,"H2",{class:!0});var Oe=a(z);q=r(Oe,"A",{id:!0,class:!0,href:!0});var dn=a(q);Se=r(dn,"SPAN",{});var mn=a(Se);y(U.$$.fragment,mn),mn.forEach(n),dn.forEach(n),$t=m(Oe),be=r(Oe,"SPAN",{});var hn=a(be);zt=i(hn,"GPTSw3Tokenizer"),hn.forEach(n),Oe.forEach(n),Ce=m(e),_=r(e,"DIV",{class:!0});var b=a(_);y(V.$$.fragment,b),Et=m(b),J=r(b,"P",{});var We=a(J);yt=i(We,"Construct an GPTSw3 tokenizer. Based on "),K=r(We,"A",{href:!0,rel:!0});var fn=a(K);xt=i(fn,"SentencePiece"),fn.forEach(n),At=i(We,"."),We.forEach(n),Gt=m(b),Q=r(b,"P",{});var He=a(Q);Mt=i(He,"This tokenizer inherits from "),ae=r(He,"A",{href:!0});var un=a(ae);It=i(un,"PreTrainedTokenizer"),un.forEach(n),jt=i(He,` which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`),He.forEach(n),Lt=m(b),y(D.$$.fragment,b),Ct=m(b),se=r(b,"DIV",{class:!0});var gn=a(se);y(X.$$.fragment,gn),gn.forEach(n),b.forEach(n),this.h()},h(){p(u,"name","hf:doc:metadata"),p(u,"content",JSON.stringify(yn)),p(g,"id","gptsw3"),p(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(g,"href","#gptsw3"),p(k,"class","relative group"),p(j,"id","overview"),p(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(j,"href","#overview"),p(P,"class","relative group"),p(B,"href","http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf"),p(B,"rel","nofollow"),p(O,"href","https://huggingface.co/AI-Sweden"),p(O,"rel","nofollow"),p(W,"href","https://huggingface.co/docs/transformers/model_doc/gpt2"),p(W,"rel","nofollow"),p(N,"id","documentation-resources"),p(N,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(N,"href","#documentation-resources"),p($,"class","relative group"),p(ne,"href","./tasks/sequence_classification"),p(oe,"href","./tasks/token_classification"),p(re,"href","./tasks/language_modeling"),p(q,"id","transformers.GPTSw3Tokenizer"),p(q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(q,"href","#transformers.GPTSw3Tokenizer"),p(z,"class","relative group"),p(K,"href","https://github.com/google/sentencepiece"),p(K,"rel","nofollow"),p(ae,"href","/docs/transformers/v4.27.1/en/main_classes/tokenizer#transformers.PreTrainedTokenizer"),p(se,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,l){t(document.head,u),c(e,I,l),c(e,k,l),t(k,g),t(g,v),x(h,v,null),t(k,T),t(k,le),t(le,Ue),c(e,$e,l),c(e,P,l),t(P,j),t(j,pe),x(F,pe,null),t(P,Ve),t(P,ce),t(ce,Je),c(e,ze,l),c(e,L,l),t(L,Ke),t(L,B),t(B,Qe),t(L,Xe),c(e,Ee,l),c(e,Z,l),t(Z,Ye),c(e,ye,l),c(e,ee,l),t(ee,Ze),c(e,xe,l),c(e,C,l),t(C,et),t(C,O),t(O,tt),t(C,nt),c(e,Ae,l),c(e,f,l),t(f,ot),t(f,W),t(W,rt),t(f,at),t(f,de),t(de,st),t(f,it),t(f,me),t(me,lt),t(f,pt),t(f,he),t(he,ct),t(f,dt),t(f,fe),t(fe,mt),t(f,ht),t(f,ue),t(ue,ft),t(f,ut),t(f,ge),t(ge,gt),c(e,Ge,l),c(e,te,l),t(te,_t),c(e,Me,l),x(H,e,l),c(e,Ie,l),c(e,$,l),t($,N),t(N,_e),x(R,_e,null),t($,wt),t($,we),t(we,kt),c(e,je,l),c(e,S,l),t(S,ke),t(ke,ne),t(ne,vt),t(S,Tt),t(S,ve),t(ve,oe),t(oe,St),t(S,bt),t(S,Te),t(Te,re),t(re,Pt),c(e,Le,l),c(e,z,l),t(z,q),t(q,Se),x(U,Se,null),t(z,$t),t(z,be),t(be,zt),c(e,Ce,l),c(e,_,l),x(V,_,null),t(_,Et),t(_,J),t(J,yt),t(J,K),t(K,xt),t(J,At),t(_,Gt),t(_,Q),t(Q,Mt),t(Q,ae),t(ae,It),t(Q,jt),t(_,Lt),x(D,_,null),t(_,Ct),t(_,se),x(X,se,null),Ne=!0},p(e,[l]){const Y={};l&2&&(Y.$$scope={dirty:l,ctx:e}),D.$set(Y)},i(e){Ne||(A(h.$$.fragment,e),A(F.$$.fragment,e),A(H.$$.fragment,e),A(R.$$.fragment,e),A(U.$$.fragment,e),A(V.$$.fragment,e),A(D.$$.fragment,e),A(X.$$.fragment,e),Ne=!0)},o(e){G(h.$$.fragment,e),G(F.$$.fragment,e),G(H.$$.fragment,e),G(R.$$.fragment,e),G(U.$$.fragment,e),G(V.$$.fragment,e),G(D.$$.fragment,e),G(X.$$.fragment,e),Ne=!1},d(e){n(u),e&&n(I),e&&n(k),M(h),e&&n($e),e&&n(P),M(F),e&&n(ze),e&&n(L),e&&n(Ee),e&&n(Z),e&&n(ye),e&&n(ee),e&&n(xe),e&&n(C),e&&n(Ae),e&&n(f),e&&n(Ge),e&&n(te),e&&n(Me),M(H,e),e&&n(Ie),e&&n($),M(R),e&&n(je),e&&n(S),e&&n(Le),e&&n(z),M(U),e&&n(Ce),e&&n(_),M(V),M(D),M(X)}}}const yn={local:"gptsw3",sections:[{local:"overview",title:"Overview"},{local:"documentation-resources",title:"Documentation resources"},{local:"transformers.GPTSw3Tokenizer",title:"GPTSw3Tokenizer"}],title:"GPT-Sw3"};function xn(Pe){return bn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ln extends kn{constructor(u){super();vn(this,u,xn,En,Tn,{})}}export{Ln as default,yn as metadata};
