import{s as ot,o as st,n as at}from"../chunks/scheduler.9bc65507.js";import{S as rt,i as lt,g as o,s as l,u as y,A as it,h as s,f as t,c as i,j as m,v as x,y as f,k as p,o as r,a,w as M,d as S,t as $,x as z}from"../chunks/index.78c82d43.js";import{D as tt}from"../chunks/Docstring.f3d04bd2.js";import{C as nt}from"../chunks/CodeBlock.48618ec2.js";import{I as xe}from"../chunks/IconCopyLink.bedaa44d.js";import{E as ct}from"../chunks/ExampleCodeBlock.872b014d.js";function pt(re){let d,j="Example usage:",w,g,u;return g=new nt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVFN3M1Rva2VuaXplciUwQSUwQXRva2VuaXplciUyMCUzRCUyMEdQVFN3M1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyQUktU3dlZGVuJTJGZ3B0LXN3My0xMjZtJTIyKSUwQXRva2VuaXplciglMjJTdmVuc2thJTIwJUMzJUE0ciUyMGt1bCElMjIpJTVCJTIyaW5wdXRfaWRzJTIyJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTSw3Tokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTSw3Tokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-126m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;Svenska är kul!&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
[<span class="hljs-number">1814</span>, <span class="hljs-number">377</span>, <span class="hljs-number">3617</span>, <span class="hljs-number">63504</span>]`}}),{c(){d=o("p"),d.textContent=j,w=l(),y(g.$$.fragment)},l(c){d=s(c,"P",{"data-svelte-h":!0}),f(d)!=="svelte-1ni337v"&&(d.textContent=j),w=i(c),x(g.$$.fragment,c)},m(c,k){a(c,d,k),a(c,w,k),M(g,c,k),u=!0},p:at,i(c){u||(S(g.$$.fragment,c),u=!0)},o(c){$(g.$$.fragment,c),u=!1},d(c){c&&(t(d),t(w)),z(g,c)}}}function dt(re){let d,j,w,g,u,c,k,E,Me,D,je="GPT-Sw3",le,_,P,ne,J,Se,Q,Ee="Overview",ie,I,Je=`The GPT-Sw3 model was first proposed in
<a href="http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf" rel="nofollow">Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish</a>
by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman,
Fredrik Carlsson, Magnus Sahlgren.`,ce,A,Ie="Since that first paper the authors have extended their work and trained new models on their new 1.2TB corpora named The Nordic Pile.",pe,N,Ae=`GPT-Sw3 is a collection of large decoder-only pretrained transformer language models that were developed by AI Sweden
in collaboration with RISE and the WASP WARA for Media and Language. GPT-Sw3 has been trained on a dataset containing
320B tokens in Swedish, Norwegian, Danish, Icelandic, English, and programming code. The model was pretrained using a
causal language modeling (CLM) objective utilizing the NeMo Megatron GPT implementation.`,de,R,Ne='This model was contributed by <a href="https://huggingface.co/AI-Sweden" rel="nofollow">AI Sweden</a>.',me,V,Re=`The implementation uses the <a href="https://huggingface.co/docs/transformers/model_doc/gpt2" rel="nofollow">GPT2Model</a> coupled
with our <code>GPTSw3Tokenizer</code>. This means that <code>AutoTokenizer</code> and <code>AutoModelForCausalLM</code> map to our tokenizer
implementation and the corresponding GPT2 model implementation respectively.
<em>Note that sentencepiece is required to use our tokenizer and can be installed with:</em> <code>pip install transformers[sentencepiece]</code> or <code>pip install sentencepiece</code>`,he,Z,Ve="Example usage:",ue,W,fe,T,C,oe,L,$e,Y,Ze="Documentation resources",ge,B,We='<li><a href="../tasks/sequence_classification">Text classification task guide</a></li> <li><a href="../tasks/token_classification">Token classification task guide</a></li> <li><a href="../tasks/language_modeling">Causal language modeling task guide</a></li>',we,b,G,se,H,ze,K,Le="GPTSw3Tokenizer",ve,h,q,Pe,O,Be='Construct an GPTSw3 tokenizer. Based on <a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a>.',Ce,ee,He=`This tokenizer inherits from <a href="/docs/transformers/v4.33.3/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`,Ge,U,Ue,te,F,ke,ae,_e;return E=new xe({}),J=new xe({}),W=new nt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMkFJLVN3ZWRlbiUyRmdwdC1zdzMtMzU2bSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJBSS1Td2VkZW4lMkZncHQtc3czLTM1Nm0lMjIpJTBBJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMlRyJUMzJUE0ZCUyMCVDMyVBNHIlMjBmaW5hJTIwZiVDMyVCNnIlMjBhdHQlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RCUwQSUwQWdlbmVyYXRlZF90b2tlbl9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZShpbnB1dHMlM0RpbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwJTJDJTIwZG9fc2FtcGxlJTNEVHJ1ZSklNUIwJTVEJTBBJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShnZW5lcmF0ZWRfdG9rZW5faWRzKSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-356m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-356m&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(<span class="hljs-string">&quot;Träd är fina för att&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_token_ids = model.generate(inputs=input_ids, max_new_tokens=<span class="hljs-number">10</span>, do_sample=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(generated_token_ids))
Träd är fina för att de är färgstarka. Men ibland är det fint`}}),L=new xe({}),H=new xe({}),q=new tt({props:{name:"class transformers.GPTSw3Tokenizer",anchor:"transformers.GPTSw3Tokenizer",parameters:[{name:"vocab_file",val:""},{name:"do_lower_case",val:" = False"},{name:"remove_space",val:" = False"},{name:"keep_accents",val:" = False"},{name:"pad_token",val:" = None"},{name:"unk_token",val:" = None"},{name:"eos_token",val:" = None"},{name:"bos_token",val:" = None"},{name:"sp_model_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTSw3Tokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
<a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a <em>.spm</em> extension) that
contains the vocabulary necessary to instantiate a tokenizer.`,name:"vocab_file"},{anchor:"transformers.GPTSw3Tokenizer.do_lower_case",description:`<strong>do_lower_case</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to lowercase the input when tokenizing.`,name:"do_lower_case"},{anchor:"transformers.GPTSw3Tokenizer.remove_space",description:`<strong>remove_space</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to strip the text when tokenizing (removing excess spaces before and after the string).`,name:"remove_space"},{anchor:"transformers.GPTSw3Tokenizer.keep_accents",description:`<strong>keep_accents</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to keep accents when tokenizing.`,name:"keep_accents"},{anchor:"transformers.GPTSw3Tokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The beginning of sequence token that can be used for downstream task, was not seen during pretraining. If
not provided, will default to &#x2019;<s>&#x2019; or &#x2019;&lt;|endoftext|&gt;&#x2019;, depending on model size.</s>`,name:"bos_token"},{anchor:"transformers.GPTSw3Tokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The end of sequence token seen during pretraining. If not provided, will default to &#x2019;&lt;|endoftext|&gt;&#x2019;`,name:"eos_token"},{anchor:"transformers.GPTSw3Tokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead. If not provided, will default to &#x2019;<unk>&#x2018;.</unk>`,name:"unk_token"},{anchor:"transformers.GPTSw3Tokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The token used for padding, for example when batching sequences of different lengths. If not provided, will
default to &#x2019;<pad>&#x2019; or &#x2019;<unk>&#x2019; depending on model size.</unk></pad>`,name:"pad_token"},{anchor:"transformers.GPTSw3Tokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Will be passed to the <code>SentencePieceProcessor.__init__()</code> method. The <a href="https://github.com/google/sentencepiece/tree/master/python" rel="nofollow">Python wrapper for
SentencePiece</a> can be used, among other things,
to set:</p>
<ul>
<li>
<p><code>enable_sampling</code>: Enable subword regularization.</p>
</li>
<li>
<p><code>nbest_size</code>: Sampling parameters for unigram. Invalid for BPE-Dropout.</p>
<ul>
<li><code>nbest_size = {0,1}</code>: No sampling is performed.</li>
<li><code>nbest_size &gt; 1</code>: samples from the nbest_size results.</li>
<li><code>nbest_size &lt; 0</code>: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
using forward-filtering-and-backward-sampling algorithm.</li>
</ul>
</li>
<li>
<p><code>alpha</code>: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
BPE-dropout.</p>
</li>
</ul>`,name:"sp_model_kwargs"},{anchor:"transformers.GPTSw3Tokenizer.sp_model",description:`<strong>sp_model</strong> (<code>SentencePieceProcessor</code>) &#x2014;
The <em>SentencePiece</em> processor that is used for every conversion (string, tokens and IDs).`,name:"sp_model"},{anchor:"transformers.GPTSw3Tokenizer.whitespaces",description:`<strong>whitespaces</strong> (<code>set</code>) &#x2014;
The whitespaces that are replaced in the whitespace normalization in preprocessing.`,name:"whitespaces"},{anchor:"transformers.GPTSw3Tokenizer.non_printing_characters_re",description:`<strong>non_printing_characters_re</strong> (<code>Pattern</code>) &#x2014;
The compiled regular expression to remove non-printing characters in preprocessing.`,name:"non_printing_characters_re"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L45"}}),U=new ct({props:{anchor:"transformers.GPTSw3Tokenizer.example",$$slots:{default:[pt]},$$scope:{ctx:re}}}),F=new tt({props:{name:"save_vocabulary",anchor:"transformers.GPTSw3Tokenizer.save_vocabulary",parameters:[{name:"save_directory",val:": str"},{name:"filename_prefix",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L258"}}),{c(){d=o("meta"),j=l(),w=o("p"),g=l(),u=o("h1"),c=o("a"),k=o("span"),y(E.$$.fragment),Me=l(),D=o("span"),D.textContent=je,le=l(),_=o("h2"),P=o("a"),ne=o("span"),y(J.$$.fragment),Se=l(),Q=o("span"),Q.textContent=Ee,ie=l(),I=o("p"),I.innerHTML=Je,ce=l(),A=o("p"),A.textContent=Ie,pe=l(),N=o("p"),N.textContent=Ae,de=l(),R=o("p"),R.innerHTML=Ne,me=l(),V=o("p"),V.innerHTML=Re,he=l(),Z=o("p"),Z.textContent=Ve,ue=l(),y(W.$$.fragment),fe=l(),T=o("h2"),C=o("a"),oe=o("span"),y(L.$$.fragment),$e=l(),Y=o("span"),Y.textContent=Ze,ge=l(),B=o("ul"),B.innerHTML=We,we=l(),b=o("h2"),G=o("a"),se=o("span"),y(H.$$.fragment),ze=l(),K=o("span"),K.textContent=Le,ve=l(),h=o("div"),y(q.$$.fragment),Pe=l(),O=o("p"),O.innerHTML=Be,Ce=l(),ee=o("p"),ee.innerHTML=He,Ge=l(),y(U.$$.fragment),Ue=l(),te=o("div"),y(F.$$.fragment),ke=l(),ae=o("p"),this.h()},l(e){const n=it("svelte-1phssyn",document.head);d=s(n,"META",{name:!0,content:!0}),n.forEach(t),j=i(e),w=s(e,"P",{}),m(w).forEach(t),g=i(e),u=s(e,"H1",{class:!0});var X=m(u);c=s(X,"A",{id:!0,class:!0,href:!0});var qe=m(c);k=s(qe,"SPAN",{});var Fe=m(k);x(E.$$.fragment,Fe),Fe.forEach(t),qe.forEach(t),Me=i(X),D=s(X,"SPAN",{"data-svelte-h":!0}),f(D)!=="svelte-1t9zr10"&&(D.textContent=je),X.forEach(t),le=i(e),_=s(e,"H2",{class:!0});var Te=m(_);P=s(Te,"A",{id:!0,class:!0,href:!0});var Xe=m(P);ne=s(Xe,"SPAN",{});var De=m(ne);x(J.$$.fragment,De),De.forEach(t),Xe.forEach(t),Se=i(Te),Q=s(Te,"SPAN",{"data-svelte-h":!0}),f(Q)!=="svelte-1jsw1pg"&&(Q.textContent=Ee),Te.forEach(t),ie=i(e),I=s(e,"P",{"data-svelte-h":!0}),f(I)!=="svelte-1qucz45"&&(I.innerHTML=Je),ce=i(e),A=s(e,"P",{"data-svelte-h":!0}),f(A)!=="svelte-1uddber"&&(A.textContent=Ie),pe=i(e),N=s(e,"P",{"data-svelte-h":!0}),f(N)!=="svelte-19zkpwq"&&(N.textContent=Ae),de=i(e),R=s(e,"P",{"data-svelte-h":!0}),f(R)!=="svelte-1u6kw5m"&&(R.innerHTML=Ne),me=i(e),V=s(e,"P",{"data-svelte-h":!0}),f(V)!=="svelte-4mp7uz"&&(V.innerHTML=Re),he=i(e),Z=s(e,"P",{"data-svelte-h":!0}),f(Z)!=="svelte-1ni337v"&&(Z.textContent=Ve),ue=i(e),x(W.$$.fragment,e),fe=i(e),T=s(e,"H2",{class:!0});var be=m(T);C=s(be,"A",{id:!0,class:!0,href:!0});var Qe=m(C);oe=s(Qe,"SPAN",{});var Ye=m(oe);x(L.$$.fragment,Ye),Ye.forEach(t),Qe.forEach(t),$e=i(be),Y=s(be,"SPAN",{"data-svelte-h":!0}),f(Y)!=="svelte-n3f0j0"&&(Y.textContent=Ze),be.forEach(t),ge=i(e),B=s(e,"UL",{"data-svelte-h":!0}),f(B)!=="svelte-v6nfyh"&&(B.innerHTML=We),we=i(e),b=s(e,"H2",{class:!0});var ye=m(b);G=s(ye,"A",{id:!0,class:!0,href:!0});var Ke=m(G);se=s(Ke,"SPAN",{});var Oe=m(se);x(H.$$.fragment,Oe),Oe.forEach(t),Ke.forEach(t),ze=i(ye),K=s(ye,"SPAN",{"data-svelte-h":!0}),f(K)!=="svelte-k6okv2"&&(K.textContent=Le),ye.forEach(t),ve=i(e),h=s(e,"DIV",{class:!0});var v=m(h);x(q.$$.fragment,v),Pe=i(v),O=s(v,"P",{"data-svelte-h":!0}),f(O)!=="svelte-19f98fg"&&(O.innerHTML=Be),Ce=i(v),ee=s(v,"P",{"data-svelte-h":!0}),f(ee)!=="svelte-14xlfoy"&&(ee.innerHTML=He),Ge=i(v),x(U.$$.fragment,v),Ue=i(v),te=s(v,"DIV",{class:!0});var et=m(te);x(F.$$.fragment,et),et.forEach(t),v.forEach(t),ke=i(e),ae=s(e,"P",{}),m(ae).forEach(t),this.h()},h(){p(d,"name","hf:doc:metadata"),p(d,"content",JSON.stringify(mt)),p(c,"id","gptsw3"),p(c,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(c,"href","#gptsw3"),p(u,"class","relative group"),p(P,"id","overview"),p(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(P,"href","#overview"),p(_,"class","relative group"),p(C,"id","documentation-resources"),p(C,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(C,"href","#documentation-resources"),p(T,"class","relative group"),p(G,"id","transformers.GPTSw3Tokenizer"),p(G,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(G,"href","#transformers.GPTSw3Tokenizer"),p(b,"class","relative group"),p(te,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(h,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,n){r(document.head,d),a(e,j,n),a(e,w,n),a(e,g,n),a(e,u,n),r(u,c),r(c,k),M(E,k,null),r(u,Me),r(u,D),a(e,le,n),a(e,_,n),r(_,P),r(P,ne),M(J,ne,null),r(_,Se),r(_,Q),a(e,ie,n),a(e,I,n),a(e,ce,n),a(e,A,n),a(e,pe,n),a(e,N,n),a(e,de,n),a(e,R,n),a(e,me,n),a(e,V,n),a(e,he,n),a(e,Z,n),a(e,ue,n),M(W,e,n),a(e,fe,n),a(e,T,n),r(T,C),r(C,oe),M(L,oe,null),r(T,$e),r(T,Y),a(e,ge,n),a(e,B,n),a(e,we,n),a(e,b,n),r(b,G),r(G,se),M(H,se,null),r(b,ze),r(b,K),a(e,ve,n),a(e,h,n),M(q,h,null),r(h,Pe),r(h,O),r(h,Ce),r(h,ee),r(h,Ge),M(U,h,null),r(h,Ue),r(h,te),M(F,te,null),a(e,ke,n),a(e,ae,n),_e=!0},p(e,[n]){const X={};n&2&&(X.$$scope={dirty:n,ctx:e}),U.$set(X)},i(e){_e||(S(E.$$.fragment,e),S(J.$$.fragment,e),S(W.$$.fragment,e),S(L.$$.fragment,e),S(H.$$.fragment,e),S(q.$$.fragment,e),S(U.$$.fragment,e),S(F.$$.fragment,e),_e=!0)},o(e){$(E.$$.fragment,e),$(J.$$.fragment,e),$(W.$$.fragment,e),$(L.$$.fragment,e),$(H.$$.fragment,e),$(q.$$.fragment,e),$(U.$$.fragment,e),$(F.$$.fragment,e),_e=!1},d(e){e&&(t(j),t(w),t(g),t(u),t(le),t(_),t(ie),t(I),t(ce),t(A),t(pe),t(N),t(de),t(R),t(me),t(V),t(he),t(Z),t(ue),t(fe),t(T),t(ge),t(B),t(we),t(b),t(ve),t(h),t(ke),t(ae)),t(d),z(E),z(J),z(W,e),z(L),z(H),z(q),z(U),z(F)}}}const mt={local:"gptsw3",sections:[{local:"overview",title:"Overview"},{local:"documentation-resources",title:"Documentation resources"},{local:"transformers.GPTSw3Tokenizer",title:"GPTSw3Tokenizer"}],title:"GPT-Sw3"};function ht(re){return st(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class _t extends rt{constructor(d){super(),lt(this,d,ht,dt,ot,{})}}export{_t as component};
