import{s as $e,o as ve,n as Te}from"../chunks/scheduler.25b97de1.js";import{S as Me,i as ye,g,s as r,r as h,A as ze,h as f,f as n,c as l,j as re,u as w,x as v,k as le,y as z,a as o,v as k,d as _,t as T,w as b}from"../chunks/index.d9030fc9.js";import{T as xe}from"../chunks/Tip.baa67368.js";import{D as _e}from"../chunks/Docstring.ffac8efa.js";import{C as be}from"../chunks/CodeBlock.e6cd0d95.js";import{E as Se}from"../chunks/ExampleCodeBlock.22dfe688.js";import{H as B,E as Pe}from"../chunks/EditOnGithub.91d95064.js";function Ge(H){let a,$=`The implementation uses the <code>GPT2Model</code> coupled with our <code>GPTSw3Tokenizer</code>. Refer to <a href="gpt2">GPT2Model documentation</a>
for API reference and examples.`,p,i,c="Note that sentencepiece is required to use our tokenizer and can be installed with <code>pip install transformers[sentencepiece]</code> or <code>pip install sentencepiece</code>";return{c(){a=g("p"),a.innerHTML=$,p=r(),i=g("p"),i.innerHTML=c},l(s){a=f(s,"P",{"data-svelte-h":!0}),v(a)!=="svelte-13njngq"&&(a.innerHTML=$),p=l(s),i=f(s,"P",{"data-svelte-h":!0}),v(i)!=="svelte-1oicrkv"&&(i.innerHTML=c)},m(s,m){o(s,a,m),o(s,p,m),o(s,i,m)},p:Te,d(s){s&&(n(a),n(p),n(i))}}}function Ue(H){let a,$="Example usage:",p,i,c;return i=new be({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVFN3M1Rva2VuaXplciUwQSUwQXRva2VuaXplciUyMCUzRCUyMEdQVFN3M1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyQUktU3dlZGVuLU1vZGVscyUyRmdwdC1zdzMtMTI2bSUyMiklMEF0b2tlbml6ZXIoJTIyU3ZlbnNrYSUyMCVDMyVBNHIlMjBrdWwhJTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTSw3Tokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTSw3Tokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden-Models/gpt-sw3-126m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;Svenska är kul!&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
[<span class="hljs-number">1814</span>, <span class="hljs-number">377</span>, <span class="hljs-number">3617</span>, <span class="hljs-number">63504</span>]`,wrap:!1}}),{c(){a=g("p"),a.textContent=$,p=r(),h(i.$$.fragment)},l(s){a=f(s,"P",{"data-svelte-h":!0}),v(a)!=="svelte-1ni337v"&&(a.textContent=$),p=l(s),w(i.$$.fragment,s)},m(s,m){o(s,a,m),o(s,p,m),k(i,s,m),c=!0},p:Te,i(s){c||(_(i.$$.fragment,s),c=!0)},o(s){T(i.$$.fragment,s),c=!1},d(s){s&&(n(a),n(p)),b(i,s)}}}function je(H){let a,$,p,i,c,s,m,A,x,de=`The GPT-Sw3 model was first proposed in
<a href="http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf" rel="nofollow">Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish</a>
by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman,
Fredrik Carlsson, Magnus Sahlgren.`,F,S,ge="Since that first paper the authors have extended their work and trained new models on their new 1.2TB corpora named The Nordic Pile.",Q,P,fe=`GPT-Sw3 is a collection of large decoder-only pretrained transformer language models that were developed by AI Sweden
in collaboration with RISE and the WASP WARA for Media and Language. GPT-Sw3 has been trained on a dataset containing
320B tokens in Swedish, Norwegian, Danish, Icelandic, English, and programming code. The model was pretrained using a
causal language modeling (CLM) objective utilizing the NeMo Megatron GPT implementation.`,N,G,ue='This model was contributed by <a href="https://huggingface.co/AI-Sweden-Models" rel="nofollow">AI Sweden Models</a>.',D,U,Y,j,K,C,O,I,he='<li><a href="../tasks/sequence_classification">Text classification task guide</a></li> <li><a href="../tasks/token_classification">Token classification task guide</a></li> <li><a href="../tasks/language_modeling">Causal language modeling task guide</a></li>',ee,M,te,V,ne,d,Z,ie,L,we='Construct an GPTSw3 tokenizer. Based on <a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a>.',pe,J,ke=`This tokenizer inherits from <a href="/docs/transformers/v4.41.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`,ce,y,me,W,E,se,R,oe,X,ae;return c=new B({props:{title:"GPT-Sw3",local:"gpt-sw3",headingTag:"h1"}}),m=new B({props:{title:"Overview",local:"overview",headingTag:"h2"}}),U=new B({props:{title:"Usage example",local:"usage-example",headingTag:"h2"}}),j=new be({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMkFJLVN3ZWRlbi1Nb2RlbHMlMkZncHQtc3czLTM1Nm0lMjIpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyQUktU3dlZGVuLU1vZGVscyUyRmdwdC1zdzMtMzU2bSUyMiklMEElMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIyVHIlQzMlQTRkJTIwJUMzJUE0ciUyMGZpbmElMjBmJUMzJUI2ciUyMGF0dCUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTBBJTBBZ2VuZXJhdGVkX3Rva2VuX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKGlucHV0cyUzRGlucHV0X2lkcyUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMTAlMkMlMjBkb19zYW1wbGUlM0RUcnVlKSU1QjAlNUQlMEElMEFwcmludCh0b2tlbml6ZXIuZGVjb2RlKGdlbmVyYXRlZF90b2tlbl9pZHMpKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden-Models/gpt-sw3-356m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;AI-Sweden-Models/gpt-sw3-356m&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(<span class="hljs-string">&quot;Träd är fina för att&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_token_ids = model.generate(inputs=input_ids, max_new_tokens=<span class="hljs-number">10</span>, do_sample=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(generated_token_ids))
Träd är fina för att de är färgstarka. Men ibland är det fint`,wrap:!1}}),C=new B({props:{title:"Resources",local:"resources",headingTag:"h2"}}),M=new xe({props:{$$slots:{default:[Ge]},$$scope:{ctx:H}}}),V=new B({props:{title:"GPTSw3Tokenizer",local:"transformers.GPTSw3Tokenizer",headingTag:"h2"}}),Z=new _e({props:{name:"class transformers.GPTSw3Tokenizer",anchor:"transformers.GPTSw3Tokenizer",parameters:[{name:"vocab_file",val:""},{name:"do_lower_case",val:" = False"},{name:"remove_space",val:" = False"},{name:"keep_accents",val:" = False"},{name:"pad_token",val:" = None"},{name:"unk_token",val:" = None"},{name:"eos_token",val:" = None"},{name:"bos_token",val:" = None"},{name:"sp_model_kwargs",val:": Optional = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTSw3Tokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
<a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a <em>.spm</em> extension) that
contains the vocabulary necessary to instantiate a tokenizer.`,name:"vocab_file"},{anchor:"transformers.GPTSw3Tokenizer.do_lower_case",description:`<strong>do_lower_case</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to lowercase the input when tokenizing.`,name:"do_lower_case"},{anchor:"transformers.GPTSw3Tokenizer.remove_space",description:`<strong>remove_space</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to strip the text when tokenizing (removing excess spaces before and after the string).`,name:"remove_space"},{anchor:"transformers.GPTSw3Tokenizer.keep_accents",description:`<strong>keep_accents</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to keep accents when tokenizing.`,name:"keep_accents"},{anchor:"transformers.GPTSw3Tokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The token used for padding, for example when batching sequences of different lengths. If not provided, will
default to &#x2019;<pad>&#x2019; or &#x2019;<unk>&#x2019; depending on model size.</unk></pad>`,name:"pad_token"},{anchor:"transformers.GPTSw3Tokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead. If not provided, will default to &#x2019;<unk>&#x2018;.</unk>`,name:"unk_token"},{anchor:"transformers.GPTSw3Tokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The end of sequence token seen during pretraining. If not provided, will default to &#x2019;&lt;|endoftext|&gt;&#x2019;`,name:"eos_token"},{anchor:"transformers.GPTSw3Tokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The beginning of sequence token that can be used for downstream task, was not seen during pretraining. If
not provided, will default to &#x2019;<s>&#x2019; or &#x2019;&lt;|endoftext|&gt;&#x2019;, depending on model size.</s>`,name:"bos_token"},{anchor:"transformers.GPTSw3Tokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Will be passed to the <code>SentencePieceProcessor.__init__()</code> method. The <a href="https://github.com/google/sentencepiece/tree/master/python" rel="nofollow">Python wrapper for
SentencePiece</a> can be used, among other things,
to set:</p>
<ul>
<li>
<p><code>enable_sampling</code>: Enable subword regularization.</p>
</li>
<li>
<p><code>nbest_size</code>: Sampling parameters for unigram. Invalid for BPE-Dropout.</p>
<ul>
<li><code>nbest_size = {0,1}</code>: No sampling is performed.</li>
<li><code>nbest_size &gt; 1</code>: samples from the nbest_size results.</li>
<li><code>nbest_size &lt; 0</code>: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
using forward-filtering-and-backward-sampling algorithm.</li>
</ul>
</li>
<li>
<p><code>alpha</code>: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
BPE-dropout.</p>
</li>
</ul>`,name:"sp_model_kwargs"},{anchor:"transformers.GPTSw3Tokenizer.sp_model",description:`<strong>sp_model</strong> (<code>SentencePieceProcessor</code>) &#x2014;
The <em>SentencePiece</em> processor that is used for every conversion (string, tokens and IDs).`,name:"sp_model"},{anchor:"transformers.GPTSw3Tokenizer.whitespaces",description:`<strong>whitespaces</strong> (<code>set</code>) &#x2014;
The whitespaces that are replaced in the whitespace normalization in preprocessing.`,name:"whitespaces"},{anchor:"transformers.GPTSw3Tokenizer.non_printing_characters_re",description:`<strong>non_printing_characters_re</strong> (<code>Pattern</code>) &#x2014;
The compiled regular expression to remove non-printing characters in preprocessing.`,name:"non_printing_characters_re"}],source:"https://github.com/huggingface/transformers/blob/v4.41.0/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L23"}}),y=new Se({props:{anchor:"transformers.GPTSw3Tokenizer.example",$$slots:{default:[Ue]},$$scope:{ctx:H}}}),E=new _e({props:{name:"save_vocabulary",anchor:"transformers.GPTSw3Tokenizer.save_vocabulary",parameters:[{name:"save_directory",val:": str"},{name:"filename_prefix",val:": Optional = None"}],source:"https://github.com/huggingface/transformers/blob/v4.41.0/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L234"}}),R=new Pe({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/gpt-sw3.md"}}),{c(){a=g("meta"),$=r(),p=g("p"),i=r(),h(c.$$.fragment),s=r(),h(m.$$.fragment),A=r(),x=g("p"),x.innerHTML=de,F=r(),S=g("p"),S.textContent=ge,Q=r(),P=g("p"),P.textContent=fe,N=r(),G=g("p"),G.innerHTML=ue,D=r(),h(U.$$.fragment),Y=r(),h(j.$$.fragment),K=r(),h(C.$$.fragment),O=r(),I=g("ul"),I.innerHTML=he,ee=r(),h(M.$$.fragment),te=r(),h(V.$$.fragment),ne=r(),d=g("div"),h(Z.$$.fragment),ie=r(),L=g("p"),L.innerHTML=we,pe=r(),J=g("p"),J.innerHTML=ke,ce=r(),h(y.$$.fragment),me=r(),W=g("div"),h(E.$$.fragment),se=r(),h(R.$$.fragment),oe=r(),X=g("p"),this.h()},l(e){const t=ze("svelte-u9bgzb",document.head);a=f(t,"META",{name:!0,content:!0}),t.forEach(n),$=l(e),p=f(e,"P",{}),re(p).forEach(n),i=l(e),w(c.$$.fragment,e),s=l(e),w(m.$$.fragment,e),A=l(e),x=f(e,"P",{"data-svelte-h":!0}),v(x)!=="svelte-1qucz45"&&(x.innerHTML=de),F=l(e),S=f(e,"P",{"data-svelte-h":!0}),v(S)!=="svelte-1uddber"&&(S.textContent=ge),Q=l(e),P=f(e,"P",{"data-svelte-h":!0}),v(P)!=="svelte-19zkpwq"&&(P.textContent=fe),N=l(e),G=f(e,"P",{"data-svelte-h":!0}),v(G)!=="svelte-p3ijej"&&(G.innerHTML=ue),D=l(e),w(U.$$.fragment,e),Y=l(e),w(j.$$.fragment,e),K=l(e),w(C.$$.fragment,e),O=l(e),I=f(e,"UL",{"data-svelte-h":!0}),v(I)!=="svelte-v6nfyh"&&(I.innerHTML=he),ee=l(e),w(M.$$.fragment,e),te=l(e),w(V.$$.fragment,e),ne=l(e),d=f(e,"DIV",{class:!0});var u=re(d);w(Z.$$.fragment,u),ie=l(u),L=f(u,"P",{"data-svelte-h":!0}),v(L)!=="svelte-19f98fg"&&(L.innerHTML=we),pe=l(u),J=f(u,"P",{"data-svelte-h":!0}),v(J)!=="svelte-qu94ym"&&(J.innerHTML=ke),ce=l(u),w(y.$$.fragment,u),me=l(u),W=f(u,"DIV",{class:!0});var q=re(W);w(E.$$.fragment,q),q.forEach(n),u.forEach(n),se=l(e),w(R.$$.fragment,e),oe=l(e),X=f(e,"P",{}),re(X).forEach(n),this.h()},h(){le(a,"name","hf:doc:metadata"),le(a,"content",Ce),le(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),le(d,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){z(document.head,a),o(e,$,t),o(e,p,t),o(e,i,t),k(c,e,t),o(e,s,t),k(m,e,t),o(e,A,t),o(e,x,t),o(e,F,t),o(e,S,t),o(e,Q,t),o(e,P,t),o(e,N,t),o(e,G,t),o(e,D,t),k(U,e,t),o(e,Y,t),k(j,e,t),o(e,K,t),k(C,e,t),o(e,O,t),o(e,I,t),o(e,ee,t),k(M,e,t),o(e,te,t),k(V,e,t),o(e,ne,t),o(e,d,t),k(Z,d,null),z(d,ie),z(d,L),z(d,pe),z(d,J),z(d,ce),k(y,d,null),z(d,me),z(d,W),k(E,W,null),o(e,se,t),k(R,e,t),o(e,oe,t),o(e,X,t),ae=!0},p(e,[t]){const u={};t&2&&(u.$$scope={dirty:t,ctx:e}),M.$set(u);const q={};t&2&&(q.$$scope={dirty:t,ctx:e}),y.$set(q)},i(e){ae||(_(c.$$.fragment,e),_(m.$$.fragment,e),_(U.$$.fragment,e),_(j.$$.fragment,e),_(C.$$.fragment,e),_(M.$$.fragment,e),_(V.$$.fragment,e),_(Z.$$.fragment,e),_(y.$$.fragment,e),_(E.$$.fragment,e),_(R.$$.fragment,e),ae=!0)},o(e){T(c.$$.fragment,e),T(m.$$.fragment,e),T(U.$$.fragment,e),T(j.$$.fragment,e),T(C.$$.fragment,e),T(M.$$.fragment,e),T(V.$$.fragment,e),T(Z.$$.fragment,e),T(y.$$.fragment,e),T(E.$$.fragment,e),T(R.$$.fragment,e),ae=!1},d(e){e&&(n($),n(p),n(i),n(s),n(A),n(x),n(F),n(S),n(Q),n(P),n(N),n(G),n(D),n(Y),n(K),n(O),n(I),n(ee),n(te),n(ne),n(d),n(se),n(oe),n(X)),n(a),b(c,e),b(m,e),b(U,e),b(j,e),b(C,e),b(M,e),b(V,e),b(Z),b(y),b(E),b(R,e)}}}const Ce='{"title":"GPT-Sw3","local":"gpt-sw3","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage example","local":"usage-example","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2},{"title":"GPTSw3Tokenizer","local":"transformers.GPTSw3Tokenizer","sections":[],"depth":2}],"depth":1}';function Ie(H){return ve(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class We extends Me{constructor(a){super(),ye(this,a,Ie,je,$e,{})}}export{We as component};
