import{s as Te,o as be,n as ke}from"../chunks/scheduler.9bc65507.js";import{S as $e,i as ve,g,s as r,r as h,A as Me,h as f,f as n,c as l,j as oe,u as w,x as v,k as ae,y as z,a as o,v as k,d as _,t as T,w as b}from"../chunks/index.707bf1b6.js";import{T as ye}from"../chunks/Tip.c2ecdbf4.js";import{D as we}from"../chunks/Docstring.17db21ae.js";import{C as _e}from"../chunks/CodeBlock.54a9f38d.js";import{E as ze}from"../chunks/ExampleCodeBlock.4f515aa9.js";import{H as B}from"../chunks/Heading.342b1fa6.js";function xe(E){let a,$=`The implementation uses the <code>GPT2Model</code> coupled with our <code>GPTSw3Tokenizer</code>. Refer to <a href="gpt2">GPT2Model documentation</a>
for API reference and examples.`,p,i,c="Note that sentencepiece is required to use our tokenizer and can be installed with <code>pip install transformers[sentencepiece]</code> or <code>pip install sentencepiece</code>";return{c(){a=g("p"),a.innerHTML=$,p=r(),i=g("p"),i.innerHTML=c},l(s){a=f(s,"P",{"data-svelte-h":!0}),v(a)!=="svelte-13njngq"&&(a.innerHTML=$),p=l(s),i=f(s,"P",{"data-svelte-h":!0}),v(i)!=="svelte-1oicrkv"&&(i.innerHTML=c)},m(s,m){o(s,a,m),o(s,p,m),o(s,i,m)},p:ke,d(s){s&&(n(a),n(p),n(i))}}}function Se(E){let a,$="Example usage:",p,i,c;return i=new _e({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVFN3M1Rva2VuaXplciUwQSUwQXRva2VuaXplciUyMCUzRCUyMEdQVFN3M1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyQUktU3dlZGVuLU1vZGVscyUyRmdwdC1zdzMtMTI2bSUyMiklMEF0b2tlbml6ZXIoJTIyU3ZlbnNrYSUyMCVDMyVBNHIlMjBrdWwhJTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTSw3Tokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTSw3Tokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden-Models/gpt-sw3-126m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;Svenska är kul!&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
[<span class="hljs-number">1814</span>, <span class="hljs-number">377</span>, <span class="hljs-number">3617</span>, <span class="hljs-number">63504</span>]`,wrap:!1}}),{c(){a=g("p"),a.textContent=$,p=r(),h(i.$$.fragment)},l(s){a=f(s,"P",{"data-svelte-h":!0}),v(a)!=="svelte-1ni337v"&&(a.textContent=$),p=l(s),w(i.$$.fragment,s)},m(s,m){o(s,a,m),o(s,p,m),k(i,s,m),c=!0},p:ke,i(s){c||(_(i.$$.fragment,s),c=!0)},o(s){T(i.$$.fragment,s),c=!1},d(s){s&&(n(a),n(p)),b(i,s)}}}function Pe(E){let a,$,p,i,c,s,m,q,x,ce=`The GPT-Sw3 model was first proposed in
<a href="http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf" rel="nofollow">Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish</a>
by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman,
Fredrik Carlsson, Magnus Sahlgren.`,A,S,me="Since that first paper the authors have extended their work and trained new models on their new 1.2TB corpora named The Nordic Pile.",F,P,de=`GPT-Sw3 is a collection of large decoder-only pretrained transformer language models that were developed by AI Sweden
in collaboration with RISE and the WASP WARA for Media and Language. GPT-Sw3 has been trained on a dataset containing
320B tokens in Swedish, Norwegian, Danish, Icelandic, English, and programming code. The model was pretrained using a
causal language modeling (CLM) objective utilizing the NeMo Megatron GPT implementation.`,Q,G,ge='This model was contributed by <a href="https://huggingface.co/AI-Sweden-Models" rel="nofollow">AI Sweden Models</a>.',N,U,D,j,Y,C,K,I,fe='<li><a href="../tasks/sequence_classification">Text classification task guide</a></li> <li><a href="../tasks/token_classification">Token classification task guide</a></li> <li><a href="../tasks/language_modeling">Causal language modeling task guide</a></li>',O,M,ee,V,te,d,Z,re,H,ue='Construct an GPTSw3 tokenizer. Based on <a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a>.',le,L,he=`This tokenizer inherits from <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`,ie,y,pe,J,R,ne,W,se;return c=new B({props:{title:"GPT-Sw3",local:"gpt-sw3",headingTag:"h1"}}),m=new B({props:{title:"Overview",local:"overview",headingTag:"h2"}}),U=new B({props:{title:"Usage example",local:"usage-example",headingTag:"h2"}}),j=new _e({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMkFJLVN3ZWRlbi1Nb2RlbHMlMkZncHQtc3czLTM1Nm0lMjIpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyQUktU3dlZGVuLU1vZGVscyUyRmdwdC1zdzMtMzU2bSUyMiklMEElMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIyVHIlQzMlQTRkJTIwJUMzJUE0ciUyMGZpbmElMjBmJUMzJUI2ciUyMGF0dCUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTBBJTBBZ2VuZXJhdGVkX3Rva2VuX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKGlucHV0cyUzRGlucHV0X2lkcyUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMTAlMkMlMjBkb19zYW1wbGUlM0RUcnVlKSU1QjAlNUQlMEElMEFwcmludCh0b2tlbml6ZXIuZGVjb2RlKGdlbmVyYXRlZF90b2tlbl9pZHMpKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden-Models/gpt-sw3-356m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;AI-Sweden-Models/gpt-sw3-356m&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(<span class="hljs-string">&quot;Träd är fina för att&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_token_ids = model.generate(inputs=input_ids, max_new_tokens=<span class="hljs-number">10</span>, do_sample=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(generated_token_ids))
Träd är fina för att de är färgstarka. Men ibland är det fint`,wrap:!1}}),C=new B({props:{title:"Resources",local:"resources",headingTag:"h2"}}),M=new ye({props:{$$slots:{default:[xe]},$$scope:{ctx:E}}}),V=new B({props:{title:"GPTSw3Tokenizer",local:"transformers.GPTSw3Tokenizer",headingTag:"h2"}}),Z=new we({props:{name:"class transformers.GPTSw3Tokenizer",anchor:"transformers.GPTSw3Tokenizer",parameters:[{name:"vocab_file",val:""},{name:"do_lower_case",val:" = False"},{name:"remove_space",val:" = False"},{name:"keep_accents",val:" = False"},{name:"pad_token",val:" = None"},{name:"unk_token",val:" = None"},{name:"eos_token",val:" = None"},{name:"bos_token",val:" = None"},{name:"sp_model_kwargs",val:": Optional = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTSw3Tokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
<a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a <em>.spm</em> extension) that
contains the vocabulary necessary to instantiate a tokenizer.`,name:"vocab_file"},{anchor:"transformers.GPTSw3Tokenizer.do_lower_case",description:`<strong>do_lower_case</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to lowercase the input when tokenizing.`,name:"do_lower_case"},{anchor:"transformers.GPTSw3Tokenizer.remove_space",description:`<strong>remove_space</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to strip the text when tokenizing (removing excess spaces before and after the string).`,name:"remove_space"},{anchor:"transformers.GPTSw3Tokenizer.keep_accents",description:`<strong>keep_accents</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to keep accents when tokenizing.`,name:"keep_accents"},{anchor:"transformers.GPTSw3Tokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The token used for padding, for example when batching sequences of different lengths. If not provided, will
default to &#x2019;<pad>&#x2019; or &#x2019;<unk>&#x2019; depending on model size.</unk></pad>`,name:"pad_token"},{anchor:"transformers.GPTSw3Tokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead. If not provided, will default to &#x2019;<unk>&#x2018;.</unk>`,name:"unk_token"},{anchor:"transformers.GPTSw3Tokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The end of sequence token seen during pretraining. If not provided, will default to &#x2019;&lt;|endoftext|&gt;&#x2019;`,name:"eos_token"},{anchor:"transformers.GPTSw3Tokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The beginning of sequence token that can be used for downstream task, was not seen during pretraining. If
not provided, will default to &#x2019;<s>&#x2019; or &#x2019;&lt;|endoftext|&gt;&#x2019;, depending on model size.</s>`,name:"bos_token"},{anchor:"transformers.GPTSw3Tokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Will be passed to the <code>SentencePieceProcessor.__init__()</code> method. The <a href="https://github.com/google/sentencepiece/tree/master/python" rel="nofollow">Python wrapper for
SentencePiece</a> can be used, among other things,
to set:</p>
<ul>
<li>
<p><code>enable_sampling</code>: Enable subword regularization.</p>
</li>
<li>
<p><code>nbest_size</code>: Sampling parameters for unigram. Invalid for BPE-Dropout.</p>
<ul>
<li><code>nbest_size = {0,1}</code>: No sampling is performed.</li>
<li><code>nbest_size &gt; 1</code>: samples from the nbest_size results.</li>
<li><code>nbest_size &lt; 0</code>: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
using forward-filtering-and-backward-sampling algorithm.</li>
</ul>
</li>
<li>
<p><code>alpha</code>: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
BPE-dropout.</p>
</li>
</ul>`,name:"sp_model_kwargs"},{anchor:"transformers.GPTSw3Tokenizer.sp_model",description:`<strong>sp_model</strong> (<code>SentencePieceProcessor</code>) &#x2014;
The <em>SentencePiece</em> processor that is used for every conversion (string, tokens and IDs).`,name:"sp_model"},{anchor:"transformers.GPTSw3Tokenizer.whitespaces",description:`<strong>whitespaces</strong> (<code>set</code>) &#x2014;
The whitespaces that are replaced in the whitespace normalization in preprocessing.`,name:"whitespaces"},{anchor:"transformers.GPTSw3Tokenizer.non_printing_characters_re",description:`<strong>non_printing_characters_re</strong> (<code>Pattern</code>) &#x2014;
The compiled regular expression to remove non-printing characters in preprocessing.`,name:"non_printing_characters_re"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L45"}}),y=new ze({props:{anchor:"transformers.GPTSw3Tokenizer.example",$$slots:{default:[Se]},$$scope:{ctx:E}}}),R=new we({props:{name:"save_vocabulary",anchor:"transformers.GPTSw3Tokenizer.save_vocabulary",parameters:[{name:"save_directory",val:": str"},{name:"filename_prefix",val:": Optional = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L258"}}),{c(){a=g("meta"),$=r(),p=g("p"),i=r(),h(c.$$.fragment),s=r(),h(m.$$.fragment),q=r(),x=g("p"),x.innerHTML=ce,A=r(),S=g("p"),S.textContent=me,F=r(),P=g("p"),P.textContent=de,Q=r(),G=g("p"),G.innerHTML=ge,N=r(),h(U.$$.fragment),D=r(),h(j.$$.fragment),Y=r(),h(C.$$.fragment),K=r(),I=g("ul"),I.innerHTML=fe,O=r(),h(M.$$.fragment),ee=r(),h(V.$$.fragment),te=r(),d=g("div"),h(Z.$$.fragment),re=r(),H=g("p"),H.innerHTML=ue,le=r(),L=g("p"),L.innerHTML=he,ie=r(),h(y.$$.fragment),pe=r(),J=g("div"),h(R.$$.fragment),ne=r(),W=g("p"),this.h()},l(e){const t=Me("svelte-u9bgzb",document.head);a=f(t,"META",{name:!0,content:!0}),t.forEach(n),$=l(e),p=f(e,"P",{}),oe(p).forEach(n),i=l(e),w(c.$$.fragment,e),s=l(e),w(m.$$.fragment,e),q=l(e),x=f(e,"P",{"data-svelte-h":!0}),v(x)!=="svelte-1qucz45"&&(x.innerHTML=ce),A=l(e),S=f(e,"P",{"data-svelte-h":!0}),v(S)!=="svelte-1uddber"&&(S.textContent=me),F=l(e),P=f(e,"P",{"data-svelte-h":!0}),v(P)!=="svelte-19zkpwq"&&(P.textContent=de),Q=l(e),G=f(e,"P",{"data-svelte-h":!0}),v(G)!=="svelte-p3ijej"&&(G.innerHTML=ge),N=l(e),w(U.$$.fragment,e),D=l(e),w(j.$$.fragment,e),Y=l(e),w(C.$$.fragment,e),K=l(e),I=f(e,"UL",{"data-svelte-h":!0}),v(I)!=="svelte-v6nfyh"&&(I.innerHTML=fe),O=l(e),w(M.$$.fragment,e),ee=l(e),w(V.$$.fragment,e),te=l(e),d=f(e,"DIV",{class:!0});var u=oe(d);w(Z.$$.fragment,u),re=l(u),H=f(u,"P",{"data-svelte-h":!0}),v(H)!=="svelte-19f98fg"&&(H.innerHTML=ue),le=l(u),L=f(u,"P",{"data-svelte-h":!0}),v(L)!=="svelte-1c3t5ty"&&(L.innerHTML=he),ie=l(u),w(y.$$.fragment,u),pe=l(u),J=f(u,"DIV",{class:!0});var X=oe(J);w(R.$$.fragment,X),X.forEach(n),u.forEach(n),ne=l(e),W=f(e,"P",{}),oe(W).forEach(n),this.h()},h(){ae(a,"name","hf:doc:metadata"),ae(a,"content",Ge),ae(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),ae(d,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){z(document.head,a),o(e,$,t),o(e,p,t),o(e,i,t),k(c,e,t),o(e,s,t),k(m,e,t),o(e,q,t),o(e,x,t),o(e,A,t),o(e,S,t),o(e,F,t),o(e,P,t),o(e,Q,t),o(e,G,t),o(e,N,t),k(U,e,t),o(e,D,t),k(j,e,t),o(e,Y,t),k(C,e,t),o(e,K,t),o(e,I,t),o(e,O,t),k(M,e,t),o(e,ee,t),k(V,e,t),o(e,te,t),o(e,d,t),k(Z,d,null),z(d,re),z(d,H),z(d,le),z(d,L),z(d,ie),k(y,d,null),z(d,pe),z(d,J),k(R,J,null),o(e,ne,t),o(e,W,t),se=!0},p(e,[t]){const u={};t&2&&(u.$$scope={dirty:t,ctx:e}),M.$set(u);const X={};t&2&&(X.$$scope={dirty:t,ctx:e}),y.$set(X)},i(e){se||(_(c.$$.fragment,e),_(m.$$.fragment,e),_(U.$$.fragment,e),_(j.$$.fragment,e),_(C.$$.fragment,e),_(M.$$.fragment,e),_(V.$$.fragment,e),_(Z.$$.fragment,e),_(y.$$.fragment,e),_(R.$$.fragment,e),se=!0)},o(e){T(c.$$.fragment,e),T(m.$$.fragment,e),T(U.$$.fragment,e),T(j.$$.fragment,e),T(C.$$.fragment,e),T(M.$$.fragment,e),T(V.$$.fragment,e),T(Z.$$.fragment,e),T(y.$$.fragment,e),T(R.$$.fragment,e),se=!1},d(e){e&&(n($),n(p),n(i),n(s),n(q),n(x),n(A),n(S),n(F),n(P),n(Q),n(G),n(N),n(D),n(Y),n(K),n(I),n(O),n(ee),n(te),n(d),n(ne),n(W)),n(a),b(c,e),b(m,e),b(U,e),b(j,e),b(C,e),b(M,e),b(V,e),b(Z),b(y),b(R)}}}const Ge='{"title":"GPT-Sw3","local":"gpt-sw3","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage example","local":"usage-example","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2},{"title":"GPTSw3Tokenizer","local":"transformers.GPTSw3Tokenizer","sections":[],"depth":2}],"depth":1}';function Ue(E){return be(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class He extends $e{constructor(a){super(),ve(this,a,Ue,Pe,Te,{})}}export{He as component};
