import{S as Us,i as Ms,s as Ns,e as n,k as p,w as y,t as r,M as $s,c as a,d as o,m as h,a as s,x as v,h as l,b as i,G as e,g,y as J,q as w,o as k,B as U,v as Gs,L as vo}from"../../chunks/vendor-hf-doc-builder.js";import{T as ks}from"../../chunks/Tip-hf-doc-builder.js";import{D as He}from"../../chunks/Docstring-hf-doc-builder.js";import{C as At}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as Te}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as yo}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function js(j){let c,T;return c=new At({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVE5lb1hKYXBhbmVzZUNvbmZpZyUyQyUyMEdQVE5lb1hKYXBhbmVzZU1vZGVsJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMEdQVE5lb1hKYXBhbmVzZSUyMGdwdC1uZW94LWphcGFuZXNlLTIuN2IlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwR1BUTmVvWEphcGFuZXNlQ29uZmlnKCklMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwbW9kZWwlMjAod2l0aCUyMHJhbmRvbSUyMHdlaWdodHMpJTIwZnJvbSUyMHRoZSUyMGdwdC1uZW94LWphcGFuZXNlLTIuN2IlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFtb2RlbCUyMCUzRCUyMEdQVE5lb1hKYXBhbmVzZU1vZGVsKGNvbmZpZ3VyYXRpb24pJTBBJTBBJTIzJTIwQWNjZXNzaW5nJTIwdGhlJTIwbW9kZWwlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMG1vZGVsLmNvbmZpZw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTNeoXJapaneseConfig, GPTNeoXJapaneseModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a GPTNeoXJapanese gpt-neox-japanese-2.7b style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = GPTNeoXJapaneseConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model (with random weights) from the gpt-neox-japanese-2.7b style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = GPTNeoXJapaneseModel(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`}}),{c(){y(c.$$.fragment)},l(u){v(c.$$.fragment,u)},m(u,f){J(c,u,f),T=!0},p:vo,i(u){T||(w(c.$$.fragment,u),T=!0)},o(u){k(c.$$.fragment,u),T=!1},d(u){U(c,u)}}}function Es(j){let c,T,u,f,b;return f=new At({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVE5lb1hKYXBhbmVzZVRva2VuaXplciUwQSUwQXRva2VuaXplciUyMCUzRCUyMEdQVE5lb1hKYXBhbmVzZVRva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQSUyMyUyMFlvdSUyMGNhbiUyMGNvbmZpcm0lMjBib3RoJTIwJUU2JTg1JUI2JUU1JUJGJTlDJTIwYW5kJTIwJUU2JTg1JUI2JUU2JTg3JTg5JTIwYXJlJTIwZW5jb2RlZCUyMHRvJTIwMTc3NDklMEF0b2tlbml6ZXIoJTIyJUU1JTkwJUJFJUU4JUJDJUE5JUUzJTgxJUFGJUU3JThDJUFCJUUzJTgxJUE3JUUzJTgxJTgyJUUzJTgyJThCJUYwJTlGJTkwJUFGJUUzJTgwJTgyJUU1JUFFJTlGJUUzJTgxJUFGJUU2JTg1JUI2JUU1JUJGJTlDKCVFNiU4NSVCNiVFNiU4NyU4OSklRTUlQTQlQTclRTUlQUQlQTYlRTUlODclQkElRTglQkElQUIlMjIpJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTBBJTBBJTIzJTIwQm90aCUyMCVFNiU4NSVCNiVFNSVCRiU5QyUyMGFuZCUyMCVFNiU4NSVCNiVFNiU4NyU4OSUyMGFyZSUyMGRlY29kZWQlMjB0byUyMCVFNiU4NSVCNiVFNSVCRiU5QyUwQXRva2VuaXplci5kZWNvZGUodG9rZW5pemVyKCUyMiVFNSU5MCVCRSVFOCVCQyVBOSVFMyU4MSVBRiVFNyU4QyVBQiVFMyU4MSVBNyVFMyU4MSU4MiVFMyU4MiU4QiVGMCU5RiU5MCVBRiVFMyU4MCU4MiVFNSVBRSU5RiVFMyU4MSVBRiVFNiU4NSVCNiVFNSVCRiU5QyglRTYlODUlQjYlRTYlODclODkpJUU1JUE0JUE3JUU1JUFEJUE2JUU1JTg3JUJBJUU4JUJBJUFCJTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTNeoXJapaneseTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTNeoXJapaneseTokenizer.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># You can confirm both \u6176\u5FDC and \u6176\u61C9 are encoded to 17749</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u{1F42F}\u3002\u5B9F\u306F\u6176\u5FDC(\u6176\u61C9)\u5927\u5B66\u51FA\u8EAB&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
[<span class="hljs-number">30014</span>, <span class="hljs-number">26883</span>, <span class="hljs-number">26638</span>, <span class="hljs-number">27228</span>, <span class="hljs-number">25</span>, <span class="hljs-number">26650</span>, <span class="hljs-number">31732</span>, <span class="hljs-number">31679</span>, <span class="hljs-number">27809</span>, <span class="hljs-number">26638</span>, <span class="hljs-number">17749</span>, <span class="hljs-number">31592</span>, <span class="hljs-number">17749</span>, <span class="hljs-number">31593</span>, <span class="hljs-number">321</span>, <span class="hljs-number">1281</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Both \u6176\u5FDC and \u6176\u61C9 are decoded to \u6176\u5FDC</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(tokenizer(<span class="hljs-string">&quot;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u{1F42F}\u3002\u5B9F\u306F\u6176\u5FDC(\u6176\u61C9)\u5927\u5B66\u51FA\u8EAB&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;\u543E\u8F29\u306F\u732B\u3067\u3042\u308B\u{1F42F}\u3002\u5B9F\u306F\u6176\u5FDC(\u6176\u5FDC)\u5927\u5B66\u51FA\u8EAB&#x27;</span>`}}),{c(){c=n("p"),T=r("Example:"),u=p(),y(f.$$.fragment)},l(d){c=a(d,"P",{});var _=s(c);T=l(_,"Example:"),_.forEach(o),u=h(d),v(f.$$.fragment,d)},m(d,_){g(d,c,_),e(c,T),g(d,u,_),J(f,d,_),b=!0},p:vo,i(d){b||(w(f.$$.fragment,d),b=!0)},o(d){k(f.$$.fragment,d),b=!1},d(d){d&&o(c),d&&o(u),U(f,d)}}}function Ps(j){let c,T,u,f,b;return{c(){c=n("p"),T=r("Although the recipe for forward pass needs to be defined within this function, one should call the "),u=n("code"),f=r("Module"),b=r(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(d){c=a(d,"P",{});var _=s(c);T=l(_,"Although the recipe for forward pass needs to be defined within this function, one should call the "),u=a(_,"CODE",{});var F=s(u);f=l(F,"Module"),F.forEach(o),b=l(_,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),_.forEach(o)},m(d,_){g(d,c,_),e(c,T),e(c,u),e(u,f),e(c,b)},d(d){d&&o(c)}}}function Xs(j){let c,T,u,f,b;return f=new At({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBHUFROZW9YSmFwYW5lc2VNb2RlbCUwQWltcG9ydCUyMHRvcmNoJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQW1vZGVsJTIwJTNEJTIwR1BUTmVvWEphcGFuZXNlTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMmFiZWphJTJGZ3B0LW5lb3gtamFwYW5lc2UtMi43YiUyMiklMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIyJUU2JTk3JUE1JUU2JTlDJUFDJUU4JUFBJTlFJUUzJTgxJUFFR1BULW5lb3glRTMlODElOENIdWdnaW5nJTIwRmFjZSVFMyU4MSVBNyVFNCVCRCVCRiVFMyU4MSU4OCVFMyU4MSVCRSVFMyU4MSU5OSVGMCU5RiU5OCU4MCUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKSUwQSUwQWxhc3RfaGlkZGVuX3N0YXRlcyUyMCUzRCUyMG91dHB1dHMubGFzdF9oaWRkZW5fc3RhdGU=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, GPTNeoXJapaneseModel
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = GPTNeoXJapaneseModel.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;\u65E5\u672C\u8A9E\u306EGPT-neox\u304CHugging Face\u3067\u4F7F\u3048\u307E\u3059\u{1F600}&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(**inputs)

<span class="hljs-meta">&gt;&gt;&gt; </span>last_hidden_states = outputs.last_hidden_state`}}),{c(){c=n("p"),T=r("Example:"),u=p(),y(f.$$.fragment)},l(d){c=a(d,"P",{});var _=s(c);T=l(_,"Example:"),_.forEach(o),u=h(d),v(f.$$.fragment,d)},m(d,_){g(d,c,_),e(c,T),g(d,u,_),J(f,d,_),b=!0},p:vo,i(d){b||(w(f.$$.fragment,d),b=!0)},o(d){k(f.$$.fragment,d),b=!1},d(d){d&&o(c),d&&o(u),U(f,d)}}}function Cs(j){let c,T,u,f,b;return{c(){c=n("p"),T=r("Although the recipe for forward pass needs to be defined within this function, one should call the "),u=n("code"),f=r("Module"),b=r(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(d){c=a(d,"P",{});var _=s(c);T=l(_,"Although the recipe for forward pass needs to be defined within this function, one should call the "),u=a(_,"CODE",{});var F=s(u);f=l(F,"Module"),F.forEach(o),b=l(_,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),_.forEach(o)},m(d,_){g(d,c,_),e(c,T),e(c,u),e(u,f),e(c,b)},d(d){d&&o(c)}}}function zs(j){let c,T,u,f,b;return f=new At({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBHUFROZW9YSmFwYW5lc2VGb3JDYXVzYWxMTSUyQyUyMEdQVE5lb1hKYXBhbmVzZUNvbmZpZyUwQWltcG9ydCUyMHRvcmNoJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQWNvbmZpZyUyMCUzRCUyMEdQVE5lb1hKYXBhbmVzZUNvbmZpZy5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQWNvbmZpZy5pc19kZWNvZGVyJTIwJTNEJTIwVHJ1ZSUwQW1vZGVsJTIwJTNEJTIwR1BUTmVvWEphcGFuZXNlRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmFiZWphJTJGZ3B0LW5lb3gtamFwYW5lc2UtMi43YiUyMiUyQyUyMGNvbmZpZyUzRGNvbmZpZyklMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIyJUU2JTk3JUE1JUU2JTlDJUFDJUU4JUFBJTlFJUUzJTgxJUFFR1BULW5lb3glRTMlODElOENIdWdnaW5nJTIwRmFjZSVFMyU4MSVBNyVFNCVCRCVCRiVFMyU4MSU4OCVFMyU4MSVCRSVFMyU4MSU5OSVGMCU5RiU5OCU4MCUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKSUwQSUwQXByZWRpY3Rpb25fbG9naXRzJTIwJTNEJTIwb3V0cHV0cy5sb2dpdHM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseConfig
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>config = GPTNeoXJapaneseConfig.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>config.is_decoder = <span class="hljs-literal">True</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = GPTNeoXJapaneseForCausalLM.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>, config=config)

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;\u65E5\u672C\u8A9E\u306EGPT-neox\u304CHugging Face\u3067\u4F7F\u3048\u307E\u3059\u{1F600}&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(**inputs)

<span class="hljs-meta">&gt;&gt;&gt; </span>prediction_logits = outputs.logits`}}),{c(){c=n("p"),T=r("Example:"),u=p(),y(f.$$.fragment)},l(d){c=a(d,"P",{});var _=s(c);T=l(_,"Example:"),_.forEach(o),u=h(d),v(f.$$.fragment,d)},m(d,_){g(d,c,_),e(c,T),g(d,u,_),J(f,d,_),b=!0},p:vo,i(d){b||(w(f.$$.fragment,d),b=!0)},o(d){k(f.$$.fragment,d),b=!1},d(d){d&&o(c),d&&o(u),U(f,d)}}}function xs(j){let c,T,u,f,b,d,_,F,Jo,Lt,W,ee,ut,be,wo,ft,ko,Qt,N,Uo,ye,Mo,No,ve,$o,Go,gt,jo,Eo,Je,Po,Xo,we,Co,zo,Ot,M,xo,ke,Fo,Vo,Ue,Zo,Io,Me,Wo,Bo,Ne,So,Ro,$e,qo,Ao,Ge,Lo,Qo,Yt,B,te,_t,je,Oo,Tt,Yo,Ht,oe,Ho,bt,Do,Ko,Dt,Ee,Kt,S,ne,yt,Pe,en,vt,tn,eo,De,Jt,Ke,on,to,R,ae,wt,Xe,nn,kt,an,oo,P,Ce,sn,q,rn,Ut,ln,dn,ze,cn,pn,hn,A,mn,et,un,fn,tt,gn,_n,Tn,se,no,L,re,Mt,xe,bn,Nt,yn,ao,$,Fe,vn,Q,Jn,ot,wn,kn,Ve,Un,Mn,Nn,G,$t,$n,Gn,Gt,jn,En,nt,Pn,jt,Xn,Cn,Et,zn,xn,Pt,Fn,Vn,Xt,Zn,In,le,Wn,ie,Ze,Bn,Ct,Sn,so,O,de,zt,Ie,Rn,xt,qn,ro,V,We,An,Be,Ln,Se,Qn,On,Yn,X,Re,Hn,Y,Dn,at,Kn,ea,Ft,ta,oa,na,ce,aa,pe,lo,H,he,Vt,qe,sa,Zt,ra,io,Z,Ae,la,D,ia,It,da,ca,Le,pa,ha,ma,C,Qe,ua,K,fa,st,ga,_a,Wt,Ta,ba,ya,me,va,ue,co;return d=new Te({}),be=new Te({}),je=new Te({}),Ee=new At({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVE5lb1hKYXBhbmVzZUZvckNhdXNhbExNJTJDJTIwR1BUTmVvWEphcGFuZXNlVG9rZW5pemVyJTBBJTBBbW9kZWwlMjAlM0QlMjBHUFROZW9YSmFwYW5lc2VGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQXRva2VuaXplciUyMCUzRCUyMEdQVE5lb1hKYXBhbmVzZVRva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyYWJlamElMkZncHQtbmVveC1qYXBhbmVzZS0yLjdiJTIyKSUwQSUwQXByb21wdCUyMCUzRCUyMCUyMiVFNCVCQSVCQSVFMyU4MSVBOEFJJUUzJTgxJThDJUU1JThEJTk0JUU4JUFBJUJGJUUzJTgxJTk5JUUzJTgyJThCJUUzJTgxJTlGJUUzJTgyJTgxJUUzJTgxJUFCJUUzJTgxJUFGJUUzJTgwJTgxJTIyJTBBJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLmlucHV0X2lkcyUwQSUwQWdlbl90b2tlbnMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSglMEElMjAlMjAlMjAlMjBpbnB1dF9pZHMlMkMlMEElMjAlMjAlMjAlMjBkb19zYW1wbGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwdGVtcGVyYXR1cmUlM0QwLjklMkMlMEElMjAlMjAlMjAlMjBtYXhfbGVuZ3RoJTNEMTAwJTJDJTBBKSUwQWdlbl90ZXh0JTIwJTNEJTIwdG9rZW5pemVyLmJhdGNoX2RlY29kZShnZW5fdG9rZW5zJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTVCMCU1RCUwQSUwQXByaW50KGdlbl90ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = GPTNeoXJapaneseForCausalLM.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTNeoXJapaneseTokenizer.from_pretrained(<span class="hljs-string">&quot;abeja/gpt-neox-japanese-2.7b&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;\u4EBA\u3068AI\u304C\u5354\u8ABF\u3059\u308B\u305F\u3081\u306B\u306F\u3001&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids

<span class="hljs-meta">&gt;&gt;&gt; </span>gen_tokens = model.generate(
<span class="hljs-meta">... </span>    input_ids,
<span class="hljs-meta">... </span>    do_sample=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    temperature=<span class="hljs-number">0.9</span>,
<span class="hljs-meta">... </span>    max_length=<span class="hljs-number">100</span>,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>gen_text = tokenizer.batch_decode(gen_tokens, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(gen_text)
\u4EBA\u3068AI\u304C\u5354\u8ABF\u3059\u308B\u305F\u3081\u306B\u306F\u3001AI\u3068\u4EBA\u304C\u5171\u5B58\u3057\u3001AI\u3092\u6B63\u3057\u304F\u7406\u89E3\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002`}}),Pe=new Te({}),Xe=new Te({}),Ce=new He({props:{name:"class transformers.GPTNeoXJapaneseConfig",anchor:"transformers.GPTNeoXJapaneseConfig",parameters:[{name:"vocab_size",val:" = 32000"},{name:"hidden_size",val:" = 2560"},{name:"num_hidden_layers",val:" = 32"},{name:"num_attention_heads",val:" = 32"},{name:"intermediate_multiple_size",val:" = 4"},{name:"hidden_act",val:" = 'gelu'"},{name:"rotary_pct",val:" = 1.0"},{name:"rotary_emb_base",val:" = 10000"},{name:"max_position_embeddings",val:" = 2048"},{name:"initializer_range",val:" = 0.02"},{name:"layer_norm_eps",val:" = 1e-05"},{name:"use_cache",val:" = True"},{name:"bos_token_id",val:" = 31996"},{name:"eos_token_id",val:" = 31999"},{name:"attention_dropout",val:" = 0.1"},{name:"hidden_dropout",val:" = 0.0"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to 32000) &#x2014;
Vocabulary size of the GPTNeoXJapanese model. Defines the number of different tokens that can be
represented by the <code>inputs_ids</code> passed when calling <code>GPTNeoXJapanese</code>.`,name:"vocab_size"},{anchor:"transformers.GPTNeoXJapaneseConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 2560) &#x2014;
Dimension of the encoder layers and the pooler layer.`,name:"hidden_size"},{anchor:"transformers.GPTNeoXJapaneseConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.GPTNeoXJapaneseConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014;
Number of attention heads for each attention layer in the Transformer encoder.`,name:"num_attention_heads"},{anchor:"transformers.GPTNeoXJapaneseConfig.intermediate_multiple_size",description:`<strong>intermediate_multiple_size</strong> (<code>int</code>, <em>optional</em>, defaults to 4) &#x2014;
Dimension of the &#x201C;intermediate&#x201D; layer in the Transformer encoder is calculated by hidden_size *
intermediate_multiple_size.`,name:"intermediate_multiple_size"},{anchor:"transformers.GPTNeoXJapaneseConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code> or <code>function</code>, <em>optional</em>, defaults to <code>&quot;gelu&quot;</code>) &#x2014;
The non-linear activation function (function or string) in the encoder and pooler.`,name:"hidden_act"},{anchor:"transformers.GPTNeoXJapaneseConfig.rotary_pct",description:`<strong>rotary_pct</strong> (<code>float</code>, <em>optional</em>, defaults to 1.00) &#x2014;
percentage of hidden dimensions to allocate to rotary embeddings`,name:"rotary_pct"},{anchor:"transformers.GPTNeoXJapaneseConfig.rotary_emb_base",description:`<strong>rotary_emb_base</strong> (<code>int</code>, <em>optional</em>, defaults to 10000) &#x2014;
base for computing rotary embeddings frequency`,name:"rotary_emb_base"},{anchor:"transformers.GPTNeoXJapaneseConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to 2048) &#x2014;
The maximum sequence length that this model might ever be used with.`,name:"max_position_embeddings"},{anchor:"transformers.GPTNeoXJapaneseConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 0.02) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.GPTNeoXJapaneseConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-5) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.GPTNeoXJapaneseConfig.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if <code>config.is_decoder=True</code>.`,name:"use_cache"},{anchor:"transformers.GPTNeoXJapaneseConfig.attention_dropout",description:`<strong>attention_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The dropout ratio for the attention.`,name:"attention_dropout"},{anchor:"transformers.GPTNeoXJapaneseConfig.hidden_dropout",description:`<strong>hidden_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The dropout ratio for the hidden layer.
Example &#x2014;`,name:"hidden_dropout"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/gpt_neox_japanese/configuration_gpt_neox_japanese.py#L28"}}),se=new yo({props:{anchor:"transformers.GPTNeoXJapaneseConfig.example",$$slots:{default:[js]},$$scope:{ctx:j}}}),xe=new Te({}),Fe=new He({props:{name:"class transformers.GPTNeoXJapaneseTokenizer",anchor:"transformers.GPTNeoXJapaneseTokenizer",parameters:[{name:"vocab_file",val:""},{name:"emoji_file",val:""},{name:"unk_token",val:" = '<|endoftext|>'"},{name:"pad_token",val:" = '<|endoftext|>'"},{name:"bos_token",val:" = '<|startoftext|>'"},{name:"eos_token",val:" = '<|endoftext|>'"},{name:"do_clean_text",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
File containing the vocabulary.`,name:"vocab_file"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.emoji_file",description:`<strong>emoji_file</strong> (<code>str</code>) &#x2014;
File containing the emoji.`,name:"emoji_file"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|endoftext|&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|endoftext|&gt;&quot;</code>) &#x2014;
The token used for padding`,name:"pad_token"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|startoftext|&gt;&quot;</code>) &#x2014;
The beginning of sequence token.`,name:"bos_token"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;|endoftext|&gt;&quot;</code>) &#x2014;
The end of sequence token.`,name:"eos_token"},{anchor:"transformers.GPTNeoXJapaneseTokenizer.do_clean_text",description:`<strong>do_clean_text</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to clean text for URL, EMAIL, TEL, Japanese DATE and Japanese PRICE.`,name:"do_clean_text"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/gpt_neox_japanese/tokenization_gpt_neox_japanese.py#L70"}}),le=new yo({props:{anchor:"transformers.GPTNeoXJapaneseTokenizer.example",$$slots:{default:[Es]},$$scope:{ctx:j}}}),Ze=new He({props:{name:"convert_tokens_to_string",anchor:"transformers.GPTNeoXJapaneseTokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/gpt_neox_japanese/tokenization_gpt_neox_japanese.py#L177"}}),Ie=new Te({}),We=new He({props:{name:"class transformers.GPTNeoXJapaneseModel",anchor:"transformers.GPTNeoXJapaneseModel",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.32.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseConfig">~GPTNeoXJapaneseConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/v4.32.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py#L440"}}),Re=new He({props:{name:"forward",anchor:"transformers.GPTNeoXJapaneseModel.forward",parameters:[{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"head_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"past_key_values",val:": typing.Optional[typing.Tuple[typing.Tuple[torch.FloatTensor]]] = None"},{name:"use_cache",val:": typing.Optional[bool] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseModel.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.32.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>.`,name:"input_ids"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p>
<ul>
<li>0 corresponds to a <em>sentence A</em> token,</li>
<li>1 corresponds to a <em>sentence B</em> token.</li>
</ul>`,name:"token_type_ids"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.max_position_embeddings - 1]</code>.`,name:"position_ids"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.head_mask",description:`<strong>head_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(num_heads,)</code> or <code>(num_layers, num_heads)</code>, <em>optional</em>) &#x2014;
Mask to nullify selected heads of the self-attention modules. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"head_mask"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <em>input_ids</em> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code> of length <code>config.n_layers</code> with each tuple having 4 tensors of shape <code>(batch_size, num_heads, sequence_length - 1, embed_size_per_head)</code>) &#x2014;
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.GPTNeoXJapaneseModel.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py#L460",returnDescription:`
<p>A <a
  href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPast"
>transformers.modeling_outputs.BaseModelOutputWithPast</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/v4.32.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseConfig"
>GPTNeoXJapaneseConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) \u2014 Sequence of hidden-states at the output of the last layer of the model.</p>
<p>If <code>past_key_values</code> is used only the last hidden-state of the sequences of shape <code>(batch_size, 1, hidden_size)</code> is output.</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) \u2014 Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and optionally if
<code>config.is_encoder_decoder=True</code> 2 additional tensors of shape <code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
<code>config.is_encoder_decoder=True</code> in the cross-attention blocks) that can be used (see <code>past_key_values</code>
input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.modeling_outputs.BaseModelOutputWithPast"
>transformers.modeling_outputs.BaseModelOutputWithPast</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),ce=new ks({props:{$$slots:{default:[Ps]},$$scope:{ctx:j}}}),pe=new yo({props:{anchor:"transformers.GPTNeoXJapaneseModel.forward.example",$$slots:{default:[Xs]},$$scope:{ctx:j}}}),qe=new Te({}),Ae=new He({props:{name:"class transformers.GPTNeoXJapaneseForCausalLM",anchor:"transformers.GPTNeoXJapaneseForCausalLM",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseForCausalLM.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.32.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseConfig">~GPTNeoXJapaneseConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/v4.32.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py#L595"}}),Qe=new He({props:{name:"forward",anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward",parameters:[{name:"input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"head_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"past_key_values",val:": typing.Optional[typing.Tuple[typing.Tuple[torch.FloatTensor]]] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"},{name:"use_cache",val:": typing.Optional[bool] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.32.0/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>.`,name:"input_ids"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.token_type_ids",description:`<strong>token_type_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Segment token indices to indicate first and second portions of the inputs. Indices are selected in <code>[0, 1]</code>:</p>
<ul>
<li>0 corresponds to a <em>sentence A</em> token,</li>
<li>1 corresponds to a <em>sentence B</em> token.</li>
</ul>`,name:"token_type_ids"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.max_position_embeddings - 1]</code>.`,name:"position_ids"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.head_mask",description:`<strong>head_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(num_heads,)</code> or <code>(num_layers, num_heads)</code>, <em>optional</em>) &#x2014;
Mask to nullify selected heads of the self-attention modules. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"head_mask"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <em>input_ids</em> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) &#x2014;
Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>. The two additional tensors are
only required when the model is used as a decoder in a Sequence to Sequence model.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
<code>[-100, 0, ..., config.vocab_size]</code> (see <code>input_ids</code> docstring) Tokens with indices set to <code>-100</code> are
ignored (masked), the loss is only computed for the tokens with labels n <code>[0, ..., config.vocab_size]</code>.`,name:"labels"},{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/gpt_neox_japanese/modeling_gpt_neox_japanese.py#L614",returnDescription:`
<p>A <a
  href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.modeling_outputs.CausalLMOutputWithPast"
>transformers.modeling_outputs.CausalLMOutputWithPast</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/v4.32.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseConfig"
>GPTNeoXJapaneseConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) \u2014 Language modeling loss (for next-token prediction).</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) \u2014 Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) \u2014 Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>)</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.modeling_outputs.CausalLMOutputWithPast"
>transformers.modeling_outputs.CausalLMOutputWithPast</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),me=new ks({props:{$$slots:{default:[Cs]},$$scope:{ctx:j}}}),ue=new yo({props:{anchor:"transformers.GPTNeoXJapaneseForCausalLM.forward.example",$$slots:{default:[zs]},$$scope:{ctx:j}}}),{c(){c=n("meta"),T=p(),u=n("h1"),f=n("a"),b=n("span"),y(d.$$.fragment),_=p(),F=n("span"),Jo=r("GPT-NeoX-Japanese"),Lt=p(),W=n("h2"),ee=n("a"),ut=n("span"),y(be.$$.fragment),wo=p(),ft=n("span"),ko=r("Overview"),Qt=p(),N=n("p"),Uo=r("We introduce GPT-NeoX-Japanese, which is an autoregressive language model for Japanese, trained on top of "),ye=n("a"),Mo=r("https://github.com/EleutherAI/gpt-neox"),No=r(`.
Japanese is a unique language with its large vocabulary and a combination of hiragana, katakana, and kanji writing scripts.
To address this distinct structure of the Japanese language, we use a `),ve=n("a"),$o=r("special sub-word tokenizer"),Go=r(". We are very grateful to "),gt=n("em"),jo=r("tanreinama"),Eo=r(` for open-sourcing this incredibly helpful tokenizer.
Following the recommendations from Google\u2019s research on `),Je=n("a"),Po=r("PaLM"),Xo=r(", we have removed bias parameters from transformer blocks, achieving better model performance. Please refer "),we=n("a"),Co=r("this article"),zo=r(" in detail."),Ot=p(),M=n("p"),xo=r("Development of the model was led by "),ke=n("a"),Fo=r("Shinya Otani"),Vo=r(", "),Ue=n("a"),Zo=r("Takayoshi Makabe"),Io=r(", "),Me=n("a"),Wo=r("Anuj Arora"),Bo=r(", and "),Ne=n("a"),So=r("Kyo Hattori"),Ro=r(" from "),$e=n("a"),qo=r("ABEJA, Inc."),Ao=r(". For more information on this model-building activity, please refer "),Ge=n("a"),Lo=r("here (ja)"),Qo=r("."),Yt=p(),B=n("h3"),te=n("a"),_t=n("span"),y(je.$$.fragment),Oo=p(),Tt=n("span"),Yo=r("Generation"),Ht=p(),oe=n("p"),Ho=r("The "),bt=n("code"),Do=r("generate()"),Ko=r(" method can be used to generate text using GPT NeoX Japanese model."),Dt=p(),y(Ee.$$.fragment),Kt=p(),S=n("h2"),ne=n("a"),yt=n("span"),y(Pe.$$.fragment),en=p(),vt=n("span"),tn=r("Documentation resources"),eo=p(),De=n("ul"),Jt=n("li"),Ke=n("a"),on=r("Causal language modeling task guide"),to=p(),R=n("h2"),ae=n("a"),wt=n("span"),y(Xe.$$.fragment),nn=p(),kt=n("span"),an=r("GPTNeoXJapaneseConfig"),oo=p(),P=n("div"),y(Ce.$$.fragment),sn=p(),q=n("p"),rn=r("This is the configuration class to store the configuration of a "),Ut=n("code"),ln=r("GPTNeoXModelJapanese"),dn=r(`. It is used to instantiate
a GPTNeoX model according to the specified arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar configuration to that of the GPTNeoXJapanese
`),ze=n("a"),cn=r("abeja/gpt-neox-japanese-2.7b"),pn=r(" architecture."),hn=p(),A=n("p"),mn=r("Configuration objects inherit from "),et=n("a"),un=r("PretrainedConfig"),fn=r(` and can be used to control the model outputs. Read the
documentation from `),tt=n("a"),gn=r("PretrainedConfig"),_n=r(" for more information. Default configs is set as 2.7B model"),Tn=p(),y(se.$$.fragment),no=p(),L=n("h2"),re=n("a"),Mt=n("span"),y(xe.$$.fragment),bn=p(),Nt=n("span"),yn=r("GPTNeoXJapaneseTokenizer"),ao=p(),$=n("div"),y(Fe.$$.fragment),vn=p(),Q=n("p"),Jn=r("This tokenizer inherits from "),ot=n("a"),wn=r("PreTrainedTokenizer"),kn=r(` and is based on Japanese special Sub-Word-Encoding that is
used in this repository (`),Ve=n("a"),Un=r("https://github.com/tanreinama/Japanese-BPEEncoder_V2"),Mn=r(`). Check the repository for details.
Japanese has a relatively large vocabulary and there is no separation between words. Furthermore, the language is a
combination of hiragana, katakana, and kanji, and variants such as \u201C1\u201D and \u201C\u2460\u201D are often used. In order to cope
with these, this tokenizer has the following features`),Nn=p(),G=n("ul"),$t=n("li"),$n=r("Subword-by-subword segmentation, which is intermediate between byte strings and morphological analysis."),Gn=p(),Gt=n("li"),jn=r(`BPEs are created for each Kanji, Hiragana, and Katakana character, and there are no BPEs that cross character
types, such as Kanji + Hiragana or Hiragana + Katakana.`),En=p(),nt=n("li"),Pn=r("All-byte encoding that does not require "),jt=n("unk"),Xn=r("."),Cn=p(),Et=n("li"),zn=r("Independent of UTF codes such as 2-byte and 3-byte characters"),xn=p(),Pt=n("li"),Fn=r("Conversion of heterographs to the same token_id"),Vn=p(),Xt=n("li"),Zn=r("Emoji and Emoticon are grouped into 12 types as special tags."),In=p(),y(le.$$.fragment),Wn=p(),ie=n("div"),y(Ze.$$.fragment),Bn=p(),Ct=n("p"),Sn=r("Converts a sequence of tokens (string) in a single string."),so=p(),O=n("h2"),de=n("a"),zt=n("span"),y(Ie.$$.fragment),Rn=p(),xt=n("span"),qn=r("GPTNeoXJapaneseModel"),ro=p(),V=n("div"),y(We.$$.fragment),An=p(),Be=n("p"),Ln=r(`The bare GPTNeoXJapanese Model transformer outputting raw hidden-states without any specific head on top.
This model is a PyTorch `),Se=n("a"),Qn=r("torch.nn.Module"),On=r(` sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),Yn=p(),X=n("div"),y(Re.$$.fragment),Hn=p(),Y=n("p"),Dn=r("The "),at=n("a"),Kn=r("GPTNeoXJapaneseModel"),ea=r(" forward method, overrides the "),Ft=n("code"),ta=r("__call__"),oa=r(" special method."),na=p(),y(ce.$$.fragment),aa=p(),y(pe.$$.fragment),lo=p(),H=n("h2"),he=n("a"),Vt=n("span"),y(qe.$$.fragment),sa=p(),Zt=n("span"),ra=r("GPTNeoXJapaneseForCausalLM"),io=p(),Z=n("div"),y(Ae.$$.fragment),la=p(),D=n("p"),ia=r("GPTNeoXJapanese Model with a "),It=n("code"),da=r("language modeling"),ca=r(` head on top for Classifier Model fine-tuning.
This model is a PyTorch `),Le=n("a"),pa=r("torch.nn.Module"),ha=r(` sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),ma=p(),C=n("div"),y(Qe.$$.fragment),ua=p(),K=n("p"),fa=r("The "),st=n("a"),ga=r("GPTNeoXJapaneseForCausalLM"),_a=r(" forward method, overrides the "),Wt=n("code"),Ta=r("__call__"),ba=r(" special method."),ya=p(),y(me.$$.fragment),va=p(),y(ue.$$.fragment),this.h()},l(t){const m=$s('[data-svelte="svelte-1phssyn"]',document.head);c=a(m,"META",{name:!0,content:!0}),m.forEach(o),T=h(t),u=a(t,"H1",{class:!0});var Oe=s(u);f=a(Oe,"A",{id:!0,class:!0,href:!0});var Bt=s(f);b=a(Bt,"SPAN",{});var St=s(b);v(d.$$.fragment,St),St.forEach(o),Bt.forEach(o),_=h(Oe),F=a(Oe,"SPAN",{});var Rt=s(F);Jo=l(Rt,"GPT-NeoX-Japanese"),Rt.forEach(o),Oe.forEach(o),Lt=h(t),W=a(t,"H2",{class:!0});var Ye=s(W);ee=a(Ye,"A",{id:!0,class:!0,href:!0});var qt=s(ee);ut=a(qt,"SPAN",{});var wa=s(ut);v(be.$$.fragment,wa),wa.forEach(o),qt.forEach(o),wo=h(Ye),ft=a(Ye,"SPAN",{});var ka=s(ft);ko=l(ka,"Overview"),ka.forEach(o),Ye.forEach(o),Qt=h(t),N=a(t,"P",{});var z=s(N);Uo=l(z,"We introduce GPT-NeoX-Japanese, which is an autoregressive language model for Japanese, trained on top of "),ye=a(z,"A",{href:!0,rel:!0});var Ua=s(ye);Mo=l(Ua,"https://github.com/EleutherAI/gpt-neox"),Ua.forEach(o),No=l(z,`.
Japanese is a unique language with its large vocabulary and a combination of hiragana, katakana, and kanji writing scripts.
To address this distinct structure of the Japanese language, we use a `),ve=a(z,"A",{href:!0,rel:!0});var Ma=s(ve);$o=l(Ma,"special sub-word tokenizer"),Ma.forEach(o),Go=l(z,". We are very grateful to "),gt=a(z,"EM",{});var Na=s(gt);jo=l(Na,"tanreinama"),Na.forEach(o),Eo=l(z,` for open-sourcing this incredibly helpful tokenizer.
Following the recommendations from Google\u2019s research on `),Je=a(z,"A",{href:!0,rel:!0});var $a=s(Je);Po=l($a,"PaLM"),$a.forEach(o),Xo=l(z,", we have removed bias parameters from transformer blocks, achieving better model performance. Please refer "),we=a(z,"A",{href:!0,rel:!0});var Ga=s(we);Co=l(Ga,"this article"),Ga.forEach(o),zo=l(z," in detail."),z.forEach(o),Ot=h(t),M=a(t,"P",{});var E=s(M);xo=l(E,"Development of the model was led by "),ke=a(E,"A",{href:!0,rel:!0});var ja=s(ke);Fo=l(ja,"Shinya Otani"),ja.forEach(o),Vo=l(E,", "),Ue=a(E,"A",{href:!0,rel:!0});var Ea=s(Ue);Zo=l(Ea,"Takayoshi Makabe"),Ea.forEach(o),Io=l(E,", "),Me=a(E,"A",{href:!0,rel:!0});var Pa=s(Me);Wo=l(Pa,"Anuj Arora"),Pa.forEach(o),Bo=l(E,", and "),Ne=a(E,"A",{href:!0,rel:!0});var Xa=s(Ne);So=l(Xa,"Kyo Hattori"),Xa.forEach(o),Ro=l(E," from "),$e=a(E,"A",{href:!0,rel:!0});var Ca=s($e);qo=l(Ca,"ABEJA, Inc."),Ca.forEach(o),Ao=l(E,". For more information on this model-building activity, please refer "),Ge=a(E,"A",{href:!0,rel:!0});var za=s(Ge);Lo=l(za,"here (ja)"),za.forEach(o),Qo=l(E,"."),E.forEach(o),Yt=h(t),B=a(t,"H3",{class:!0});var po=s(B);te=a(po,"A",{id:!0,class:!0,href:!0});var xa=s(te);_t=a(xa,"SPAN",{});var Fa=s(_t);v(je.$$.fragment,Fa),Fa.forEach(o),xa.forEach(o),Oo=h(po),Tt=a(po,"SPAN",{});var Va=s(Tt);Yo=l(Va,"Generation"),Va.forEach(o),po.forEach(o),Ht=h(t),oe=a(t,"P",{});var ho=s(oe);Ho=l(ho,"The "),bt=a(ho,"CODE",{});var Za=s(bt);Do=l(Za,"generate()"),Za.forEach(o),Ko=l(ho," method can be used to generate text using GPT NeoX Japanese model."),ho.forEach(o),Dt=h(t),v(Ee.$$.fragment,t),Kt=h(t),S=a(t,"H2",{class:!0});var mo=s(S);ne=a(mo,"A",{id:!0,class:!0,href:!0});var Ia=s(ne);yt=a(Ia,"SPAN",{});var Wa=s(yt);v(Pe.$$.fragment,Wa),Wa.forEach(o),Ia.forEach(o),en=h(mo),vt=a(mo,"SPAN",{});var Ba=s(vt);tn=l(Ba,"Documentation resources"),Ba.forEach(o),mo.forEach(o),eo=h(t),De=a(t,"UL",{});var Sa=s(De);Jt=a(Sa,"LI",{});var Ra=s(Jt);Ke=a(Ra,"A",{href:!0});var qa=s(Ke);on=l(qa,"Causal language modeling task guide"),qa.forEach(o),Ra.forEach(o),Sa.forEach(o),to=h(t),R=a(t,"H2",{class:!0});var uo=s(R);ae=a(uo,"A",{id:!0,class:!0,href:!0});var Aa=s(ae);wt=a(Aa,"SPAN",{});var La=s(wt);v(Xe.$$.fragment,La),La.forEach(o),Aa.forEach(o),nn=h(uo),kt=a(uo,"SPAN",{});var Qa=s(kt);an=l(Qa,"GPTNeoXJapaneseConfig"),Qa.forEach(o),uo.forEach(o),oo=h(t),P=a(t,"DIV",{class:!0});var fe=s(P);v(Ce.$$.fragment,fe),sn=h(fe),q=a(fe,"P",{});var rt=s(q);rn=l(rt,"This is the configuration class to store the configuration of a "),Ut=a(rt,"CODE",{});var Oa=s(Ut);ln=l(Oa,"GPTNeoXModelJapanese"),Oa.forEach(o),dn=l(rt,`. It is used to instantiate
a GPTNeoX model according to the specified arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar configuration to that of the GPTNeoXJapanese
`),ze=a(rt,"A",{href:!0,rel:!0});var Ya=s(ze);cn=l(Ya,"abeja/gpt-neox-japanese-2.7b"),Ya.forEach(o),pn=l(rt," architecture."),rt.forEach(o),hn=h(fe),A=a(fe,"P",{});var lt=s(A);mn=l(lt,"Configuration objects inherit from "),et=a(lt,"A",{href:!0});var Ha=s(et);un=l(Ha,"PretrainedConfig"),Ha.forEach(o),fn=l(lt,` and can be used to control the model outputs. Read the
documentation from `),tt=a(lt,"A",{href:!0});var Da=s(tt);gn=l(Da,"PretrainedConfig"),Da.forEach(o),_n=l(lt," for more information. Default configs is set as 2.7B model"),lt.forEach(o),Tn=h(fe),v(se.$$.fragment,fe),fe.forEach(o),no=h(t),L=a(t,"H2",{class:!0});var fo=s(L);re=a(fo,"A",{id:!0,class:!0,href:!0});var Ka=s(re);Mt=a(Ka,"SPAN",{});var es=s(Mt);v(xe.$$.fragment,es),es.forEach(o),Ka.forEach(o),bn=h(fo),Nt=a(fo,"SPAN",{});var ts=s(Nt);yn=l(ts,"GPTNeoXJapaneseTokenizer"),ts.forEach(o),fo.forEach(o),ao=h(t),$=a(t,"DIV",{class:!0});var I=s($);v(Fe.$$.fragment,I),vn=h(I),Q=a(I,"P",{});var it=s(Q);Jn=l(it,"This tokenizer inherits from "),ot=a(it,"A",{href:!0});var os=s(ot);wn=l(os,"PreTrainedTokenizer"),os.forEach(o),kn=l(it,` and is based on Japanese special Sub-Word-Encoding that is
used in this repository (`),Ve=a(it,"A",{href:!0,rel:!0});var ns=s(Ve);Un=l(ns,"https://github.com/tanreinama/Japanese-BPEEncoder_V2"),ns.forEach(o),Mn=l(it,`). Check the repository for details.
Japanese has a relatively large vocabulary and there is no separation between words. Furthermore, the language is a
combination of hiragana, katakana, and kanji, and variants such as \u201C1\u201D and \u201C\u2460\u201D are often used. In order to cope
with these, this tokenizer has the following features`),it.forEach(o),Nn=h(I),G=a(I,"UL",{});var x=s(G);$t=a(x,"LI",{});var as=s($t);$n=l(as,"Subword-by-subword segmentation, which is intermediate between byte strings and morphological analysis."),as.forEach(o),Gn=h(x),Gt=a(x,"LI",{});var ss=s(Gt);jn=l(ss,`BPEs are created for each Kanji, Hiragana, and Katakana character, and there are no BPEs that cross character
types, such as Kanji + Hiragana or Hiragana + Katakana.`),ss.forEach(o),En=h(x),nt=a(x,"LI",{});var Ja=s(nt);Pn=l(Ja,"All-byte encoding that does not require "),jt=a(Ja,"UNK",{});var rs=s(jt);Xn=l(rs,"."),rs.forEach(o),Ja.forEach(o),Cn=h(x),Et=a(x,"LI",{});var ls=s(Et);zn=l(ls,"Independent of UTF codes such as 2-byte and 3-byte characters"),ls.forEach(o),xn=h(x),Pt=a(x,"LI",{});var is=s(Pt);Fn=l(is,"Conversion of heterographs to the same token_id"),is.forEach(o),Vn=h(x),Xt=a(x,"LI",{});var ds=s(Xt);Zn=l(ds,"Emoji and Emoticon are grouped into 12 types as special tags."),ds.forEach(o),x.forEach(o),In=h(I),v(le.$$.fragment,I),Wn=h(I),ie=a(I,"DIV",{class:!0});var go=s(ie);v(Ze.$$.fragment,go),Bn=h(go),Ct=a(go,"P",{});var cs=s(Ct);Sn=l(cs,"Converts a sequence of tokens (string) in a single string."),cs.forEach(o),go.forEach(o),I.forEach(o),so=h(t),O=a(t,"H2",{class:!0});var _o=s(O);de=a(_o,"A",{id:!0,class:!0,href:!0});var ps=s(de);zt=a(ps,"SPAN",{});var hs=s(zt);v(Ie.$$.fragment,hs),hs.forEach(o),ps.forEach(o),Rn=h(_o),xt=a(_o,"SPAN",{});var ms=s(xt);qn=l(ms,"GPTNeoXJapaneseModel"),ms.forEach(o),_o.forEach(o),ro=h(t),V=a(t,"DIV",{class:!0});var dt=s(V);v(We.$$.fragment,dt),An=h(dt),Be=a(dt,"P",{});var To=s(Be);Ln=l(To,`The bare GPTNeoXJapanese Model transformer outputting raw hidden-states without any specific head on top.
This model is a PyTorch `),Se=a(To,"A",{href:!0,rel:!0});var us=s(Se);Qn=l(us,"torch.nn.Module"),us.forEach(o),On=l(To,` sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),To.forEach(o),Yn=h(dt),X=a(dt,"DIV",{class:!0});var ge=s(X);v(Re.$$.fragment,ge),Hn=h(ge),Y=a(ge,"P",{});var ct=s(Y);Dn=l(ct,"The "),at=a(ct,"A",{href:!0});var fs=s(at);Kn=l(fs,"GPTNeoXJapaneseModel"),fs.forEach(o),ea=l(ct," forward method, overrides the "),Ft=a(ct,"CODE",{});var gs=s(Ft);ta=l(gs,"__call__"),gs.forEach(o),oa=l(ct," special method."),ct.forEach(o),na=h(ge),v(ce.$$.fragment,ge),aa=h(ge),v(pe.$$.fragment,ge),ge.forEach(o),dt.forEach(o),lo=h(t),H=a(t,"H2",{class:!0});var bo=s(H);he=a(bo,"A",{id:!0,class:!0,href:!0});var _s=s(he);Vt=a(_s,"SPAN",{});var Ts=s(Vt);v(qe.$$.fragment,Ts),Ts.forEach(o),_s.forEach(o),sa=h(bo),Zt=a(bo,"SPAN",{});var bs=s(Zt);ra=l(bs,"GPTNeoXJapaneseForCausalLM"),bs.forEach(o),bo.forEach(o),io=h(t),Z=a(t,"DIV",{class:!0});var pt=s(Z);v(Ae.$$.fragment,pt),la=h(pt),D=a(pt,"P",{});var ht=s(D);ia=l(ht,"GPTNeoXJapanese Model with a "),It=a(ht,"CODE",{});var ys=s(It);da=l(ys,"language modeling"),ys.forEach(o),ca=l(ht,` head on top for Classifier Model fine-tuning.
This model is a PyTorch `),Le=a(ht,"A",{href:!0,rel:!0});var vs=s(Le);pa=l(vs,"torch.nn.Module"),vs.forEach(o),ha=l(ht,` sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`),ht.forEach(o),ma=h(pt),C=a(pt,"DIV",{class:!0});var _e=s(C);v(Qe.$$.fragment,_e),ua=h(_e),K=a(_e,"P",{});var mt=s(K);fa=l(mt,"The "),st=a(mt,"A",{href:!0});var Js=s(st);ga=l(Js,"GPTNeoXJapaneseForCausalLM"),Js.forEach(o),_a=l(mt," forward method, overrides the "),Wt=a(mt,"CODE",{});var ws=s(Wt);Ta=l(ws,"__call__"),ws.forEach(o),ba=l(mt," special method."),mt.forEach(o),ya=h(_e),v(me.$$.fragment,_e),va=h(_e),v(ue.$$.fragment,_e),_e.forEach(o),pt.forEach(o),this.h()},h(){i(c,"name","hf:doc:metadata"),i(c,"content",JSON.stringify(Fs)),i(f,"id","gptneoxjapanese"),i(f,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(f,"href","#gptneoxjapanese"),i(u,"class","relative group"),i(ee,"id","overview"),i(ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(ee,"href","#overview"),i(W,"class","relative group"),i(ye,"href","https://github.com/EleutherAI/gpt-neox"),i(ye,"rel","nofollow"),i(ve,"href","https://github.com/tanreinama/Japanese-BPEEncoder_V2"),i(ve,"rel","nofollow"),i(Je,"href","https://ai.googleblog.com/2022/04/pathways-language-model-palm-scaling-to.html"),i(Je,"rel","nofollow"),i(we,"href","https://medium.com/ml-abeja/training-a-better-gpt-2-93b157662ae4"),i(we,"rel","nofollow"),i(ke,"href","https://github.com/SO0529"),i(ke,"rel","nofollow"),i(Ue,"href","https://github.com/spider-man-tm"),i(Ue,"rel","nofollow"),i(Me,"href","https://github.com/Anuj040"),i(Me,"rel","nofollow"),i(Ne,"href","https://github.com/go5paopao"),i(Ne,"rel","nofollow"),i($e,"href","https://www.abejainc.com/"),i($e,"rel","nofollow"),i(Ge,"href","https://tech-blog.abeja.asia/entry/abeja-gpt-project-202207"),i(Ge,"rel","nofollow"),i(te,"id","generation"),i(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(te,"href","#generation"),i(B,"class","relative group"),i(ne,"id","documentation-resources"),i(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(ne,"href","#documentation-resources"),i(S,"class","relative group"),i(Ke,"href","../tasks/language_modeling"),i(ae,"id","transformers.GPTNeoXJapaneseConfig"),i(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(ae,"href","#transformers.GPTNeoXJapaneseConfig"),i(R,"class","relative group"),i(ze,"href","https://huggingface.co/abeja/gpt-neox-japanese-2.7b"),i(ze,"rel","nofollow"),i(et,"href","/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig"),i(tt,"href","/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig"),i(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(re,"id","transformers.GPTNeoXJapaneseTokenizer"),i(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(re,"href","#transformers.GPTNeoXJapaneseTokenizer"),i(L,"class","relative group"),i(ot,"href","/docs/transformers/v4.32.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizer"),i(Ve,"href","https://github.com/tanreinama/Japanese-BPEEncoder_V2"),i(Ve,"rel","nofollow"),i(ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(de,"id","transformers.GPTNeoXJapaneseModel"),i(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(de,"href","#transformers.GPTNeoXJapaneseModel"),i(O,"class","relative group"),i(Se,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),i(Se,"rel","nofollow"),i(at,"href","/docs/transformers/v4.32.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseModel"),i(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(he,"id","transformers.GPTNeoXJapaneseForCausalLM"),i(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(he,"href","#transformers.GPTNeoXJapaneseForCausalLM"),i(H,"class","relative group"),i(Le,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),i(Le,"rel","nofollow"),i(st,"href","/docs/transformers/v4.32.0/en/model_doc/gpt_neox_japanese#transformers.GPTNeoXJapaneseForCausalLM"),i(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),i(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,m){e(document.head,c),g(t,T,m),g(t,u,m),e(u,f),e(f,b),J(d,b,null),e(u,_),e(u,F),e(F,Jo),g(t,Lt,m),g(t,W,m),e(W,ee),e(ee,ut),J(be,ut,null),e(W,wo),e(W,ft),e(ft,ko),g(t,Qt,m),g(t,N,m),e(N,Uo),e(N,ye),e(ye,Mo),e(N,No),e(N,ve),e(ve,$o),e(N,Go),e(N,gt),e(gt,jo),e(N,Eo),e(N,Je),e(Je,Po),e(N,Xo),e(N,we),e(we,Co),e(N,zo),g(t,Ot,m),g(t,M,m),e(M,xo),e(M,ke),e(ke,Fo),e(M,Vo),e(M,Ue),e(Ue,Zo),e(M,Io),e(M,Me),e(Me,Wo),e(M,Bo),e(M,Ne),e(Ne,So),e(M,Ro),e(M,$e),e($e,qo),e(M,Ao),e(M,Ge),e(Ge,Lo),e(M,Qo),g(t,Yt,m),g(t,B,m),e(B,te),e(te,_t),J(je,_t,null),e(B,Oo),e(B,Tt),e(Tt,Yo),g(t,Ht,m),g(t,oe,m),e(oe,Ho),e(oe,bt),e(bt,Do),e(oe,Ko),g(t,Dt,m),J(Ee,t,m),g(t,Kt,m),g(t,S,m),e(S,ne),e(ne,yt),J(Pe,yt,null),e(S,en),e(S,vt),e(vt,tn),g(t,eo,m),g(t,De,m),e(De,Jt),e(Jt,Ke),e(Ke,on),g(t,to,m),g(t,R,m),e(R,ae),e(ae,wt),J(Xe,wt,null),e(R,nn),e(R,kt),e(kt,an),g(t,oo,m),g(t,P,m),J(Ce,P,null),e(P,sn),e(P,q),e(q,rn),e(q,Ut),e(Ut,ln),e(q,dn),e(q,ze),e(ze,cn),e(q,pn),e(P,hn),e(P,A),e(A,mn),e(A,et),e(et,un),e(A,fn),e(A,tt),e(tt,gn),e(A,_n),e(P,Tn),J(se,P,null),g(t,no,m),g(t,L,m),e(L,re),e(re,Mt),J(xe,Mt,null),e(L,bn),e(L,Nt),e(Nt,yn),g(t,ao,m),g(t,$,m),J(Fe,$,null),e($,vn),e($,Q),e(Q,Jn),e(Q,ot),e(ot,wn),e(Q,kn),e(Q,Ve),e(Ve,Un),e(Q,Mn),e($,Nn),e($,G),e(G,$t),e($t,$n),e(G,Gn),e(G,Gt),e(Gt,jn),e(G,En),e(G,nt),e(nt,Pn),e(nt,jt),e(jt,Xn),e(G,Cn),e(G,Et),e(Et,zn),e(G,xn),e(G,Pt),e(Pt,Fn),e(G,Vn),e(G,Xt),e(Xt,Zn),e($,In),J(le,$,null),e($,Wn),e($,ie),J(Ze,ie,null),e(ie,Bn),e(ie,Ct),e(Ct,Sn),g(t,so,m),g(t,O,m),e(O,de),e(de,zt),J(Ie,zt,null),e(O,Rn),e(O,xt),e(xt,qn),g(t,ro,m),g(t,V,m),J(We,V,null),e(V,An),e(V,Be),e(Be,Ln),e(Be,Se),e(Se,Qn),e(Be,On),e(V,Yn),e(V,X),J(Re,X,null),e(X,Hn),e(X,Y),e(Y,Dn),e(Y,at),e(at,Kn),e(Y,ea),e(Y,Ft),e(Ft,ta),e(Y,oa),e(X,na),J(ce,X,null),e(X,aa),J(pe,X,null),g(t,lo,m),g(t,H,m),e(H,he),e(he,Vt),J(qe,Vt,null),e(H,sa),e(H,Zt),e(Zt,ra),g(t,io,m),g(t,Z,m),J(Ae,Z,null),e(Z,la),e(Z,D),e(D,ia),e(D,It),e(It,da),e(D,ca),e(D,Le),e(Le,pa),e(D,ha),e(Z,ma),e(Z,C),J(Qe,C,null),e(C,ua),e(C,K),e(K,fa),e(K,st),e(st,ga),e(K,_a),e(K,Wt),e(Wt,Ta),e(K,ba),e(C,ya),J(me,C,null),e(C,va),J(ue,C,null),co=!0},p(t,[m]){const Oe={};m&2&&(Oe.$$scope={dirty:m,ctx:t}),se.$set(Oe);const Bt={};m&2&&(Bt.$$scope={dirty:m,ctx:t}),le.$set(Bt);const St={};m&2&&(St.$$scope={dirty:m,ctx:t}),ce.$set(St);const Rt={};m&2&&(Rt.$$scope={dirty:m,ctx:t}),pe.$set(Rt);const Ye={};m&2&&(Ye.$$scope={dirty:m,ctx:t}),me.$set(Ye);const qt={};m&2&&(qt.$$scope={dirty:m,ctx:t}),ue.$set(qt)},i(t){co||(w(d.$$.fragment,t),w(be.$$.fragment,t),w(je.$$.fragment,t),w(Ee.$$.fragment,t),w(Pe.$$.fragment,t),w(Xe.$$.fragment,t),w(Ce.$$.fragment,t),w(se.$$.fragment,t),w(xe.$$.fragment,t),w(Fe.$$.fragment,t),w(le.$$.fragment,t),w(Ze.$$.fragment,t),w(Ie.$$.fragment,t),w(We.$$.fragment,t),w(Re.$$.fragment,t),w(ce.$$.fragment,t),w(pe.$$.fragment,t),w(qe.$$.fragment,t),w(Ae.$$.fragment,t),w(Qe.$$.fragment,t),w(me.$$.fragment,t),w(ue.$$.fragment,t),co=!0)},o(t){k(d.$$.fragment,t),k(be.$$.fragment,t),k(je.$$.fragment,t),k(Ee.$$.fragment,t),k(Pe.$$.fragment,t),k(Xe.$$.fragment,t),k(Ce.$$.fragment,t),k(se.$$.fragment,t),k(xe.$$.fragment,t),k(Fe.$$.fragment,t),k(le.$$.fragment,t),k(Ze.$$.fragment,t),k(Ie.$$.fragment,t),k(We.$$.fragment,t),k(Re.$$.fragment,t),k(ce.$$.fragment,t),k(pe.$$.fragment,t),k(qe.$$.fragment,t),k(Ae.$$.fragment,t),k(Qe.$$.fragment,t),k(me.$$.fragment,t),k(ue.$$.fragment,t),co=!1},d(t){o(c),t&&o(T),t&&o(u),U(d),t&&o(Lt),t&&o(W),U(be),t&&o(Qt),t&&o(N),t&&o(Ot),t&&o(M),t&&o(Yt),t&&o(B),U(je),t&&o(Ht),t&&o(oe),t&&o(Dt),U(Ee,t),t&&o(Kt),t&&o(S),U(Pe),t&&o(eo),t&&o(De),t&&o(to),t&&o(R),U(Xe),t&&o(oo),t&&o(P),U(Ce),U(se),t&&o(no),t&&o(L),U(xe),t&&o(ao),t&&o($),U(Fe),U(le),U(Ze),t&&o(so),t&&o(O),U(Ie),t&&o(ro),t&&o(V),U(We),U(Re),U(ce),U(pe),t&&o(lo),t&&o(H),U(qe),t&&o(io),t&&o(Z),U(Ae),U(Qe),U(me),U(ue)}}}const Fs={local:"gptneoxjapanese",sections:[{local:"overview",sections:[{local:"generation",title:"Generation"}],title:"Overview"},{local:"documentation-resources",title:"Documentation resources"},{local:"transformers.GPTNeoXJapaneseConfig",title:"GPTNeoXJapaneseConfig"},{local:"transformers.GPTNeoXJapaneseTokenizer",title:"GPTNeoXJapaneseTokenizer"},{local:"transformers.GPTNeoXJapaneseModel",title:"GPTNeoXJapaneseModel"},{local:"transformers.GPTNeoXJapaneseForCausalLM",title:"GPTNeoXJapaneseForCausalLM"}],title:"GPT-NeoX-Japanese"};function Vs(j){return Gs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class qs extends Us{constructor(c){super();Ms(this,c,Vs,xs,Ns,{})}}export{qs as default,Fs as metadata};
