import{s as Bt,o as qt,n as Lt}from"../chunks/scheduler.9bc65507.js";import{S as Ut,i as Dt,g as s,s as l,u as b,A as At,h as o,f as t,c as i,j as u,v,y as c,k as m,o as a,a as r,w,d as T,t as M,x as y}from"../chunks/index.78c82d43.js";import{D as be}from"../chunks/Docstring.f3d04bd2.js";import{C as at}from"../chunks/CodeBlock.48618ec2.js";import{I as ot}from"../chunks/IconCopyLink.bedaa44d.js";import{E as Rt}from"../chunks/ExampleCodeBlock.872b014d.js";function Nt(ve){let h,U="pair mask has the following format:",j,g,_;return g=new at({props:{code:"MCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMEElN0MlMjBmaXJzdCUyMHNlcXVlbmNlJTIwJTIwJTIwJTIwJTdDJTIwc2Vjb25kJTIwc2VxdWVuY2UlMjAlN0M=",highlighted:`0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1 1
| first sequence    | second sequence |`}}),{c(){h=s("p"),h.textContent=U,j=l(),b(g.$$.fragment)},l(p){h=o(p,"P",{"data-svelte-h":!0}),c(h)!=="svelte-qjgeij"&&(h.textContent=U),j=i(p),v(g.$$.fragment,p)},m(p,J){r(p,h,J),r(p,j,J),w(g,p,J),_=!0},p:Lt,i(p){_||(T(g.$$.fragment,p),_=!0)},o(p){M(g.$$.fragment,p),_=!1},d(p){p&&(t(h),t(j)),y(g,p)}}}function Pt(ve){let h,U,j,g,_,p,J,D,He,oe,rt="BertJapanese",we,$,z,fe,A,Se,ae,lt="Overview",Te,R,it="The BERT models trained on Japanese text.",Me,N,pt="There are models with two different tokenization methods:",ye,P,ct='<li>Tokenize with MeCab and WordPiece. This requires some extra dependencies, <a href="https://github.com/polm/fugashi" rel="nofollow">fugashi</a> which is a wrapper around <a href="https://taku910.github.io/mecab/" rel="nofollow">MeCab</a>.</li> <li>Tokenize into characters.</li>',je,H,dt=`To use <em>MecabTokenizer</em>, you should <code>pip install transformers[&quot;ja&quot;]</code> (or <code>pip install -e .[&quot;ja&quot;]</code> if you install
from source) to install dependencies.`,xe,S,mt='See <a href="https://github.com/cl-tohoku/bert-japanese" rel="nofollow">details on cl-tohoku repository</a>.',Je,Z,ht="Example of using a model with MeCab and WordPiece tokenization:",$e,Q,Ce,V,ut="Example of using a model with Character tokenization:",ze,X,Ee,O,_t="Tips:",Ie,W,ft='<li>This implementation is the same as BERT, except for tokenization method. Refer to the <a href="bert">documentation of BERT</a> for more usage examples.</li>',Be,G,gt='This model was contributed by <a href="https://huggingface.co/cl-tohoku" rel="nofollow">cl-tohoku</a>.',qe,C,E,ge,F,Ze,re,kt="BertJapaneseTokenizer",Le,d,Y,Qe,le,bt="Construct a BERT tokenizer for Japanese text.",Ve,ie,vt=`This tokenizer inherits from <a href="/docs/transformers/v4.33.3/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> which contains most of the main methods. Users should refer
to: this superclass for more information regarding those methods.`,Xe,x,K,Oe,pe,wt=`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A BERT sequence has the following format:`,We,ce,Tt="<li>single sequence: <code>[CLS] X [SEP]</code></li> <li>pair of sequences: <code>[CLS] A [SEP] B [SEP]</code></li>",Ge,I,ee,Fe,de,Mt="Converts a sequence of tokens (string) in a single string.",Ye,k,te,Ke,me,yt="Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence",et,B,tt,he,jt="If <code>token_ids_1</code> is <code>None</code>, this method only returns the first portion of the mask (0s).",nt,q,ne,st,ue,xt=`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> method.`,Ue,ke,De;return D=new ot({}),A=new ot({}),Q=new at({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQWJlcnRqYXBhbmVzZSUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIyY2wtdG9ob2t1JTJGYmVydC1iYXNlLWphcGFuZXNlJTIyKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmNsLXRvaG9rdSUyRmJlcnQtYmFzZS1qYXBhbmVzZSUyMiklMEElMEElMjMlMjMlMjBJbnB1dCUyMEphcGFuZXNlJTIwVGV4dCUwQWxpbmUlMjAlM0QlMjAlMjIlRTUlOTAlQkUlRTglQkMlQTklRTMlODElQUYlRTclOEMlQUIlRTMlODElQTclRTMlODElODIlRTMlODIlOEIlRTMlODAlODIlMjIlMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIobGluZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQlNUIwJTVEKSklMEElMEFvdXRwdXRzJTIwJTNEJTIwYmVydGphcGFuZXNlKCoqaW5wdXRzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>bertjapanese = AutoModel.from_pretrained(<span class="hljs-string">&quot;cl-tohoku/bert-base-japanese&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;cl-tohoku/bert-base-japanese&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment">## Input Japanese Text</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>line = <span class="hljs-string">&quot;吾輩は猫である。&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(line, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>][<span class="hljs-number">0</span>]))
[CLS] 吾輩 は 猫 で ある 。 [SEP]

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = bertjapanese(**inputs)`}}),X=new at({props:{code:"YmVydGphcGFuZXNlJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJjbC10b2hva3UlMkZiZXJ0LWJhc2UtamFwYW5lc2UtY2hhciUyMiklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJjbC10b2hva3UlMkZiZXJ0LWJhc2UtamFwYW5lc2UtY2hhciUyMiklMEElMEElMjMlMjMlMjBJbnB1dCUyMEphcGFuZXNlJTIwVGV4dCUwQWxpbmUlMjAlM0QlMjAlMjIlRTUlOTAlQkUlRTglQkMlQTklRTMlODElQUYlRTclOEMlQUIlRTMlODElQTclRTMlODElODIlRTMlODIlOEIlRTMlODAlODIlMjIlMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIobGluZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQlNUIwJTVEKSklMEElMEFvdXRwdXRzJTIwJTNEJTIwYmVydGphcGFuZXNlKCoqaW5wdXRzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>bertjapanese = AutoModel.from_pretrained(<span class="hljs-string">&quot;cl-tohoku/bert-base-japanese-char&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;cl-tohoku/bert-base-japanese-char&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment">## Input Japanese Text</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>line = <span class="hljs-string">&quot;吾輩は猫である。&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(line, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>][<span class="hljs-number">0</span>]))
[CLS] 吾 輩 は 猫 で あ る 。 [SEP]

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = bertjapanese(**inputs)`}}),F=new ot({}),Y=new be({props:{name:"class transformers.BertJapaneseTokenizer",anchor:"transformers.BertJapaneseTokenizer",parameters:[{name:"vocab_file",val:""},{name:"spm_file",val:" = None"},{name:"do_lower_case",val:" = False"},{name:"do_word_tokenize",val:" = True"},{name:"do_subword_tokenize",val:" = True"},{name:"word_tokenizer_type",val:" = 'basic'"},{name:"subword_tokenizer_type",val:" = 'wordpiece'"},{name:"never_split",val:" = None"},{name:"unk_token",val:" = '[UNK]'"},{name:"sep_token",val:" = '[SEP]'"},{name:"pad_token",val:" = '[PAD]'"},{name:"cls_token",val:" = '[CLS]'"},{name:"mask_token",val:" = '[MASK]'"},{name:"mecab_kwargs",val:" = None"},{name:"sudachi_kwargs",val:" = None"},{name:"jumanpp_kwargs",val:" = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BertJapaneseTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
Path to a one-wordpiece-per-line vocabulary file.`,name:"vocab_file"},{anchor:"transformers.BertJapaneseTokenizer.spm_file",description:`<strong>spm_file</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Path to <a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a .spm or .model
extension) that contains the vocabulary.`,name:"spm_file"},{anchor:"transformers.BertJapaneseTokenizer.do_lower_case",description:`<strong>do_lower_case</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to lower case the input. Only has an effect when do_basic_tokenize=True.`,name:"do_lower_case"},{anchor:"transformers.BertJapaneseTokenizer.do_word_tokenize",description:`<strong>do_word_tokenize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to do word tokenization.`,name:"do_word_tokenize"},{anchor:"transformers.BertJapaneseTokenizer.do_subword_tokenize",description:`<strong>do_subword_tokenize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to do subword tokenization.`,name:"do_subword_tokenize"},{anchor:"transformers.BertJapaneseTokenizer.word_tokenizer_type",description:`<strong>word_tokenizer_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;basic&quot;</code>) &#x2014;
Type of word tokenizer. Choose from [&#x201C;basic&#x201D;, &#x201C;mecab&#x201D;, &#x201C;sudachi&#x201D;, &#x201C;jumanpp&#x201D;].`,name:"word_tokenizer_type"},{anchor:"transformers.BertJapaneseTokenizer.subword_tokenizer_type",description:`<strong>subword_tokenizer_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;wordpiece&quot;</code>) &#x2014;
Type of subword tokenizer. Choose from [&#x201C;wordpiece&#x201D;, &#x201C;character&#x201D;, &#x201C;sentencepiece&#x201D;,].`,name:"subword_tokenizer_type"},{anchor:"transformers.BertJapaneseTokenizer.mecab_kwargs",description:`<strong>mecab_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary passed to the <code>MecabTokenizer</code> constructor.`,name:"mecab_kwargs"},{anchor:"transformers.BertJapaneseTokenizer.sudachi_kwargs",description:`<strong>sudachi_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary passed to the <code>SudachiTokenizer</code> constructor.`,name:"sudachi_kwargs"},{anchor:"transformers.BertJapaneseTokenizer.jumanpp_kwargs",description:`<strong>jumanpp_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Dictionary passed to the <code>JumanppTokenizer</code> constructor.`,name:"jumanpp_kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/bert_japanese/tokenization_bert_japanese.py#L107"}}),K=new be({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.BertJapaneseTokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.BertJapaneseTokenizer.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.BertJapaneseTokenizer.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/bert_japanese/tokenization_bert_japanese.py#L308",returnDescription:`
<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),ee=new be({props:{name:"convert_tokens_to_string",anchor:"transformers.BertJapaneseTokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/bert_japanese/tokenization_bert_japanese.py#L300"}}),te=new be({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.BertJapaneseTokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.BertJapaneseTokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.BertJapaneseTokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/bert_japanese/tokenization_bert_japanese.py#L363",returnDescription:`
<p>List of <a href="../glossary#token-type-ids">token type IDs</a> according to the given sequence(s).</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),B=new Rt({props:{anchor:"transformers.BertJapaneseTokenizer.create_token_type_ids_from_sequences.example",$$slots:{default:[Nt]},$$scope:{ctx:ve}}}),ne=new be({props:{name:"get_special_tokens_mask",anchor:"transformers.BertJapaneseTokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.BertJapaneseTokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.BertJapaneseTokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.BertJapaneseTokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/v4.33.3/src/transformers/models/bert_japanese/tokenization_bert_japanese.py#L334",returnDescription:`
<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),{c(){h=s("meta"),U=l(),j=s("p"),g=l(),_=s("h1"),p=s("a"),J=s("span"),b(D.$$.fragment),He=l(),oe=s("span"),oe.textContent=rt,we=l(),$=s("h2"),z=s("a"),fe=s("span"),b(A.$$.fragment),Se=l(),ae=s("span"),ae.textContent=lt,Te=l(),R=s("p"),R.textContent=it,Me=l(),N=s("p"),N.textContent=pt,ye=l(),P=s("ul"),P.innerHTML=ct,je=l(),H=s("p"),H.innerHTML=dt,xe=l(),S=s("p"),S.innerHTML=mt,Je=l(),Z=s("p"),Z.textContent=ht,$e=l(),b(Q.$$.fragment),Ce=l(),V=s("p"),V.textContent=ut,ze=l(),b(X.$$.fragment),Ee=l(),O=s("p"),O.textContent=_t,Ie=l(),W=s("ul"),W.innerHTML=ft,Be=l(),G=s("p"),G.innerHTML=gt,qe=l(),C=s("h2"),E=s("a"),ge=s("span"),b(F.$$.fragment),Ze=l(),re=s("span"),re.textContent=kt,Le=l(),d=s("div"),b(Y.$$.fragment),Qe=l(),le=s("p"),le.textContent=bt,Ve=l(),ie=s("p"),ie.innerHTML=vt,Xe=l(),x=s("div"),b(K.$$.fragment),Oe=l(),pe=s("p"),pe.textContent=wt,We=l(),ce=s("ul"),ce.innerHTML=Tt,Ge=l(),I=s("div"),b(ee.$$.fragment),Fe=l(),de=s("p"),de.textContent=Mt,Ye=l(),k=s("div"),b(te.$$.fragment),Ke=l(),me=s("p"),me.textContent=yt,et=l(),b(B.$$.fragment),tt=l(),he=s("p"),he.innerHTML=jt,nt=l(),q=s("div"),b(ne.$$.fragment),st=l(),ue=s("p"),ue.innerHTML=xt,Ue=l(),ke=s("p"),this.h()},l(e){const n=At("svelte-1phssyn",document.head);h=o(n,"META",{name:!0,content:!0}),n.forEach(t),U=i(e),j=o(e,"P",{}),u(j).forEach(t),g=i(e),_=o(e,"H1",{class:!0});var se=u(_);p=o(se,"A",{id:!0,class:!0,href:!0});var Jt=u(p);J=o(Jt,"SPAN",{});var $t=u(J);v(D.$$.fragment,$t),$t.forEach(t),Jt.forEach(t),He=i(se),oe=o(se,"SPAN",{"data-svelte-h":!0}),c(oe)!=="svelte-swoogh"&&(oe.textContent=rt),se.forEach(t),we=i(e),$=o(e,"H2",{class:!0});var Ae=u($);z=o(Ae,"A",{id:!0,class:!0,href:!0});var Ct=u(z);fe=o(Ct,"SPAN",{});var zt=u(fe);v(A.$$.fragment,zt),zt.forEach(t),Ct.forEach(t),Se=i(Ae),ae=o(Ae,"SPAN",{"data-svelte-h":!0}),c(ae)!=="svelte-1jsw1pg"&&(ae.textContent=lt),Ae.forEach(t),Te=i(e),R=o(e,"P",{"data-svelte-h":!0}),c(R)!=="svelte-q6gk8k"&&(R.textContent=it),Me=i(e),N=o(e,"P",{"data-svelte-h":!0}),c(N)!=="svelte-130rfh4"&&(N.textContent=pt),ye=i(e),P=o(e,"UL",{"data-svelte-h":!0}),c(P)!=="svelte-1nk5iha"&&(P.innerHTML=ct),je=i(e),H=o(e,"P",{"data-svelte-h":!0}),c(H)!=="svelte-1dsac8d"&&(H.innerHTML=dt),xe=i(e),S=o(e,"P",{"data-svelte-h":!0}),c(S)!=="svelte-u7nt0b"&&(S.innerHTML=mt),Je=i(e),Z=o(e,"P",{"data-svelte-h":!0}),c(Z)!=="svelte-vqnu8f"&&(Z.textContent=ht),$e=i(e),v(Q.$$.fragment,e),Ce=i(e),V=o(e,"P",{"data-svelte-h":!0}),c(V)!=="svelte-m6bjin"&&(V.textContent=ut),ze=i(e),v(X.$$.fragment,e),Ee=i(e),O=o(e,"P",{"data-svelte-h":!0}),c(O)!=="svelte-axv494"&&(O.textContent=_t),Ie=i(e),W=o(e,"UL",{"data-svelte-h":!0}),c(W)!=="svelte-14vlmtl"&&(W.innerHTML=ft),Be=i(e),G=o(e,"P",{"data-svelte-h":!0}),c(G)!=="svelte-1ykd1l9"&&(G.innerHTML=gt),qe=i(e),C=o(e,"H2",{class:!0});var Re=u(C);E=o(Re,"A",{id:!0,class:!0,href:!0});var Et=u(E);ge=o(Et,"SPAN",{});var It=u(ge);v(F.$$.fragment,It),It.forEach(t),Et.forEach(t),Ze=i(Re),re=o(Re,"SPAN",{"data-svelte-h":!0}),c(re)!=="svelte-18b117u"&&(re.textContent=kt),Re.forEach(t),Le=i(e),d=o(e,"DIV",{class:!0});var f=u(d);v(Y.$$.fragment,f),Qe=i(f),le=o(f,"P",{"data-svelte-h":!0}),c(le)!=="svelte-1tm7ou1"&&(le.textContent=bt),Ve=i(f),ie=o(f,"P",{"data-svelte-h":!0}),c(ie)!=="svelte-1nrgg26"&&(ie.innerHTML=vt),Xe=i(f),x=o(f,"DIV",{class:!0});var _e=u(x);v(K.$$.fragment,_e),Oe=i(_e),pe=o(_e,"P",{"data-svelte-h":!0}),c(pe)!=="svelte-t7qurq"&&(pe.textContent=wt),We=i(_e),ce=o(_e,"UL",{"data-svelte-h":!0}),c(ce)!=="svelte-xi6653"&&(ce.innerHTML=Tt),_e.forEach(t),Ge=i(f),I=o(f,"DIV",{class:!0});var Ne=u(I);v(ee.$$.fragment,Ne),Fe=i(Ne),de=o(Ne,"P",{"data-svelte-h":!0}),c(de)!=="svelte-b3k2yi"&&(de.textContent=Mt),Ne.forEach(t),Ye=i(f),k=o(f,"DIV",{class:!0});var L=u(k);v(te.$$.fragment,L),Ke=i(L),me=o(L,"P",{"data-svelte-h":!0}),c(me)!=="svelte-gn6wi7"&&(me.textContent=yt),et=i(L),v(B.$$.fragment,L),tt=i(L),he=o(L,"P",{"data-svelte-h":!0}),c(he)!=="svelte-owoxgn"&&(he.innerHTML=jt),L.forEach(t),nt=i(f),q=o(f,"DIV",{class:!0});var Pe=u(q);v(ne.$$.fragment,Pe),st=i(Pe),ue=o(Pe,"P",{"data-svelte-h":!0}),c(ue)!=="svelte-1f4f5kp"&&(ue.innerHTML=xt),Pe.forEach(t),f.forEach(t),Ue=i(e),ke=o(e,"P",{}),u(ke).forEach(t),this.h()},h(){m(h,"name","hf:doc:metadata"),m(h,"content",JSON.stringify(Ht)),m(p,"id","bertjapanese"),m(p,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(p,"href","#bertjapanese"),m(_,"class","relative group"),m(z,"id","overview"),m(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(z,"href","#overview"),m($,"class","relative group"),m(E,"id","transformers.BertJapaneseTokenizer"),m(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(E,"href","#transformers.BertJapaneseTokenizer"),m(C,"class","relative group"),m(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(d,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,n){a(document.head,h),r(e,U,n),r(e,j,n),r(e,g,n),r(e,_,n),a(_,p),a(p,J),w(D,J,null),a(_,He),a(_,oe),r(e,we,n),r(e,$,n),a($,z),a(z,fe),w(A,fe,null),a($,Se),a($,ae),r(e,Te,n),r(e,R,n),r(e,Me,n),r(e,N,n),r(e,ye,n),r(e,P,n),r(e,je,n),r(e,H,n),r(e,xe,n),r(e,S,n),r(e,Je,n),r(e,Z,n),r(e,$e,n),w(Q,e,n),r(e,Ce,n),r(e,V,n),r(e,ze,n),w(X,e,n),r(e,Ee,n),r(e,O,n),r(e,Ie,n),r(e,W,n),r(e,Be,n),r(e,G,n),r(e,qe,n),r(e,C,n),a(C,E),a(E,ge),w(F,ge,null),a(C,Ze),a(C,re),r(e,Le,n),r(e,d,n),w(Y,d,null),a(d,Qe),a(d,le),a(d,Ve),a(d,ie),a(d,Xe),a(d,x),w(K,x,null),a(x,Oe),a(x,pe),a(x,We),a(x,ce),a(d,Ge),a(d,I),w(ee,I,null),a(I,Fe),a(I,de),a(d,Ye),a(d,k),w(te,k,null),a(k,Ke),a(k,me),a(k,et),w(B,k,null),a(k,tt),a(k,he),a(d,nt),a(d,q),w(ne,q,null),a(q,st),a(q,ue),r(e,Ue,n),r(e,ke,n),De=!0},p(e,[n]){const se={};n&2&&(se.$$scope={dirty:n,ctx:e}),B.$set(se)},i(e){De||(T(D.$$.fragment,e),T(A.$$.fragment,e),T(Q.$$.fragment,e),T(X.$$.fragment,e),T(F.$$.fragment,e),T(Y.$$.fragment,e),T(K.$$.fragment,e),T(ee.$$.fragment,e),T(te.$$.fragment,e),T(B.$$.fragment,e),T(ne.$$.fragment,e),De=!0)},o(e){M(D.$$.fragment,e),M(A.$$.fragment,e),M(Q.$$.fragment,e),M(X.$$.fragment,e),M(F.$$.fragment,e),M(Y.$$.fragment,e),M(K.$$.fragment,e),M(ee.$$.fragment,e),M(te.$$.fragment,e),M(B.$$.fragment,e),M(ne.$$.fragment,e),De=!1},d(e){e&&(t(U),t(j),t(g),t(_),t(we),t($),t(Te),t(R),t(Me),t(N),t(ye),t(P),t(je),t(H),t(xe),t(S),t(Je),t(Z),t($e),t(Ce),t(V),t(ze),t(Ee),t(O),t(Ie),t(W),t(Be),t(G),t(qe),t(C),t(Le),t(d),t(Ue),t(ke)),t(h),y(D),y(A),y(Q,e),y(X,e),y(F),y(Y),y(K),y(ee),y(te),y(B),y(ne)}}}const Ht={local:"bertjapanese",sections:[{local:"overview",title:"Overview"},{local:"transformers.BertJapaneseTokenizer",title:"BertJapaneseTokenizer"}],title:"BertJapanese"};function St(ve){return qt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Gt extends Ut{constructor(h){super(),Dt(this,h,St,Pt,Bt,{})}}export{Gt as component};
