import{S as gn,i as bn,s as vn,e as r,k as l,w as v,t as o,M as wn,c as n,d as s,m as c,a,x as w,h as i,b as p,G as e,g as f,y,q as T,o as $,B as E,v as yn,L as kn}from"../../chunks/vendor-hf-doc-builder.js";import{D as P}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Er}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as Ct}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as _n}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function Tn(we){let u,I,g,h,J;return h=new Er({props:{code:"MCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMEElN0MlMjBmaXJzdCUyMHNlcXVlbmNlJTIwJTIwJTIwJTIwJTdDJTIwc2Vjb25kJTIwc2VxdWVuY2UlMjAlN0M=",highlighted:`0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1 1
| first sequence    | second sequence |`}}),{c(){u=r("p"),I=o("pair mask has the following format:"),g=l(),v(h.$$.fragment)},l(d){u=n(d,"P",{});var M=a(u);I=i(M,"pair mask has the following format:"),M.forEach(s),g=c(d),w(h.$$.fragment,d)},m(d,M){f(d,u,M),e(u,I),f(d,g,M),y(h,d,M),J=!0},p:kn,i(d){J||(T(h.$$.fragment,d),J=!0)},o(d){$(h.$$.fragment,d),J=!1},d(d){d&&s(u),d&&s(g),E(h,d)}}}function $n(we){let u,I,g,h,J;return h=new Er({props:{code:"MCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMEElN0MlMjBmaXJzdCUyMHNlcXVlbmNlJTIwJTIwJTIwJTIwJTdDJTIwc2Vjb25kJTIwc2VxdWVuY2UlMjAlN0M=",highlighted:`0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1 1
| first sequence    | second sequence |`}}),{c(){u=r("p"),I=o("BERT sequence pair mask has the following format:"),g=l(),v(h.$$.fragment)},l(d){u=n(d,"P",{});var M=a(u);I=i(M,"BERT sequence pair mask has the following format:"),M.forEach(s),g=c(d),w(h.$$.fragment,d)},m(d,M){f(d,u,M),e(u,I),f(d,g,M),y(h,d,M),J=!0},p:kn,i(d){J||(T(h.$$.fragment,d),J=!0)},o(d){$(h.$$.fragment,d),J=!1},d(d){d&&s(u),d&&s(g),E(h,d)}}}function En(we){let u,I,g,h,J,d,M,Ue,Vt,ht,D,R,Ae,O,Nt,Be,Ft,ut,C,Xt,Y,Zt,Wt,_t,ye,Gt,kt,Te,Pe,St,gt,$e,Ot,bt,Q,vt,q,Yt,K,Qt,Kt,ee,es,ts,wt,U,V,Re,te,ss,Ce,rs,yt,_,se,ns,Ve,as,os,Ne,is,ls,re,Fe,Xe,cs,ds,Ze,We,ps,ms,ne,fs,Ee,hs,us,_s,H,ae,ks,Ge,gs,bs,oe,Me,vs,Se,ws,ys,ze,Ts,Oe,$s,Es,N,ie,Ms,Ye,zs,Js,L,le,Is,Qe,Ls,qs,F,Hs,A,js,Ke,xs,Ds,et,Us,As,Bs,X,ce,Ps,de,Rs,tt,Cs,Vs,Tt,B,Z,st,pe,Ns,rt,Fs,$t,k,me,Xs,fe,Zs,nt,Ws,Gs,Ss,at,Os,Ys,ot,it,Qs,Ks,he,er,Je,tr,sr,rr,j,ue,nr,lt,ar,or,_e,Ie,ir,ct,lr,cr,Le,dr,dt,pr,mr,x,ke,fr,pt,hr,ur,W,_r,G,ge,kr,be,gr,mt,br,vr,Et;return d=new Ct({}),O=new Ct({}),Q=new Er({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEhlcmJlcnRUb2tlbml6ZXIlMkMlMjBSb2JlcnRhTW9kZWwlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBIZXJiZXJ0VG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJhbGxlZ3JvJTJGaGVyYmVydC1rbGVqLWNhc2VkLXRva2VuaXplci12MSUyMiklMEFtb2RlbCUyMCUzRCUyMFJvYmVydGFNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIyYWxsZWdybyUyRmhlcmJlcnQta2xlai1jYXNlZC12MSUyMiklMEElMEFlbmNvZGVkX2lucHV0JTIwJTNEJTIwdG9rZW5pemVyLmVuY29kZSglMjJLdG8lMjBtYSUyMGxlcHN6JUM0JTg1JTIwc3p0dWslQzQlOTklMkMlMjBtYSUyMGxlcHN6eSUyMHJ6JUM0JTg1ZCUyMCVFMiU4MCU5MyUyMHRvJTIwamFzbmUuJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwoZW5jb2RlZF9pbnB1dCklMEElMEElMjMlMjBIZXJCRVJUJTIwY2FuJTIwYWxzbyUyMGJlJTIwbG9hZGVkJTIwdXNpbmclMjBBdXRvVG9rZW5pemVyJTIwYW5kJTIwQXV0b01vZGVsJTNBJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmFsbGVncm8lMkZoZXJiZXJ0LWtsZWotY2FzZWQtdG9rZW5pemVyLXYxJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJhbGxlZ3JvJTJGaGVyYmVydC1rbGVqLWNhc2VkLXYxJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> HerbertTokenizer, RobertaModel

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = HerbertTokenizer.from_pretrained(<span class="hljs-string">&quot;allegro/herbert-klej-cased-tokenizer-v1&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = RobertaModel.from_pretrained(<span class="hljs-string">&quot;allegro/herbert-klej-cased-v1&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer.encode(<span class="hljs-string">&quot;Kto ma lepsz\u0105 sztuk\u0119, ma lepszy rz\u0105d \u2013 to jasne.&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(encoded_input)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># HerBERT can also be loaded using AutoTokenizer and AutoModel:</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;allegro/herbert-klej-cased-tokenizer-v1&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModel.from_pretrained(<span class="hljs-string">&quot;allegro/herbert-klej-cased-v1&quot;</span>)`}}),te=new Ct({}),se=new P({props:{name:"class transformers.HerbertTokenizer",anchor:"transformers.HerbertTokenizer",parameters:[{name:"vocab_file",val:""},{name:"merges_file",val:""},{name:"tokenizer_file",val:" = None"},{name:"cls_token",val:" = '<s>'"},{name:"unk_token",val:" = '<unk>'"},{name:"pad_token",val:" = '<pad>'"},{name:"mask_token",val:" = '<mask>'"},{name:"sep_token",val:" = '</s>'"},{name:"bos_token",val:" = '<s>'"},{name:"do_lowercase_and_remove_accent",val:" = False"},{name:"additional_special_tokens",val:" = ['<special0>', '<special1>', '<special2>', '<special3>', '<special4>', '<special5>', '<special6>', '<special7>', '<special8>', '<special9>']"},{name:"lang2id",val:" = None"},{name:"id2lang",val:" = None"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/herbert/tokenization_herbert.py#L277"}}),ae=new P({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.HerbertTokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.HerbertTokenizer.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.HerbertTokenizer.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/herbert/tokenization_herbert.py#L514",returnDescription:`
<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),ie=new P({props:{name:"convert_tokens_to_string",anchor:"transformers.HerbertTokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/herbert/tokenization_herbert.py#L508"}}),le=new P({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.HerbertTokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.HerbertTokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.HerbertTokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/herbert/tokenization_herbert.py#L571",returnDescription:`
<p>List of <a href="../glossary#token-type-ids">token type IDs</a> according to the given sequence(s).</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),F=new _n({props:{anchor:"transformers.HerbertTokenizer.create_token_type_ids_from_sequences.example",$$slots:{default:[Tn]},$$scope:{ctx:we}}}),ce=new P({props:{name:"get_special_tokens_mask",anchor:"transformers.HerbertTokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.HerbertTokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.HerbertTokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.HerbertTokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/herbert/tokenization_herbert.py#L542",returnDescription:`
<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),pe=new Ct({}),me=new P({props:{name:"class transformers.HerbertTokenizerFast",anchor:"transformers.HerbertTokenizerFast",parameters:[{name:"vocab_file",val:" = None"},{name:"merges_file",val:" = None"},{name:"tokenizer_file",val:" = None"},{name:"cls_token",val:" = '<s>'"},{name:"unk_token",val:" = '<unk>'"},{name:"pad_token",val:" = '<pad>'"},{name:"mask_token",val:" = '<mask>'"},{name:"sep_token",val:" = '</s>'"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.HerbertTokenizerFast.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
Path to the vocabulary file.`,name:"vocab_file"},{anchor:"transformers.HerbertTokenizerFast.merges_file",description:`<strong>merges_file</strong> (<code>str</code>) &#x2014;
Path to the merges file.`,name:"merges_file"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/herbert/tokenization_herbert_fast.py#L40"}}),ue=new P({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.HerbertTokenizerFast.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.HerbertTokenizerFast.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.HerbertTokenizerFast.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/herbert/tokenization_herbert_fast.py#L89",returnDescription:`
<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),ke=new P({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.HerbertTokenizerFast.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.HerbertTokenizerFast.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.HerbertTokenizerFast.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/herbert/tokenization_herbert_fast.py#L143",returnDescription:`
<p>List of <a href="../glossary#token-type-ids">token type IDs</a> according to the given sequence(s).</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),W=new _n({props:{anchor:"transformers.HerbertTokenizerFast.create_token_type_ids_from_sequences.example",$$slots:{default:[$n]},$$scope:{ctx:we}}}),ge=new P({props:{name:"get_special_tokens_mask",anchor:"transformers.HerbertTokenizerFast.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.HerbertTokenizerFast.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.HerbertTokenizerFast.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.HerbertTokenizerFast.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/herbert/tokenization_herbert_fast.py#L116",returnDescription:`
<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),{c(){u=r("meta"),I=l(),g=r("h1"),h=r("a"),J=r("span"),v(d.$$.fragment),M=l(),Ue=r("span"),Vt=o("HerBERT"),ht=l(),D=r("h2"),R=r("a"),Ae=r("span"),v(O.$$.fragment),Nt=l(),Be=r("span"),Ft=o("Overview"),ut=l(),C=r("p"),Xt=o("The HerBERT model was proposed in "),Y=r("a"),Zt=o("KLEJ: Comprehensive Benchmark for Polish Language Understanding"),Wt=o(` by Piotr Rybak, Robert Mroczkowski, Janusz Tracz, and
Ireneusz Gawlik. It is a BERT-based Language Model trained on Polish Corpora using only MLM objective with dynamic
masking of whole words.`),_t=l(),ye=r("p"),Gt=o("The abstract from the paper is the following:"),kt=l(),Te=r("p"),Pe=r("em"),St=o(`In recent years, a series of Transformer-based models unlocked major improvements in general natural language
understanding (NLU) tasks. Such a fast pace of research would not be possible without general NLU benchmarks, which
allow for a fair comparison of the proposed methods. However, such benchmarks are available only for a handful of
languages. To alleviate this issue, we introduce a comprehensive multi-task benchmark for the Polish language
understanding, accompanied by an online leaderboard. It consists of a diverse set of tasks, adopted from existing
datasets for named entity recognition, question-answering, textual entailment, and others. We also introduce a new
sentiment analysis task for the e-commerce domain, named Allegro Reviews (AR). To ensure a common evaluation scheme and
promote models that generalize to different NLU tasks, the benchmark includes datasets from varying domains and
applications. Additionally, we release HerBERT, a Transformer-based model trained specifically for the Polish language,
which has the best average performance and obtains the best results for three out of nine tasks. Finally, we provide an
extensive evaluation, including several standard baselines and recently proposed, multilingual Transformer-based
models.`),gt=l(),$e=r("p"),Ot=o("Examples of use:"),bt=l(),v(Q.$$.fragment),vt=l(),q=r("p"),Yt=o("This model was contributed by "),K=r("a"),Qt=o("rmroczkowski"),Kt=o(`. The original code can be found
`),ee=r("a"),es=o("here"),ts=o("."),wt=l(),U=r("h2"),V=r("a"),Re=r("span"),v(te.$$.fragment),ss=l(),Ce=r("span"),rs=o("HerbertTokenizer"),yt=l(),_=r("div"),v(se.$$.fragment),ns=l(),Ve=r("p"),as=o("Construct a BPE tokenizer for HerBERT."),os=l(),Ne=r("p"),is=o("Peculiarities:"),ls=l(),re=r("ul"),Fe=r("li"),Xe=r("p"),cs=o(`uses BERT\u2019s pre-tokenizer: BaseTokenizer splits tokens on spaces, and also on punctuation. Each occurrence of a
punctuation character will be treated separately.`),ds=l(),Ze=r("li"),We=r("p"),ps=o("Such pretokenized input is BPE subtokenized"),ms=l(),ne=r("p"),fs=o("This tokenizer inherits from "),Ee=r("a"),hs=o("XLMTokenizer"),us=o(` which contains most of the methods. Users should refer to the
superclass for more information regarding methods.`),_s=l(),H=r("div"),v(ae.$$.fragment),ks=l(),Ge=r("p"),gs=o(`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. An XLM sequence has the following format:`),bs=l(),oe=r("ul"),Me=r("li"),vs=o("single sequence: "),Se=r("code"),ws=o("<s> X </s>"),ys=l(),ze=r("li"),Ts=o("pair of sequences: "),Oe=r("code"),$s=o("<s> A </s> B </s>"),Es=l(),N=r("div"),v(ie.$$.fragment),Ms=l(),Ye=r("p"),zs=o("Converts a sequence of tokens (string) in a single string."),Js=l(),L=r("div"),v(le.$$.fragment),Is=l(),Qe=r("p"),Ls=o("Create a mask from the two sequences passed to be used in a sequence-pair classification task. An XLM sequence"),qs=l(),v(F.$$.fragment),Hs=l(),A=r("p"),js=o("If "),Ke=r("code"),xs=o("token_ids_1"),Ds=o(" is "),et=r("code"),Us=o("None"),As=o(", this method only returns the first portion of the mask (0s)."),Bs=l(),X=r("div"),v(ce.$$.fragment),Ps=l(),de=r("p"),Rs=o(`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `),tt=r("code"),Cs=o("prepare_for_model"),Vs=o(" method."),Tt=l(),B=r("h2"),Z=r("a"),st=r("span"),v(pe.$$.fragment),Ns=l(),rt=r("span"),Fs=o("HerbertTokenizerFast"),$t=l(),k=r("div"),v(me.$$.fragment),Xs=l(),fe=r("p"),Zs=o("Construct a \u201CFast\u201D BPE tokenizer for HerBERT (backed by HuggingFace\u2019s "),nt=r("em"),Ws=o("tokenizers"),Gs=o(" library)."),Ss=l(),at=r("p"),Os=o("Peculiarities:"),Ys=l(),ot=r("ul"),it=r("li"),Qs=o(`uses BERT\u2019s pre-tokenizer: BertPreTokenizer splits tokens on spaces, and also on punctuation. Each occurrence of
a punctuation character will be treated separately.`),Ks=l(),he=r("p"),er=o("This tokenizer inherits from "),Je=r("a"),tr=o("PreTrainedTokenizer"),sr=o(` which contains most of the methods. Users should refer to the
superclass for more information regarding methods.`),rr=l(),j=r("div"),v(ue.$$.fragment),nr=l(),lt=r("p"),ar=o(`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. An HerBERT, like BERT sequence has the following format:`),or=l(),_e=r("ul"),Ie=r("li"),ir=o("single sequence: "),ct=r("code"),lr=o("<s> X </s>"),cr=l(),Le=r("li"),dr=o("pair of sequences: "),dt=r("code"),pr=o("<s> A </s> B </s>"),mr=l(),x=r("div"),v(ke.$$.fragment),fr=l(),pt=r("p"),hr=o("Create a mask from the two sequences passed to be used in a sequence-pair classification task. HerBERT, like"),ur=l(),v(W.$$.fragment),_r=l(),G=r("div"),v(ge.$$.fragment),kr=l(),be=r("p"),gr=o(`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `),mt=r("code"),br=o("prepare_for_model"),vr=o(" method."),this.h()},l(t){const m=wn('[data-svelte="svelte-1phssyn"]',document.head);u=n(m,"META",{name:!0,content:!0}),m.forEach(s),I=c(t),g=n(t,"H1",{class:!0});var ve=a(g);h=n(ve,"A",{id:!0,class:!0,href:!0});var ft=a(h);J=n(ft,"SPAN",{});var Mr=a(J);w(d.$$.fragment,Mr),Mr.forEach(s),ft.forEach(s),M=c(ve),Ue=n(ve,"SPAN",{});var zr=a(Ue);Vt=i(zr,"HerBERT"),zr.forEach(s),ve.forEach(s),ht=c(t),D=n(t,"H2",{class:!0});var Mt=a(D);R=n(Mt,"A",{id:!0,class:!0,href:!0});var Jr=a(R);Ae=n(Jr,"SPAN",{});var Ir=a(Ae);w(O.$$.fragment,Ir),Ir.forEach(s),Jr.forEach(s),Nt=c(Mt),Be=n(Mt,"SPAN",{});var Lr=a(Be);Ft=i(Lr,"Overview"),Lr.forEach(s),Mt.forEach(s),ut=c(t),C=n(t,"P",{});var zt=a(C);Xt=i(zt,"The HerBERT model was proposed in "),Y=n(zt,"A",{href:!0,rel:!0});var qr=a(Y);Zt=i(qr,"KLEJ: Comprehensive Benchmark for Polish Language Understanding"),qr.forEach(s),Wt=i(zt,` by Piotr Rybak, Robert Mroczkowski, Janusz Tracz, and
Ireneusz Gawlik. It is a BERT-based Language Model trained on Polish Corpora using only MLM objective with dynamic
masking of whole words.`),zt.forEach(s),_t=c(t),ye=n(t,"P",{});var Hr=a(ye);Gt=i(Hr,"The abstract from the paper is the following:"),Hr.forEach(s),kt=c(t),Te=n(t,"P",{});var jr=a(Te);Pe=n(jr,"EM",{});var xr=a(Pe);St=i(xr,`In recent years, a series of Transformer-based models unlocked major improvements in general natural language
understanding (NLU) tasks. Such a fast pace of research would not be possible without general NLU benchmarks, which
allow for a fair comparison of the proposed methods. However, such benchmarks are available only for a handful of
languages. To alleviate this issue, we introduce a comprehensive multi-task benchmark for the Polish language
understanding, accompanied by an online leaderboard. It consists of a diverse set of tasks, adopted from existing
datasets for named entity recognition, question-answering, textual entailment, and others. We also introduce a new
sentiment analysis task for the e-commerce domain, named Allegro Reviews (AR). To ensure a common evaluation scheme and
promote models that generalize to different NLU tasks, the benchmark includes datasets from varying domains and
applications. Additionally, we release HerBERT, a Transformer-based model trained specifically for the Polish language,
which has the best average performance and obtains the best results for three out of nine tasks. Finally, we provide an
extensive evaluation, including several standard baselines and recently proposed, multilingual Transformer-based
models.`),xr.forEach(s),jr.forEach(s),gt=c(t),$e=n(t,"P",{});var Dr=a($e);Ot=i(Dr,"Examples of use:"),Dr.forEach(s),bt=c(t),w(Q.$$.fragment,t),vt=c(t),q=n(t,"P",{});var qe=a(q);Yt=i(qe,"This model was contributed by "),K=n(qe,"A",{href:!0,rel:!0});var Ur=a(K);Qt=i(Ur,"rmroczkowski"),Ur.forEach(s),Kt=i(qe,`. The original code can be found
`),ee=n(qe,"A",{href:!0,rel:!0});var Ar=a(ee);es=i(Ar,"here"),Ar.forEach(s),ts=i(qe,"."),qe.forEach(s),wt=c(t),U=n(t,"H2",{class:!0});var Jt=a(U);V=n(Jt,"A",{id:!0,class:!0,href:!0});var Br=a(V);Re=n(Br,"SPAN",{});var Pr=a(Re);w(te.$$.fragment,Pr),Pr.forEach(s),Br.forEach(s),ss=c(Jt),Ce=n(Jt,"SPAN",{});var Rr=a(Ce);rs=i(Rr,"HerbertTokenizer"),Rr.forEach(s),Jt.forEach(s),yt=c(t),_=n(t,"DIV",{class:!0});var b=a(_);w(se.$$.fragment,b),ns=c(b),Ve=n(b,"P",{});var Cr=a(Ve);as=i(Cr,"Construct a BPE tokenizer for HerBERT."),Cr.forEach(s),os=c(b),Ne=n(b,"P",{});var Vr=a(Ne);is=i(Vr,"Peculiarities:"),Vr.forEach(s),ls=c(b),re=n(b,"UL",{});var It=a(re);Fe=n(It,"LI",{});var Nr=a(Fe);Xe=n(Nr,"P",{});var Fr=a(Xe);cs=i(Fr,`uses BERT\u2019s pre-tokenizer: BaseTokenizer splits tokens on spaces, and also on punctuation. Each occurrence of a
punctuation character will be treated separately.`),Fr.forEach(s),Nr.forEach(s),ds=c(It),Ze=n(It,"LI",{});var Xr=a(Ze);We=n(Xr,"P",{});var Zr=a(We);ps=i(Zr,"Such pretokenized input is BPE subtokenized"),Zr.forEach(s),Xr.forEach(s),It.forEach(s),ms=c(b),ne=n(b,"P",{});var Lt=a(ne);fs=i(Lt,"This tokenizer inherits from "),Ee=n(Lt,"A",{href:!0});var Wr=a(Ee);hs=i(Wr,"XLMTokenizer"),Wr.forEach(s),us=i(Lt,` which contains most of the methods. Users should refer to the
superclass for more information regarding methods.`),Lt.forEach(s),_s=c(b),H=n(b,"DIV",{class:!0});var He=a(H);w(ae.$$.fragment,He),ks=c(He),Ge=n(He,"P",{});var Gr=a(Ge);gs=i(Gr,`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. An XLM sequence has the following format:`),Gr.forEach(s),bs=c(He),oe=n(He,"UL",{});var qt=a(oe);Me=n(qt,"LI",{});var wr=a(Me);vs=i(wr,"single sequence: "),Se=n(wr,"CODE",{});var Sr=a(Se);ws=i(Sr,"<s> X </s>"),Sr.forEach(s),wr.forEach(s),ys=c(qt),ze=n(qt,"LI",{});var yr=a(ze);Ts=i(yr,"pair of sequences: "),Oe=n(yr,"CODE",{});var Or=a(Oe);$s=i(Or,"<s> A </s> B </s>"),Or.forEach(s),yr.forEach(s),qt.forEach(s),He.forEach(s),Es=c(b),N=n(b,"DIV",{class:!0});var Ht=a(N);w(ie.$$.fragment,Ht),Ms=c(Ht),Ye=n(Ht,"P",{});var Yr=a(Ye);zs=i(Yr,"Converts a sequence of tokens (string) in a single string."),Yr.forEach(s),Ht.forEach(s),Js=c(b),L=n(b,"DIV",{class:!0});var S=a(L);w(le.$$.fragment,S),Is=c(S),Qe=n(S,"P",{});var Qr=a(Qe);Ls=i(Qr,"Create a mask from the two sequences passed to be used in a sequence-pair classification task. An XLM sequence"),Qr.forEach(s),qs=c(S),w(F.$$.fragment,S),Hs=c(S),A=n(S,"P",{});var je=a(A);js=i(je,"If "),Ke=n(je,"CODE",{});var Kr=a(Ke);xs=i(Kr,"token_ids_1"),Kr.forEach(s),Ds=i(je," is "),et=n(je,"CODE",{});var en=a(et);Us=i(en,"None"),en.forEach(s),As=i(je,", this method only returns the first portion of the mask (0s)."),je.forEach(s),S.forEach(s),Bs=c(b),X=n(b,"DIV",{class:!0});var jt=a(X);w(ce.$$.fragment,jt),Ps=c(jt),de=n(jt,"P",{});var xt=a(de);Rs=i(xt,`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `),tt=n(xt,"CODE",{});var tn=a(tt);Cs=i(tn,"prepare_for_model"),tn.forEach(s),Vs=i(xt," method."),xt.forEach(s),jt.forEach(s),b.forEach(s),Tt=c(t),B=n(t,"H2",{class:!0});var Dt=a(B);Z=n(Dt,"A",{id:!0,class:!0,href:!0});var sn=a(Z);st=n(sn,"SPAN",{});var rn=a(st);w(pe.$$.fragment,rn),rn.forEach(s),sn.forEach(s),Ns=c(Dt),rt=n(Dt,"SPAN",{});var nn=a(rt);Fs=i(nn,"HerbertTokenizerFast"),nn.forEach(s),Dt.forEach(s),$t=c(t),k=n(t,"DIV",{class:!0});var z=a(k);w(me.$$.fragment,z),Xs=c(z),fe=n(z,"P",{});var Ut=a(fe);Zs=i(Ut,"Construct a \u201CFast\u201D BPE tokenizer for HerBERT (backed by HuggingFace\u2019s "),nt=n(Ut,"EM",{});var an=a(nt);Ws=i(an,"tokenizers"),an.forEach(s),Gs=i(Ut," library)."),Ut.forEach(s),Ss=c(z),at=n(z,"P",{});var on=a(at);Os=i(on,"Peculiarities:"),on.forEach(s),Ys=c(z),ot=n(z,"UL",{});var ln=a(ot);it=n(ln,"LI",{});var cn=a(it);Qs=i(cn,`uses BERT\u2019s pre-tokenizer: BertPreTokenizer splits tokens on spaces, and also on punctuation. Each occurrence of
a punctuation character will be treated separately.`),cn.forEach(s),ln.forEach(s),Ks=c(z),he=n(z,"P",{});var At=a(he);er=i(At,"This tokenizer inherits from "),Je=n(At,"A",{href:!0});var dn=a(Je);tr=i(dn,"PreTrainedTokenizer"),dn.forEach(s),sr=i(At,` which contains most of the methods. Users should refer to the
superclass for more information regarding methods.`),At.forEach(s),rr=c(z),j=n(z,"DIV",{class:!0});var xe=a(j);w(ue.$$.fragment,xe),nr=c(xe),lt=n(xe,"P",{});var pn=a(lt);ar=i(pn,`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. An HerBERT, like BERT sequence has the following format:`),pn.forEach(s),or=c(xe),_e=n(xe,"UL",{});var Bt=a(_e);Ie=n(Bt,"LI",{});var Tr=a(Ie);ir=i(Tr,"single sequence: "),ct=n(Tr,"CODE",{});var mn=a(ct);lr=i(mn,"<s> X </s>"),mn.forEach(s),Tr.forEach(s),cr=c(Bt),Le=n(Bt,"LI",{});var $r=a(Le);dr=i($r,"pair of sequences: "),dt=n($r,"CODE",{});var fn=a(dt);pr=i(fn,"<s> A </s> B </s>"),fn.forEach(s),$r.forEach(s),Bt.forEach(s),xe.forEach(s),mr=c(z),x=n(z,"DIV",{class:!0});var De=a(x);w(ke.$$.fragment,De),fr=c(De),pt=n(De,"P",{});var hn=a(pt);hr=i(hn,"Create a mask from the two sequences passed to be used in a sequence-pair classification task. HerBERT, like"),hn.forEach(s),ur=c(De),w(W.$$.fragment,De),De.forEach(s),_r=c(z),G=n(z,"DIV",{class:!0});var Pt=a(G);w(ge.$$.fragment,Pt),kr=c(Pt),be=n(Pt,"P",{});var Rt=a(be);gr=i(Rt,`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `),mt=n(Rt,"CODE",{});var un=a(mt);br=i(un,"prepare_for_model"),un.forEach(s),vr=i(Rt," method."),Rt.forEach(s),Pt.forEach(s),z.forEach(s),this.h()},h(){p(u,"name","hf:doc:metadata"),p(u,"content",JSON.stringify(Mn)),p(h,"id","herbert"),p(h,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(h,"href","#herbert"),p(g,"class","relative group"),p(R,"id","overview"),p(R,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(R,"href","#overview"),p(D,"class","relative group"),p(Y,"href","https://www.aclweb.org/anthology/2020.acl-main.111.pdf"),p(Y,"rel","nofollow"),p(K,"href","https://huggingface.co/rmroczkowski"),p(K,"rel","nofollow"),p(ee,"href","https://github.com/allegro/HerBERT"),p(ee,"rel","nofollow"),p(V,"id","transformers.HerbertTokenizer"),p(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(V,"href","#transformers.HerbertTokenizer"),p(U,"class","relative group"),p(Ee,"href","/docs/transformers/v4.30.0/en/model_doc/xlm#transformers.XLMTokenizer"),p(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(Z,"id","transformers.HerbertTokenizerFast"),p(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Z,"href","#transformers.HerbertTokenizerFast"),p(B,"class","relative group"),p(Je,"href","/docs/transformers/v4.30.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizer"),p(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),p(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,m){e(document.head,u),f(t,I,m),f(t,g,m),e(g,h),e(h,J),y(d,J,null),e(g,M),e(g,Ue),e(Ue,Vt),f(t,ht,m),f(t,D,m),e(D,R),e(R,Ae),y(O,Ae,null),e(D,Nt),e(D,Be),e(Be,Ft),f(t,ut,m),f(t,C,m),e(C,Xt),e(C,Y),e(Y,Zt),e(C,Wt),f(t,_t,m),f(t,ye,m),e(ye,Gt),f(t,kt,m),f(t,Te,m),e(Te,Pe),e(Pe,St),f(t,gt,m),f(t,$e,m),e($e,Ot),f(t,bt,m),y(Q,t,m),f(t,vt,m),f(t,q,m),e(q,Yt),e(q,K),e(K,Qt),e(q,Kt),e(q,ee),e(ee,es),e(q,ts),f(t,wt,m),f(t,U,m),e(U,V),e(V,Re),y(te,Re,null),e(U,ss),e(U,Ce),e(Ce,rs),f(t,yt,m),f(t,_,m),y(se,_,null),e(_,ns),e(_,Ve),e(Ve,as),e(_,os),e(_,Ne),e(Ne,is),e(_,ls),e(_,re),e(re,Fe),e(Fe,Xe),e(Xe,cs),e(re,ds),e(re,Ze),e(Ze,We),e(We,ps),e(_,ms),e(_,ne),e(ne,fs),e(ne,Ee),e(Ee,hs),e(ne,us),e(_,_s),e(_,H),y(ae,H,null),e(H,ks),e(H,Ge),e(Ge,gs),e(H,bs),e(H,oe),e(oe,Me),e(Me,vs),e(Me,Se),e(Se,ws),e(oe,ys),e(oe,ze),e(ze,Ts),e(ze,Oe),e(Oe,$s),e(_,Es),e(_,N),y(ie,N,null),e(N,Ms),e(N,Ye),e(Ye,zs),e(_,Js),e(_,L),y(le,L,null),e(L,Is),e(L,Qe),e(Qe,Ls),e(L,qs),y(F,L,null),e(L,Hs),e(L,A),e(A,js),e(A,Ke),e(Ke,xs),e(A,Ds),e(A,et),e(et,Us),e(A,As),e(_,Bs),e(_,X),y(ce,X,null),e(X,Ps),e(X,de),e(de,Rs),e(de,tt),e(tt,Cs),e(de,Vs),f(t,Tt,m),f(t,B,m),e(B,Z),e(Z,st),y(pe,st,null),e(B,Ns),e(B,rt),e(rt,Fs),f(t,$t,m),f(t,k,m),y(me,k,null),e(k,Xs),e(k,fe),e(fe,Zs),e(fe,nt),e(nt,Ws),e(fe,Gs),e(k,Ss),e(k,at),e(at,Os),e(k,Ys),e(k,ot),e(ot,it),e(it,Qs),e(k,Ks),e(k,he),e(he,er),e(he,Je),e(Je,tr),e(he,sr),e(k,rr),e(k,j),y(ue,j,null),e(j,nr),e(j,lt),e(lt,ar),e(j,or),e(j,_e),e(_e,Ie),e(Ie,ir),e(Ie,ct),e(ct,lr),e(_e,cr),e(_e,Le),e(Le,dr),e(Le,dt),e(dt,pr),e(k,mr),e(k,x),y(ke,x,null),e(x,fr),e(x,pt),e(pt,hr),e(x,ur),y(W,x,null),e(k,_r),e(k,G),y(ge,G,null),e(G,kr),e(G,be),e(be,gr),e(be,mt),e(mt,br),e(be,vr),Et=!0},p(t,[m]){const ve={};m&2&&(ve.$$scope={dirty:m,ctx:t}),F.$set(ve);const ft={};m&2&&(ft.$$scope={dirty:m,ctx:t}),W.$set(ft)},i(t){Et||(T(d.$$.fragment,t),T(O.$$.fragment,t),T(Q.$$.fragment,t),T(te.$$.fragment,t),T(se.$$.fragment,t),T(ae.$$.fragment,t),T(ie.$$.fragment,t),T(le.$$.fragment,t),T(F.$$.fragment,t),T(ce.$$.fragment,t),T(pe.$$.fragment,t),T(me.$$.fragment,t),T(ue.$$.fragment,t),T(ke.$$.fragment,t),T(W.$$.fragment,t),T(ge.$$.fragment,t),Et=!0)},o(t){$(d.$$.fragment,t),$(O.$$.fragment,t),$(Q.$$.fragment,t),$(te.$$.fragment,t),$(se.$$.fragment,t),$(ae.$$.fragment,t),$(ie.$$.fragment,t),$(le.$$.fragment,t),$(F.$$.fragment,t),$(ce.$$.fragment,t),$(pe.$$.fragment,t),$(me.$$.fragment,t),$(ue.$$.fragment,t),$(ke.$$.fragment,t),$(W.$$.fragment,t),$(ge.$$.fragment,t),Et=!1},d(t){s(u),t&&s(I),t&&s(g),E(d),t&&s(ht),t&&s(D),E(O),t&&s(ut),t&&s(C),t&&s(_t),t&&s(ye),t&&s(kt),t&&s(Te),t&&s(gt),t&&s($e),t&&s(bt),E(Q,t),t&&s(vt),t&&s(q),t&&s(wt),t&&s(U),E(te),t&&s(yt),t&&s(_),E(se),E(ae),E(ie),E(le),E(F),E(ce),t&&s(Tt),t&&s(B),E(pe),t&&s($t),t&&s(k),E(me),E(ue),E(ke),E(W),E(ge)}}}const Mn={local:"herbert",sections:[{local:"overview",title:"Overview"},{local:"transformers.HerbertTokenizer",title:"HerbertTokenizer"},{local:"transformers.HerbertTokenizerFast",title:"HerbertTokenizerFast"}],title:"HerBERT"};function zn(we){return yn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class jn extends gn{constructor(u){super();bn(this,u,zn,En,vn,{})}}export{jn as default,Mn as metadata};
