import{S as $s,i as zs,s as Js,e as n,k as a,w as u,t as i,M as Ms,c as r,d as s,m as l,a as o,x as g,h as d,b as c,G as e,g as f,y as k,L as qs,q as _,o as w,B as b,v as Is}from"../../chunks/vendor-hf-doc-builder.js";import{D as Z}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Rs}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as Yt}from"../../chunks/IconCopyLink-hf-doc-builder.js";function Us(Ot){let y,ze,E,z,ce,L,Ye,pe,Oe,Je,B,J,me,C,Ke,he,et,Me,M,tt,F,st,nt,qe,se,rt,Ie,ne,fe,ot,Re,re,at,Ue,A,xe,v,it,N,lt,dt,D,ct,pt,Ve,$,q,ue,P,mt,ge,ht,je,p,X,ft,ke,ut,gt,S,kt,oe,_t,wt,bt,I,W,vt,_e,Tt,yt,T,Q,Et,we,Bt,$t,H,ae,zt,be,Jt,Mt,ie,qt,ve,It,Rt,R,G,Ut,Te,xt,Vt,U,Y,jt,ye,Zt,Lt,x,O,Ct,K,Ft,Ee,At,Nt,Dt,V,ee,Pt,Be,Xt,St,j,te,Wt,$e,Qt,Ze;return L=new Yt({}),C=new Yt({}),A=new Rs({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQWJlcnR3ZWV0JTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJ2aW5haSUyRmJlcnR3ZWV0LWJhc2UlMjIpJTBBJTBBJTIzJTIwRm9yJTIwdHJhbnNmb3JtZXJzJTIwdjQueCUyQiUzQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMnZpbmFpJTJGYmVydHdlZXQtYmFzZSUyMiUyQyUyMHVzZV9mYXN0JTNERmFsc2UpJTBBJTBBJTIzJTIwRm9yJTIwdHJhbnNmb3JtZXJzJTIwdjMueCUzQSUwQSUyMyUyMHRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMnZpbmFpJTJGYmVydHdlZXQtYmFzZSUyMiklMEElMEElMjMlMjBJTlBVVCUyMFRXRUVUJTIwSVMlMjBBTFJFQURZJTIwTk9STUFMSVpFRCElMEFsaW5lJTIwJTNEJTIwJTIyU0MlMjBoYXMlMjBmaXJzdCUyMHR3byUyMHByZXN1bXB0aXZlJTIwY2FzZXMlMjBvZiUyMGNvcm9uYXZpcnVzJTIwJTJDJTIwREhFQyUyMGNvbmZpcm1zJTIwSFRUUFVSTCUyMHZpYSUyMCU0MFVTRVIlMjAlM0FjcnklM0ElMjIlMEElMEFpbnB1dF9pZHMlMjAlM0QlMjB0b3JjaC50ZW5zb3IoJTVCdG9rZW5pemVyLmVuY29kZShsaW5lKSU1RCklMEElMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwZmVhdHVyZXMlMjAlM0QlMjBiZXJ0d2VldChpbnB1dF9pZHMpJTIwJTIwJTIzJTIwTW9kZWxzJTIwb3V0cHV0cyUyMGFyZSUyMG5vdyUyMHR1cGxlcyUwQSUwQSUyMyUyMFdpdGglMjBUZW5zb3JGbG93JTIwMi4wJTJCJTNBJTBBJTIzJTIwZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsJTBBJTIzJTIwYmVydHdlZXQlMjAlM0QlMjBURkF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIydmluYWklMkZiZXJ0d2VldC1iYXNlJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>bertweet = AutoModel.from_pretrained(<span class="hljs-string">&quot;vinai/bertweet-base&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># For transformers v4.x+:</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;vinai/bertweet-base&quot;</span>, use_fast=<span class="hljs-literal">False</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># For transformers v3.x:</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># tokenizer = AutoTokenizer.from_pretrained(&quot;vinai/bertweet-base&quot;)</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># INPUT TWEET IS ALREADY NORMALIZED!</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>line = <span class="hljs-string">&quot;SC has first two presumptive cases of coronavirus , DHEC confirms HTTPURL via @USER :cry:&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = torch.tensor([tokenizer.encode(line)])

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    features = bertweet(input_ids)  <span class="hljs-comment"># Models outputs are now tuples</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># With TensorFlow 2.0+:</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># from transformers import TFAutoModel</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># bertweet = TFAutoModel.from_pretrained(&quot;vinai/bertweet-base&quot;)</span>`}}),P=new Yt({}),X=new Z({props:{name:"class transformers.BertweetTokenizer",anchor:"transformers.BertweetTokenizer",parameters:[{name:"vocab_file",val:""},{name:"merges_file",val:""},{name:"normalization",val:" = False"},{name:"bos_token",val:" = '<s>'"},{name:"eos_token",val:" = '</s>'"},{name:"sep_token",val:" = '</s>'"},{name:"cls_token",val:" = '<s>'"},{name:"unk_token",val:" = '<unk>'"},{name:"pad_token",val:" = '<pad>'"},{name:"mask_token",val:" = '<mask>'"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BertweetTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
Path to the vocabulary file.`,name:"vocab_file"},{anchor:"transformers.BertweetTokenizer.merges_file",description:`<strong>merges_file</strong> (<code>str</code>) &#x2014;
Path to the merges file.`,name:"merges_file"},{anchor:"transformers.BertweetTokenizer.normalization",description:`<strong>normalization</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to apply a normalization preprocess.`,name:"normalization"},{anchor:"transformers.BertweetTokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the beginning of
sequence. The token used is the <code>cls_token</code>.</p>

					</div>`,name:"bos_token"},{anchor:"transformers.BertweetTokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the end of sequence.
The token used is the <code>sep_token</code>.</p>

					</div>`,name:"eos_token"},{anchor:"transformers.BertweetTokenizer.sep_token",description:`<strong>sep_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
sequence classification or for a text and a question for question answering. It is also used as the last
token of a sequence built with special tokens.`,name:"sep_token"},{anchor:"transformers.BertweetTokenizer.cls_token",description:`<strong>cls_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The classifier token which is used when doing sequence classification (classification of the whole sequence
instead of per-token classification). It is the first token of the sequence when built with special tokens.`,name:"cls_token"},{anchor:"transformers.BertweetTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.BertweetTokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;pad&gt;&quot;</code>) &#x2014;
The token used for padding, for example when batching sequences of different lengths.`,name:"pad_token"},{anchor:"transformers.BertweetTokenizer.mask_token",description:`<strong>mask_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;mask&gt;&quot;</code>) &#x2014;
The token used for masking values. This is the token used when training this model with masked language
modeling. This is the token which the model will try to predict.`,name:"mask_token"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/bertweet/tokenization_bertweet.py#L68"}}),W=new Z({props:{name:"add_from_file",anchor:"transformers.BertweetTokenizer.add_from_file",parameters:[{name:"f",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/bertweet/tokenization_bertweet.py#L415"}}),Q=new Z({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.BertweetTokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.BertweetTokenizer.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.BertweetTokenizer.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/bertweet/tokenization_bertweet.py#L184",returnDescription:`
<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),G=new Z({props:{name:"convert_tokens_to_string",anchor:"transformers.BertweetTokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/bertweet/tokenization_bertweet.py#L385"}}),Y=new Z({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.BertweetTokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.BertweetTokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.BertweetTokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/bertweet/tokenization_bertweet.py#L238",returnDescription:`
<p>List of zeros.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),O=new Z({props:{name:"get_special_tokens_mask",anchor:"transformers.BertweetTokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.BertweetTokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.BertweetTokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.BertweetTokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/bertweet/tokenization_bertweet.py#L210",returnDescription:`
<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`
<p><code>List[int]</code></p>
`}}),ee=new Z({props:{name:"normalizeToken",anchor:"transformers.BertweetTokenizer.normalizeToken",parameters:[{name:"token",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/bertweet/tokenization_bertweet.py#L358"}}),te=new Z({props:{name:"normalizeTweet",anchor:"transformers.BertweetTokenizer.normalizeTweet",parameters:[{name:"tweet",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.30.0/src/transformers/models/bertweet/tokenization_bertweet.py#L324"}}),{c(){y=n("meta"),ze=a(),E=n("h1"),z=n("a"),ce=n("span"),u(L.$$.fragment),Ye=a(),pe=n("span"),Oe=i("BERTweet"),Je=a(),B=n("h2"),J=n("a"),me=n("span"),u(C.$$.fragment),Ke=a(),he=n("span"),et=i("Overview"),Me=a(),M=n("p"),tt=i("The BERTweet model was proposed in "),F=n("a"),st=i("BERTweet: A pre-trained language model for English Tweets"),nt=i(" by Dat Quoc Nguyen, Thanh Vu, Anh Tuan Nguyen."),qe=a(),se=n("p"),rt=i("The abstract from the paper is the following:"),Ie=a(),ne=n("p"),fe=n("em"),ot=i(`We present BERTweet, the first public large-scale pre-trained language model for English Tweets. Our BERTweet, having
the same architecture as BERT-base (Devlin et al., 2019), is trained using the RoBERTa pre-training procedure (Liu et
al., 2019). Experiments show that BERTweet outperforms strong baselines RoBERTa-base and XLM-R-base (Conneau et al.,
2020), producing better performance results than the previous state-of-the-art models on three Tweet NLP tasks:
Part-of-speech tagging, Named-entity recognition and text classification.`),Re=a(),re=n("p"),at=i("Example of use:"),Ue=a(),u(A.$$.fragment),xe=a(),v=n("p"),it=i("This model was contributed by "),N=n("a"),lt=i("dqnguyen"),dt=i(". The original code can be found "),D=n("a"),ct=i("here"),pt=i("."),Ve=a(),$=n("h2"),q=n("a"),ue=n("span"),u(P.$$.fragment),mt=a(),ge=n("span"),ht=i("BertweetTokenizer"),je=a(),p=n("div"),u(X.$$.fragment),ft=a(),ke=n("p"),ut=i("Constructs a BERTweet tokenizer, using Byte-Pair-Encoding."),gt=a(),S=n("p"),kt=i("This tokenizer inherits from "),oe=n("a"),_t=i("PreTrainedTokenizer"),wt=i(` which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`),bt=a(),I=n("div"),u(W.$$.fragment),vt=a(),_e=n("p"),Tt=i("Loads a pre-existing dictionary from a text file and adds its symbols to this instance."),yt=a(),T=n("div"),u(Q.$$.fragment),Et=a(),we=n("p"),Bt=i(`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A BERTweet sequence has the following format:`),$t=a(),H=n("ul"),ae=n("li"),zt=i("single sequence: "),be=n("code"),Jt=i("<s> X </s>"),Mt=a(),ie=n("li"),qt=i("pair of sequences: "),ve=n("code"),It=i("<s> A </s></s> B </s>"),Rt=a(),R=n("div"),u(G.$$.fragment),Ut=a(),Te=n("p"),xt=i("Converts a sequence of tokens (string) in a single string."),Vt=a(),U=n("div"),u(Y.$$.fragment),jt=a(),ye=n("p"),Zt=i(`Create a mask from the two sequences passed to be used in a sequence-pair classification task. BERTweet does
not make use of token type ids, therefore a list of zeros is returned.`),Lt=a(),x=n("div"),u(O.$$.fragment),Ct=a(),K=n("p"),Ft=i(`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `),Ee=n("code"),At=i("prepare_for_model"),Nt=i(" method."),Dt=a(),V=n("div"),u(ee.$$.fragment),Pt=a(),Be=n("p"),Xt=i("Normalize tokens in a Tweet"),St=a(),j=n("div"),u(te.$$.fragment),Wt=a(),$e=n("p"),Qt=i("Normalize a raw Tweet"),this.h()},l(t){const m=Ms('[data-svelte="svelte-1phssyn"]',document.head);y=r(m,"META",{name:!0,content:!0}),m.forEach(s),ze=l(t),E=r(t,"H1",{class:!0});var Le=o(E);z=r(Le,"A",{id:!0,class:!0,href:!0});var Kt=o(z);ce=r(Kt,"SPAN",{});var es=o(ce);g(L.$$.fragment,es),es.forEach(s),Kt.forEach(s),Ye=l(Le),pe=r(Le,"SPAN",{});var ts=o(pe);Oe=d(ts,"BERTweet"),ts.forEach(s),Le.forEach(s),Je=l(t),B=r(t,"H2",{class:!0});var Ce=o(B);J=r(Ce,"A",{id:!0,class:!0,href:!0});var ss=o(J);me=r(ss,"SPAN",{});var ns=o(me);g(C.$$.fragment,ns),ns.forEach(s),ss.forEach(s),Ke=l(Ce),he=r(Ce,"SPAN",{});var rs=o(he);et=d(rs,"Overview"),rs.forEach(s),Ce.forEach(s),Me=l(t),M=r(t,"P",{});var Fe=o(M);tt=d(Fe,"The BERTweet model was proposed in "),F=r(Fe,"A",{href:!0,rel:!0});var os=o(F);st=d(os,"BERTweet: A pre-trained language model for English Tweets"),os.forEach(s),nt=d(Fe," by Dat Quoc Nguyen, Thanh Vu, Anh Tuan Nguyen."),Fe.forEach(s),qe=l(t),se=r(t,"P",{});var as=o(se);rt=d(as,"The abstract from the paper is the following:"),as.forEach(s),Ie=l(t),ne=r(t,"P",{});var is=o(ne);fe=r(is,"EM",{});var ls=o(fe);ot=d(ls,`We present BERTweet, the first public large-scale pre-trained language model for English Tweets. Our BERTweet, having
the same architecture as BERT-base (Devlin et al., 2019), is trained using the RoBERTa pre-training procedure (Liu et
al., 2019). Experiments show that BERTweet outperforms strong baselines RoBERTa-base and XLM-R-base (Conneau et al.,
2020), producing better performance results than the previous state-of-the-art models on three Tweet NLP tasks:
Part-of-speech tagging, Named-entity recognition and text classification.`),ls.forEach(s),is.forEach(s),Re=l(t),re=r(t,"P",{});var ds=o(re);at=d(ds,"Example of use:"),ds.forEach(s),Ue=l(t),g(A.$$.fragment,t),xe=l(t),v=r(t,"P",{});var le=o(v);it=d(le,"This model was contributed by "),N=r(le,"A",{href:!0,rel:!0});var cs=o(N);lt=d(cs,"dqnguyen"),cs.forEach(s),dt=d(le,". The original code can be found "),D=r(le,"A",{href:!0,rel:!0});var ps=o(D);ct=d(ps,"here"),ps.forEach(s),pt=d(le,"."),le.forEach(s),Ve=l(t),$=r(t,"H2",{class:!0});var Ae=o($);q=r(Ae,"A",{id:!0,class:!0,href:!0});var ms=o(q);ue=r(ms,"SPAN",{});var hs=o(ue);g(P.$$.fragment,hs),hs.forEach(s),ms.forEach(s),mt=l(Ae),ge=r(Ae,"SPAN",{});var fs=o(ge);ht=d(fs,"BertweetTokenizer"),fs.forEach(s),Ae.forEach(s),je=l(t),p=r(t,"DIV",{class:!0});var h=o(p);g(X.$$.fragment,h),ft=l(h),ke=r(h,"P",{});var us=o(ke);ut=d(us,"Constructs a BERTweet tokenizer, using Byte-Pair-Encoding."),us.forEach(s),gt=l(h),S=r(h,"P",{});var Ne=o(S);kt=d(Ne,"This tokenizer inherits from "),oe=r(Ne,"A",{href:!0});var gs=o(oe);_t=d(gs,"PreTrainedTokenizer"),gs.forEach(s),wt=d(Ne,` which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`),Ne.forEach(s),bt=l(h),I=r(h,"DIV",{class:!0});var De=o(I);g(W.$$.fragment,De),vt=l(De),_e=r(De,"P",{});var ks=o(_e);Tt=d(ks,"Loads a pre-existing dictionary from a text file and adds its symbols to this instance."),ks.forEach(s),De.forEach(s),yt=l(h),T=r(h,"DIV",{class:!0});var de=o(T);g(Q.$$.fragment,de),Et=l(de),we=r(de,"P",{});var _s=o(we);Bt=d(_s,`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A BERTweet sequence has the following format:`),_s.forEach(s),$t=l(de),H=r(de,"UL",{});var Pe=o(H);ae=r(Pe,"LI",{});var Ht=o(ae);zt=d(Ht,"single sequence: "),be=r(Ht,"CODE",{});var ws=o(be);Jt=d(ws,"<s> X </s>"),ws.forEach(s),Ht.forEach(s),Mt=l(Pe),ie=r(Pe,"LI",{});var Gt=o(ie);qt=d(Gt,"pair of sequences: "),ve=r(Gt,"CODE",{});var bs=o(ve);It=d(bs,"<s> A </s></s> B </s>"),bs.forEach(s),Gt.forEach(s),Pe.forEach(s),de.forEach(s),Rt=l(h),R=r(h,"DIV",{class:!0});var Xe=o(R);g(G.$$.fragment,Xe),Ut=l(Xe),Te=r(Xe,"P",{});var vs=o(Te);xt=d(vs,"Converts a sequence of tokens (string) in a single string."),vs.forEach(s),Xe.forEach(s),Vt=l(h),U=r(h,"DIV",{class:!0});var Se=o(U);g(Y.$$.fragment,Se),jt=l(Se),ye=r(Se,"P",{});var Ts=o(ye);Zt=d(Ts,`Create a mask from the two sequences passed to be used in a sequence-pair classification task. BERTweet does
not make use of token type ids, therefore a list of zeros is returned.`),Ts.forEach(s),Se.forEach(s),Lt=l(h),x=r(h,"DIV",{class:!0});var We=o(x);g(O.$$.fragment,We),Ct=l(We),K=r(We,"P",{});var Qe=o(K);Ft=d(Qe,`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer `),Ee=r(Qe,"CODE",{});var ys=o(Ee);At=d(ys,"prepare_for_model"),ys.forEach(s),Nt=d(Qe," method."),Qe.forEach(s),We.forEach(s),Dt=l(h),V=r(h,"DIV",{class:!0});var He=o(V);g(ee.$$.fragment,He),Pt=l(He),Be=r(He,"P",{});var Es=o(Be);Xt=d(Es,"Normalize tokens in a Tweet"),Es.forEach(s),He.forEach(s),St=l(h),j=r(h,"DIV",{class:!0});var Ge=o(j);g(te.$$.fragment,Ge),Wt=l(Ge),$e=r(Ge,"P",{});var Bs=o($e);Qt=d(Bs,"Normalize a raw Tweet"),Bs.forEach(s),Ge.forEach(s),h.forEach(s),this.h()},h(){c(y,"name","hf:doc:metadata"),c(y,"content",JSON.stringify(xs)),c(z,"id","bertweet"),c(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(z,"href","#bertweet"),c(E,"class","relative group"),c(J,"id","overview"),c(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(J,"href","#overview"),c(B,"class","relative group"),c(F,"href","https://www.aclweb.org/anthology/2020.emnlp-demos.2.pdf"),c(F,"rel","nofollow"),c(N,"href","https://huggingface.co/dqnguyen"),c(N,"rel","nofollow"),c(D,"href","https://github.com/VinAIResearch/BERTweet"),c(D,"rel","nofollow"),c(q,"id","transformers.BertweetTokenizer"),c(q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(q,"href","#transformers.BertweetTokenizer"),c($,"class","relative group"),c(oe,"href","/docs/transformers/v4.30.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizer"),c(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(p,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,m){e(document.head,y),f(t,ze,m),f(t,E,m),e(E,z),e(z,ce),k(L,ce,null),e(E,Ye),e(E,pe),e(pe,Oe),f(t,Je,m),f(t,B,m),e(B,J),e(J,me),k(C,me,null),e(B,Ke),e(B,he),e(he,et),f(t,Me,m),f(t,M,m),e(M,tt),e(M,F),e(F,st),e(M,nt),f(t,qe,m),f(t,se,m),e(se,rt),f(t,Ie,m),f(t,ne,m),e(ne,fe),e(fe,ot),f(t,Re,m),f(t,re,m),e(re,at),f(t,Ue,m),k(A,t,m),f(t,xe,m),f(t,v,m),e(v,it),e(v,N),e(N,lt),e(v,dt),e(v,D),e(D,ct),e(v,pt),f(t,Ve,m),f(t,$,m),e($,q),e(q,ue),k(P,ue,null),e($,mt),e($,ge),e(ge,ht),f(t,je,m),f(t,p,m),k(X,p,null),e(p,ft),e(p,ke),e(ke,ut),e(p,gt),e(p,S),e(S,kt),e(S,oe),e(oe,_t),e(S,wt),e(p,bt),e(p,I),k(W,I,null),e(I,vt),e(I,_e),e(_e,Tt),e(p,yt),e(p,T),k(Q,T,null),e(T,Et),e(T,we),e(we,Bt),e(T,$t),e(T,H),e(H,ae),e(ae,zt),e(ae,be),e(be,Jt),e(H,Mt),e(H,ie),e(ie,qt),e(ie,ve),e(ve,It),e(p,Rt),e(p,R),k(G,R,null),e(R,Ut),e(R,Te),e(Te,xt),e(p,Vt),e(p,U),k(Y,U,null),e(U,jt),e(U,ye),e(ye,Zt),e(p,Lt),e(p,x),k(O,x,null),e(x,Ct),e(x,K),e(K,Ft),e(K,Ee),e(Ee,At),e(K,Nt),e(p,Dt),e(p,V),k(ee,V,null),e(V,Pt),e(V,Be),e(Be,Xt),e(p,St),e(p,j),k(te,j,null),e(j,Wt),e(j,$e),e($e,Qt),Ze=!0},p:qs,i(t){Ze||(_(L.$$.fragment,t),_(C.$$.fragment,t),_(A.$$.fragment,t),_(P.$$.fragment,t),_(X.$$.fragment,t),_(W.$$.fragment,t),_(Q.$$.fragment,t),_(G.$$.fragment,t),_(Y.$$.fragment,t),_(O.$$.fragment,t),_(ee.$$.fragment,t),_(te.$$.fragment,t),Ze=!0)},o(t){w(L.$$.fragment,t),w(C.$$.fragment,t),w(A.$$.fragment,t),w(P.$$.fragment,t),w(X.$$.fragment,t),w(W.$$.fragment,t),w(Q.$$.fragment,t),w(G.$$.fragment,t),w(Y.$$.fragment,t),w(O.$$.fragment,t),w(ee.$$.fragment,t),w(te.$$.fragment,t),Ze=!1},d(t){s(y),t&&s(ze),t&&s(E),b(L),t&&s(Je),t&&s(B),b(C),t&&s(Me),t&&s(M),t&&s(qe),t&&s(se),t&&s(Ie),t&&s(ne),t&&s(Re),t&&s(re),t&&s(Ue),b(A,t),t&&s(xe),t&&s(v),t&&s(Ve),t&&s($),b(P),t&&s(je),t&&s(p),b(X),b(W),b(Q),b(G),b(Y),b(O),b(ee),b(te)}}}const xs={local:"bertweet",sections:[{local:"overview",title:"Overview"},{local:"transformers.BertweetTokenizer",title:"BertweetTokenizer"}],title:"BERTweet"};function Vs(Ot){return Is(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fs extends $s{constructor(y){super();zs(this,y,Vs,Us,Js,{})}}export{Fs as default,xs as metadata};
