import{s as Oe,n as Ke,o as et}from"../chunks/scheduler.01eeda35.js";import{S as tt,i as nt,g as i,s,r as m,A as st,h as l,f as n,c as o,j as Q,u as h,x as d,k as w,y as r,a,v as g,d as u,t as f,w as b}from"../chunks/index.6dd51b66.js";import{D as te}from"../chunks/Docstring.76e6b3cf.js";import{C as Ye}from"../chunks/CodeBlock.864da1b0.js";import{H as ne,E as ot}from"../chunks/EditOnGithub.7faefd25.js";function at($e){let _,se,K,oe,J,ae,T,Ze=`<img alt="PyTorch" src="https://img.shields.io/badge/PyTorch-DE3412?style=flat&amp;logo=pytorch&amp;logoColor=white"/> <img alt="TensorFlow" src="https://img.shields.io/badge/TensorFlow-FF6F00?style=flat&amp;logo=tensorflow&amp;logoColor=white"/> <img alt="Flax" src="https://img.shields.io/badge/Flax-29a79b.svg?style=flat&amp;logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC0AAAAtCAMAAAANxBKoAAAC7lBMVEUAAADg5vYHPVgAoJH+/v76+v39/f9JbLP///9+AIgAnY3///+mcqzt8fXy9fgkXa3Ax9709fr+///9/f8qXq49qp5AaLGMwrv8/P0eW60VWawxYq8yqJzG2dytt9Wyu9elzci519Lf3O3S2efY3OrY0+Xp7PT///////+dqNCexMc6Z7AGpJeGvbenstPZ5ejQ1OfJzOLa7ejh4+/r8fT29vpccbklWK8PVa0AS6ghW63O498vYa+lsdKz1NDRt9Kw1c672tbD3tnAxt7R6OHp5vDe7OrDyuDn6vLl6/EAQKak0MgATakkppo3ZK/Bz9y8w9yzu9jey97axdvHzeG21NHH4trTwthKZrVGZLSUSpuPQJiGAI+GAI8SWKydycLL4d7f2OTi1+S9xNzL0ePT6OLGzeEAo5U0qJw/aLEAo5JFa7JBabEAp5Y4qZ2QxLyKmsm3kL2xoMOehrRNb7RIbbOZgrGre68AUqwAqZqNN5aKJ5N/lMq+qsd8kMa4pcWzh7muhLMEV69juq2kbKqgUaOTR5uMMZWLLZSGAI5VAIdEAH+ovNDHuNCnxcy3qcaYx8K8msGplrx+wLahjbYdXrV6vbMvYK9DrZ8QrZ8tqJuFms+Sos6sw8ecy8RffsNVeMCvmb43aLltv7Q4Y7EZWK4QWa1gt6meZKUdr6GOAZVeA4xPAISyveLUwtivxtKTpNJ2jcqfvcltiMiwwcfAoMVxhL+Kx7xjdrqTe60tsaNQs6KaRKACrJ6UTZwkqpqTL5pkHY4AloSgsd2ptNXPvNOOncuxxsqFl8lmg8apt8FJcr9EbryGxLqlkrkrY7dRa7ZGZLQ5t6iXUZ6PPpgVpZeJCJFKAIGareTa0+KJod3H0deY2M+esM25usmYu8d2zsJOdcBVvrCLbqcAOaaHaKQAMaScWqKBXqCXMJ2RHpiLF5NmJZAdAHN2kta11dKu1M+DkcZLdb+Mcql3TppyRJdzQ5ZtNZNlIY+DF4+voCOQAAAAZ3RSTlMABAT+MEEJ/RH+/TP+Zlv+pUo6Ifz8+fco/fz6+evr39S9nJmOilQaF/7+/f38+smmoYp6b1T+/v7++vj189zU0tDJxsGzsrKSfv34+Pf27dDOysG9t6+n/vv6+vr59uzr1tG+tZ6Qg9Ym3QAABR5JREFUSMeNlVVUG1EQhpcuxEspXqS0SKEtxQp1d3d332STTRpIQhIISQgJhODu7lAoDoUCpe7u7u7+1puGpqnCPOyZvffbOXPm/PsP9JfQgyCC+tmTABTOcbxDz/heENS7/1F+9nhvkHePG0wNDLbGWwdXL+rbLWvpmZHXD8+gMfBjTh+aSe6Gnn7lwQIOTR0c8wfX3PWgv7avbdKwf/ZoBp1Gp/PvuvXW3vw5ib7emnTW4OR+3D4jB9vjNJ/7gNvfWWeH/TO/JyYrsiKCRjVEZA3UB+96kON+DxOQ/NLE8PE5iUYgIXjFnCOlxEQMaSGVxjg4gxOnEycGz8bptuNjVx08LscIgrzH3umcn+KKtiBIyvzOO2O99aAdR8cF19oZalnCtvREUw79tCd5sow1g1UKM6kXqUx4T8wsi3sTjJ3yzDmmhenLXLpo8u45eG5y4Vvbk6kkC4LLtJMowkSQxmk4ggVJEG+7c6QpHT8vvW9X7/o7+3ELmiJi2mEzZJiz8cT6TBlanBk70cB5GGIGC1gRDdZ00yADLW1FL6gqhtvNXNG5S9gdSrk4M1qu7JAsmYshzDS4peoMrU/gT7qQdqYGZaYhxZmVbGJAm/CS/HloWyhRUlknQ9KYcExTwS80d3VNOxUZJpITYyspl0LbhArhpZCD9cRWEQuhYkNGMHToQ/2Cs6swJlb39CsllxdXX6IUKh/H5jbnSsPKjgmoaFQ1f8wRLR0UnGE/RcDEjj2jXG1WVTwUs8+zxfcrVO+vSsuOpVKxCfYZiQ0/aPKuxQbQ8lIz+DClxC8u+snlcJ7Yr1z1JPqUH0V+GDXbOwAib931Y4Imaq0NTIXPXY+N5L18GJ37SVWu+hwXff8l72Ds9XuwYIBaXPq6Shm4l+Vl/5QiOlV+uTk6YR9PxKsI9xNJny31ygK1e+nIRC1N97EGkFPI+jCpiHe5PCEy7oWqWSwRrpOvhFzcbTWMbm3ZJAOn1rUKpYIt/lDhW/5RHHteeWFN60qo98YJuoq1nK3uW5AabyspC1BcIEpOhft+SZAShYoLSvnmSfnYADUERP5jJn2h5XtsgCRuhYQqAvwTwn33+YWEKUI72HX5AtfSAZDe8F2DtPPm77afhl0EkthzuCQU0BWApgQIH9+KB0JhopMM7bJrdTRoleM2JAVNMyPF+wdoaz+XJpGoVAQ7WXUkcV7gT3oUZyi/ISIJAVKhgNp+4b4veCFhYVJw4locdSjZCp9cPUhLF9EZ3KKzURepMEtCDPP3VcWFx4UIiZIklIpFNfHpdEafIF2aRmOcrUmjohbT2WUllbmRvgfbythbQO3222fpDJoufaQPncYYuqoGtUEsCJZL6/3PR5b4syeSjZMQG/T2maGANlXT2v8S4AULWaUkCxfLyW8iW4kdka+nEMjxpL2NCwsYNBp+Q61PF43zyDg9Bm9+3NNySn78jMZUUkumqE4Gp7JmFOdP1vc8PpRrzj9+wPinCy8K1PiJ4aYbnTYpCCbDkBSbzhu2QJ1Gd82t8jI8TH51+OzvXoWbnXUOBkNW+0mWFwGcGOUVpU81/n3TOHb5oMt2FgYGjzau0Nif0Ss7Q3XB33hjjQHjHA5E5aOyIQc8CBrLdQSs3j92VG+3nNEjbkbdbBr9zm04ruvw37vh0QKOdeGIkckc80fX3KH/h7PT4BOjgCty8VZ5ux1MoO5Cf5naca2LAsEgehI+drX8o/0Nu+W0m6K/I9gGPd/dfx/EN/wN62AhsBWuAAAAAElFTkSuQmCC
"/>`,re,z,ie,B,Ie='The BARTpho model was proposed in <a href="https://arxiv.org/abs/2109.09701" rel="nofollow">BARTpho: Pre-trained Sequence-to-Sequence Models for Vietnamese</a> by Nguyen Luong Tran, Duong Minh Le and Dat Quoc Nguyen.',le,x,We="The abstract from the paper is the following:",pe,A,Ve=`<em>We present BARTpho with two versions — BARTpho_word and BARTpho_syllable — the first public large-scale monolingual
sequence-to-sequence models pre-trained for Vietnamese. Our BARTpho uses the “large” architecture and pre-training
scheme of the sequence-to-sequence denoising model BART, thus especially suitable for generative NLP tasks. Experiments
on a downstream task of Vietnamese text summarization show that in both automatic and human evaluations, our BARTpho
outperforms the strong baseline mBART and improves the state-of-the-art. We release BARTpho to facilitate future
research and applications of generative Vietnamese NLP tasks.</em>`,ce,U,Ee='This model was contributed by <a href="https://huggingface.co/dqnguyen" rel="nofollow">dqnguyen</a>. The original code can be found <a href="https://github.com/VinAIResearch/BARTpho" rel="nofollow">here</a>.',de,q,me,j,he,C,ge,R,He=`<li>Following mBART, BARTpho uses the “large” architecture of BART with an additional layer-normalization layer on top of
both the encoder and decoder. Thus, usage examples in the <a href="bart">documentation of BART</a>, when adapting to use
with BARTpho, should be adjusted by replacing the BART-specialized classes with the mBART-specialized counterparts.
For example:</li>`,ue,L,fe,$,Ge=`<li>This implementation is only for tokenization: “monolingual_vocab_file” consists of Vietnamese-specialized types
extracted from the pre-trained SentencePiece model “vocab_file” that is available from the multilingual XLM-RoBERTa.
Other languages, if employing this pre-trained multilingual SentencePiece model “vocab_file” for subword
segmentation, can reuse BartphoTokenizer with their own language-specialized “monolingual_vocab_file”.</li>`,be,Z,ke,p,I,Je,X,Qe='Adapted from <a href="/docs/transformers/main/en/model_doc/xlm-roberta#transformers.XLMRobertaTokenizer">XLMRobertaTokenizer</a>. Based on <a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a>.',ze,P,Xe=`This tokenizer inherits from <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`,Be,k,W,xe,S,Pe=`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. An BARTPho sequence has the following format:`,Ae,N,Se="<li>single sequence: <code>&lt;s&gt; X &lt;/s&gt;</code></li> <li>pair of sequences: <code>&lt;s&gt; A &lt;/s&gt;&lt;/s&gt; B &lt;/s&gt;</code></li>",Ue,y,V,qe,F,Ne="Converts a sequence of tokens (strings for sub-words) in a single string.",je,v,E,Ce,D,Fe=`Create a mask from the two sequences passed to be used in a sequence-pair classification task. BARTPho does not
make use of token type ids, therefore a list of zeros is returned.`,Re,M,H,Le,Y,De=`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> method.`,_e,G,Te,ee,ye;return J=new ne({props:{title:"BARTpho",local:"bartpho",headingTag:"h1"}}),z=new ne({props:{title:"Overview",local:"overview",headingTag:"h2"}}),q=new ne({props:{title:"Usage example",local:"usage-example",headingTag:"h2"}}),j=new Ye({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQWJhcnRwaG8lMjAlM0QlMjBBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMnZpbmFpJTJGYmFydHBoby1zeWxsYWJsZSUyMiklMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJ2aW5haSUyRmJhcnRwaG8tc3lsbGFibGUlMjIpJTBBJTBBbGluZSUyMCUzRCUyMCUyMkNoJUMzJUJBbmclMjB0JUMzJUI0aSUyMGwlQzMlQTAlMjBuaCVFMSVCQiVBRm5nJTIwbmdoaSVDMyVBQW4lMjBjJUUxJUJCJUE5dSUyMHZpJUMzJUFBbi4lMjIlMEElMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIobGluZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBd2l0aCUyMHRvcmNoLm5vX2dyYWQoKSUzQSUwQSUyMCUyMCUyMCUyMGZlYXR1cmVzJTIwJTNEJTIwYmFydHBobygqKmlucHV0X2lkcyklMjAlMjAlMjMlMjBNb2RlbHMlMjBvdXRwdXRzJTIwYXJlJTIwbm93JTIwdHVwbGVzJTBBJTBBJTIzJTIwV2l0aCUyMFRlbnNvckZsb3clMjAyLjAlMkIlM0ElMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVEZBdXRvTW9kZWwlMEElMEFiYXJ0cGhvJTIwJTNEJTIwVEZBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMnZpbmFpJTJGYmFydHBoby1zeWxsYWJsZSUyMiklMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIobGluZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIpJTBBZmVhdHVyZXMlMjAlM0QlMjBiYXJ0cGhvKCoqaW5wdXRfaWRzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>bartpho = AutoModel.from_pretrained(<span class="hljs-string">&quot;vinai/bartpho-syllable&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;vinai/bartpho-syllable&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>line = <span class="hljs-string">&quot;Chúng tôi là những nghiên cứu viên.&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(line, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    features = bartpho(**input_ids)  <span class="hljs-comment"># Models outputs are now tuples</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># With TensorFlow 2.0+:</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModel

<span class="hljs-meta">&gt;&gt;&gt; </span>bartpho = TFAutoModel.from_pretrained(<span class="hljs-string">&quot;vinai/bartpho-syllable&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(line, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>features = bartpho(**input_ids)`,wrap:!1}}),C=new ne({props:{title:"Usage tips",local:"usage-tips",headingTag:"h2"}}),L=new Ye({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyME1CYXJ0Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTBBJTBBYmFydHBobyUyMCUzRCUyME1CYXJ0Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uLmZyb21fcHJldHJhaW5lZCglMjJ2aW5haSUyRmJhcnRwaG8tc3lsbGFibGUlMjIpJTBBVFhUJTIwJTNEJTIwJTIyQ2glQzMlQkFuZyUyMHQlQzMlQjRpJTIwbCVDMyVBMCUyMCUzQ21hc2slM0UlMjBuZ2hpJUMzJUFBbiUyMGMlRTElQkIlQTl1JTIwdmklQzMlQUFuLiUyMiUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplciglNUJUWFQlNUQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RCUwQWxvZ2l0cyUyMCUzRCUyMGJhcnRwaG8oaW5wdXRfaWRzKS5sb2dpdHMlMEFtYXNrZWRfaW5kZXglMjAlM0QlMjAoaW5wdXRfaWRzJTVCMCU1RCUyMCUzRCUzRCUyMHRva2VuaXplci5tYXNrX3Rva2VuX2lkKS5ub256ZXJvKCkuaXRlbSgpJTBBcHJvYnMlMjAlM0QlMjBsb2dpdHMlNUIwJTJDJTIwbWFza2VkX2luZGV4JTVELnNvZnRtYXgoZGltJTNEMCklMEF2YWx1ZXMlMkMlMjBwcmVkaWN0aW9ucyUyMCUzRCUyMHByb2JzLnRvcGsoNSklMEFwcmludCh0b2tlbml6ZXIuZGVjb2RlKHByZWRpY3Rpb25zKS5zcGxpdCgpKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> MBartForConditionalGeneration

<span class="hljs-meta">&gt;&gt;&gt; </span>bartpho = MBartForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;vinai/bartpho-syllable&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>TXT = <span class="hljs-string">&quot;Chúng tôi là &lt;mask&gt; nghiên cứu viên.&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer([TXT], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>logits = bartpho(input_ids).logits
<span class="hljs-meta">&gt;&gt;&gt; </span>masked_index = (input_ids[<span class="hljs-number">0</span>] == tokenizer.mask_token_id).nonzero().item()
<span class="hljs-meta">&gt;&gt;&gt; </span>probs = logits[<span class="hljs-number">0</span>, masked_index].softmax(dim=<span class="hljs-number">0</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>values, predictions = probs.topk(<span class="hljs-number">5</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(predictions).split())`,wrap:!1}}),Z=new ne({props:{title:"BartphoTokenizer",local:"transformers.BartphoTokenizer",headingTag:"h2"}}),I=new te({props:{name:"class transformers.BartphoTokenizer",anchor:"transformers.BartphoTokenizer",parameters:[{name:"vocab_file",val:""},{name:"monolingual_vocab_file",val:""},{name:"bos_token",val:" = '<s>'"},{name:"eos_token",val:" = '</s>'"},{name:"sep_token",val:" = '</s>'"},{name:"cls_token",val:" = '<s>'"},{name:"unk_token",val:" = '<unk>'"},{name:"pad_token",val:" = '<pad>'"},{name:"mask_token",val:" = '<mask>'"},{name:"sp_model_kwargs",val:": typing.Optional[typing.Dict[str, typing.Any]] = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BartphoTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
Path to the vocabulary file. This vocabulary is the pre-trained SentencePiece model available from the
multilingual XLM-RoBERTa, also used in mBART, consisting of 250K types.`,name:"vocab_file"},{anchor:"transformers.BartphoTokenizer.monolingual_vocab_file",description:`<strong>monolingual_vocab_file</strong> (<code>str</code>) &#x2014;
Path to the monolingual vocabulary file. This monolingual vocabulary consists of Vietnamese-specialized
types extracted from the multilingual vocabulary vocab_file of 250K types.`,name:"monolingual_vocab_file"},{anchor:"transformers.BartphoTokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the beginning of
sequence. The token used is the <code>cls_token</code>.</p>

					</div>`,name:"bos_token"},{anchor:"transformers.BartphoTokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the end of sequence.
The token used is the <code>sep_token</code>.</p>

					</div>`,name:"eos_token"},{anchor:"transformers.BartphoTokenizer.sep_token",description:`<strong>sep_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
sequence classification or for a text and a question for question answering. It is also used as the last
token of a sequence built with special tokens.`,name:"sep_token"},{anchor:"transformers.BartphoTokenizer.cls_token",description:`<strong>cls_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The classifier token which is used when doing sequence classification (classification of the whole sequence
instead of per-token classification). It is the first token of the sequence when built with special tokens.`,name:"cls_token"},{anchor:"transformers.BartphoTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.BartphoTokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;pad&gt;&quot;</code>) &#x2014;
The token used for padding, for example when batching sequences of different lengths.`,name:"pad_token"},{anchor:"transformers.BartphoTokenizer.mask_token",description:`<strong>mask_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;mask&gt;&quot;</code>) &#x2014;
The token used for masking values. This is the token used when training this model with masked language
modeling. This is the token which the model will try to predict.`,name:"mask_token"},{anchor:"transformers.BartphoTokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Will be passed to the <code>SentencePieceProcessor.__init__()</code> method. The <a href="https://github.com/google/sentencepiece/tree/master/python" rel="nofollow">Python wrapper for
SentencePiece</a> can be used, among other things,
to set:</p>
<ul>
<li>
<p><code>enable_sampling</code>: Enable subword regularization.</p>
</li>
<li>
<p><code>nbest_size</code>: Sampling parameters for unigram. Invalid for BPE-Dropout.</p>
<ul>
<li><code>nbest_size = {0,1}</code>: No sampling is performed.</li>
<li><code>nbest_size &gt; 1</code>: samples from the nbest_size results.</li>
<li><code>nbest_size &lt; 0</code>: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
using forward-filtering-and-backward-sampling algorithm.</li>
</ul>
</li>
<li>
<p><code>alpha</code>: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
BPE-dropout.</p>
</li>
</ul>`,name:"sp_model_kwargs"},{anchor:"transformers.BartphoTokenizer.sp_model",description:`<strong>sp_model</strong> (<code>SentencePieceProcessor</code>) &#x2014;
The <em>SentencePiece</em> processor that is used for every conversion (string, tokens and IDs).`,name:"sp_model"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bartpho/tokenization_bartpho.py#L34"}}),W=new te({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.BartphoTokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.BartphoTokenizer.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.BartphoTokenizer.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bartpho/tokenization_bartpho.py#L177",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),V=new te({props:{name:"convert_tokens_to_string",anchor:"transformers.BartphoTokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bartpho/tokenization_bartpho.py#L279"}}),E=new te({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.BartphoTokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.BartphoTokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.BartphoTokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bartpho/tokenization_bartpho.py#L231",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of zeros.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),H=new te({props:{name:"get_special_tokens_mask",anchor:"transformers.BartphoTokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.BartphoTokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.BartphoTokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.BartphoTokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bartpho/tokenization_bartpho.py#L203",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),G=new ot({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/bartpho.md"}}),{c(){_=i("meta"),se=s(),K=i("p"),oe=s(),m(J.$$.fragment),ae=s(),T=i("div"),T.innerHTML=Ze,re=s(),m(z.$$.fragment),ie=s(),B=i("p"),B.innerHTML=Ie,le=s(),x=i("p"),x.textContent=We,pe=s(),A=i("p"),A.innerHTML=Ve,ce=s(),U=i("p"),U.innerHTML=Ee,de=s(),m(q.$$.fragment),me=s(),m(j.$$.fragment),he=s(),m(C.$$.fragment),ge=s(),R=i("ul"),R.innerHTML=He,ue=s(),m(L.$$.fragment),fe=s(),$=i("ul"),$.innerHTML=Ge,be=s(),m(Z.$$.fragment),ke=s(),p=i("div"),m(I.$$.fragment),Je=s(),X=i("p"),X.innerHTML=Qe,ze=s(),P=i("p"),P.innerHTML=Xe,Be=s(),k=i("div"),m(W.$$.fragment),xe=s(),S=i("p"),S.textContent=Pe,Ae=s(),N=i("ul"),N.innerHTML=Se,Ue=s(),y=i("div"),m(V.$$.fragment),qe=s(),F=i("p"),F.textContent=Ne,je=s(),v=i("div"),m(E.$$.fragment),Ce=s(),D=i("p"),D.textContent=Fe,Re=s(),M=i("div"),m(H.$$.fragment),Le=s(),Y=i("p"),Y.innerHTML=De,_e=s(),m(G.$$.fragment),Te=s(),ee=i("p"),this.h()},l(e){const t=st("svelte-u9bgzb",document.head);_=l(t,"META",{name:!0,content:!0}),t.forEach(n),se=o(e),K=l(e,"P",{}),Q(K).forEach(n),oe=o(e),h(J.$$.fragment,e),ae=o(e),T=l(e,"DIV",{class:!0,"data-svelte-h":!0}),d(T)!=="svelte-1px1zvi"&&(T.innerHTML=Ze),re=o(e),h(z.$$.fragment,e),ie=o(e),B=l(e,"P",{"data-svelte-h":!0}),d(B)!=="svelte-1nncmhi"&&(B.innerHTML=Ie),le=o(e),x=l(e,"P",{"data-svelte-h":!0}),d(x)!=="svelte-vfdo9a"&&(x.textContent=We),pe=o(e),A=l(e,"P",{"data-svelte-h":!0}),d(A)!=="svelte-1bdfxqp"&&(A.innerHTML=Ve),ce=o(e),U=l(e,"P",{"data-svelte-h":!0}),d(U)!=="svelte-puazo9"&&(U.innerHTML=Ee),de=o(e),h(q.$$.fragment,e),me=o(e),h(j.$$.fragment,e),he=o(e),h(C.$$.fragment,e),ge=o(e),R=l(e,"UL",{"data-svelte-h":!0}),d(R)!=="svelte-19b1na2"&&(R.innerHTML=He),ue=o(e),h(L.$$.fragment,e),fe=o(e),$=l(e,"UL",{"data-svelte-h":!0}),d($)!=="svelte-kcbp2n"&&($.innerHTML=Ge),be=o(e),h(Z.$$.fragment,e),ke=o(e),p=l(e,"DIV",{class:!0});var c=Q(p);h(I.$$.fragment,c),Je=o(c),X=l(c,"P",{"data-svelte-h":!0}),d(X)!=="svelte-19zf7wc"&&(X.innerHTML=Qe),ze=o(c),P=l(c,"P",{"data-svelte-h":!0}),d(P)!=="svelte-1c3t5ty"&&(P.innerHTML=Xe),Be=o(c),k=l(c,"DIV",{class:!0});var O=Q(k);h(W.$$.fragment,O),xe=o(O),S=l(O,"P",{"data-svelte-h":!0}),d(S)!=="svelte-hazsjn"&&(S.textContent=Pe),Ae=o(O),N=l(O,"UL",{"data-svelte-h":!0}),d(N)!=="svelte-rq8uot"&&(N.innerHTML=Se),O.forEach(n),Ue=o(c),y=l(c,"DIV",{class:!0});var ve=Q(y);h(V.$$.fragment,ve),qe=o(ve),F=l(ve,"P",{"data-svelte-h":!0}),d(F)!=="svelte-1ne8awa"&&(F.textContent=Ne),ve.forEach(n),je=o(c),v=l(c,"DIV",{class:!0});var Me=Q(v);h(E.$$.fragment,Me),Ce=o(Me),D=l(Me,"P",{"data-svelte-h":!0}),d(D)!=="svelte-1t18cnp"&&(D.textContent=Fe),Me.forEach(n),Re=o(c),M=l(c,"DIV",{class:!0});var we=Q(M);h(H.$$.fragment,we),Le=o(we),Y=l(we,"P",{"data-svelte-h":!0}),d(Y)!=="svelte-1f4f5kp"&&(Y.innerHTML=De),we.forEach(n),c.forEach(n),_e=o(e),h(G.$$.fragment,e),Te=o(e),ee=l(e,"P",{}),Q(ee).forEach(n),this.h()},h(){w(_,"name","hf:doc:metadata"),w(_,"content",rt),w(T,"class","flex flex-wrap space-x-1"),w(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(p,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){r(document.head,_),a(e,se,t),a(e,K,t),a(e,oe,t),g(J,e,t),a(e,ae,t),a(e,T,t),a(e,re,t),g(z,e,t),a(e,ie,t),a(e,B,t),a(e,le,t),a(e,x,t),a(e,pe,t),a(e,A,t),a(e,ce,t),a(e,U,t),a(e,de,t),g(q,e,t),a(e,me,t),g(j,e,t),a(e,he,t),g(C,e,t),a(e,ge,t),a(e,R,t),a(e,ue,t),g(L,e,t),a(e,fe,t),a(e,$,t),a(e,be,t),g(Z,e,t),a(e,ke,t),a(e,p,t),g(I,p,null),r(p,Je),r(p,X),r(p,ze),r(p,P),r(p,Be),r(p,k),g(W,k,null),r(k,xe),r(k,S),r(k,Ae),r(k,N),r(p,Ue),r(p,y),g(V,y,null),r(y,qe),r(y,F),r(p,je),r(p,v),g(E,v,null),r(v,Ce),r(v,D),r(p,Re),r(p,M),g(H,M,null),r(M,Le),r(M,Y),a(e,_e,t),g(G,e,t),a(e,Te,t),a(e,ee,t),ye=!0},p:Ke,i(e){ye||(u(J.$$.fragment,e),u(z.$$.fragment,e),u(q.$$.fragment,e),u(j.$$.fragment,e),u(C.$$.fragment,e),u(L.$$.fragment,e),u(Z.$$.fragment,e),u(I.$$.fragment,e),u(W.$$.fragment,e),u(V.$$.fragment,e),u(E.$$.fragment,e),u(H.$$.fragment,e),u(G.$$.fragment,e),ye=!0)},o(e){f(J.$$.fragment,e),f(z.$$.fragment,e),f(q.$$.fragment,e),f(j.$$.fragment,e),f(C.$$.fragment,e),f(L.$$.fragment,e),f(Z.$$.fragment,e),f(I.$$.fragment,e),f(W.$$.fragment,e),f(V.$$.fragment,e),f(E.$$.fragment,e),f(H.$$.fragment,e),f(G.$$.fragment,e),ye=!1},d(e){e&&(n(se),n(K),n(oe),n(ae),n(T),n(re),n(ie),n(B),n(le),n(x),n(pe),n(A),n(ce),n(U),n(de),n(me),n(he),n(ge),n(R),n(ue),n(fe),n($),n(be),n(ke),n(p),n(_e),n(Te),n(ee)),n(_),b(J,e),b(z,e),b(q,e),b(j,e),b(C,e),b(L,e),b(Z,e),b(I),b(W),b(V),b(E),b(H),b(G,e)}}}const rt='{"title":"BARTpho","local":"bartpho","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage example","local":"usage-example","sections":[],"depth":2},{"title":"Usage tips","local":"usage-tips","sections":[],"depth":2},{"title":"BartphoTokenizer","local":"transformers.BartphoTokenizer","sections":[],"depth":2}],"depth":1}';function it($e){return et(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ht extends tt{constructor(_){super(),nt(this,_,it,at,Oe,{})}}export{ht as component};
