import{s as wn,o as zn,n as bt}from"../chunks/scheduler.9bc65507.js";import{S as jn,i as qn,g as l,s as o,r as f,A as Fn,h as i,f as t,c as s,j as z,u,x as c,k as j,y as a,a as r,v as _,d as h,t as g,w as k}from"../chunks/index.707bf1b6.js";import{T as Mn}from"../chunks/Tip.c2ecdbf4.js";import{D as U}from"../chunks/Docstring.17db21ae.js";import{C as Ne}from"../chunks/CodeBlock.54a9f38d.js";import{E as Cn}from"../chunks/ExampleCodeBlock.4f515aa9.js";import{H as Ve}from"../chunks/Heading.342b1fa6.js";function Un(I){let d,L="Code Llama のベースとなる<code>Llama2</code>ファミリー モデルは、<code>bfloat16</code>を使用してトレーニングされましたが、元の推論では<code>float16</code>を使用します。さまざまな精度を見てみましょう。",m,b,C="<li><code>float32</code>: モデルの初期化に関する PyTorch の規約では、モデルの重みがどの <code>dtype</code> で格納されたかに関係なく、モデルを <code>float32</code> にロードします。 「transformers」も、PyTorch との一貫性を保つためにこの規則に従っています。これはデフォルトで選択されます。 <code>AutoModel</code> API でストレージの重み付けタイプを使用してチェックポイントのロードをキャストする場合は、<code>torch_dtype=&quot;auto&quot;</code> を指定する必要があります。 <code>model = AutoModelForCausalLM.from_pretrained(&quot;path&quot;, torch_dtype = &quot;auto&quot;)</code>。</li> <li><code>bfloat16</code>: コード Llama はこの精度でトレーニングされているため、さらなるトレーニングや微調整に使用することをお勧めします。</li> <li><code>float16</code>: この精度を使用して推論を実行することをお勧めします。通常は <code>bfloat16</code> より高速であり、評価メトリクスには <code>bfloat16</code> と比べて明らかな低下が見られないためです。 bfloat16 を使用して推論を実行することもできます。微調整後、float16 と bfloat16 の両方で推論結果を確認することをお勧めします。</li>",v,x,N="上で述べたように、モデルを初期化するときに <code>torch_dtype=&quot;auto&quot;</code> を使用しない限り、ストレージの重みの <code>dtype</code> はほとんど無関係です。その理由は、モデルが最初にダウンロードされ (オンラインのチェックポイントの <code>dtype</code> を使用)、次に <code>torch</code> のデフォルトの <code>dtype</code> にキャストされるためです (<code>torch.float32</code> になります)。指定された <code>torch_dtype</code> がある場合は、代わりにそれが使用されます。";return{c(){d=l("p"),d.innerHTML=L,m=o(),b=l("ul"),b.innerHTML=C,v=o(),x=l("p"),x.innerHTML=N},l(T){d=i(T,"P",{"data-svelte-h":!0}),c(d)!=="svelte-roqng7"&&(d.innerHTML=L),m=s(T),b=i(T,"UL",{"data-svelte-h":!0}),c(b)!=="svelte-71l69f"&&(b.innerHTML=C),v=s(T),x=i(T,"P",{"data-svelte-h":!0}),c(x)!=="svelte-1g8mroy"&&(x.innerHTML=N)},m(T,P){r(T,d,P),r(T,m,P),r(T,b,P),r(T,v,P),r(T,x,P)},p:bt,d(T){T&&(t(d),t(m),t(b),t(v),t(x))}}}function In(I){let d,L=`コード Llama は、<code>Llama2</code> モデルと同じアーキテクチャを持っています。API リファレンスについては、<a href="llama2">Llama2 のドキュメント ページ</a> を参照してください。
以下の Code Llama トークナイザーのリファレンスを見つけてください。`;return{c(){d=l("p"),d.innerHTML=L},l(m){d=i(m,"P",{"data-svelte-h":!0}),c(d)!=="svelte-cvveze"&&(d.innerHTML=L)},m(m,b){r(m,d,b)},p:bt,d(m){m&&t(d)}}}function Jn(I){let d,L="sequence pair mask has the following format:",m,b,C;return b=new Ne({props:{code:"MCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMEElN0MlMjBmaXJzdCUyMHNlcXVlbmNlJTIwJTIwJTIwJTIwJTdDJTIwc2Vjb25kJTIwc2VxdWVuY2UlMjAlN0M=",highlighted:`0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1 1
| first sequence    | second sequence |`,wrap:!1}}),{c(){d=l("p"),d.textContent=L,m=o(),f(b.$$.fragment)},l(v){d=i(v,"P",{"data-svelte-h":!0}),c(d)!=="svelte-16klr56"&&(d.textContent=L),m=s(v),u(b.$$.fragment,v)},m(v,x){r(v,d,x),r(v,m,x),_(b,v,x),C=!0},p:bt,i(v){C||(h(b.$$.fragment,v),C=!0)},o(v){g(b.$$.fragment,v),C=!1},d(v){v&&(t(d),t(m)),k(b,v)}}}function En(I){let d,L;return d=new Ne({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMENvZGVMbGFtYVRva2VuaXplckZhc3QlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBDb2RlTGxhbWFUb2tlbml6ZXJGYXN0LmZyb21fcHJldHJhaW5lZCglMjJoZi1pbnRlcm5hbC10ZXN0aW5nJTJGbGxhbWEtdG9rZW5pemVyJTIyKSUwQXRva2VuaXplci5lbmNvZGUoJTIySGVsbG8lMjB0aGlzJTIwaXMlMjBhJTIwdGVzdCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> CodeLlamaTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = CodeLlamaTokenizerFast.from_pretrained(<span class="hljs-string">&quot;hf-internal-testing/llama-tokenizer&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.encode(<span class="hljs-string">&quot;Hello this is a test&quot;</span>)
[<span class="hljs-number">1</span>, <span class="hljs-number">15043</span>, <span class="hljs-number">445</span>, <span class="hljs-number">338</span>, <span class="hljs-number">263</span>, <span class="hljs-number">1243</span>]`,wrap:!1}}),{c(){f(d.$$.fragment)},l(m){u(d.$$.fragment,m)},m(m,b){_(d,m,b),L=!0},p:bt,i(m){L||(h(d.$$.fragment,m),L=!0)},o(m){g(d.$$.fragment,m),L=!1},d(m){k(d,m)}}}function Pn(I){let d,L,m,b,C,v,x,N,T,P=`Code Llama モデルはによって <a href="https://ai.meta.com/research/publications/code-llama-open-foundation-models-for-code/" rel="nofollow">Code Llama: Open Foundation Models for Code</a> で提案されました。 Baptiste Rozière, Jonas Gehring, Fabian Gloeckle, Sten Sootla, Itai Gat, Xiaoqing Ellen Tan, Yossi Adi, Jingyu Liu, Tal Remez, Jérémy Rapin, Artyom Kozhevnikov, Ivan Evtimov, Joanna Bitton, Manish Bhatt, Cristian Canton Ferrer, Aaron Grattafiori, Wenhan Xiong, Alexandre Défossez, Jade Copet, Faisal Azhar, Hugo Touvron, Louis Martin, Nicolas Usunier, Thomas Scialom, Gabriel Synnaeve.
論文の要約は次のとおりです。`,We,W,Xt="<em>私たちは Code Llama をリリースします。これは Llama 2 に基づくコードの大規模言語モデル ファミリであり、オープン モデルの中で最先端のパフォーマンス、埋め込み機能、大規模な入力コンテキストのサポート、プログラミング タスクのゼロショット命令追従機能を提供します。 。幅広いアプリケーションをカバーするための複数のフレーバーを提供しています。基盤モデル (Code Llama)、Python 特化 (Code Llama - Python)、およびそれぞれ 7B、13B、および 34B パラメーターを備えた命令追従モデル (Code Llama - Instruct) です。すべてのモデルは 16,000 トークンのシーケンスでトレーニングされ、最大 100,000 トークンの入力で改善が見られます。 7B および 13B コード ラマとコード ラマ - 命令バリアントは、周囲のコンテンツに基づいた埋め込みをサポートします。 Code Llama は、いくつかのコード ベンチマークでオープン モデルの中で最先端のパフォーマンスに達し、HumanEval と MBPP でそれぞれ最大 53% と 55% のスコアを獲得しました。特に、Code Llama - Python 7B は HumanEval および MBPP 上で Llama 2 70B よりも優れたパフォーマンスを示し、すべてのモデルは MultiPL-E 上で公開されている他のすべてのモデルよりも優れています。私たちは、研究と商業利用の両方を許可する寛容なライセンスに基づいて Code Llama をリリースしています。</em>",Se,S,Qt='すべての Code Llama モデル チェックポイントを <a href="https://huggingface.co/models?search=code_llama" rel="nofollow">こちら</a> で確認し、<a href="https://huggingface.co/codellama" rel="nofollow">codellama org</a> で正式にリリースされたチェックポイントを確認してください。',Xe,X,Yt='このモデルは <a href="https://huggingface.co/ArthurZ" rel="nofollow">ArthurZucker</a> によって提供されました。著者のオリジナルのコードは <a href="https://github.com/facebookresearch/llama" rel="nofollow">こちら</a> にあります。',Qe,Q,Ye,R,Oe,Y,Ot="チップ：",Ke,O,Kt="<li>充填タスクはすぐにサポートされます。入力を埋めたい場所には <code>tokenizer.fill_token</code> を使用する必要があります。</li> <li>モデル変換スクリプトは、<code>Llama2</code> ファミリの場合と同じです。</li>",et,K,en="使用例は次のとおりです。",tt,ee,nt,te,tn=`スクリプトを実行するには、(最大のバージョンであっても) float16 精度でモデル全体をホストするのに十分な CPU RAM が必要であることに注意してください。
いくつかのチェックポイントがあり、それぞれにモデルの各重みの一部が含まれているため、すべてを RAM にロードする必要があります)。`,ot,ne,nn="変換後、モデルとトークナイザーは次の方法でロードできます。",st,oe,at,se,on="塗りつぶされた部分だけが必要な場合:",rt,ae,lt,re,sn='内部では、トークナイザーが <a href="https://huggingface.co/docs/transformers/main/model_doc/code_llama#transformers.CodeLlamaTokenizer.fill_token" rel="nofollow"><code>&lt;FILL_ME&gt;</code> によって自動的に分割</a> して、<a href="https://github.com/facebookresearch/codellama/blob/cb51c14ec761370ba2e2bc351374a79265d0465e/llama/generation.py#L402" rel="nofollow">に続く書式設定された入力文字列を作成します。オリジナルのトレーニング パターン</a>。これは、パターンを自分で準備するよりも堅牢です。トークンの接着など、デバッグが非常に難しい落とし穴を回避できます。このモデルまたは他のモデルに必要な CPU および GPU メモリの量を確認するには、その値を決定するのに役立つ <a href="https://huggingface.co/spaces/hf-accelerate/model-memory-usage" rel="nofollow">この計算ツール</a> を試してください。',it,le,an='LLaMA トークナイザーは、<a href="https://github.com/google/sentencepiece" rel="nofollow">sentencepiece</a> に基づく BPE モデルです。センテンスピースの癖の 1 つは、シーケンスをデコードするときに、最初のトークンが単語の先頭 (例: 「Banana」) である場合、トークナイザーは文字列の先頭にプレフィックス スペースを追加しないことです。',dt,H,ct,ie,mt,$,de,vt,Te,rn=`Construct a CodeLlama tokenizer. Based on byte-level Byte-Pair-Encoding. The default padding token is unset as
there is no padding token in the original model.`,yt,xe,ln=`The default configuration match that of
<a href="https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf/blob/main/tokenizer_config.json" rel="nofollow">codellama/CodeLlama-7b-Instruct-hf</a>
which supports prompt infilling.`,Tt,$e,ce,xt,Z,me,$t,Le,dn=`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> method.`,Lt,q,pe,Mt,Me,cn="Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT",Ct,B,wt,Ce,mn="if token_ids_1 is None, only returns the first portion of the mask (0s).",zt,A,fe,jt,we,pn="Save the vocabulary and special tokens file to a directory.",pt,ue,ft,p,_e,qt,ze,fn="Construct a Llama tokenizer. Based on byte-level Byte-Pair-Encoding.",Ft,je,un="This uses notably ByteFallback and no normalization.",Ut,D,It,qe,_n=`If you want to change the <code>bos_token</code> or the <code>eos_token</code>, make sure to specify them when initializing the model, or
call <code>tokenizer.update_post_processor()</code> to make sure that the post-processing is correctly done (otherwise the
values of the first token and final token of an encoded sequence will not be correct). For more details, checkout
[post-processors] (<a href="https://huggingface.co/docs/tokenizers/api/post-processors" rel="nofollow">https://huggingface.co/docs/tokenizers/api/post-processors</a>) documentation.`,Jt,Fe,hn=`This tokenizer inherits from <a href="/docs/transformers/v4.36.1/ja/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods. The default configuration match that of
<a href="https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf/blob/main/tokenizer_config.json" rel="nofollow">codellama/CodeLlama-7b-Instruct-hf</a>
which supports prompt infilling.`,Et,w,he,Pt,Ue,gn=`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. The special tokens depend on calling set_lang.`,Rt,Ie,kn="An NLLB sequence has the following format, where <code>X</code> represents the sequence:",Ht,Je,bn="<li><code>input_ids</code> (for encoder) <code>X [eos, src_lang_code]</code></li> <li><code>decoder_input_ids</code>: (for decoder) <code>X [eos, tgt_lang_code]</code></li>",Zt,Ee,vn=`BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
separator.`,Bt,G,ge,At,Pe,yn=`Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> or <code>encode_plus</code> methods.`,Dt,J,ke,Gt,Re,Tn=`Create the token type IDs corresponding to the sequences passed. <a href="../glossary#token-type-ids">What are token type
IDs?</a>`,Vt,He,xn="Should be overridden in a subclass if the model has a special way of building those.",Nt,V,be,Wt,Ze,$n="Updates the underlying post processor with the current <code>bos_token</code> and <code>eos_token</code>.",St,Be,ve,ut,De,_t;return C=new Ve({props:{title:"CodeLlama",local:"codellama",headingTag:"h1"}}),x=new Ve({props:{title:"Overview",local:"overview",headingTag:"h2"}}),Q=new Ve({props:{title:"Usage tips and examples",local:"usage-tips-and-examples",headingTag:"h2"}}),R=new Mn({props:{warning:!0,$$slots:{default:[Un]},$$scope:{ctx:I}}}),ee=new Ne({props:{code:"cHl0aG9uJTIwc3JjJTJGdHJhbnNmb3JtZXJzJTJGbW9kZWxzJTJGbGxhbWElMkZjb252ZXJ0X2xsYW1hX3dlaWdodHNfdG9faGYucHklMjAlNUMlMEElMjAlMjAlMjAlMjAtLWlucHV0X2RpciUyMCUyRnBhdGglMkZ0byUyRmRvd25sb2FkZWQlMkZsbGFtYSUyRndlaWdodHMlMjAtLW1vZGVsX3NpemUlMjA3QiUyMC0tb3V0cHV0X2RpciUyMCUyRm91dHB1dCUyRnBhdGg=",highlighted:`python src/transformers/models/llama/convert_llama_weights_to_hf.py \\
    --input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path`,wrap:!1}}),oe=new Ne({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMExsYW1hRm9yQ2F1c2FsTE0lMkMlMjBDb2RlTGxhbWFUb2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBDb2RlTGxhbWFUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmNvZGVsbGFtYSUyRkNvZGVMbGFtYS03Yi1oZiUyMiklMEFtb2RlbCUyMCUzRCUyMExsYW1hRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmNvZGVsbGFtYSUyRkNvZGVMbGFtYS03Yi1oZiUyMiklMEFQUk9NUFQlMjAlM0QlMjAnJydkZWYlMjByZW1vdmVfbm9uX2FzY2lpKHMlM0ElMjBzdHIpJTIwLSUzRSUyMHN0ciUzQSUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplcihQUk9NUFQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RCUwQWdlbmVyYXRlZF9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZShpbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEyOCklMEElMEFmaWxsaW5nJTIwJTNEJTIwdG9rZW5pemVyLmJhdGNoX2RlY29kZShnZW5lcmF0ZWRfaWRzJTVCJTNBJTJDJTIwaW5wdXRfaWRzLnNoYXBlJTVCMSU1RCUzQSU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlMjAlM0QlMjBUcnVlKSU1QjAlNUQlMEFwcmludChQUk9NUFQucmVwbGFjZSglMjIlM0NGSUxMX01FJTNFJTIyJTJDJTIwZmlsbGluZykpJTBBJTBB",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlamaForCausalLM, CodeLlamaTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = CodeLlamaTokenizer.from_pretrained(<span class="hljs-string">&quot;codellama/CodeLlama-7b-hf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = LlamaForCausalLM.from_pretrained(<span class="hljs-string">&quot;codellama/CodeLlama-7b-hf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>PROMPT = <span class="hljs-string">&#x27;&#x27;&#x27;def remove_non_ascii(s: str) -&gt; str:
    &quot;&quot;&quot; &lt;FILL_ME&gt;
    return result
&#x27;&#x27;&#x27;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(PROMPT, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(input_ids, max_new_tokens=<span class="hljs-number">128</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>filling = tokenizer.batch_decode(generated_ids[:, input_ids.shape[<span class="hljs-number">1</span>]:], skip_special_tokens = <span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(PROMPT.replace(<span class="hljs-string">&quot;&lt;FILL_ME&gt;&quot;</span>, filling))
<span class="hljs-keyword">def</span> <span class="hljs-title function_">remove_non_ascii</span>(<span class="hljs-params">s: <span class="hljs-built_in">str</span></span>) -&gt; <span class="hljs-built_in">str</span>:
    <span class="hljs-string">&quot;&quot;&quot; Remove non-ASCII characters from a string.

    Args:
        s: The string to remove non-ASCII characters from.

    Returns:
        The string with non-ASCII characters removed.
    &quot;&quot;&quot;</span>
    result = <span class="hljs-string">&quot;&quot;</span>
    <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> s:
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">ord</span>(c) &lt; <span class="hljs-number">128</span>:
            result += c
    <span class="hljs-keyword">return</span> result`,wrap:!1}}),ae=new Ne({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBaW1wb3J0JTIwdG9yY2glMEElMEFnZW5lcmF0b3IlMjAlM0QlMjBwaXBlbGluZSglMjJ0ZXh0LWdlbmVyYXRpb24lMjIlMkNtb2RlbCUzRCUyMmNvZGVsbGFtYSUyRkNvZGVMbGFtYS03Yi1oZiUyMiUyQ3RvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyKSUwQWdlbmVyYXRvcignZGVmJTIwcmVtb3ZlX25vbl9hc2NpaShzJTNBJTIwc3RyKSUyMC0lM0UlMjBzdHIlM0ElNUNuJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTIwJTNDRklMTF9NRSUzRSU1Q24lMjAlMjAlMjAlMjByZXR1cm4lMjByZXN1bHQnJTJDJTIwbWF4X25ld190b2tlbnMlMjAlM0QlMjAxMjglMkMlMjByZXR1cm5fdHlwZSUyMCUzRCUyMDEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>generator = pipeline(<span class="hljs-string">&quot;text-generation&quot;</span>,model=<span class="hljs-string">&quot;codellama/CodeLlama-7b-hf&quot;</span>,torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generator(<span class="hljs-string">&#x27;def remove_non_ascii(s: str) -&gt; str:\\n    &quot;&quot;&quot; &lt;FILL_ME&gt;\\n    return result&#x27;</span>, max_new_tokens = <span class="hljs-number">128</span>, return_type = <span class="hljs-number">1</span>)`,wrap:!1}}),H=new Mn({props:{$$slots:{default:[In]},$$scope:{ctx:I}}}),ie=new Ve({props:{title:"CodeLlamaTokenizer",local:"transformers.CodeLlamaTokenizer",headingTag:"h2"}}),de=new U({props:{name:"class transformers.CodeLlamaTokenizer",anchor:"transformers.CodeLlamaTokenizer",parameters:[{name:"vocab_file",val:""},{name:"unk_token",val:" = '<unk>'"},{name:"bos_token",val:" = '<s>'"},{name:"eos_token",val:" = '</s>'"},{name:"prefix_token",val:" = '▁<PRE>'"},{name:"middle_token",val:" = '▁<MID>'"},{name:"suffix_token",val:" = '▁<SUF>'"},{name:"eot_token",val:" = '▁<EOT>'"},{name:"fill_token",val:" = '<FILL_ME>'"},{name:"suffix_first",val:" = False"},{name:"sp_model_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"add_bos_token",val:" = True"},{name:"add_eos_token",val:" = False"},{name:"clean_up_tokenization_spaces",val:" = False"},{name:"additional_special_tokens",val:" = None"},{name:"use_default_system_prompt",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
Path to the vocabulary file.`,name:"vocab_file"},{anchor:"transformers.CodeLlamaTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.CodeLlamaTokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.`,name:"bos_token"},{anchor:"transformers.CodeLlamaTokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the end of sequence.
The token used is the <code>sep_token</code>.</p>

					</div>`,name:"eos_token"},{anchor:"transformers.CodeLlamaTokenizer.prefix_token",description:`<strong>prefix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;PRE&gt;&quot;</code>) &#x2014;
Prefix token used for infilling.`,name:"prefix_token"},{anchor:"transformers.CodeLlamaTokenizer.middle_token",description:`<strong>middle_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;MID&gt;&quot;</code>) &#x2014;
Middle token used for infilling.`,name:"middle_token"},{anchor:"transformers.CodeLlamaTokenizer.suffix_token",description:`<strong>suffix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;SUF&gt;&quot;</code>) &#x2014;
Suffix token used for infilling.`,name:"suffix_token"},{anchor:"transformers.CodeLlamaTokenizer.eot_token",description:`<strong>eot_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;EOT&gt;&quot;</code>) &#x2014;
End of text token used for infilling.`,name:"eot_token"},{anchor:"transformers.CodeLlamaTokenizer.fill_token",description:`<strong>fill_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;FILL_ME&gt;&quot;</code>) &#x2014;
The token used to split the input between the prefix and suffix.`,name:"fill_token"},{anchor:"transformers.CodeLlamaTokenizer.suffix_first",description:`<strong>suffix_first</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the input prompt and suffix should be formatted with the suffix first.`,name:"suffix_first"},{anchor:"transformers.CodeLlamaTokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Will be passed to the <code>SentencePieceProcessor.__init__()</code> method. The <a href="https://github.com/google/sentencepiece/tree/master/python" rel="nofollow">Python wrapper for
SentencePiece</a> can be used, among other things,
to set:</p>
<ul>
<li>
<p><code>enable_sampling</code>: Enable subword regularization.</p>
</li>
<li>
<p><code>nbest_size</code>: Sampling parameters for unigram. Invalid for BPE-Dropout.</p>
<ul>
<li><code>nbest_size = {0,1}</code>: No sampling is performed.</li>
<li><code>nbest_size &gt; 1</code>: samples from the nbest_size results.</li>
<li><code>nbest_size &lt; 0</code>: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
using forward-filtering-and-backward-sampling algorithm.</li>
</ul>
</li>
<li>
<p><code>alpha</code>: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
BPE-dropout.</p>
</li>
</ul>`,name:"sp_model_kwargs"},{anchor:"transformers.CodeLlamaTokenizer.add_bos_token",description:`<strong>add_bos_token</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to add a beginning of sequence token at the start of sequences.`,name:"add_bos_token"},{anchor:"transformers.CodeLlamaTokenizer.add_eos_token",description:`<strong>add_eos_token</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to add an end of sequence token at the end of sequences.`,name:"add_eos_token"},{anchor:"transformers.CodeLlamaTokenizer.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to clean up the tokenization spaces.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.CodeLlamaTokenizer.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
Additional special tokens used by the tokenizer.`,name:"additional_special_tokens"},{anchor:"transformers.CodeLlamaTokenizer.use_default_system_prompt",description:`<strong>use_default_system_prompt</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the default system prompt for Llama should be used.`,name:"use_default_system_prompt"}],source:"https://github.com/huggingface/transformers/blob/v4.36.1/src/transformers/models/code_llama/tokenization_code_llama.py#L59"}}),ce=new U({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.CodeLlamaTokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:""},{name:"token_ids_1",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/v4.36.1/src/transformers/models/code_llama/tokenization_code_llama.py#L369"}}),me=new U({props:{name:"get_special_tokens_mask",anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/v4.36.1/src/transformers/models/code_llama/tokenization_code_llama.py#L381",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),pe=new U({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of ids.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.36.1/src/transformers/models/code_llama/tokenization_code_llama.py#L419",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of <a href="../glossary#token-type-ids">token type IDs</a> according to the given sequence(s).</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),B=new Cn({props:{anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences.example",$$slots:{default:[Jn]},$$scope:{ctx:I}}}),fe=new U({props:{name:"save_vocabulary",anchor:"transformers.CodeLlamaTokenizer.save_vocabulary",parameters:[{name:"save_directory",val:""},{name:"filename_prefix",val:": typing.Optional[str] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.save_vocabulary.save_directory",description:`<strong>save_directory</strong> (<code>str</code>) &#x2014;
The directory in which to save the vocabulary.`,name:"save_directory"}],source:"https://github.com/huggingface/transformers/blob/v4.36.1/src/transformers/models/code_llama/tokenization_code_llama.py#L341",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Paths to the files saved.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Tuple(str)</code></p>
`}}),ue=new Ve({props:{title:"CodeLlamaTokenizerFast",local:"transformers.CodeLlamaTokenizerFast",headingTag:"h2"}}),_e=new U({props:{name:"class transformers.CodeLlamaTokenizerFast",anchor:"transformers.CodeLlamaTokenizerFast",parameters:[{name:"vocab_file",val:" = None"},{name:"tokenizer_file",val:" = None"},{name:"clean_up_tokenization_spaces",val:" = False"},{name:"unk_token",val:" = '<unk>'"},{name:"bos_token",val:" = '<s>'"},{name:"eos_token",val:" = '</s>'"},{name:"prefix_token",val:" = '▁<PRE>'"},{name:"middle_token",val:" = '▁<MID>'"},{name:"suffix_token",val:" = '▁<SUF>'"},{name:"eot_token",val:" = '▁<EOT>'"},{name:"fill_token",val:" = '<FILL_ME>'"},{name:"additional_special_tokens",val:" = None"},{name:"add_bos_token",val:" = True"},{name:"add_eos_token",val:" = False"},{name:"use_default_system_prompt",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>, <em>optional</em>) &#x2014;
<a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a .model extension) that
contains the vocabulary necessary to instantiate a tokenizer.`,name:"vocab_file"},{anchor:"transformers.CodeLlamaTokenizerFast.tokenizer_file",description:`<strong>tokenizer_file</strong> (<code>str</code>, <em>optional</em>) &#x2014;
<a href="https://github.com/huggingface/tokenizers" rel="nofollow">tokenizers</a> file (generally has a .json extension) that
contains everything needed to load the tokenizer.`,name:"tokenizer_file"},{anchor:"transformers.CodeLlamaTokenizerFast.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>str</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Wether to cleanup spaces after decoding, cleanup consists in removing potential artifacts like extra
spaces.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.CodeLlamaTokenizerFast.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.CodeLlamaTokenizerFast.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.`,name:"bos_token"},{anchor:"transformers.CodeLlamaTokenizerFast.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.`,name:"eos_token"},{anchor:"transformers.CodeLlamaTokenizerFast.prefix_token",description:`<strong>prefix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;PRE&gt;&quot;</code>) &#x2014;
Prefix token used for infilling.`,name:"prefix_token"},{anchor:"transformers.CodeLlamaTokenizerFast.middle_token",description:`<strong>middle_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;MID&gt;&quot;</code>) &#x2014;
Middle token used for infilling.`,name:"middle_token"},{anchor:"transformers.CodeLlamaTokenizerFast.suffix_token",description:`<strong>suffix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;SUF&gt;&quot;</code>) &#x2014;
Suffix token used for infilling.`,name:"suffix_token"},{anchor:"transformers.CodeLlamaTokenizerFast.eot_token",description:`<strong>eot_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;EOT&gt;&quot;</code>) &#x2014;
End of text token used for infilling.`,name:"eot_token"},{anchor:"transformers.CodeLlamaTokenizerFast.fill_token",description:`<strong>fill_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;FILL_ME&gt;&quot;</code>) &#x2014;
The token used to split the input between the prefix and suffix.`,name:"fill_token"},{anchor:"transformers.CodeLlamaTokenizerFast.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
Additional special tokens used by the tokenizer.`,name:"additional_special_tokens"},{anchor:"transformers.CodeLlamaTokenizerFast.add_bos_token",description:`<strong>add_bos_token</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to add a beginning of sequence token at the start of sequences.`,name:"add_bos_token"},{anchor:"transformers.CodeLlamaTokenizerFast.add_eos_token",description:`<strong>add_eos_token</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to add an end of sequence token at the end of sequences.`,name:"add_eos_token"},{anchor:"transformers.CodeLlamaTokenizerFast.use_default_system_prompt",description:`<strong>use_default_system_prompt</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the default system prompt for Llama should be used.`,name:"use_default_system_prompt"}],source:"https://github.com/huggingface/transformers/blob/v4.36.1/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L52"}}),D=new Cn({props:{anchor:"transformers.CodeLlamaTokenizerFast.example",$$slots:{default:[En]},$$scope:{ctx:I}}}),he=new U({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.CodeLlamaTokenizerFast.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizerFast.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.36.1/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L413",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),ge=new U({props:{name:"get_special_tokens_mask",anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of ids of the first sequence.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
List of ids of the second sequence.`,name:"token_ids_1"},{anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/v4.36.1/src/transformers/tokenization_utils_base.py#L3766",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>1 for a special token, 0 for a sequence token.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of integers in the range [0, 1]</p>
`}}),ke=new U({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.CodeLlamaTokenizerFast.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.create_token_type_ids_from_sequences.token_ids_0",description:"<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014; The first tokenized sequence.",name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizerFast.create_token_type_ids_from_sequences.token_ids_1",description:"<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014; The second tokenized sequence.",name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/v4.36.1/src/transformers/tokenization_utils_base.py#L3301",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The token type ids.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),be=new U({props:{name:"update_post_processor",anchor:"transformers.CodeLlamaTokenizerFast.update_post_processor",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.36.1/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L179"}}),ve=new U({props:{name:"save_vocabulary",anchor:"transformers.CodeLlamaTokenizerFast.save_vocabulary",parameters:[{name:"save_directory",val:": str"},{name:"filename_prefix",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/transformers/blob/v4.36.1/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L333"}}),{c(){d=l("meta"),L=o(),m=l("p"),b=o(),f(C.$$.fragment),v=o(),f(x.$$.fragment),N=o(),T=l("p"),T.innerHTML=P,We=o(),W=l("p"),W.innerHTML=Xt,Se=o(),S=l("p"),S.innerHTML=Qt,Xe=o(),X=l("p"),X.innerHTML=Yt,Qe=o(),f(Q.$$.fragment),Ye=o(),f(R.$$.fragment),Oe=o(),Y=l("p"),Y.textContent=Ot,Ke=o(),O=l("ul"),O.innerHTML=Kt,et=o(),K=l("p"),K.textContent=en,tt=o(),f(ee.$$.fragment),nt=o(),te=l("p"),te.textContent=tn,ot=o(),ne=l("p"),ne.textContent=nn,st=o(),f(oe.$$.fragment),at=o(),se=l("p"),se.textContent=on,rt=o(),f(ae.$$.fragment),lt=o(),re=l("p"),re.innerHTML=sn,it=o(),le=l("p"),le.innerHTML=an,dt=o(),f(H.$$.fragment),ct=o(),f(ie.$$.fragment),mt=o(),$=l("div"),f(de.$$.fragment),vt=o(),Te=l("p"),Te.textContent=rn,yt=o(),xe=l("p"),xe.innerHTML=ln,Tt=o(),$e=l("div"),f(ce.$$.fragment),xt=o(),Z=l("div"),f(me.$$.fragment),$t=o(),Le=l("p"),Le.innerHTML=dn,Lt=o(),q=l("div"),f(pe.$$.fragment),Mt=o(),Me=l("p"),Me.textContent=cn,Ct=o(),f(B.$$.fragment),wt=o(),Ce=l("p"),Ce.textContent=mn,zt=o(),A=l("div"),f(fe.$$.fragment),jt=o(),we=l("p"),we.textContent=pn,pt=o(),f(ue.$$.fragment),ft=o(),p=l("div"),f(_e.$$.fragment),qt=o(),ze=l("p"),ze.textContent=fn,Ft=o(),je=l("p"),je.textContent=un,Ut=o(),f(D.$$.fragment),It=o(),qe=l("p"),qe.innerHTML=_n,Jt=o(),Fe=l("p"),Fe.innerHTML=hn,Et=o(),w=l("div"),f(he.$$.fragment),Pt=o(),Ue=l("p"),Ue.textContent=gn,Rt=o(),Ie=l("p"),Ie.innerHTML=kn,Ht=o(),Je=l("ul"),Je.innerHTML=bn,Zt=o(),Ee=l("p"),Ee.textContent=vn,Bt=o(),G=l("div"),f(ge.$$.fragment),At=o(),Pe=l("p"),Pe.innerHTML=yn,Dt=o(),J=l("div"),f(ke.$$.fragment),Gt=o(),Re=l("p"),Re.innerHTML=Tn,Vt=o(),He=l("p"),He.textContent=xn,Nt=o(),V=l("div"),f(be.$$.fragment),Wt=o(),Ze=l("p"),Ze.innerHTML=$n,St=o(),Be=l("div"),f(ve.$$.fragment),ut=o(),De=l("p"),this.h()},l(e){const n=Fn("svelte-u9bgzb",document.head);d=i(n,"META",{name:!0,content:!0}),n.forEach(t),L=s(e),m=i(e,"P",{}),z(m).forEach(t),b=s(e),u(C.$$.fragment,e),v=s(e),u(x.$$.fragment,e),N=s(e),T=i(e,"P",{"data-svelte-h":!0}),c(T)!=="svelte-1njx1ag"&&(T.innerHTML=P),We=s(e),W=i(e,"P",{"data-svelte-h":!0}),c(W)!=="svelte-sy6as4"&&(W.innerHTML=Xt),Se=s(e),S=i(e,"P",{"data-svelte-h":!0}),c(S)!=="svelte-1re3wdp"&&(S.innerHTML=Qt),Xe=s(e),X=i(e,"P",{"data-svelte-h":!0}),c(X)!=="svelte-2m739y"&&(X.innerHTML=Yt),Qe=s(e),u(Q.$$.fragment,e),Ye=s(e),u(R.$$.fragment,e),Oe=s(e),Y=i(e,"P",{"data-svelte-h":!0}),c(Y)!=="svelte-fhhf9b"&&(Y.textContent=Ot),Ke=s(e),O=i(e,"UL",{"data-svelte-h":!0}),c(O)!=="svelte-1bm3iwk"&&(O.innerHTML=Kt),et=s(e),K=i(e,"P",{"data-svelte-h":!0}),c(K)!=="svelte-1l145o"&&(K.textContent=en),tt=s(e),u(ee.$$.fragment,e),nt=s(e),te=i(e,"P",{"data-svelte-h":!0}),c(te)!=="svelte-8qdh8l"&&(te.textContent=tn),ot=s(e),ne=i(e,"P",{"data-svelte-h":!0}),c(ne)!=="svelte-1otl2iw"&&(ne.textContent=nn),st=s(e),u(oe.$$.fragment,e),at=s(e),se=i(e,"P",{"data-svelte-h":!0}),c(se)!=="svelte-1fz6bbs"&&(se.textContent=on),rt=s(e),u(ae.$$.fragment,e),lt=s(e),re=i(e,"P",{"data-svelte-h":!0}),c(re)!=="svelte-1eh4tus"&&(re.innerHTML=sn),it=s(e),le=i(e,"P",{"data-svelte-h":!0}),c(le)!=="svelte-53awnm"&&(le.innerHTML=an),dt=s(e),u(H.$$.fragment,e),ct=s(e),u(ie.$$.fragment,e),mt=s(e),$=i(e,"DIV",{class:!0});var M=z($);u(de.$$.fragment,M),vt=s(M),Te=i(M,"P",{"data-svelte-h":!0}),c(Te)!=="svelte-1vmwvh5"&&(Te.textContent=rn),yt=s(M),xe=i(M,"P",{"data-svelte-h":!0}),c(xe)!=="svelte-1thmlao"&&(xe.innerHTML=ln),Tt=s(M),$e=i(M,"DIV",{class:!0});var Ge=z($e);u(ce.$$.fragment,Ge),Ge.forEach(t),xt=s(M),Z=i(M,"DIV",{class:!0});var ye=z(Z);u(me.$$.fragment,ye),$t=s(ye),Le=i(ye,"P",{"data-svelte-h":!0}),c(Le)!=="svelte-1f4f5kp"&&(Le.innerHTML=dn),ye.forEach(t),Lt=s(M),q=i(M,"DIV",{class:!0});var F=z(q);u(pe.$$.fragment,F),Mt=s(F),Me=i(F,"P",{"data-svelte-h":!0}),c(Me)!=="svelte-13bfd60"&&(Me.textContent=cn),Ct=s(F),u(B.$$.fragment,F),wt=s(F),Ce=i(F,"P",{"data-svelte-h":!0}),c(Ce)!=="svelte-wtrslu"&&(Ce.textContent=mn),F.forEach(t),zt=s(M),A=i(M,"DIV",{class:!0});var ht=z(A);u(fe.$$.fragment,ht),jt=s(ht),we=i(ht,"P",{"data-svelte-h":!0}),c(we)!=="svelte-1slb66l"&&(we.textContent=pn),ht.forEach(t),M.forEach(t),pt=s(e),u(ue.$$.fragment,e),ft=s(e),p=i(e,"DIV",{class:!0});var y=z(p);u(_e.$$.fragment,y),qt=s(y),ze=i(y,"P",{"data-svelte-h":!0}),c(ze)!=="svelte-15tdcz8"&&(ze.textContent=fn),Ft=s(y),je=i(y,"P",{"data-svelte-h":!0}),c(je)!=="svelte-llhmpa"&&(je.textContent=un),Ut=s(y),u(D.$$.fragment,y),It=s(y),qe=i(y,"P",{"data-svelte-h":!0}),c(qe)!=="svelte-cnb6q1"&&(qe.innerHTML=_n),Jt=s(y),Fe=i(y,"P",{"data-svelte-h":!0}),c(Fe)!=="svelte-ba41lz"&&(Fe.innerHTML=hn),Et=s(y),w=i(y,"DIV",{class:!0});var E=z(w);u(he.$$.fragment,E),Pt=s(E),Ue=i(E,"P",{"data-svelte-h":!0}),c(Ue)!=="svelte-1vll0v2"&&(Ue.textContent=gn),Rt=s(E),Ie=i(E,"P",{"data-svelte-h":!0}),c(Ie)!=="svelte-90np8u"&&(Ie.innerHTML=kn),Ht=s(E),Je=i(E,"UL",{"data-svelte-h":!0}),c(Je)!=="svelte-mlrsks"&&(Je.innerHTML=bn),Zt=s(E),Ee=i(E,"P",{"data-svelte-h":!0}),c(Ee)!=="svelte-46aam0"&&(Ee.textContent=vn),E.forEach(t),Bt=s(y),G=i(y,"DIV",{class:!0});var gt=z(G);u(ge.$$.fragment,gt),At=s(gt),Pe=i(gt,"P",{"data-svelte-h":!0}),c(Pe)!=="svelte-1wmjg8a"&&(Pe.innerHTML=yn),gt.forEach(t),Dt=s(y),J=i(y,"DIV",{class:!0});var Ae=z(J);u(ke.$$.fragment,Ae),Gt=s(Ae),Re=i(Ae,"P",{"data-svelte-h":!0}),c(Re)!=="svelte-zj1vf1"&&(Re.innerHTML=Tn),Vt=s(Ae),He=i(Ae,"P",{"data-svelte-h":!0}),c(He)!=="svelte-9vptpw"&&(He.textContent=xn),Ae.forEach(t),Nt=s(y),V=i(y,"DIV",{class:!0});var kt=z(V);u(be.$$.fragment,kt),Wt=s(kt),Ze=i(kt,"P",{"data-svelte-h":!0}),c(Ze)!=="svelte-nfci2w"&&(Ze.innerHTML=$n),kt.forEach(t),St=s(y),Be=i(y,"DIV",{class:!0});var Ln=z(Be);u(ve.$$.fragment,Ln),Ln.forEach(t),y.forEach(t),ut=s(e),De=i(e,"P",{}),z(De).forEach(t),this.h()},h(){j(d,"name","hf:doc:metadata"),j(d,"content",Rn),j($e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(Be,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(p,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,n){a(document.head,d),r(e,L,n),r(e,m,n),r(e,b,n),_(C,e,n),r(e,v,n),_(x,e,n),r(e,N,n),r(e,T,n),r(e,We,n),r(e,W,n),r(e,Se,n),r(e,S,n),r(e,Xe,n),r(e,X,n),r(e,Qe,n),_(Q,e,n),r(e,Ye,n),_(R,e,n),r(e,Oe,n),r(e,Y,n),r(e,Ke,n),r(e,O,n),r(e,et,n),r(e,K,n),r(e,tt,n),_(ee,e,n),r(e,nt,n),r(e,te,n),r(e,ot,n),r(e,ne,n),r(e,st,n),_(oe,e,n),r(e,at,n),r(e,se,n),r(e,rt,n),_(ae,e,n),r(e,lt,n),r(e,re,n),r(e,it,n),r(e,le,n),r(e,dt,n),_(H,e,n),r(e,ct,n),_(ie,e,n),r(e,mt,n),r(e,$,n),_(de,$,null),a($,vt),a($,Te),a($,yt),a($,xe),a($,Tt),a($,$e),_(ce,$e,null),a($,xt),a($,Z),_(me,Z,null),a(Z,$t),a(Z,Le),a($,Lt),a($,q),_(pe,q,null),a(q,Mt),a(q,Me),a(q,Ct),_(B,q,null),a(q,wt),a(q,Ce),a($,zt),a($,A),_(fe,A,null),a(A,jt),a(A,we),r(e,pt,n),_(ue,e,n),r(e,ft,n),r(e,p,n),_(_e,p,null),a(p,qt),a(p,ze),a(p,Ft),a(p,je),a(p,Ut),_(D,p,null),a(p,It),a(p,qe),a(p,Jt),a(p,Fe),a(p,Et),a(p,w),_(he,w,null),a(w,Pt),a(w,Ue),a(w,Rt),a(w,Ie),a(w,Ht),a(w,Je),a(w,Zt),a(w,Ee),a(p,Bt),a(p,G),_(ge,G,null),a(G,At),a(G,Pe),a(p,Dt),a(p,J),_(ke,J,null),a(J,Gt),a(J,Re),a(J,Vt),a(J,He),a(p,Nt),a(p,V),_(be,V,null),a(V,Wt),a(V,Ze),a(p,St),a(p,Be),_(ve,Be,null),r(e,ut,n),r(e,De,n),_t=!0},p(e,[n]){const M={};n&2&&(M.$$scope={dirty:n,ctx:e}),R.$set(M);const Ge={};n&2&&(Ge.$$scope={dirty:n,ctx:e}),H.$set(Ge);const ye={};n&2&&(ye.$$scope={dirty:n,ctx:e}),B.$set(ye);const F={};n&2&&(F.$$scope={dirty:n,ctx:e}),D.$set(F)},i(e){_t||(h(C.$$.fragment,e),h(x.$$.fragment,e),h(Q.$$.fragment,e),h(R.$$.fragment,e),h(ee.$$.fragment,e),h(oe.$$.fragment,e),h(ae.$$.fragment,e),h(H.$$.fragment,e),h(ie.$$.fragment,e),h(de.$$.fragment,e),h(ce.$$.fragment,e),h(me.$$.fragment,e),h(pe.$$.fragment,e),h(B.$$.fragment,e),h(fe.$$.fragment,e),h(ue.$$.fragment,e),h(_e.$$.fragment,e),h(D.$$.fragment,e),h(he.$$.fragment,e),h(ge.$$.fragment,e),h(ke.$$.fragment,e),h(be.$$.fragment,e),h(ve.$$.fragment,e),_t=!0)},o(e){g(C.$$.fragment,e),g(x.$$.fragment,e),g(Q.$$.fragment,e),g(R.$$.fragment,e),g(ee.$$.fragment,e),g(oe.$$.fragment,e),g(ae.$$.fragment,e),g(H.$$.fragment,e),g(ie.$$.fragment,e),g(de.$$.fragment,e),g(ce.$$.fragment,e),g(me.$$.fragment,e),g(pe.$$.fragment,e),g(B.$$.fragment,e),g(fe.$$.fragment,e),g(ue.$$.fragment,e),g(_e.$$.fragment,e),g(D.$$.fragment,e),g(he.$$.fragment,e),g(ge.$$.fragment,e),g(ke.$$.fragment,e),g(be.$$.fragment,e),g(ve.$$.fragment,e),_t=!1},d(e){e&&(t(L),t(m),t(b),t(v),t(N),t(T),t(We),t(W),t(Se),t(S),t(Xe),t(X),t(Qe),t(Ye),t(Oe),t(Y),t(Ke),t(O),t(et),t(K),t(tt),t(nt),t(te),t(ot),t(ne),t(st),t(at),t(se),t(rt),t(lt),t(re),t(it),t(le),t(dt),t(ct),t(mt),t($),t(pt),t(ft),t(p),t(ut),t(De)),t(d),k(C,e),k(x,e),k(Q,e),k(R,e),k(ee,e),k(oe,e),k(ae,e),k(H,e),k(ie,e),k(de),k(ce),k(me),k(pe),k(B),k(fe),k(ue,e),k(_e),k(D),k(he),k(ge),k(ke),k(be),k(ve)}}}const Rn='{"title":"CodeLlama","local":"codellama","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage tips and examples","local":"usage-tips-and-examples","sections":[],"depth":2},{"title":"CodeLlamaTokenizer","local":"transformers.CodeLlamaTokenizer","sections":[],"depth":2},{"title":"CodeLlamaTokenizerFast","local":"transformers.CodeLlamaTokenizerFast","sections":[],"depth":2}],"depth":1}';function Hn(I){return zn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wn extends jn{constructor(d){super(),qn(this,d,Hn,Pn,wn,{})}}export{Wn as component};
