import{s as Se,n as Le,o as Fe}from"../chunks/scheduler.25b97de1.js";import{S as Pe,i as Ke,g as a,s,r as U,A as De,h as i,f as M,c as n,j as He,u as o,x as y,k as Qe,y as qe,a as t,v as j,d as C,t as p,w}from"../chunks/index.d9030fc9.js";import{C as Cl}from"../chunks/CodeBlock.e6cd0d95.js";import{H as m,E as Oe}from"../chunks/EditOnGithub.91d95064.js";function lM(pe){let r,ml,pl,rl,T,Tl,d,we='<code>transformers</code> is an opinionated framework; our philosophy is defined in the following <a href="./philosophy">conceptual guide</a>.',dl,J,me=`The core of that philosophy is exemplified by the <a href="https://huggingface.co/blog/transformers-design-philosophy" rel="nofollow">single model, single file</a>
aspect of the library. This component’s downside is that it limits the inheritance and importability of components from
files to others in the toolkit.`,Jl,c,re=`As a result, model components tend to be repeated across many files. There are as many attention layers defined
in <code>transformers</code> as there are models, and a significant number of those are identical to each other.
The unfortunate consequence is that independent implementations tend to diverge as fixes and changes get applied
to specific parts of the code.`,cl,A,Te=`In order to balance this issue, we introduced the concept of “copies” across the library. By adding a comment indicating
that code is a copy of another, we can enforce through CI and local commands that copies do not diverge. However,
while the complexity is low, this is often quite tedious to do.`,Al,f,de=`And, finally, this contributes to adding a significant overhead to contributing models which we would like to remove.
This approach often requires model contributions to add modeling code (~1k lines), processor (~500 lines), tests, docs,
etc. Model contribution PRs rarely add less than 3-5k lines of code, with much of this code being boilerplate.`,fl,h,Je=`This raises the bar for contributions, and with Modular Transformers, we’re aiming to lower the bar to a much more
acceptable point.`,hl,I,Il,u,ce=`Modular Transformers introduces the concept of a “modular” file to a model folder. This modular file accepts code
that isn’t typically accepted in modeling/processing files, as it allows importing from neighbouring models as well
as inheritance from classes to others.`,ul,b,Ae=`This modular file defines models, processors, and the configuration class that would otherwise be defined in their
respective modules.`,bl,_,fe=`Finally, this feature introduces a new <code>linter</code> which will “unravel” the modular file into the “single model, single
file” directory structure. These files will get auto-generated every time the script is run; reducing the required
contributions to the modular file, and therefore only to the changes between the contributed model and others.`,_l,Z,he=`Model users will end up importing and using the single-file interface, so no change is expected here. Doing this, we
hope to combine the best of both worlds: enabling simple contributions while sticking to our philosophy.`,Zl,g,Ie=`This is therefore a replacement for the <code># Copied from</code> markers, and previously contributed models can be expected to
be moved to the new Modular Transformers format in the coming months.`,gl,k,kl,v,ue=`The “linter”, which unravels the inheritance and creates all single-files from the modular file, will flatten the
inheritance while trying to be invisible to Python users. At this time, the linter flattens a <strong>single</strong> level of
inheritance.`,vl,W,be="For example:",Wl,B,_e=`<li>If a configuration class inherits from another and adds/deletes an argument, the generated file will either directly
reference it (in case of addition) or completely remove it (in case of deletion).</li> <li>If a class inherits from another, for example: class GemmaModel(LlamaModel):, dependencies are automatically
inferred. All submodules will be automatically inferred from the superclass.</li> <li>If you define new functions in the <code>modular</code> and use them inside classes, the linter will automatically infer the</li>`,Bl,$,Ze=`You should be able to write everything (the tokenizer, the image processor, the model, the config) in this <code>modular</code>
file, and the corresponding files will be created for you.`,$l,G,Gl,R,ge="[TODO] We are introducing a new test, that makes sure the generated content matches what is present in the <code>modular_xxxx.py</code>",Rl,x,xl,V,ke=`Here is a quick example with BERT and RoBERTa. The two models are intimately related: their modeling implementation
differs solely by a change in the embedding layer.`,Vl,E,ve=`Instead of redefining the model entirely, here is what the <code>modular_roberta.py</code> file looks like for the modeling &amp;
configuration classes (for the sake of the example, the tokenizer is ignored at this time as very different).`,El,X,Xl,Y,We="Note that if you do not use the dependency that you defined, you will have the following error:",Yl,N,Nl,z,Be="Additionally, you may find a list of examples here:",zl,H,Hl,Q,$e="It is not a replacement for the modeling code (yet?), and if your model is not based on anything else that ever existed, then you can add a <code>modeling</code> file as usual.",Ql,S,Sl,L,Ll,F,Ge="To remove attributes that are not used in your modular model, and that you don’t want to see in the unravelled modeling:",Fl,P,Pl,K,Re="If you check the original <code>LlamaModel</code>, it has a <code>embed_tokens</code> which was removed here (as you would expect!)",Kl,D,xe="Removing a function is pretty similar, you just need to write it with a <code>raise ValueError(&quot;&quot;)</code> to mimick the behaviour you actually want when you remove a parent function in python.",Dl,q,ql,O,Ol,ll,Ve="If you define a new function in the <code>modular</code> file to be used inside a class, say",le,el,ee,Ml,Ee=`the <code>my_new_function</code> function (and, recursively, any other new functions called in its body) will be automatically copy-pasted
in the file where it is used.`,Me,tl,te,sl,Xe="We recently shipped a few features that allow you to go from:",se,nl,ne,al,Ye="This is useful want you <strong>don’t</strong> want to unravel the call to <code>super()</code>, and you want to differentiate which super init call you are doing!",ae,il,ie,yl,Ne="We now also support special cases like",ye,Ul,Ue,ol,ze="where the name of your class <code>GemmaVision</code> is not the same as the modular <code>Gemma</code>. This is super useful for composite models.",oe,jl,je,wl,Ce;return T=new m({props:{title:"Modular transformers",local:"modular-transformers",headingTag:"h1"}}),I=new m({props:{title:"What is it?",local:"what-is-it",headingTag:"h2"}}),k=new m({props:{title:"Details",local:"details",headingTag:"h3"}}),G=new m({props:{title:"Enforcement",local:"enforcement",headingTag:"h3"}}),x=new m({props:{title:"Examples",local:"examples",headingTag:"h3"}}),X=new Cl({props:{code:"ZnJvbSUyMHRvcmNoJTIwaW1wb3J0JTIwbm4lMEFmcm9tJTIwLi5iZXJ0LmNvbmZpZ3VyYXRpb25fYmVydCUyMGltcG9ydCUyMEJlcnRDb25maWclMEFmcm9tJTIwLi5iZXJ0Lm1vZGVsaW5nX2JlcnQlMjBpbXBvcnQlMjAoJTBBJTIwJTIwJTIwJTIwQmVydE1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwQmVydEVtYmVkZGluZ3MlMkMlMEElMjAlMjAlMjAlMjBCZXJ0Rm9yTWFza2VkTE0lMEEpJTBBJTBBJTIzJTIwVGhlJTIwUm9CRVJUYSUyMGNvbmZpZyUyMGlzJTIwaWRlbnRpY2FsJTIwdG8lMjBCRVJUJ3MlMjBjb25maWclMEFjbGFzcyUyMFJvYmVydGFDb25maWcoQmVydENvbmZpZyklM0ElMEElMjAlMjBtb2RlbF90eXBlJTIwJTNEJTIwJ3JvYmVydGEnJTBBJTBBJTIzJTIwV2UlMjByZWRlZmluZSUyMHRoZSUyMGVtYmVkZGluZ3MlMjBoZXJlJTIwdG8lMjBoaWdobGlnaHQlMjB0aGUlMjBwYWRkaW5nJTIwSUQlMjBkaWZmZXJlbmNlJTJDJTIwYW5kJTIwd2UlMjByZWRlZmluZSUyMHRoZSUyMHBvc2l0aW9uJTIwZW1iZWRkaW5ncyUwQWNsYXNzJTIwUm9iZXJ0YUVtYmVkZGluZ3MoQmVydEVtYmVkZGluZ3MpJTNBJTBBJTIwJTIwJTIwJTIwZGVmJTIwX19pbml0X18oc2VsZiUyQyUyMGNvbmZpZyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdXBlcigpLl9faW5pdF9fKGNvbmZpZygpKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYucGFkZGluZ19pZHglMjAlM0QlMjBjb25maWcucGFkX3Rva2VuX2lkJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5wb3NpdGlvbl9lbWJlZGRpbmdzJTIwJTNEJTIwbm4uRW1iZWRkaW5nKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbmZpZy5tYXhfcG9zaXRpb25fZW1iZWRkaW5ncyUyQyUyMGNvbmZpZy5oaWRkZW5fc2l6ZSUyQyUyMHBhZGRpbmdfaWR4JTNEc2VsZi5wYWRkaW5nX2lkeCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMEElMjMlMjBUaGUlMjBSb0JFUlRhJTIwbW9kZWwlMjBpcyUyMGlkZW50aWNhbCUyMHRvJTIwdGhlJTIwQkVSVCUyMG1vZGVsJTJDJTIwZXhjZXB0JTIwZm9yJTIwdGhlJTIwZW1iZWRkaW5nJTIwbGF5ZXIuJTIwJTBBJTIzJTIwV2UlMjByZWRlZmluZSUyMHRoZSUyMGVtYmVkZGluZ3MlMjBhYm92ZSUyQyUyMHNvJTIwaGVyZSUyMHRoZXJlJTIwaXMlMjBubyUyMG5lZWQlMjB0byUyMGRvJTIwYWRkaXRpb25hbCUyMHdvcmslMEFjbGFzcyUyMFJvYmVydGFNb2RlbChCZXJ0TW9kZWwpJTNBJTBBJTIwJTIwZGVmJTIwX19pbml0X18oc2VsZiUyQyUyMGNvbmZpZyklM0ElMEElMjAlMjAlMjAlMjBzdXBlcigpLl9faW5pdF9fKGNvbmZpZyklMEElMjAlMjAlMjAlMjBzZWxmLmVtYmVkZGluZ3MlMjAlM0QlMjBSb2JlcnRhRW1iZWRkaW5ncyhjb25maWcpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTBBJTIzJTIwVGhlJTIwaGVhZHMlMjBub3clMjBvbmx5JTIwbmVlZCUyMHRvJTIwcmVkZWZpbmUlMjB0aGUlMjBtb2RlbCUyMGluc2lkZSUyMHRvJTIwdGhlJTIwY29ycmVjdCUyMCU2MFJvYmVydGFNb2RlbCU2MCUwQWNsYXNzJTIwUm9iZXJ0YUZvck1hc2tlZExNKEJlcnRGb3JNYXNrZWRMTSklM0ElMEElMjAlMjBkZWYlMjBfX2luaXRfXyhzZWxmJTJDJTIwY29uZmlnKSUzQSUwQSUyMCUyMCUyMCUyMHN1cGVyKCkuX19pbml0X18oY29uZmlnKSUwQSUyMCUyMCUyMCUyMHNlbGYubW9kZWwlMjAlM0QlMjBSb2JlcnRhTW9kZWwoY29uZmlnKQ==",highlighted:`<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> nn
<span class="hljs-keyword">from</span> ..bert.configuration_bert <span class="hljs-keyword">import</span> BertConfig
<span class="hljs-keyword">from</span> ..bert.modeling_bert <span class="hljs-keyword">import</span> (
    BertModel,
    BertEmbeddings,
    BertForMaskedLM
)

<span class="hljs-comment"># The RoBERTa config is identical to BERT&#x27;s config</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">RobertaConfig</span>(<span class="hljs-title class_ inherited__">BertConfig</span>):
  model_type = <span class="hljs-string">&#x27;roberta&#x27;</span>

<span class="hljs-comment"># We redefine the embeddings here to highlight the padding ID difference, and we redefine the position embeddings</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">RobertaEmbeddings</span>(<span class="hljs-title class_ inherited__">BertEmbeddings</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>):
        <span class="hljs-built_in">super</span>().__init__(config())

        self.padding_idx = config.pad_token_id
        self.position_embeddings = nn.Embedding(
            config.max_position_embeddings, config.hidden_size, padding_idx=self.padding_idx
        )

<span class="hljs-comment"># The RoBERTa model is identical to the BERT model, except for the embedding layer. </span>
<span class="hljs-comment"># We redefine the embeddings above, so here there is no need to do additional work</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">RobertaModel</span>(<span class="hljs-title class_ inherited__">BertModel</span>):
  <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>):
    <span class="hljs-built_in">super</span>().__init__(config)
    self.embeddings = RobertaEmbeddings(config)

      
<span class="hljs-comment"># The heads now only need to redefine the model inside to the correct \`RobertaModel\`</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">RobertaForMaskedLM</span>(<span class="hljs-title class_ inherited__">BertForMaskedLM</span>):
  <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>):
    <span class="hljs-built_in">super</span>().__init__(config)
    self.model = RobertaModel(config)`,wrap:!1}}),N=new Cl({props:{code:"VmFsdWVFcnJvciUzQSUyMFlvdSUyMGRlZmluZWQlMjAlNjBSb2JlcnRhRW1iZWRkaW5ncyU2MCUyMGluJTIwdGhlJTIwbW9kdWxhcl9yb2JlcnRhLnB5JTJDJTIwaXQlMjBzaG91bGQlMjBiZSUyMHVzZWQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3aGVuJTIweW91JTIwZGVmaW5lJTIwJTYwQmVydE1vZGVsJTYwJTJDJTIwYXMlMjBpdCUyMGlzJTIwb25lJTIwb2YlMjBpdCdzJTIwZGlyZWN0JTIwZGVwZW5kZW5jaWVzLiUyME1ha2UlMjBzdXJlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIweW91JTIwdXNlJTIwaXQlMjBpbiUyMHRoZSUyMCU2MF9faW5pdF9fJTYwJTIwZnVuY3Rpb24u",highlighted:'ValueError: You defined `RobertaEmbeddings` <span class="hljs-keyword">in</span> the modular_roberta.py, it should be used\n                                    when you define `BertModel`, as it is one of it<span class="hljs-string">&#x27;s direct dependencies. Make sure\n                                    you use it in the `__init__` function.</span>',wrap:!1}}),H=new m({props:{title:"What it is not",local:"what-it-is-not",headingTag:"h2"}}),S=new m({props:{title:"Advanced usage",local:"advanced-usage",headingTag:"h2"}}),L=new m({props:{title:"Removing attributes and functions",local:"removing-attributes-and-functions",headingTag:"h3"}}),P=new Cl({props:{code:" ",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">GemmaModel</span>(<span class="hljs-title class_ inherited__">LlamaModel</span>):                 |           <span class="hljs-keyword">class</span> <span class="hljs-title class_">GemmaModel</span>(<span class="hljs-title class_ inherited__">PreTrainedModel</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>):               |              <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>):
        <span class="hljs-built_in">super</span>().__init__(self, eos_token)     |                 <span class="hljs-built_in">super</span>().__init__(config)
        <span class="hljs-keyword">del</span> self.embed_tokens                 |                 self.padding_idx = config.pad_token_id
                                              |                 self.vocab_size = config.vocab_size
                                              |
                                              |                 self.layers = nn.ModuleList(
                                              |                     [LlamaDecoderLayer(config, layer_idx) <span class="hljs-keyword">for</span> layer_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(config.num_hidden_layers)]
                                              |                 )
                                              |                 self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
                                              |                 self.rotary_emb = LlamaRotaryEmbedding(config=config)
                                              |                 self.gradient_checkpointing = <span class="hljs-literal">False</span>
                                              |                 
                                              |                 <span class="hljs-comment"># Initialize weights and apply final processing</span>
                                              |                 self.post_init()`,wrap:!1}}),q=new Cl({props:{code:"Y2xhc3MlMjBHZW1tYVRva2VuaXplcihMbGFtYVRva2VuaXplciklM0ElMEElMjAlMjAlMjAlMjAuLi4lMEElMEElMjAlMjAlMjAlMjBkZWYlMjBnZXRfc3BtX3Byb2Nlc3NvcihzZWxmKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJhaXNlJTIwQXR0cmlidXRlRXJyb3IoJTIyTm90JTIwbmVlZGVkJTIwZm9yJTIwR2VtbWElMjIpJTBBJTBBJTIwJTIwJTIwJTIwZGVmJTIwdW5rX3Rva2VuX2xlbmd0aChzZWxmKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJhaXNlJTIwQXR0cmlidXRlRXJyb3IoJTIyTm90JTIwbmVlZGVkJTIwZm9yJTIwR2VtbWElMjIp",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">GemmaTokenizer</span>(<span class="hljs-title class_ inherited__">LlamaTokenizer</span>):
    ...

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_spm_processor</span>(<span class="hljs-params">self</span>):
        <span class="hljs-keyword">raise</span> AttributeError(<span class="hljs-string">&quot;Not needed for Gemma&quot;</span>)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">unk_token_length</span>(<span class="hljs-params">self</span>):
        <span class="hljs-keyword">raise</span> AttributeError(<span class="hljs-string">&quot;Not needed for Gemma&quot;</span>)`,wrap:!1}}),O=new m({props:{title:"Define new functions",local:"define-new-functions",headingTag:"h3"}}),el=new Cl({props:{code:"ZGVmJTIwbXlfbmV3X2Z1bmN0aW9uKCphcmdzJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIzJTIwRG8lMjBzb21ldGhpbmclMjBoZXJlJTBBJTIwJTIwcGFzcyUwQSUwQWNsYXNzJTIwR2VtbWFNb2RlbChMbGFtYU1vZGVsKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoKmFyZ3MlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBDYWxsJTIwdGhlJTIwZnVuY3Rpb24lMEElMjAlMjAlMjAlMjAlMjAlMjBleGFtcGxlJTIwJTNEJTIwbXlfbmV3X2Z1bmN0aW9uKCphcmdzJTJDJTIwKiprd2FyZ3MpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwY29udGludWUlMjBoZXJl",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">my_new_function</span>(<span class="hljs-params">*args, **kwargs</span>):
  <span class="hljs-comment"># Do something here</span>
  <span class="hljs-keyword">pass</span>

<span class="hljs-keyword">class</span> <span class="hljs-title class_">GemmaModel</span>(<span class="hljs-title class_ inherited__">LlamaModel</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">*args, **kwargs</span>):
      <span class="hljs-comment"># Call the function</span>
      example = my_new_function(*args, **kwargs)
      <span class="hljs-comment"># continue here</span>`,wrap:!1}}),tl=new m({props:{title:"Calling super()",local:"calling-super",headingTag:"h3"}}),nl=new Cl({props:{code:"Y2xhc3MlMjBHZW1tYVRva2VuaXplcihMbGFtYVRva2VuaXplciUyQyUyMFByZXRyYWluZWRUb2tlbml6ZXJGYXN0KSUzQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNsYXNzJTIwR2VtbWFNb2RlbChubi5Nb2R1bGUpJTNBJTBBJTIwJTIwJTIwJTIwZGVmJTIwX19pbml0X18oc2VsZiUyQyUyMGVvc190b2tlbiUzRCUyMiUzQyUyRnMlM0UlMjIpJTNBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdDJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZGVmJTIwX19pbml0X18oc2VsZiklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlb3NfdG9rZW4lMjAlM0QlMjBBZGRlZFRva2VuKGVvc190b2tlbiklMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0MlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlb3NfdG9rZW4lMjAlM0QlMjBBZGRlZFRva2VuKGVvc190b2tlbiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBQcmV0cmFpbmVkVG9rZW5pemVyRmFzdC5fX2luaXRfXyhzZWxmJTJDJTIwZW9zX3Rva2VuKSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN1cGVyKCkuX19pbml0X18oZW9zX3Rva2VuKQ==",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">GemmaTokenizer</span>(LlamaTokenizer, PretrainedTokenizerFast):         |           <span class="hljs-keyword">class</span> <span class="hljs-title class_">GemmaModel</span>(nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, eos_token=<span class="hljs-string">&quot;&lt;/s&gt;&quot;</span></span>):                              |             <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        eos_token = AddedToken(eos_token)                              |                eos_token = AddedToken(eos_token)
        PretrainedTokenizerFast.__init__(self, eos_token)              |                <span class="hljs-built_in">super</span>().__init__(eos_token)`,wrap:!1}}),il=new m({props:{title:"Special naming",local:"special-naming",headingTag:"h3"}}),Ul=new Cl({props:{code:"Y2xhc3MlMjBHZW1tYVZpc2lvbk1vZGVsKENMSVBNb2RlbCklM0ElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMEElMjAlMjAlMjAlMjBwYXNz",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">GemmaVisionModel</span>(<span class="hljs-title class_ inherited__">CLIPModel</span>):                                 
    <span class="hljs-keyword">pass</span>`,wrap:!1}}),jl=new Oe({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/modular_transformers.md"}}),{c(){r=a("meta"),ml=s(),pl=a("p"),rl=s(),U(T.$$.fragment),Tl=s(),d=a("p"),d.innerHTML=we,dl=s(),J=a("p"),J.innerHTML=me,Jl=s(),c=a("p"),c.innerHTML=re,cl=s(),A=a("p"),A.textContent=Te,Al=s(),f=a("p"),f.textContent=de,fl=s(),h=a("p"),h.textContent=Je,hl=s(),U(I.$$.fragment),Il=s(),u=a("p"),u.textContent=ce,ul=s(),b=a("p"),b.textContent=Ae,bl=s(),_=a("p"),_.innerHTML=fe,_l=s(),Z=a("p"),Z.textContent=he,Zl=s(),g=a("p"),g.innerHTML=Ie,gl=s(),U(k.$$.fragment),kl=s(),v=a("p"),v.innerHTML=ue,vl=s(),W=a("p"),W.textContent=be,Wl=s(),B=a("ul"),B.innerHTML=_e,Bl=s(),$=a("p"),$.innerHTML=Ze,$l=s(),U(G.$$.fragment),Gl=s(),R=a("p"),R.innerHTML=ge,Rl=s(),U(x.$$.fragment),xl=s(),V=a("p"),V.textContent=ke,Vl=s(),E=a("p"),E.innerHTML=ve,El=s(),U(X.$$.fragment),Xl=s(),Y=a("p"),Y.textContent=We,Yl=s(),U(N.$$.fragment),Nl=s(),z=a("p"),z.textContent=Be,zl=s(),U(H.$$.fragment),Hl=s(),Q=a("p"),Q.innerHTML=$e,Ql=s(),U(S.$$.fragment),Sl=s(),U(L.$$.fragment),Ll=s(),F=a("p"),F.textContent=Ge,Fl=s(),U(P.$$.fragment),Pl=s(),K=a("p"),K.innerHTML=Re,Kl=s(),D=a("p"),D.innerHTML=xe,Dl=s(),U(q.$$.fragment),ql=s(),U(O.$$.fragment),Ol=s(),ll=a("p"),ll.innerHTML=Ve,le=s(),U(el.$$.fragment),ee=s(),Ml=a("p"),Ml.innerHTML=Ee,Me=s(),U(tl.$$.fragment),te=s(),sl=a("p"),sl.textContent=Xe,se=s(),U(nl.$$.fragment),ne=s(),al=a("p"),al.innerHTML=Ye,ae=s(),U(il.$$.fragment),ie=s(),yl=a("p"),yl.textContent=Ne,ye=s(),U(Ul.$$.fragment),Ue=s(),ol=a("p"),ol.innerHTML=ze,oe=s(),U(jl.$$.fragment),je=s(),wl=a("p"),this.h()},l(l){const e=De("svelte-u9bgzb",document.head);r=i(e,"META",{name:!0,content:!0}),e.forEach(M),ml=n(l),pl=i(l,"P",{}),He(pl).forEach(M),rl=n(l),o(T.$$.fragment,l),Tl=n(l),d=i(l,"P",{"data-svelte-h":!0}),y(d)!=="svelte-1qsl2az"&&(d.innerHTML=we),dl=n(l),J=i(l,"P",{"data-svelte-h":!0}),y(J)!=="svelte-1qcwnc7"&&(J.innerHTML=me),Jl=n(l),c=i(l,"P",{"data-svelte-h":!0}),y(c)!=="svelte-hcbv0p"&&(c.innerHTML=re),cl=n(l),A=i(l,"P",{"data-svelte-h":!0}),y(A)!=="svelte-14lpvvv"&&(A.textContent=Te),Al=n(l),f=i(l,"P",{"data-svelte-h":!0}),y(f)!=="svelte-1tehds2"&&(f.textContent=de),fl=n(l),h=i(l,"P",{"data-svelte-h":!0}),y(h)!=="svelte-5u9x31"&&(h.textContent=Je),hl=n(l),o(I.$$.fragment,l),Il=n(l),u=i(l,"P",{"data-svelte-h":!0}),y(u)!=="svelte-1qqnnub"&&(u.textContent=ce),ul=n(l),b=i(l,"P",{"data-svelte-h":!0}),y(b)!=="svelte-huwsbv"&&(b.textContent=Ae),bl=n(l),_=i(l,"P",{"data-svelte-h":!0}),y(_)!=="svelte-dk7wye"&&(_.innerHTML=fe),_l=n(l),Z=i(l,"P",{"data-svelte-h":!0}),y(Z)!=="svelte-12pm85e"&&(Z.textContent=he),Zl=n(l),g=i(l,"P",{"data-svelte-h":!0}),y(g)!=="svelte-ah1dr9"&&(g.innerHTML=Ie),gl=n(l),o(k.$$.fragment,l),kl=n(l),v=i(l,"P",{"data-svelte-h":!0}),y(v)!=="svelte-xkvkhs"&&(v.innerHTML=ue),vl=n(l),W=i(l,"P",{"data-svelte-h":!0}),y(W)!=="svelte-1gkqha7"&&(W.textContent=be),Wl=n(l),B=i(l,"UL",{"data-svelte-h":!0}),y(B)!=="svelte-wnqir2"&&(B.innerHTML=_e),Bl=n(l),$=i(l,"P",{"data-svelte-h":!0}),y($)!=="svelte-12b6859"&&($.innerHTML=Ze),$l=n(l),o(G.$$.fragment,l),Gl=n(l),R=i(l,"P",{"data-svelte-h":!0}),y(R)!=="svelte-drmr0e"&&(R.innerHTML=ge),Rl=n(l),o(x.$$.fragment,l),xl=n(l),V=i(l,"P",{"data-svelte-h":!0}),y(V)!=="svelte-1o8u693"&&(V.textContent=ke),Vl=n(l),E=i(l,"P",{"data-svelte-h":!0}),y(E)!=="svelte-si4m91"&&(E.innerHTML=ve),El=n(l),o(X.$$.fragment,l),Xl=n(l),Y=i(l,"P",{"data-svelte-h":!0}),y(Y)!=="svelte-tpa7s6"&&(Y.textContent=We),Yl=n(l),o(N.$$.fragment,l),Nl=n(l),z=i(l,"P",{"data-svelte-h":!0}),y(z)!=="svelte-834mi0"&&(z.textContent=Be),zl=n(l),o(H.$$.fragment,l),Hl=n(l),Q=i(l,"P",{"data-svelte-h":!0}),y(Q)!=="svelte-1cj74u8"&&(Q.innerHTML=$e),Ql=n(l),o(S.$$.fragment,l),Sl=n(l),o(L.$$.fragment,l),Ll=n(l),F=i(l,"P",{"data-svelte-h":!0}),y(F)!=="svelte-1iowufa"&&(F.textContent=Ge),Fl=n(l),o(P.$$.fragment,l),Pl=n(l),K=i(l,"P",{"data-svelte-h":!0}),y(K)!=="svelte-py9in9"&&(K.innerHTML=Re),Kl=n(l),D=i(l,"P",{"data-svelte-h":!0}),y(D)!=="svelte-m2hnwf"&&(D.innerHTML=xe),Dl=n(l),o(q.$$.fragment,l),ql=n(l),o(O.$$.fragment,l),Ol=n(l),ll=i(l,"P",{"data-svelte-h":!0}),y(ll)!=="svelte-6t8mis"&&(ll.innerHTML=Ve),le=n(l),o(el.$$.fragment,l),ee=n(l),Ml=i(l,"P",{"data-svelte-h":!0}),y(Ml)!=="svelte-9rp4ko"&&(Ml.innerHTML=Ee),Me=n(l),o(tl.$$.fragment,l),te=n(l),sl=i(l,"P",{"data-svelte-h":!0}),y(sl)!=="svelte-twcsb7"&&(sl.textContent=Xe),se=n(l),o(nl.$$.fragment,l),ne=n(l),al=i(l,"P",{"data-svelte-h":!0}),y(al)!=="svelte-1aokivf"&&(al.innerHTML=Ye),ae=n(l),o(il.$$.fragment,l),ie=n(l),yl=i(l,"P",{"data-svelte-h":!0}),y(yl)!=="svelte-m0n5a3"&&(yl.textContent=Ne),ye=n(l),o(Ul.$$.fragment,l),Ue=n(l),ol=i(l,"P",{"data-svelte-h":!0}),y(ol)!=="svelte-zxzljf"&&(ol.innerHTML=ze),oe=n(l),o(jl.$$.fragment,l),je=n(l),wl=i(l,"P",{}),He(wl).forEach(M),this.h()},h(){Qe(r,"name","hf:doc:metadata"),Qe(r,"content",eM)},m(l,e){qe(document.head,r),t(l,ml,e),t(l,pl,e),t(l,rl,e),j(T,l,e),t(l,Tl,e),t(l,d,e),t(l,dl,e),t(l,J,e),t(l,Jl,e),t(l,c,e),t(l,cl,e),t(l,A,e),t(l,Al,e),t(l,f,e),t(l,fl,e),t(l,h,e),t(l,hl,e),j(I,l,e),t(l,Il,e),t(l,u,e),t(l,ul,e),t(l,b,e),t(l,bl,e),t(l,_,e),t(l,_l,e),t(l,Z,e),t(l,Zl,e),t(l,g,e),t(l,gl,e),j(k,l,e),t(l,kl,e),t(l,v,e),t(l,vl,e),t(l,W,e),t(l,Wl,e),t(l,B,e),t(l,Bl,e),t(l,$,e),t(l,$l,e),j(G,l,e),t(l,Gl,e),t(l,R,e),t(l,Rl,e),j(x,l,e),t(l,xl,e),t(l,V,e),t(l,Vl,e),t(l,E,e),t(l,El,e),j(X,l,e),t(l,Xl,e),t(l,Y,e),t(l,Yl,e),j(N,l,e),t(l,Nl,e),t(l,z,e),t(l,zl,e),j(H,l,e),t(l,Hl,e),t(l,Q,e),t(l,Ql,e),j(S,l,e),t(l,Sl,e),j(L,l,e),t(l,Ll,e),t(l,F,e),t(l,Fl,e),j(P,l,e),t(l,Pl,e),t(l,K,e),t(l,Kl,e),t(l,D,e),t(l,Dl,e),j(q,l,e),t(l,ql,e),j(O,l,e),t(l,Ol,e),t(l,ll,e),t(l,le,e),j(el,l,e),t(l,ee,e),t(l,Ml,e),t(l,Me,e),j(tl,l,e),t(l,te,e),t(l,sl,e),t(l,se,e),j(nl,l,e),t(l,ne,e),t(l,al,e),t(l,ae,e),j(il,l,e),t(l,ie,e),t(l,yl,e),t(l,ye,e),j(Ul,l,e),t(l,Ue,e),t(l,ol,e),t(l,oe,e),j(jl,l,e),t(l,je,e),t(l,wl,e),Ce=!0},p:Le,i(l){Ce||(C(T.$$.fragment,l),C(I.$$.fragment,l),C(k.$$.fragment,l),C(G.$$.fragment,l),C(x.$$.fragment,l),C(X.$$.fragment,l),C(N.$$.fragment,l),C(H.$$.fragment,l),C(S.$$.fragment,l),C(L.$$.fragment,l),C(P.$$.fragment,l),C(q.$$.fragment,l),C(O.$$.fragment,l),C(el.$$.fragment,l),C(tl.$$.fragment,l),C(nl.$$.fragment,l),C(il.$$.fragment,l),C(Ul.$$.fragment,l),C(jl.$$.fragment,l),Ce=!0)},o(l){p(T.$$.fragment,l),p(I.$$.fragment,l),p(k.$$.fragment,l),p(G.$$.fragment,l),p(x.$$.fragment,l),p(X.$$.fragment,l),p(N.$$.fragment,l),p(H.$$.fragment,l),p(S.$$.fragment,l),p(L.$$.fragment,l),p(P.$$.fragment,l),p(q.$$.fragment,l),p(O.$$.fragment,l),p(el.$$.fragment,l),p(tl.$$.fragment,l),p(nl.$$.fragment,l),p(il.$$.fragment,l),p(Ul.$$.fragment,l),p(jl.$$.fragment,l),Ce=!1},d(l){l&&(M(ml),M(pl),M(rl),M(Tl),M(d),M(dl),M(J),M(Jl),M(c),M(cl),M(A),M(Al),M(f),M(fl),M(h),M(hl),M(Il),M(u),M(ul),M(b),M(bl),M(_),M(_l),M(Z),M(Zl),M(g),M(gl),M(kl),M(v),M(vl),M(W),M(Wl),M(B),M(Bl),M($),M($l),M(Gl),M(R),M(Rl),M(xl),M(V),M(Vl),M(E),M(El),M(Xl),M(Y),M(Yl),M(Nl),M(z),M(zl),M(Hl),M(Q),M(Ql),M(Sl),M(Ll),M(F),M(Fl),M(Pl),M(K),M(Kl),M(D),M(Dl),M(ql),M(Ol),M(ll),M(le),M(ee),M(Ml),M(Me),M(te),M(sl),M(se),M(ne),M(al),M(ae),M(ie),M(yl),M(ye),M(Ue),M(ol),M(oe),M(je),M(wl)),M(r),w(T,l),w(I,l),w(k,l),w(G,l),w(x,l),w(X,l),w(N,l),w(H,l),w(S,l),w(L,l),w(P,l),w(q,l),w(O,l),w(el,l),w(tl,l),w(nl,l),w(il,l),w(Ul,l),w(jl,l)}}}const eM='{"title":"Modular transformers","local":"modular-transformers","sections":[{"title":"What is it?","local":"what-is-it","sections":[{"title":"Details","local":"details","sections":[],"depth":3},{"title":"Enforcement","local":"enforcement","sections":[],"depth":3},{"title":"Examples","local":"examples","sections":[],"depth":3}],"depth":2},{"title":"What it is not","local":"what-it-is-not","sections":[],"depth":2},{"title":"Advanced usage","local":"advanced-usage","sections":[{"title":"Removing attributes and functions","local":"removing-attributes-and-functions","sections":[],"depth":3},{"title":"Define new functions","local":"define-new-functions","sections":[],"depth":3},{"title":"Calling super()","local":"calling-super","sections":[],"depth":3},{"title":"Special naming","local":"special-naming","sections":[],"depth":3}],"depth":2}],"depth":1}';function MM(pe){return Fe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class iM extends Pe{constructor(r){super(),Ke(this,r,MM,lM,Se,{})}}export{iM as component};
