import{s as ua,o as wa,n as $}from"../chunks/scheduler.9bc65507.js";import{S as ja,i as Ja,g as f,s as r,r as d,A as ya,h as T,f as a,c as i,j as Ta,u,x as h,k as ha,y as _a,a as t,v as w,d as j,t as J,w as y}from"../chunks/index.707bf1b6.js";import{T as Bs}from"../chunks/Tip.c2ecdbf4.js";import{C as I}from"../chunks/CodeBlock.54a9f38d.js";import{H as N}from"../chunks/Heading.342b1fa6.js";import{H as da,a as Ce}from"../chunks/HfOption.6d864328.js";function ga(_){let l,o='In addition to the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> class, Transformers also provides a <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Seq2SeqTrainer">Seq2SeqTrainer</a> class for sequence-to-sequence tasks like translation or summarization. There is also the <a href="https://huggingface.co/docs/trl/main/en/sft_trainer#trl.SFTTrainer" rel="nofollow">SFTTrainer</a> class from the <a href="https://hf.co/docs/trl" rel="nofollow">TRL</a> library which wraps the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> class and is optimized for training language models like Llama-2 and Mistral with autoregressive techniques. <a href="https://huggingface.co/docs/trl/main/en/sft_trainer#trl.SFTTrainer" rel="nofollow">SFTTrainer</a> also supports features like sequence packing, LoRA, quantization, and DeepSpeed for efficiently scaling to any model size.',n,m,c,M,g='Feel free to check out the <a href="./main_classes/trainer">API reference</a> for these other <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a>-type classes to learn more about when to use which one. In general, <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> is the most versatile option and is appropriate for a broad spectrum of tasks. <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Seq2SeqTrainer">Seq2SeqTrainer</a> is designed for sequence-to-sequence tasks and <a href="https://huggingface.co/docs/trl/main/en/sft_trainer#trl.SFTTrainer" rel="nofollow">SFTTrainer</a> is designed for training language models.';return{c(){l=f("p"),l.innerHTML=o,n=r(),m=f("br"),c=r(),M=f("p"),M.innerHTML=g},l(U){l=T(U,"P",{"data-svelte-h":!0}),h(l)!=="svelte-4t04ja"&&(l.innerHTML=o),n=i(U),m=T(U,"BR",{}),c=i(U),M=T(U,"P",{"data-svelte-h":!0}),h(M)!=="svelte-jprrv"&&(M.innerHTML=g)},m(U,p){t(U,l,p),t(U,n,p),t(U,m,p),t(U,c,p),t(U,M,p)},p:$,d(U){U&&(a(l),a(n),a(m),a(c),a(M))}}}function ba(_){let l,o='Check out the <a href="./main_classes/logging">logging</a> API reference for more information about the different logging levels.';return{c(){l=f("p"),l.innerHTML=o},l(n){l=T(n,"P",{"data-svelte-h":!0}),h(l)!=="svelte-6jeeaq"&&(l.innerHTML=o)},m(n,m){t(n,l,m)},p:$,d(n){n&&a(l)}}}function Ua(_){let l,o='<a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> sets the log level separately for each node in the <code>Trainer.__init__()</code> method, so you may want to consider setting this sooner if you’re using other Transformers functionalities before creating the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> object.';return{c(){l=f("p"),l.innerHTML=o},l(n){l=T(n,"P",{"data-svelte-h":!0}),h(l)!=="svelte-1s8mp4t"&&(l.innerHTML=o)},m(n,m){t(n,l,m)},p:$,d(n){n&&a(l)}}}function Ia(_){let l,o;return l=new I({props:{code:"bXlfYXBwLnB5JTIwLi4uJTIwLS1sb2dfbGV2ZWwlMjB3YXJuaW5nJTIwLS1sb2dfbGV2ZWxfcmVwbGljYSUyMGVycm9y",highlighted:"my_app.py ... --log_level warning --log_level_replica error",wrap:!1}}),{c(){d(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,m){w(l,n,m),o=!0},p:$,i(n){o||(j(l.$$.fragment,n),o=!0)},o(n){J(l.$$.fragment,n),o=!1},d(n){y(l,n)}}}function Aa(_){let l,o="Add the <code>log_on_each_node 0</code> parameter for multi-node environments.",n,m,c;return m=new I({props:{code:"bXlfYXBwLnB5JTIwLi4uJTIwLS1sb2dfbGV2ZWwlMjB3YXJuaW5nJTIwLS1sb2dfbGV2ZWxfcmVwbGljYSUyMGVycm9yJTIwLS1sb2dfb25fZWFjaF9ub2RlJTIwMCUwQSUwQSUyMyUyMHNldCUyMHRvJTIwb25seSUyMHJlcG9ydCUyMGVycm9ycyUwQW15X2FwcC5weSUyMC4uLiUyMC0tbG9nX2xldmVsJTIwZXJyb3IlMjAtLWxvZ19sZXZlbF9yZXBsaWNhJTIwZXJyb3IlMjAtLWxvZ19vbl9lYWNoX25vZGUlMjAw",highlighted:`my_app.py ... --log_level warning --log_level_replica error --log_on_each_node 0

<span class="hljs-comment"># set to only report errors</span>
my_app.py ... --log_level error --log_level_replica error --log_on_each_node 0`,wrap:!1}}),{c(){l=f("p"),l.innerHTML=o,n=r(),d(m.$$.fragment)},l(M){l=T(M,"P",{"data-svelte-h":!0}),h(l)!=="svelte-86ubk0"&&(l.innerHTML=o),n=i(M),u(m.$$.fragment,M)},m(M,g){t(M,l,g),t(M,n,g),w(m,M,g),c=!0},p:$,i(M){c||(j(m.$$.fragment,M),c=!0)},o(M){J(m.$$.fragment,M),c=!1},d(M){M&&(a(l),a(n)),y(m,M)}}}function $a(_){let l,o,n,m;return l=new Ce({props:{id:"logging",option:"single node",$$slots:{default:[Ia]},$$scope:{ctx:_}}}),n=new Ce({props:{id:"logging",option:"multi-node",$$slots:{default:[Aa]},$$scope:{ctx:_}}}),{c(){d(l.$$.fragment),o=r(),d(n.$$.fragment)},l(c){u(l.$$.fragment,c),o=i(c),u(n.$$.fragment,c)},m(c,M){w(l,c,M),t(c,o,M),w(n,c,M),m=!0},p(c,M){const g={};M&2&&(g.$$scope={dirty:M,ctx:c}),l.$set(g);const U={};M&2&&(U.$$scope={dirty:M,ctx:c}),n.$set(U)},i(c){m||(j(l.$$.fragment,c),j(n.$$.fragment,c),m=!0)},o(c){J(l.$$.fragment,c),J(n.$$.fragment,c),m=!1},d(c){c&&a(o),y(l,c),y(n,c)}}}function Za(_){let l,o='Learn more about FSDP sharding strategies, CPU offloading, and more with the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> in the <a href="fsdp">Fully Sharded Data Parallel</a> guide.';return{c(){l=f("p"),l.innerHTML=o},l(n){l=T(n,"P",{"data-svelte-h":!0}),h(l)!=="svelte-1d131ns"&&(l.innerHTML=o)},m(n,m){t(n,l,m)},p:$,d(n){n&&a(l)}}}function Ca(_){let l,o;return l=new I({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMEFkaXN0cmlidXRlZF90eXBlJTNBJTIwTVVMVElfR1BVJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTBBZG93bmNhc3RfYmYxNiUzQSUyMCdubyclMEFncHVfaWRzJTNBJTIwYWxsJTBBbWFjaGluZV9yYW5rJTNBJTIwMCUyMCUyM2NoYW5nZSUyMHJhbmslMjBhcyUyMHBlciUyMHRoZSUyMG5vZGUlMEFtYWluX3Byb2Nlc3NfaXAlM0ElMjAxOTIuMTY4LjIwLjElMEFtYWluX3Byb2Nlc3NfcG9ydCUzQSUyMDk4OTglMEFtYWluX3RyYWluaW5nX2Z1bmN0aW9uJTNBJTIwbWFpbiUwQW1peGVkX3ByZWNpc2lvbiUzQSUyMGZwMTYlMEFudW1fbWFjaGluZXMlM0ElMjAyJTBBbnVtX3Byb2Nlc3NlcyUzQSUyMDglMEFyZHp2X2JhY2tlbmQlM0ElMjBzdGF0aWMlMEFzYW1lX25ldHdvcmslM0ElMjB0cnVlJTBBdHB1X2VudiUzQSUyMCU1QiU1RCUwQXRwdV91c2VfY2x1c3RlciUzQSUyMGZhbHNlJTBBdHB1X3VzZV9zdWRvJTNBJTIwZmFsc2UlMEF1c2VfY3B1JTNBJTIwZmFsc2U=",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>                                                                                             
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">MULTI_GPU</span>                                                                                                    
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">gpu_ids:</span> <span class="hljs-string">all</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span> <span class="hljs-comment">#change rank as per the node</span>
<span class="hljs-attr">main_process_ip:</span> <span class="hljs-number">192.168</span><span class="hljs-number">.20</span><span class="hljs-number">.1</span>
<span class="hljs-attr">main_process_port:</span> <span class="hljs-number">9898</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">fp16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">8</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){d(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,m){w(l,n,m),o=!0},p:$,i(n){o||(j(l.$$.fragment,n),o=!0)},o(n){J(l.$$.fragment,n),o=!1},d(n){y(l,n)}}}function Xa(_){let l,o;return l=new I({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkaXN0cmlidXRlZF90eXBlJTNBJTIwRlNEUCUwQWRvd25jYXN0X2JmMTYlM0ElMjAnbm8nJTBBZnNkcF9jb25maWclM0ElMEElMjAlMjBmc2RwX2F1dG9fd3JhcF9wb2xpY3klM0ElMjBUUkFOU0ZPUk1FUl9CQVNFRF9XUkFQJTBBJTIwJTIwZnNkcF9iYWNrd2FyZF9wcmVmZXRjaF9wb2xpY3klM0ElMjBCQUNLV0FSRF9QUkUlMEElMjAlMjBmc2RwX2ZvcndhcmRfcHJlZmV0Y2glM0ElMjB0cnVlJTBBJTIwJTIwZnNkcF9vZmZsb2FkX3BhcmFtcyUzQSUyMGZhbHNlJTBBJTIwJTIwZnNkcF9zaGFyZGluZ19zdHJhdGVneSUzQSUyMDElMEElMjAlMjBmc2RwX3N0YXRlX2RpY3RfdHlwZSUzQSUyMEZVTExfU1RBVEVfRElDVCUwQSUyMCUyMGZzZHBfc3luY19tb2R1bGVfc3RhdGVzJTNBJTIwdHJ1ZSUwQSUyMCUyMGZzZHBfdHJhbnNmb3JtZXJfbGF5ZXJfY2xzX3RvX3dyYXAlM0ElMjBCZXJ0TGF5ZXIlMEElMjAlMjBmc2RwX3VzZV9vcmlnX3BhcmFtcyUzQSUyMHRydWUlMEFtYWNoaW5lX3JhbmslM0ElMjAwJTBBbWFpbl90cmFpbmluZ19mdW5jdGlvbiUzQSUyMG1haW4lMEFtaXhlZF9wcmVjaXNpb24lM0ElMjBiZjE2JTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjAyJTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9lbnYlM0ElMjAlNUIlNUQlMEF0cHVfdXNlX2NsdXN0ZXIlM0ElMjBmYWxzZSUwQXRwdV91c2Vfc3VkbyUzQSUyMGZhbHNlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">FSDP</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">fsdp_config:</span>
  <span class="hljs-attr">fsdp_auto_wrap_policy:</span> <span class="hljs-string">TRANSFORMER_BASED_WRAP</span>
  <span class="hljs-attr">fsdp_backward_prefetch_policy:</span> <span class="hljs-string">BACKWARD_PRE</span>
  <span class="hljs-attr">fsdp_forward_prefetch:</span> <span class="hljs-literal">true</span>
  <span class="hljs-attr">fsdp_offload_params:</span> <span class="hljs-literal">false</span>
  <span class="hljs-attr">fsdp_sharding_strategy:</span> <span class="hljs-number">1</span>
  <span class="hljs-attr">fsdp_state_dict_type:</span> <span class="hljs-string">FULL_STATE_DICT</span>
  <span class="hljs-attr">fsdp_sync_module_states:</span> <span class="hljs-literal">true</span>
  <span class="hljs-attr">fsdp_transformer_layer_cls_to_wrap:</span> <span class="hljs-string">BertLayer</span>
  <span class="hljs-attr">fsdp_use_orig_params:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">bf16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){d(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,m){w(l,n,m),o=!0},p:$,i(n){o||(j(l.$$.fragment,n),o=!0)},o(n){J(l.$$.fragment,n),o=!1},d(n){y(l,n)}}}function Ba(_){let l,o;return l=new I({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTBBJTIwJTIwZGVlcHNwZWVkX2NvbmZpZ19maWxlJTNBJTIwJTJGaG9tZSUyRnVzZXIlMkZjb25maWdzJTJGZHNfemVybzNfY29uZmlnLmpzb24lMEElMjAlMjB6ZXJvM19pbml0X2ZsYWclM0ElMjB0cnVlJTBBZGlzdHJpYnV0ZWRfdHlwZSUzQSUyMERFRVBTUEVFRCUwQWRvd25jYXN0X2JmMTYlM0ElMjAnbm8nJTBBbWFjaGluZV9yYW5rJTNBJTIwMCUwQW1haW5fdHJhaW5pbmdfZnVuY3Rpb24lM0ElMjBtYWluJTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjA0JTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9lbnYlM0ElMjAlNUIlNUQlMEF0cHVfdXNlX2NsdXN0ZXIlM0ElMjBmYWxzZSUwQXRwdV91c2Vfc3VkbyUzQSUyMGZhbHNlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">deepspeed_config:</span>
  <span class="hljs-attr">deepspeed_config_file:</span> <span class="hljs-string">/home/user/configs/ds_zero3_config.json</span>
  <span class="hljs-attr">zero3_init_flag:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">DEEPSPEED</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">4</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){d(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,m){w(l,n,m),o=!0},p:$,i(n){o||(j(l.$$.fragment,n),o=!0)},o(n){J(l.$$.fragment,n),o=!1},d(n){y(l,n)}}}function va(_){let l,o;return l=new I({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTBBJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNBJTIwMSUwQSUyMCUyMGdyYWRpZW50X2NsaXBwaW5nJTNBJTIwMC43JTBBJTIwJTIwb2ZmbG9hZF9vcHRpbWl6ZXJfZGV2aWNlJTNBJTIwY3B1JTBBJTIwJTIwb2ZmbG9hZF9wYXJhbV9kZXZpY2UlM0ElMjBjcHUlMEElMjAlMjB6ZXJvM19pbml0X2ZsYWclM0ElMjB0cnVlJTBBJTIwJTIwemVyb19zdGFnZSUzQSUyMDIlMEFkaXN0cmlidXRlZF90eXBlJTNBJTIwREVFUFNQRUVEJTBBZG93bmNhc3RfYmYxNiUzQSUyMCdubyclMEFtYWNoaW5lX3JhbmslM0ElMjAwJTBBbWFpbl90cmFpbmluZ19mdW5jdGlvbiUzQSUyMG1haW4lMEFtaXhlZF9wcmVjaXNpb24lM0ElMjBiZjE2JTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjA0JTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9lbnYlM0ElMjAlNUIlNUQlMEF0cHVfdXNlX2NsdXN0ZXIlM0ElMjBmYWxzZSUwQXRwdV91c2Vfc3VkbyUzQSUyMGZhbHNlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>                                                                                             
<span class="hljs-attr">deepspeed_config:</span>                                                                                                              
  <span class="hljs-attr">gradient_accumulation_steps:</span> <span class="hljs-number">1</span>
  <span class="hljs-attr">gradient_clipping:</span> <span class="hljs-number">0.7</span>
  <span class="hljs-attr">offload_optimizer_device:</span> <span class="hljs-string">cpu</span>
  <span class="hljs-attr">offload_param_device:</span> <span class="hljs-string">cpu</span>
  <span class="hljs-attr">zero3_init_flag:</span> <span class="hljs-literal">true</span>
  <span class="hljs-attr">zero_stage:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">DEEPSPEED</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">bf16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">4</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){d(l.$$.fragment)},l(n){u(l.$$.fragment,n)},m(n,m){w(l,n,m),o=!0},p:$,i(n){o||(j(l.$$.fragment,n),o=!0)},o(n){J(l.$$.fragment,n),o=!1},d(n){y(l,n)}}}function Na(_){let l,o,n,m,c,M,g,U;return l=new Ce({props:{id:"config",option:"DistributedDataParallel",$$slots:{default:[Ca]},$$scope:{ctx:_}}}),n=new Ce({props:{id:"config",option:"FSDP",$$slots:{default:[Xa]},$$scope:{ctx:_}}}),c=new Ce({props:{id:"config",option:"DeepSpeed",$$slots:{default:[Ba]},$$scope:{ctx:_}}}),g=new Ce({props:{id:"config",option:"DeepSpeed with Accelerate plugin",$$slots:{default:[va]},$$scope:{ctx:_}}}),{c(){d(l.$$.fragment),o=r(),d(n.$$.fragment),m=r(),d(c.$$.fragment),M=r(),d(g.$$.fragment)},l(p){u(l.$$.fragment,p),o=i(p),u(n.$$.fragment,p),m=i(p),u(c.$$.fragment,p),M=i(p),u(g.$$.fragment,p)},m(p,b){w(l,p,b),t(p,o,b),w(n,p,b),t(p,m,b),w(c,p,b),t(p,M,b),w(g,p,b),U=!0},p(p,b){const W={};b&2&&(W.$$scope={dirty:b,ctx:p}),l.$set(W);const A={};b&2&&(A.$$scope={dirty:b,ctx:p}),n.$set(A);const Xe={};b&2&&(Xe.$$scope={dirty:b,ctx:p}),c.$set(Xe);const V={};b&2&&(V.$$scope={dirty:b,ctx:p}),g.$set(V)},i(p){U||(j(l.$$.fragment,p),j(n.$$.fragment,p),j(c.$$.fragment,p),j(g.$$.fragment,p),U=!0)},o(p){J(l.$$.fragment,p),J(n.$$.fragment,p),J(c.$$.fragment,p),J(g.$$.fragment,p),U=!1},d(p){p&&(a(o),a(m),a(M)),y(l,p),y(n,p),y(c,p),y(g,p)}}}function Wa(_){let l,o,n,m,c,M,g,U='The <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> is a complete training and evaluation loop for PyTorch models implemented in the Transformers library. You only need to pass it the necessary pieces for training (model, tokenizer, dataset, evaluation function, training hyperparameters, etc.), and the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> class takes care of the rest. This makes it easier to start training faster without manually writing your own training loop. But at the same time, <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> is very customizable and offers a ton of training options so you can tailor it to your exact training needs.',p,b,W,A,Xe='Before you start, make sure <a href="https://hf.co/docs/accelerate" rel="nofollow">Accelerate</a> - a library for enabling and running PyTorch training across distributed environments - is installed.',V,k,ve,E,vs='This guide provides an overview of the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> class.',Ne,G,We,F,Ns='<a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> includes all the code you’ll find in a basic training loop:',Ve,R,Ws='<li>perform a training step to calculate the loss</li> <li>calculate the gradients with the <a href="https://huggingface.co/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.backward" rel="nofollow">backward</a> method</li> <li>update the weights based on the gradients</li> <li>repeat this process until you’ve reached a predetermined number of epochs</li>',ke,S,Vs='The <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> class abstracts all of this code away so you don’t have to worry about manually writing a training loop every time or if you’re just getting started with PyTorch and training. You only need to provide the essential components required for training, such as a model and a dataset, and the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> class handles everything else.',Ee,Y,ks='If you want to specify any training options or hyperparameters, you can find them in the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a> class. For example, let’s define where to save the model in <code>output_dir</code> and push the model to the Hub after training with <code>push_to_hub=True</code>.',Ge,H,Fe,z,Es='Pass <code>training_args</code> to the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> along with a model, dataset, something to preprocess the dataset with (depending on your data type it could be a tokenizer, feature extractor or image processor), a data collator, and a function to compute the metrics you want to track during training.',Re,Q,Gs='Finally, call <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.train">train()</a> to start training!',Se,L,Ye,x,He,P,Fs='The <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> class saves your model checkpoints to the directory specified in the <code>output_dir</code> parameter of <a href="/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>. You’ll find the checkpoints saved in a <code>checkpoint-000</code> subfolder where the numbers at the end correspond to the training step. Saving checkpoints are useful for resuming training later.',ze,D,Qe,q,Rs='You can save your checkpoints (the optimizer state is not saved by default) to the Hub by setting <code>push_to_hub=True</code> in <a href="/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a> to commit and push them. Other options for deciding how your checkpoints are saved are set up in the <a href="https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments.hub_strategy" rel="nofollow"><code>hub_strategy</code></a> parameter:',Le,K,Ss="<li><code>hub_strategy=&quot;checkpoint&quot;</code> pushes the latest checkpoint to a subfolder named “last-checkpoint” from which you can resume training</li> <li><code>hub_strategy=&quot;all_checkpoints&quot;</code> pushes all checkpoints to the directory defined in <code>output_dir</code> (you’ll see one checkpoint per folder in your model repository)</li>",xe,O,Ys='When you resume training from a checkpoint, the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> tries to keep the Python, NumPy, and PyTorch RNG states the same as they were when the checkpoint was saved. But because PyTorch has various non-deterministic default settings, the RNG states aren’t guaranteed to be the same. If you want to enable full determinism, take a look at the <a href="https://pytorch.org/docs/stable/notes/randomness#controlling-sources-of-randomness" rel="nofollow">Controlling sources of randomness</a> guide to learn what you can enable to make your training fully deterministic. Keep in mind though that by making certain settings deterministic, training may be slower.',Pe,ee,De,se,Hs='While the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> class is designed to be accessible and easy-to-use, it also offers a lot of customizability for more adventurous users. Many of the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a>’s method can be subclassed and overridden to support the functionality you want, without having to rewrite the entire training loop from scratch to accommodate it. These methods include:',qe,ae,zs='<li><a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.get_train_dataloader">get_train_dataloader()</a> creates a training DataLoader</li> <li><a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.get_eval_dataloader">get_eval_dataloader()</a> creates an evaluation DataLoader</li> <li><a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.get_test_dataloader">get_test_dataloader()</a> creates a test DataLoader</li> <li><a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.log">log()</a> logs information on the various objects that watch training</li> <li><a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.create_optimizer_and_scheduler">create_optimizer_and_scheduler()</a> creates an optimizer and learning rate scheduler if they weren’t passed in the <code>__init__</code>; these can also be separately customized with <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.create_optimizer">create_optimizer()</a> and <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.create_scheduler">create_scheduler()</a> respectively</li> <li><a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.compute_loss">compute_loss()</a> computes the loss on a batch of training inputs</li> <li><a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.training_step">training_step()</a> performs the training step</li> <li><a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.prediction_step">prediction_step()</a> performs the prediction and test step</li> <li><a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.evaluate">evaluate()</a> evaluates the model and returns the evaluation metrics</li> <li><a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.predict">predict()</a> makes predictions (with metrics if labels are available) on the test set</li>',Ke,le,Qs='For example, if you want to customize the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.compute_loss">compute_loss()</a> method to use a weighted loss instead.',Oe,te,es,ne,ss,re,Ls='Another option for customizing the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> is to use <a href="callbacks">callbacks</a>. Callbacks <em>don’t change</em> anything in the training loop. They inspect the training loop state and then execute some action (early stopping, logging results, etc.) depending on the state. In other words, a callback can’t be used to implement something like a custom loss function and you’ll need to subclass and override the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.compute_loss">compute_loss()</a> method for that.',as,ie,xs="For example, if you want to add an early stopping callback to the training loop after 10 steps.",ls,oe,ts,ce,Ps='Then pass it to the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a>’s <code>callback</code> parameter.',ns,pe,rs,me,is,Z,os,Me,Ds='The <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> is set to <code>logging.INFO</code> by default which reports errors, warnings, and other basic information. A <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> replica - in distributed environments - is set to <code>logging.WARNING</code> which only reports errors and warnings. You can change the logging level with the <a href="https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments.log_level" rel="nofollow"><code>log_level</code></a> and <a href="https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments.log_level_replica" rel="nofollow"><code>log_level_replica</code></a> parameters in <a href="/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>.',cs,fe,qs='To configure the log level setting for each node, use the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments.log_on_each_node" rel="nofollow"><code>log_on_each_node</code></a> parameter to determine whether to use the log level on each node or only on the main node.',ps,C,ms,Te,Ks="For example, to set your main code and modules to use the same log level according to each node:",Ms,he,fs,de,Os="Use different combinations of <code>log_level</code> and <code>log_level_replica</code> to configure what gets logged on each of the nodes.",Ts,X,hs,ue,ds,we,ea='<a href="https://hf.co/papers/2310.05914" rel="nofollow">NEFTune</a> is a technique that can improve performance by adding noise to the embedding vectors during training. To enable it in <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a>, set the <code>neftune_noise_alpha</code> parameter in <a href="/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a> to control how much noise is added.',us,je,ws,Je,sa="NEFTune is disabled after training to restore the original embedding layer to avoid any unexpected behavior.",js,ye,Js,_e,aa='The <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> class is powered by <a href="https://hf.co/docs/accelerate" rel="nofollow">Accelerate</a>, a library for easily training PyTorch models in distributed environments with support for integrations such as <a href="https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api/" rel="nofollow">FullyShardedDataParallel (FSDP)</a> and <a href="https://www.deepspeed.ai/" rel="nofollow">DeepSpeed</a>.',ys,B,_s,ge,la='To use Accelerate with <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a>, run the <a href="https://huggingface.co/docs/accelerate/package_reference/cli#accelerate-config" rel="nofollow"><code>accelerate.config</code></a> command to set up training for your training environment. This command creates a <code>config_file.yaml</code> that’ll be used when you launch your training script. For example, some example configurations you can setup are:',gs,v,bs,be,ta='The <a href="https://huggingface.co/docs/accelerate/package_reference/cli#accelerate-launch" rel="nofollow"><code>accelerate_launch</code></a> command is the recommended way to launch your training script on a distributed system with Accelerate and <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> with the parameters specified in <code>config_file.yaml</code>. This file is saved to the Accelerate cache folder and automatically loaded when you run <code>accelerate_launch</code>.',Us,Ue,na='For example, to run the <a href="https://github.com/huggingface/transformers/blob/f4db565b695582891e43a5e042e5d318e28f20b8/examples/pytorch/text-classification/run_glue.py#L4" rel="nofollow">run_glue.py</a> training script with the FSDP configuration:',Is,Ie,As,Ae,ra="You could also specify the parameters from the <code>config_file.yaml</code> file directly in the command line:",$s,$e,Zs,Ze,ia='Check out the <a href="https://huggingface.co/docs/accelerate/basic_tutorials/launch" rel="nofollow">Launching your Accelerate scripts</a> tutorial to learn more about <code>accelerate_launch</code> and custom configurations.',Cs,Be,Xs;return c=new N({props:{title:"Trainer",local:"trainer",headingTag:"h1"}}),b=new Bs({props:{$$slots:{default:[ga]},$$scope:{ctx:_}}}),k=new I({props:{code:"cGlwJTIwaW5zdGFsbCUyMGFjY2VsZXJhdGUlMEElMEElMjMlMjB1cGdyYWRlJTBBcGlwJTIwaW5zdGFsbCUyMGFjY2VsZXJhdGUlMjAtLXVwZ3JhZGU=",highlighted:`pip install accelerate

<span class="hljs-comment"># upgrade</span>
pip install accelerate --upgrade`,wrap:!1}}),G=new N({props:{title:"Basic usage",local:"basic-usage",headingTag:"h2"}}),H=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJ5b3VyLW1vZGVsJTIyJTJDJTBBJTIwJTIwJTIwJTIwbGVhcm5pbmdfcmF0ZSUzRDJlLTUlMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfZXZhbF9iYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMiUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRDAuMDElMkMlMEElMjAlMjAlMjAlMjBldmFsdWF0aW9uX3N0cmF0ZWd5JTNEJTIyZXBvY2glMjIlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3N0cmF0ZWd5JTNEJTIyZXBvY2glMjIlMkMlMEElMjAlMjAlMjAlMjBsb2FkX2Jlc3RfbW9kZWxfYXRfZW5kJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHB1c2hfdG9faHViJTNEVHJ1ZSUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

training_args = TrainingArguments(
    output_dir=<span class="hljs-string">&quot;your-model&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    per_device_train_batch_size=<span class="hljs-number">16</span>,
    per_device_eval_batch_size=<span class="hljs-number">16</span>,
    num_train_epochs=<span class="hljs-number">2</span>,
    weight_decay=<span class="hljs-number">0.01</span>,
    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    save_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    load_best_model_at_end=<span class="hljs-literal">True</span>,
    push_to_hub=<span class="hljs-literal">True</span>,
)`,wrap:!1}}),L=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRlc3QlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2NvbGxhdG9yJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMGNvbXB1dGVfbWV0cmljcyUzRGNvbXB1dGVfbWV0cmljcyUyQyUwQSklMEElMEF0cmFpbmVyLnRyYWluKCk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=dataset[<span class="hljs-string">&quot;test&quot;</span>],
    tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_metrics,
)

trainer.train()`,wrap:!1}}),x=new N({props:{title:"Checkpoints",local:"checkpoints",headingTag:"h3"}}),D=new I({props:{code:"JTIzJTIwcmVzdW1lJTIwZnJvbSUyMGxhdGVzdCUyMGNoZWNrcG9pbnQlMEF0cmFpbmVyLnRyYWluKHJlc3VtZV9mcm9tX2NoZWNrcG9pbnQlM0RUcnVlKSUwQSUwQSUyMyUyMHJlc3VtZSUyMGZyb20lMjBzcGVjaWZpYyUyMGNoZWNrcG9pbnQlMjBzYXZlZCUyMGluJTIwb3V0cHV0JTIwZGlyZWN0b3J5JTBBdHJhaW5lci50cmFpbihyZXN1bWVfZnJvbV9jaGVja3BvaW50JTNEJTIyeW91ci1tb2RlbCUyRmNoZWNrcG9pbnQtMTAwMCUyMik=",highlighted:`<span class="hljs-comment"># resume from latest checkpoint</span>
trainer.train(resume_from_checkpoint=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># resume from specific checkpoint saved in output directory</span>
trainer.train(resume_from_checkpoint=<span class="hljs-string">&quot;your-model/checkpoint-1000&quot;</span>)`,wrap:!1}}),ee=new N({props:{title:"Customize the Trainer",local:"customize-the-trainer",headingTag:"h2"}}),te=new I({props:{code:"ZnJvbSUyMHRvcmNoJTIwaW1wb3J0JTIwbm4lMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVHJhaW5lciUwQSUwQWNsYXNzJTIwQ3VzdG9tVHJhaW5lcihUcmFpbmVyKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGNvbXB1dGVfbG9zcyhzZWxmJTJDJTIwbW9kZWwlMkMlMjBpbnB1dHMlMkMlMjByZXR1cm5fb3V0cHV0cyUzREZhbHNlKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhYmVscyUyMCUzRCUyMGlucHV0cy5wb3AoJTIybGFiZWxzJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMGZvcndhcmQlMjBwYXNzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvZ2l0cyUyMCUzRCUyMG91dHB1dHMuZ2V0KCUyMmxvZ2l0cyUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBjb21wdXRlJTIwY3VzdG9tJTIwbG9zcyUyMGZvciUyMDMlMjBsYWJlbHMlMjB3aXRoJTIwZGlmZmVyZW50JTIwd2VpZ2h0cyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvc3NfZmN0JTIwJTNEJTIwbm4uQ3Jvc3NFbnRyb3B5TG9zcyh3ZWlnaHQlM0R0b3JjaC50ZW5zb3IoJTVCMS4wJTJDJTIwMi4wJTJDJTIwMy4wJTVEJTJDJTIwZGV2aWNlJTNEbW9kZWwuZGV2aWNlKSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mY3QobG9naXRzLnZpZXcoLTElMkMlMjBzZWxmLm1vZGVsLmNvbmZpZy5udW1fbGFiZWxzKSUyQyUyMGxhYmVscy52aWV3KC0xKSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAobG9zcyUyQyUyMG91dHB1dHMpJTIwaWYlMjByZXR1cm5fb3V0cHV0cyUyMGVsc2UlMjBsb3Nz",highlighted:`<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> nn
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

<span class="hljs-keyword">class</span> <span class="hljs-title class_">CustomTrainer</span>(<span class="hljs-title class_ inherited__">Trainer</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_loss</span>(<span class="hljs-params">self, model, inputs, return_outputs=<span class="hljs-literal">False</span></span>):
        labels = inputs.pop(<span class="hljs-string">&quot;labels&quot;</span>)
        <span class="hljs-comment"># forward pass</span>
        outputs = model(**inputs)
        logits = outputs.get(<span class="hljs-string">&quot;logits&quot;</span>)
        <span class="hljs-comment"># compute custom loss for 3 labels with different weights</span>
        loss_fct = nn.CrossEntropyLoss(weight=torch.tensor([<span class="hljs-number">1.0</span>, <span class="hljs-number">2.0</span>, <span class="hljs-number">3.0</span>], device=model.device))
        loss = loss_fct(logits.view(-<span class="hljs-number">1</span>, self.model.config.num_labels), labels.view(-<span class="hljs-number">1</span>))
        <span class="hljs-keyword">return</span> (loss, outputs) <span class="hljs-keyword">if</span> return_outputs <span class="hljs-keyword">else</span> loss`,wrap:!1}}),ne=new N({props:{title:"Callbacks",local:"callbacks",headingTag:"h3"}}),oe=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXJDYWxsYmFjayUwQSUwQWNsYXNzJTIwRWFybHlTdG9wcGluZ0NhbGxiYWNrKFRyYWluZXJDYWxsYmFjayklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBfX2luaXRfXyhzZWxmJTJDJTIwbnVtX3N0ZXBzJTNEMTApJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5udW1fc3RlcHMlMjAlM0QlMjBudW1fc3RlcHMlMEElMjAlMjAlMjAlMjAlMEElMjAlMjAlMjAlMjBkZWYlMjBvbl9zdGVwX2VuZChzZWxmJTJDJTIwYXJncyUyQyUyMHN0YXRlJTJDJTIwY29udHJvbCUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwc3RhdGUuZ2xvYmFsX3N0ZXAlMjAlM0UlM0QlMjBzZWxmLm51bV9zdGVwcyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMCU3QiUyMnNob3VsZF90cmFpbmluZ19zdG9wJTIyJTNBJTIwVHJ1ZSU3RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAlN0IlN0Q=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainerCallback

<span class="hljs-keyword">class</span> <span class="hljs-title class_">EarlyStoppingCallback</span>(<span class="hljs-title class_ inherited__">TrainerCallback</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, num_steps=<span class="hljs-number">10</span></span>):
        self.num_steps = num_steps
    
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">on_step_end</span>(<span class="hljs-params">self, args, state, control, **kwargs</span>):
        <span class="hljs-keyword">if</span> state.global_step &gt;= self.num_steps:
            <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;should_training_stop&quot;</span>: <span class="hljs-literal">True</span>}
        <span class="hljs-keyword">else</span>:
            <span class="hljs-keyword">return</span> {}`,wrap:!1}}),pe=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRlc3QlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2NvbGxhdG9yJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMGNvbXB1dGVfbWV0cmljcyUzRGNvbXB1dGVfbWV0cmljcyUyQyUwQSUyMCUyMCUyMCUyMGNhbGxiYWNrJTNEJTVCRWFybHlTdG9wcGluZ0NhbGxiYWNrKCklNUQlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=dataset[<span class="hljs-string">&quot;test&quot;</span>],
    tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_metrics,
    callback=[EarlyStoppingCallback()],
)`,wrap:!1}}),me=new N({props:{title:"Logging",local:"logging",headingTag:"h2"}}),Z=new Bs({props:{$$slots:{default:[ba]},$$scope:{ctx:_}}}),C=new Bs({props:{$$slots:{default:[Ua]},$$scope:{ctx:_}}}),he=new I({props:{code:"bG9nZ2VyJTIwJTNEJTIwbG9nZ2luZy5nZXRMb2dnZXIoX19uYW1lX18pJTBBJTBBbG9nZ2luZy5iYXNpY0NvbmZpZyglMEElMjAlMjAlMjAlMjBmb3JtYXQlM0QlMjIlMjUoYXNjdGltZSlzJTIwLSUyMCUyNShsZXZlbG5hbWUpcyUyMC0lMjAlMjUobmFtZSlzJTIwLSUyMCUyNShtZXNzYWdlKXMlMjIlMkMlMEElMjAlMjAlMjAlMjBkYXRlZm10JTNEJTIyJTI1bSUyRiUyNWQlMkYlMjVZJTIwJTI1SCUzQSUyNU0lM0ElMjVTJTIyJTJDJTBBJTIwJTIwJTIwJTIwaGFuZGxlcnMlM0QlNUJsb2dnaW5nLlN0cmVhbUhhbmRsZXIoc3lzLnN0ZG91dCklNUQlMkMlMEEpJTBBJTBBbG9nX2xldmVsJTIwJTNEJTIwdHJhaW5pbmdfYXJncy5nZXRfcHJvY2Vzc19sb2dfbGV2ZWwoKSUwQWxvZ2dlci5zZXRMZXZlbChsb2dfbGV2ZWwpJTBBZGF0YXNldHMudXRpbHMubG9nZ2luZy5zZXRfdmVyYm9zaXR5KGxvZ19sZXZlbCklMEF0cmFuc2Zvcm1lcnMudXRpbHMubG9nZ2luZy5zZXRfdmVyYm9zaXR5KGxvZ19sZXZlbCklMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciguLi4p",highlighted:`logger = logging.getLogger(__name__)

logging.basicConfig(
    <span class="hljs-built_in">format</span>=<span class="hljs-string">&quot;%(asctime)s - %(levelname)s - %(name)s - %(message)s&quot;</span>,
    datefmt=<span class="hljs-string">&quot;%m/%d/%Y %H:%M:%S&quot;</span>,
    handlers=[logging.StreamHandler(sys.stdout)],
)

log_level = training_args.get_process_log_level()
logger.setLevel(log_level)
datasets.utils.logging.set_verbosity(log_level)
transformers.utils.logging.set_verbosity(log_level)

trainer = Trainer(...)`,wrap:!1}}),X=new da({props:{id:"logging",options:["single node","multi-node"],$$slots:{default:[$a]},$$scope:{ctx:_}}}),ue=new N({props:{title:"NEFTune",local:"neftune",headingTag:"h2"}}),je=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTJDJTIwVHJhaW5lciUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyguLi4lMkMlMjBuZWZ0dW5lX25vaXNlX2FscGhhJTNEMC4xKSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKC4uLiUyQyUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, Trainer

training_args = TrainingArguments(..., neftune_noise_alpha=<span class="hljs-number">0.1</span>)
trainer = Trainer(..., args=training_args)`,wrap:!1}}),ye=new N({props:{title:"Accelerate and Trainer",local:"accelerate-and-trainer",headingTag:"h2"}}),B=new Bs({props:{$$slots:{default:[Za]},$$scope:{ctx:_}}}),v=new da({props:{id:"config",options:["DistributedDataParallel","FSDP","DeepSpeed","DeepSpeed with Accelerate plugin"],$$slots:{default:[Na]},$$scope:{ctx:_}}}),Ie=new I({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMCU1QyUwQSUyMCUyMCUyMCUyMC4lMkZleGFtcGxlcyUyRnB5dG9yY2glMkZ0ZXh0LWNsYXNzaWZpY2F0aW9uJTJGcnVuX2dsdWUucHklMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1vZGVsX25hbWVfb3JfcGF0aCUyMGdvb2dsZS1iZXJ0JTJGYmVydC1iYXNlLWNhc2VkJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS10YXNrX25hbWUlMjAlMjRUQVNLX05BTUUlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWRvX3RyYWluJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kb19ldmFsJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tYXhfc2VxX2xlbmd0aCUyMDEyOCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwMTYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxlYXJuaW5nX3JhdGUlMjA1ZS01JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1udW1fdHJhaW5fZXBvY2hzJTIwMyUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tb3V0cHV0X2RpciUyMCUyRnRtcCUyRiUyNFRBU0tfTkFNRSUyRiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tb3ZlcndyaXRlX291dHB1dF9kaXI=",highlighted:`accelerate launch \\
    ./examples/pytorch/text-classification/run_glue.py \\
    --model_name_or_path google-bert/bert-base-cased \\
    --task_name <span class="hljs-variable">$TASK_NAME</span> \\
    --do_train \\
    --do_eval \\
    --max_seq_length 128 \\
    --per_device_train_batch_size 16 \\
    --learning_rate 5e-5 \\
    --num_train_epochs 3 \\
    --output_dir /tmp/<span class="hljs-variable">$TASK_NAME</span>/ \\
    --overwrite_output_dir`,wrap:!1}}),$e=new I({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tbnVtX3Byb2Nlc3NlcyUzRDIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXVzZV9mc2RwJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1taXhlZF9wcmVjaXNpb24lM0RiZjE2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1mc2RwX2F1dG9fd3JhcF9wb2xpY3klM0RUUkFOU0ZPUk1FUl9CQVNFRF9XUkFQJTIwJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1mc2RwX3RyYW5zZm9ybWVyX2xheWVyX2Nsc190b193cmFwJTNEJTIyQmVydExheWVyJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1mc2RwX3NoYXJkaW5nX3N0cmF0ZWd5JTNEMSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZnNkcF9zdGF0ZV9kaWN0X3R5cGUlM0RGVUxMX1NUQVRFX0RJQ1QlMjAlNUMlMEElMjAlMjAlMjAlMjAuJTJGZXhhbXBsZXMlMkZweXRvcmNoJTJGdGV4dC1jbGFzc2lmaWNhdGlvbiUyRnJ1bl9nbHVlLnB5JTBBJTIwJTIwJTIwJTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBnb29nbGUtYmVydCUyRmJlcnQtYmFzZS1jYXNlZCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tdGFza19uYW1lJTIwJTI0VEFTS19OQU1FJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kb190cmFpbiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZG9fZXZhbCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbWF4X3NlcV9sZW5ndGglMjAxMjglMjAlNUMlMEElMjAlMjAlMjAlMjAtLXBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUyMDE2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sZWFybmluZ19yYXRlJTIwNWUtNSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbnVtX3RyYWluX2Vwb2NocyUyMDMlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW91dHB1dF9kaXIlMjAlMkZ0bXAlMkYlMjRUQVNLX05BTUUlMkYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW92ZXJ3cml0ZV9vdXRwdXRfZGly",highlighted:`accelerate launch --num_processes=2 \\
    --use_fsdp \\
    --mixed_precision=bf16 \\
    --fsdp_auto_wrap_policy=TRANSFORMER_BASED_WRAP  \\
    --fsdp_transformer_layer_cls_to_wrap=<span class="hljs-string">&quot;BertLayer&quot;</span> \\
    --fsdp_sharding_strategy=1 \\
    --fsdp_state_dict_type=FULL_STATE_DICT \\
    ./examples/pytorch/text-classification/run_glue.py
    --model_name_or_path google-bert/bert-base-cased \\
    --task_name <span class="hljs-variable">$TASK_NAME</span> \\
    --do_train \\
    --do_eval \\
    --max_seq_length 128 \\
    --per_device_train_batch_size 16 \\
    --learning_rate 5e-5 \\
    --num_train_epochs 3 \\
    --output_dir /tmp/<span class="hljs-variable">$TASK_NAME</span>/ \\
    --overwrite_output_dir`,wrap:!1}}),{c(){l=f("meta"),o=r(),n=f("p"),m=r(),d(c.$$.fragment),M=r(),g=f("p"),g.innerHTML=U,p=r(),d(b.$$.fragment),W=r(),A=f("p"),A.innerHTML=Xe,V=r(),d(k.$$.fragment),ve=r(),E=f("p"),E.innerHTML=vs,Ne=r(),d(G.$$.fragment),We=r(),F=f("p"),F.innerHTML=Ns,Ve=r(),R=f("ol"),R.innerHTML=Ws,ke=r(),S=f("p"),S.innerHTML=Vs,Ee=r(),Y=f("p"),Y.innerHTML=ks,Ge=r(),d(H.$$.fragment),Fe=r(),z=f("p"),z.innerHTML=Es,Re=r(),Q=f("p"),Q.innerHTML=Gs,Se=r(),d(L.$$.fragment),Ye=r(),d(x.$$.fragment),He=r(),P=f("p"),P.innerHTML=Fs,ze=r(),d(D.$$.fragment),Qe=r(),q=f("p"),q.innerHTML=Rs,Le=r(),K=f("ul"),K.innerHTML=Ss,xe=r(),O=f("p"),O.innerHTML=Ys,Pe=r(),d(ee.$$.fragment),De=r(),se=f("p"),se.innerHTML=Hs,qe=r(),ae=f("ul"),ae.innerHTML=zs,Ke=r(),le=f("p"),le.innerHTML=Qs,Oe=r(),d(te.$$.fragment),es=r(),d(ne.$$.fragment),ss=r(),re=f("p"),re.innerHTML=Ls,as=r(),ie=f("p"),ie.textContent=xs,ls=r(),d(oe.$$.fragment),ts=r(),ce=f("p"),ce.innerHTML=Ps,ns=r(),d(pe.$$.fragment),rs=r(),d(me.$$.fragment),is=r(),d(Z.$$.fragment),os=r(),Me=f("p"),Me.innerHTML=Ds,cs=r(),fe=f("p"),fe.innerHTML=qs,ps=r(),d(C.$$.fragment),ms=r(),Te=f("p"),Te.textContent=Ks,Ms=r(),d(he.$$.fragment),fs=r(),de=f("p"),de.innerHTML=Os,Ts=r(),d(X.$$.fragment),hs=r(),d(ue.$$.fragment),ds=r(),we=f("p"),we.innerHTML=ea,us=r(),d(je.$$.fragment),ws=r(),Je=f("p"),Je.textContent=sa,js=r(),d(ye.$$.fragment),Js=r(),_e=f("p"),_e.innerHTML=aa,ys=r(),d(B.$$.fragment),_s=r(),ge=f("p"),ge.innerHTML=la,gs=r(),d(v.$$.fragment),bs=r(),be=f("p"),be.innerHTML=ta,Us=r(),Ue=f("p"),Ue.innerHTML=na,Is=r(),d(Ie.$$.fragment),As=r(),Ae=f("p"),Ae.innerHTML=ra,$s=r(),d($e.$$.fragment),Zs=r(),Ze=f("p"),Ze.innerHTML=ia,Cs=r(),Be=f("p"),this.h()},l(e){const s=ya("svelte-u9bgzb",document.head);l=T(s,"META",{name:!0,content:!0}),s.forEach(a),o=i(e),n=T(e,"P",{}),Ta(n).forEach(a),m=i(e),u(c.$$.fragment,e),M=i(e),g=T(e,"P",{"data-svelte-h":!0}),h(g)!=="svelte-j9m5mw"&&(g.innerHTML=U),p=i(e),u(b.$$.fragment,e),W=i(e),A=T(e,"P",{"data-svelte-h":!0}),h(A)!=="svelte-17cwfvo"&&(A.innerHTML=Xe),V=i(e),u(k.$$.fragment,e),ve=i(e),E=T(e,"P",{"data-svelte-h":!0}),h(E)!=="svelte-k1ibm0"&&(E.innerHTML=vs),Ne=i(e),u(G.$$.fragment,e),We=i(e),F=T(e,"P",{"data-svelte-h":!0}),h(F)!=="svelte-ix9zcf"&&(F.innerHTML=Ns),Ve=i(e),R=T(e,"OL",{"data-svelte-h":!0}),h(R)!=="svelte-300ub0"&&(R.innerHTML=Ws),ke=i(e),S=T(e,"P",{"data-svelte-h":!0}),h(S)!=="svelte-1u9dh7r"&&(S.innerHTML=Vs),Ee=i(e),Y=T(e,"P",{"data-svelte-h":!0}),h(Y)!=="svelte-1x9mezm"&&(Y.innerHTML=ks),Ge=i(e),u(H.$$.fragment,e),Fe=i(e),z=T(e,"P",{"data-svelte-h":!0}),h(z)!=="svelte-10gnbw8"&&(z.innerHTML=Es),Re=i(e),Q=T(e,"P",{"data-svelte-h":!0}),h(Q)!=="svelte-1nfe8g7"&&(Q.innerHTML=Gs),Se=i(e),u(L.$$.fragment,e),Ye=i(e),u(x.$$.fragment,e),He=i(e),P=T(e,"P",{"data-svelte-h":!0}),h(P)!=="svelte-rqef3x"&&(P.innerHTML=Fs),ze=i(e),u(D.$$.fragment,e),Qe=i(e),q=T(e,"P",{"data-svelte-h":!0}),h(q)!=="svelte-pouhxr"&&(q.innerHTML=Rs),Le=i(e),K=T(e,"UL",{"data-svelte-h":!0}),h(K)!=="svelte-1f7sie2"&&(K.innerHTML=Ss),xe=i(e),O=T(e,"P",{"data-svelte-h":!0}),h(O)!=="svelte-g35nbb"&&(O.innerHTML=Ys),Pe=i(e),u(ee.$$.fragment,e),De=i(e),se=T(e,"P",{"data-svelte-h":!0}),h(se)!=="svelte-v8jct0"&&(se.innerHTML=Hs),qe=i(e),ae=T(e,"UL",{"data-svelte-h":!0}),h(ae)!=="svelte-jlx7vb"&&(ae.innerHTML=zs),Ke=i(e),le=T(e,"P",{"data-svelte-h":!0}),h(le)!=="svelte-q8uvua"&&(le.innerHTML=Qs),Oe=i(e),u(te.$$.fragment,e),es=i(e),u(ne.$$.fragment,e),ss=i(e),re=T(e,"P",{"data-svelte-h":!0}),h(re)!=="svelte-1gmmzdh"&&(re.innerHTML=Ls),as=i(e),ie=T(e,"P",{"data-svelte-h":!0}),h(ie)!=="svelte-ckk1ok"&&(ie.textContent=xs),ls=i(e),u(oe.$$.fragment,e),ts=i(e),ce=T(e,"P",{"data-svelte-h":!0}),h(ce)!=="svelte-1xnk79s"&&(ce.innerHTML=Ps),ns=i(e),u(pe.$$.fragment,e),rs=i(e),u(me.$$.fragment,e),is=i(e),u(Z.$$.fragment,e),os=i(e),Me=T(e,"P",{"data-svelte-h":!0}),h(Me)!=="svelte-ryvkp4"&&(Me.innerHTML=Ds),cs=i(e),fe=T(e,"P",{"data-svelte-h":!0}),h(fe)!=="svelte-3l1nfv"&&(fe.innerHTML=qs),ps=i(e),u(C.$$.fragment,e),ms=i(e),Te=T(e,"P",{"data-svelte-h":!0}),h(Te)!=="svelte-14qffqk"&&(Te.textContent=Ks),Ms=i(e),u(he.$$.fragment,e),fs=i(e),de=T(e,"P",{"data-svelte-h":!0}),h(de)!=="svelte-fy4z00"&&(de.innerHTML=Os),Ts=i(e),u(X.$$.fragment,e),hs=i(e),u(ue.$$.fragment,e),ds=i(e),we=T(e,"P",{"data-svelte-h":!0}),h(we)!=="svelte-qcr4kb"&&(we.innerHTML=ea),us=i(e),u(je.$$.fragment,e),ws=i(e),Je=T(e,"P",{"data-svelte-h":!0}),h(Je)!=="svelte-1tyg2gl"&&(Je.textContent=sa),js=i(e),u(ye.$$.fragment,e),Js=i(e),_e=T(e,"P",{"data-svelte-h":!0}),h(_e)!=="svelte-5w6u54"&&(_e.innerHTML=aa),ys=i(e),u(B.$$.fragment,e),_s=i(e),ge=T(e,"P",{"data-svelte-h":!0}),h(ge)!=="svelte-15ach0g"&&(ge.innerHTML=la),gs=i(e),u(v.$$.fragment,e),bs=i(e),be=T(e,"P",{"data-svelte-h":!0}),h(be)!=="svelte-1tpra1j"&&(be.innerHTML=ta),Us=i(e),Ue=T(e,"P",{"data-svelte-h":!0}),h(Ue)!=="svelte-cneufk"&&(Ue.innerHTML=na),Is=i(e),u(Ie.$$.fragment,e),As=i(e),Ae=T(e,"P",{"data-svelte-h":!0}),h(Ae)!=="svelte-13yow9q"&&(Ae.innerHTML=ra),$s=i(e),u($e.$$.fragment,e),Zs=i(e),Ze=T(e,"P",{"data-svelte-h":!0}),h(Ze)!=="svelte-zjyylh"&&(Ze.innerHTML=ia),Cs=i(e),Be=T(e,"P",{}),Ta(Be).forEach(a),this.h()},h(){ha(l,"name","hf:doc:metadata"),ha(l,"content",Va)},m(e,s){_a(document.head,l),t(e,o,s),t(e,n,s),t(e,m,s),w(c,e,s),t(e,M,s),t(e,g,s),t(e,p,s),w(b,e,s),t(e,W,s),t(e,A,s),t(e,V,s),w(k,e,s),t(e,ve,s),t(e,E,s),t(e,Ne,s),w(G,e,s),t(e,We,s),t(e,F,s),t(e,Ve,s),t(e,R,s),t(e,ke,s),t(e,S,s),t(e,Ee,s),t(e,Y,s),t(e,Ge,s),w(H,e,s),t(e,Fe,s),t(e,z,s),t(e,Re,s),t(e,Q,s),t(e,Se,s),w(L,e,s),t(e,Ye,s),w(x,e,s),t(e,He,s),t(e,P,s),t(e,ze,s),w(D,e,s),t(e,Qe,s),t(e,q,s),t(e,Le,s),t(e,K,s),t(e,xe,s),t(e,O,s),t(e,Pe,s),w(ee,e,s),t(e,De,s),t(e,se,s),t(e,qe,s),t(e,ae,s),t(e,Ke,s),t(e,le,s),t(e,Oe,s),w(te,e,s),t(e,es,s),w(ne,e,s),t(e,ss,s),t(e,re,s),t(e,as,s),t(e,ie,s),t(e,ls,s),w(oe,e,s),t(e,ts,s),t(e,ce,s),t(e,ns,s),w(pe,e,s),t(e,rs,s),w(me,e,s),t(e,is,s),w(Z,e,s),t(e,os,s),t(e,Me,s),t(e,cs,s),t(e,fe,s),t(e,ps,s),w(C,e,s),t(e,ms,s),t(e,Te,s),t(e,Ms,s),w(he,e,s),t(e,fs,s),t(e,de,s),t(e,Ts,s),w(X,e,s),t(e,hs,s),w(ue,e,s),t(e,ds,s),t(e,we,s),t(e,us,s),w(je,e,s),t(e,ws,s),t(e,Je,s),t(e,js,s),w(ye,e,s),t(e,Js,s),t(e,_e,s),t(e,ys,s),w(B,e,s),t(e,_s,s),t(e,ge,s),t(e,gs,s),w(v,e,s),t(e,bs,s),t(e,be,s),t(e,Us,s),t(e,Ue,s),t(e,Is,s),w(Ie,e,s),t(e,As,s),t(e,Ae,s),t(e,$s,s),w($e,e,s),t(e,Zs,s),t(e,Ze,s),t(e,Cs,s),t(e,Be,s),Xs=!0},p(e,[s]){const oa={};s&2&&(oa.$$scope={dirty:s,ctx:e}),b.$set(oa);const ca={};s&2&&(ca.$$scope={dirty:s,ctx:e}),Z.$set(ca);const pa={};s&2&&(pa.$$scope={dirty:s,ctx:e}),C.$set(pa);const ma={};s&2&&(ma.$$scope={dirty:s,ctx:e}),X.$set(ma);const Ma={};s&2&&(Ma.$$scope={dirty:s,ctx:e}),B.$set(Ma);const fa={};s&2&&(fa.$$scope={dirty:s,ctx:e}),v.$set(fa)},i(e){Xs||(j(c.$$.fragment,e),j(b.$$.fragment,e),j(k.$$.fragment,e),j(G.$$.fragment,e),j(H.$$.fragment,e),j(L.$$.fragment,e),j(x.$$.fragment,e),j(D.$$.fragment,e),j(ee.$$.fragment,e),j(te.$$.fragment,e),j(ne.$$.fragment,e),j(oe.$$.fragment,e),j(pe.$$.fragment,e),j(me.$$.fragment,e),j(Z.$$.fragment,e),j(C.$$.fragment,e),j(he.$$.fragment,e),j(X.$$.fragment,e),j(ue.$$.fragment,e),j(je.$$.fragment,e),j(ye.$$.fragment,e),j(B.$$.fragment,e),j(v.$$.fragment,e),j(Ie.$$.fragment,e),j($e.$$.fragment,e),Xs=!0)},o(e){J(c.$$.fragment,e),J(b.$$.fragment,e),J(k.$$.fragment,e),J(G.$$.fragment,e),J(H.$$.fragment,e),J(L.$$.fragment,e),J(x.$$.fragment,e),J(D.$$.fragment,e),J(ee.$$.fragment,e),J(te.$$.fragment,e),J(ne.$$.fragment,e),J(oe.$$.fragment,e),J(pe.$$.fragment,e),J(me.$$.fragment,e),J(Z.$$.fragment,e),J(C.$$.fragment,e),J(he.$$.fragment,e),J(X.$$.fragment,e),J(ue.$$.fragment,e),J(je.$$.fragment,e),J(ye.$$.fragment,e),J(B.$$.fragment,e),J(v.$$.fragment,e),J(Ie.$$.fragment,e),J($e.$$.fragment,e),Xs=!1},d(e){e&&(a(o),a(n),a(m),a(M),a(g),a(p),a(W),a(A),a(V),a(ve),a(E),a(Ne),a(We),a(F),a(Ve),a(R),a(ke),a(S),a(Ee),a(Y),a(Ge),a(Fe),a(z),a(Re),a(Q),a(Se),a(Ye),a(He),a(P),a(ze),a(Qe),a(q),a(Le),a(K),a(xe),a(O),a(Pe),a(De),a(se),a(qe),a(ae),a(Ke),a(le),a(Oe),a(es),a(ss),a(re),a(as),a(ie),a(ls),a(ts),a(ce),a(ns),a(rs),a(is),a(os),a(Me),a(cs),a(fe),a(ps),a(ms),a(Te),a(Ms),a(fs),a(de),a(Ts),a(hs),a(ds),a(we),a(us),a(ws),a(Je),a(js),a(Js),a(_e),a(ys),a(_s),a(ge),a(gs),a(bs),a(be),a(Us),a(Ue),a(Is),a(As),a(Ae),a($s),a(Zs),a(Ze),a(Cs),a(Be)),a(l),y(c,e),y(b,e),y(k,e),y(G,e),y(H,e),y(L,e),y(x,e),y(D,e),y(ee,e),y(te,e),y(ne,e),y(oe,e),y(pe,e),y(me,e),y(Z,e),y(C,e),y(he,e),y(X,e),y(ue,e),y(je,e),y(ye,e),y(B,e),y(v,e),y(Ie,e),y($e,e)}}}const Va='{"title":"Trainer","local":"trainer","sections":[{"title":"Basic usage","local":"basic-usage","sections":[{"title":"Checkpoints","local":"checkpoints","sections":[],"depth":3}],"depth":2},{"title":"Customize the Trainer","local":"customize-the-trainer","sections":[{"title":"Callbacks","local":"callbacks","sections":[],"depth":3}],"depth":2},{"title":"Logging","local":"logging","sections":[],"depth":2},{"title":"NEFTune","local":"neftune","sections":[],"depth":2},{"title":"Accelerate and Trainer","local":"accelerate-and-trainer","sections":[],"depth":2}],"depth":1}';function ka(_){return wa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ha extends ja{constructor(l){super(),Ja(this,l,ka,Wa,ua,{})}}export{Ha as component};
