import{s as ve,n as _e,o as ke}from"../chunks/scheduler.9991993c.js";import{S as Ne,i as Ve,g as y,s,r as p,A as Re,h as o,f as t,c as n,j as Ae,u as M,x as w,k as Ye,y as Qe,a,v as i,d as m,t as c,w as r}from"../chunks/index.7fc9a5e7.js";import{C as j}from"../chunks/CodeBlock.e11cba92.js";import{H as E}from"../chunks/Heading.e3de321f.js";function xe(be){let J,S,F,z,T,K,d,he='随着模型变得越来越大，并行性已经成为在有限硬件上训练更大模型和加速训练速度的策略，增加了数个数量级。在Hugging Face，我们创建了<a href="https://huggingface.co/docs/accelerate" rel="nofollow">🤗 加速</a>库，以帮助用户在任何类型的分布式设置上轻松训练🤗 Transformers模型，无论是在一台机器上的多个GPU还是在多个机器上的多个GPU。在本教程中，了解如何自定义您的原生PyTorch训练循环，以启用分布式环境中的训练。',L,U,P,f,$e="通过安装🤗 加速开始:",q,b,D,h,Ie="然后导入并创建<code>Accelerator</code>对象。<code>Accelerator</code>将自动检测您的分布式设置类型，并初始化所有必要的训练组件。您不需要显式地将模型放在设备上。",O,$,ee,I,le,C,Ce="下一步是将所有相关的训练对象传递给<code>prepare</code>方法。这包括您的训练和评估DataLoader、一个模型和一个优化器:",te,u,ae,B,se,Z,ue="最后一步是用🤗 加速的<code>backward</code>方法替换训练循环中的典型<code>loss.backward()</code>:",ne,g,pe,W,Be="如您在下面的代码中所见，您只需要添加四行额外的代码到您的训练循环中即可启用分布式训练！",Me,X,ie,G,me,A,Ze="在添加了相关代码行后，可以在脚本或笔记本（如Colaboratory）中启动训练。",ce,Y,re,v,ge="如果您从脚本中运行训练，请运行以下命令以创建和保存配置文件:",ye,_,oe,k,We="然后使用以下命令启动训练:",we,N,Je,V,je,R,Xe="🤗 加速还可以在笔记本中运行，如果您计划使用Colaboratory的TPU，则可在其中运行。将负责训练的所有代码包装在一个函数中，并将其传递给<code>notebook_launcher</code>:",Te,Q,de,x,Ge='有关🤗 加速及其丰富功能的更多信息，请参阅<a href="https://huggingface.co/docs/accelerate" rel="nofollow">文档</a>。',Ue,H,fe;return T=new E({props:{title:"🤗 加速分布式训练",local:"-加速分布式训练",headingTag:"h1"}}),U=new E({props:{title:"设置",local:"设置",headingTag:"h2"}}),b=new j({props:{code:"cGlwJTIwaW5zdGFsbCUyMGFjY2VsZXJhdGU=",highlighted:"pip install accelerate",wrap:!1}}),$=new j({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

<span class="hljs-meta">&gt;&gt;&gt; </span>accelerator = Accelerator()`,wrap:!1}}),I=new E({props:{title:"准备加速",local:"准备加速",headingTag:"h2"}}),u=new j({props:{code:"dHJhaW5fZGF0YWxvYWRlciUyQyUyMGV2YWxfZGF0YWxvYWRlciUyQyUyMG1vZGVsJTJDJTIwb3B0aW1pemVyJTIwJTNEJTIwYWNjZWxlcmF0b3IucHJlcGFyZSglMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhbG9hZGVyJTJDJTIwZXZhbF9kYXRhbG9hZGVyJTJDJTIwbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare(
<span class="hljs-meta">... </span>    train_dataloader, eval_dataloader, model, optimizer
<span class="hljs-meta">... </span>)`,wrap:!1}}),B=new E({props:{title:"反向传播",local:"反向传播",headingTag:"h2"}}),g=new j({props:{code:"Zm9yJTIwZXBvY2glMjBpbiUyMHJhbmdlKG51bV9lcG9jaHMpJTNBJTBBJTIwJTIwJTIwJTIwZm9yJTIwYmF0Y2glMjBpbiUyMHRyYWluX2RhdGFsb2FkZXIlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKipiYXRjaCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwb3V0cHV0cy5sb3NzJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3IuYmFja3dhcmQobG9zcyklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbHJfc2NoZWR1bGVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplci56ZXJvX2dyYWQoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByb2dyZXNzX2Jhci51cGRhdGUoMSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_epochs):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
<span class="hljs-meta">... </span>        outputs = model(**batch)
<span class="hljs-meta">... </span>        loss = outputs.loss
<span class="hljs-meta">... </span>        accelerator.backward(loss)

<span class="hljs-meta">... </span>        optimizer.step()
<span class="hljs-meta">... </span>        lr_scheduler.step()
<span class="hljs-meta">... </span>        optimizer.zero_grad()
<span class="hljs-meta">... </span>        progress_bar.update(<span class="hljs-number">1</span>)`,wrap:!1}}),X=new j({props:{code:"JTJCJTIwZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUyMCUyMGZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBZGFtVyUyQyUyMEF1dG9Nb2RlbEZvclNlcXVlbmNlQ2xhc3NpZmljYXRpb24lMkMlMjBnZXRfc2NoZWR1bGVyJTBBJTBBJTJCJTIwYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcigpJTBBJTBBJTIwJTIwbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JTZXF1ZW5jZUNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZChjaGVja3BvaW50JTJDJTIwbnVtX2xhYmVscyUzRDIpJTBBJTIwJTIwb3B0aW1pemVyJTIwJTNEJTIwQWRhbVcobW9kZWwucGFyYW1ldGVycygpJTJDJTIwbHIlM0QzZS01KSUwQSUwQS0lMjBkZXZpY2UlMjAlM0QlMjB0b3JjaC5kZXZpY2UoJTIyY3VkYSUyMiklMjBpZiUyMHRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCklMjBlbHNlJTIwdG9yY2guZGV2aWNlKCUyMmNwdSUyMiklMEEtJTIwbW9kZWwudG8oZGV2aWNlKSUwQSUwQSUyQiUyMHRyYWluX2RhdGFsb2FkZXIlMkMlMjBldmFsX2RhdGFsb2FkZXIlMkMlMjBtb2RlbCUyQyUyMG9wdGltaXplciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUoJTBBJTJCJTIwJTIwJTIwJTIwJTIwdHJhaW5fZGF0YWxvYWRlciUyQyUyMGV2YWxfZGF0YWxvYWRlciUyQyUyMG1vZGVsJTJDJTIwb3B0aW1pemVyJTBBJTJCJTIwKSUwQSUwQSUyMCUyMG51bV9lcG9jaHMlMjAlM0QlMjAzJTBBJTIwJTIwbnVtX3RyYWluaW5nX3N0ZXBzJTIwJTNEJTIwbnVtX2Vwb2NocyUyMColMjBsZW4odHJhaW5fZGF0YWxvYWRlciklMEElMjAlMjBscl9zY2hlZHVsZXIlMjAlM0QlMjBnZXRfc2NoZWR1bGVyKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMmxpbmVhciUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplciUzRG9wdGltaXplciUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMG51bV93YXJtdXBfc3RlcHMlM0QwJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwbnVtX3RyYWluaW5nX3N0ZXBzJTNEbnVtX3RyYWluaW5nX3N0ZXBzJTBBJTIwJTIwKSUwQSUwQSUyMCUyMHByb2dyZXNzX2JhciUyMCUzRCUyMHRxZG0ocmFuZ2UobnVtX3RyYWluaW5nX3N0ZXBzKSklMEElMEElMjAlMjBtb2RlbC50cmFpbigpJTBBJTIwJTIwZm9yJTIwZXBvY2glMjBpbiUyMHJhbmdlKG51bV9lcG9jaHMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwYmF0Y2glMjBpbiUyMHRyYWluX2RhdGFsb2FkZXIlM0ElMEEtJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYmF0Y2glMjAlM0QlMjAlN0JrJTNBJTIwdi50byhkZXZpY2UpJTIwZm9yJTIwayUyQyUyMHYlMjBpbiUyMGJhdGNoLml0ZW1zKCklN0QlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKipiYXRjaCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwb3V0cHV0cy5sb3NzJTBBLSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvc3MuYmFja3dhcmQoKSUwQSUyQiUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxyX3NjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuemVyb19ncmFkKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcm9ncmVzc19iYXIudXBkYXRlKDEp",highlighted:`<span class="hljs-addition">+ from accelerate import Accelerator</span>
  from transformers import AdamW, AutoModelForSequenceClassification, get_scheduler

<span class="hljs-addition">+ accelerator = Accelerator()</span>

  model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)
  optimizer = AdamW(model.parameters(), lr=3e-5)

<span class="hljs-deletion">- device = torch.device(&quot;cuda&quot;) if torch.cuda.is_available() else torch.device(&quot;cpu&quot;)</span>
<span class="hljs-deletion">- model.to(device)</span>

<span class="hljs-addition">+ train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare(</span>
<span class="hljs-addition">+     train_dataloader, eval_dataloader, model, optimizer</span>
<span class="hljs-addition">+ )</span>

  num_epochs = 3
  num_training_steps = num_epochs * len(train_dataloader)
  lr_scheduler = get_scheduler(
      &quot;linear&quot;,
      optimizer=optimizer,
      num_warmup_steps=0,
      num_training_steps=num_training_steps
  )

  progress_bar = tqdm(range(num_training_steps))

  model.train()
  for epoch in range(num_epochs):
      for batch in train_dataloader:
<span class="hljs-deletion">-         batch = {k: v.to(device) for k, v in batch.items()}</span>
          outputs = model(**batch)
          loss = outputs.loss
<span class="hljs-deletion">-         loss.backward()</span>
<span class="hljs-addition">+         accelerator.backward(loss)</span>

          optimizer.step()
          lr_scheduler.step()
          optimizer.zero_grad()
          progress_bar.update(1)`,wrap:!1}}),G=new E({props:{title:"训练",local:"训练",headingTag:"h2"}}),Y=new E({props:{title:"用脚本训练",local:"用脚本训练",headingTag:"h3"}}),_=new j({props:{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZw==",highlighted:"accelerate config",wrap:!1}}),N=new j({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluLnB5",highlighted:"accelerate launch train.py",wrap:!1}}),V=new E({props:{title:"用笔记本训练",local:"用笔记本训练",headingTag:"h3"}}),Q=new j({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBub3RlYm9va19sYXVuY2hlciUwQSUwQW5vdGVib29rX2xhdW5jaGVyKHRyYWluaW5nX2Z1bmN0aW9uKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> notebook_launcher

<span class="hljs-meta">&gt;&gt;&gt; </span>notebook_launcher(training_function)`,wrap:!1}}),{c(){J=y("meta"),S=s(),F=y("p"),z=s(),p(T.$$.fragment),K=s(),d=y("p"),d.innerHTML=he,L=s(),p(U.$$.fragment),P=s(),f=y("p"),f.textContent=$e,q=s(),p(b.$$.fragment),D=s(),h=y("p"),h.innerHTML=Ie,O=s(),p($.$$.fragment),ee=s(),p(I.$$.fragment),le=s(),C=y("p"),C.innerHTML=Ce,te=s(),p(u.$$.fragment),ae=s(),p(B.$$.fragment),se=s(),Z=y("p"),Z.innerHTML=ue,ne=s(),p(g.$$.fragment),pe=s(),W=y("p"),W.textContent=Be,Me=s(),p(X.$$.fragment),ie=s(),p(G.$$.fragment),me=s(),A=y("p"),A.textContent=Ze,ce=s(),p(Y.$$.fragment),re=s(),v=y("p"),v.textContent=ge,ye=s(),p(_.$$.fragment),oe=s(),k=y("p"),k.textContent=We,we=s(),p(N.$$.fragment),Je=s(),p(V.$$.fragment),je=s(),R=y("p"),R.innerHTML=Xe,Te=s(),p(Q.$$.fragment),de=s(),x=y("p"),x.innerHTML=Ge,Ue=s(),H=y("p"),this.h()},l(e){const l=Re("svelte-u9bgzb",document.head);J=o(l,"META",{name:!0,content:!0}),l.forEach(t),S=n(e),F=o(e,"P",{}),Ae(F).forEach(t),z=n(e),M(T.$$.fragment,e),K=n(e),d=o(e,"P",{"data-svelte-h":!0}),w(d)!=="svelte-101ov1a"&&(d.innerHTML=he),L=n(e),M(U.$$.fragment,e),P=n(e),f=o(e,"P",{"data-svelte-h":!0}),w(f)!=="svelte-tw1ztk"&&(f.textContent=$e),q=n(e),M(b.$$.fragment,e),D=n(e),h=o(e,"P",{"data-svelte-h":!0}),w(h)!=="svelte-fuwluk"&&(h.innerHTML=Ie),O=n(e),M($.$$.fragment,e),ee=n(e),M(I.$$.fragment,e),le=n(e),C=o(e,"P",{"data-svelte-h":!0}),w(C)!=="svelte-rz36qo"&&(C.innerHTML=Ce),te=n(e),M(u.$$.fragment,e),ae=n(e),M(B.$$.fragment,e),se=n(e),Z=o(e,"P",{"data-svelte-h":!0}),w(Z)!=="svelte-1gfcp9e"&&(Z.innerHTML=ue),ne=n(e),M(g.$$.fragment,e),pe=n(e),W=o(e,"P",{"data-svelte-h":!0}),w(W)!=="svelte-13i7565"&&(W.textContent=Be),Me=n(e),M(X.$$.fragment,e),ie=n(e),M(G.$$.fragment,e),me=n(e),A=o(e,"P",{"data-svelte-h":!0}),w(A)!=="svelte-1b13pra"&&(A.textContent=Ze),ce=n(e),M(Y.$$.fragment,e),re=n(e),v=o(e,"P",{"data-svelte-h":!0}),w(v)!=="svelte-1wt7rny"&&(v.textContent=ge),ye=n(e),M(_.$$.fragment,e),oe=n(e),k=o(e,"P",{"data-svelte-h":!0}),w(k)!=="svelte-1jwcwo5"&&(k.textContent=We),we=n(e),M(N.$$.fragment,e),Je=n(e),M(V.$$.fragment,e),je=n(e),R=o(e,"P",{"data-svelte-h":!0}),w(R)!=="svelte-1a5o0v"&&(R.innerHTML=Xe),Te=n(e),M(Q.$$.fragment,e),de=n(e),x=o(e,"P",{"data-svelte-h":!0}),w(x)!=="svelte-1mt0oay"&&(x.innerHTML=Ge),Ue=n(e),H=o(e,"P",{}),Ae(H).forEach(t),this.h()},h(){Ye(J,"name","hf:doc:metadata"),Ye(J,"content",Ee)},m(e,l){Qe(document.head,J),a(e,S,l),a(e,F,l),a(e,z,l),i(T,e,l),a(e,K,l),a(e,d,l),a(e,L,l),i(U,e,l),a(e,P,l),a(e,f,l),a(e,q,l),i(b,e,l),a(e,D,l),a(e,h,l),a(e,O,l),i($,e,l),a(e,ee,l),i(I,e,l),a(e,le,l),a(e,C,l),a(e,te,l),i(u,e,l),a(e,ae,l),i(B,e,l),a(e,se,l),a(e,Z,l),a(e,ne,l),i(g,e,l),a(e,pe,l),a(e,W,l),a(e,Me,l),i(X,e,l),a(e,ie,l),i(G,e,l),a(e,me,l),a(e,A,l),a(e,ce,l),i(Y,e,l),a(e,re,l),a(e,v,l),a(e,ye,l),i(_,e,l),a(e,oe,l),a(e,k,l),a(e,we,l),i(N,e,l),a(e,Je,l),i(V,e,l),a(e,je,l),a(e,R,l),a(e,Te,l),i(Q,e,l),a(e,de,l),a(e,x,l),a(e,Ue,l),a(e,H,l),fe=!0},p:_e,i(e){fe||(m(T.$$.fragment,e),m(U.$$.fragment,e),m(b.$$.fragment,e),m($.$$.fragment,e),m(I.$$.fragment,e),m(u.$$.fragment,e),m(B.$$.fragment,e),m(g.$$.fragment,e),m(X.$$.fragment,e),m(G.$$.fragment,e),m(Y.$$.fragment,e),m(_.$$.fragment,e),m(N.$$.fragment,e),m(V.$$.fragment,e),m(Q.$$.fragment,e),fe=!0)},o(e){c(T.$$.fragment,e),c(U.$$.fragment,e),c(b.$$.fragment,e),c($.$$.fragment,e),c(I.$$.fragment,e),c(u.$$.fragment,e),c(B.$$.fragment,e),c(g.$$.fragment,e),c(X.$$.fragment,e),c(G.$$.fragment,e),c(Y.$$.fragment,e),c(_.$$.fragment,e),c(N.$$.fragment,e),c(V.$$.fragment,e),c(Q.$$.fragment,e),fe=!1},d(e){e&&(t(S),t(F),t(z),t(K),t(d),t(L),t(P),t(f),t(q),t(D),t(h),t(O),t(ee),t(le),t(C),t(te),t(ae),t(se),t(Z),t(ne),t(pe),t(W),t(Me),t(ie),t(me),t(A),t(ce),t(re),t(v),t(ye),t(oe),t(k),t(we),t(Je),t(je),t(R),t(Te),t(de),t(x),t(Ue),t(H)),t(J),r(T,e),r(U,e),r(b,e),r($,e),r(I,e),r(u,e),r(B,e),r(g,e),r(X,e),r(G,e),r(Y,e),r(_,e),r(N,e),r(V,e),r(Q,e)}}}const Ee='{"title":"🤗 加速分布式训练","local":"-加速分布式训练","sections":[{"title":"设置","local":"设置","sections":[],"depth":2},{"title":"准备加速","local":"准备加速","sections":[],"depth":2},{"title":"反向传播","local":"反向传播","sections":[],"depth":2},{"title":"训练","local":"训练","sections":[{"title":"用脚本训练","local":"用脚本训练","sections":[],"depth":3},{"title":"用笔记本训练","local":"用笔记本训练","sections":[],"depth":3}],"depth":2}],"depth":1}';function Fe(be){return ke(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Le extends Ne{constructor(J){super(),Ve(this,J,Fe,xe,ve,{})}}export{Le as component};
