import{S as ly,i as ay,s as oy,e as a,k as p,w as u,t as n,M as sy,c as o,d as t,m as f,a as s,x as m,h as i,b as c,N as js,G as l,g as h,y as d,L as ry,q as v,o as y,B as w,v as ny}from"../../chunks/vendor-hf-doc-builder.js";import{I as b}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as _}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as iy}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function hy($u){let W,Is,B,ie,va,qe,Ui,ya,Ti,Cs,De,Gs,he,Ji,wa,$i,ki,Ss,Tl,ba,ji,Zs,Jl,Ii,Ws,$l,ga,Oe,Ci,Bs,kl,Gi,As,jl,Il,_a,Si,Zi,Ns,A,Wi,Ke,Bi,Ai,Ma,Ni,Ps,et,Pi,tt,Vi,Vs,lt,Ea,Ri,Qi,Rs,Cl,at,Gl,ku,Qs,N,pe,Ua,ot,Fi,Ta,Yi,Fs,P,fe,Ja,st,zi,$a,Xi,Ys,Sl,ka,rt,Hi,zs,V,ce,ja,nt,Li,Ia,xi,Xs,Zl,Ca,it,qi,Hs,$,Di,Ga,Oi,Ki,ht,eh,th,Ls,R,ue,Sa,pt,lh,Za,ah,xs,Wl,oh,qs,T,ft,sh,Wa,rh,nh,ih,ct,hh,Ba,ph,fh,ch,ut,uh,Aa,mh,dh,vh,mt,yh,Na,wh,bh,Ds,Q,me,Pa,dt,gh,Va,_h,Os,Bl,Mh,Ks,de,Eh,vt,Uh,Th,er,F,ve,Ra,yt,Jh,Qa,$h,tr,Y,ye,Fa,wt,kh,Ya,jh,lr,Al,we,Ih,za,Ch,Gh,Xa,Sh,ar,bt,ju,or,Nl,Ha,La,Zh,sr,gt,Iu,rr,z,be,xa,_t,Wh,qa,Bh,nr,ge,Ah,Da,Nh,Ph,ir,Pl,Vh,hr,Mt,pr,Et,fr,X,_e,Oa,Ut,Rh,Ka,Qh,cr,Vl,Fh,ur,E,eo,to,Yh,zh,Rl,lo,Xh,Hh,Lh,Ql,ao,xh,qh,Dh,Fl,oo,Oh,Kh,ep,Yl,so,tp,lp,mr,Tt,dr,H,Me,ro,Jt,ap,no,op,vr,$t,yr,L,Ee,io,kt,sp,ho,rp,wr,zl,np,br,Ue,ip,po,hp,pp,gr,Te,fp,fo,cp,up,_r,U,mp,co,dp,vp,uo,yp,wp,mo,bp,gp,vo,_p,Mp,Mr,k,Ep,yo,Up,Tp,wo,Jp,$p,Er,jt,Cu,Ur,Je,kp,bo,jp,Ip,Tr,x,$e,go,It,Cp,_o,Gp,Jr,q,ke,Mo,Ct,Sp,Eo,Zp,$r,je,Wp,Uo,Bp,Ap,kr,Gt,jr,St,Ir,Ie,Np,To,Pp,Vp,Cr,j,Xl,Jo,Rp,Qp,Fp,Hl,$o,Yp,zp,Xp,Ll,ko,Hp,Lp,Gr,Ce,xp,jo,qp,Dp,Sr,Zt,Zr,xl,Op,Wr,ql,Io,Kp,Br,D,Ge,Co,Wt,ef,Go,tf,Ar,Se,lf,Bt,af,of,Nr,Dl,sf,Pr,At,rf,So,nf,Vr,Ol,Nt,hf,Rr,Pt,Qr,O,Ze,Zo,Vt,pf,Wo,ff,Fr,Rt,Yr,K,We,Bo,Qt,cf,Ao,uf,zr,Ft,mf,Yt,df,Xr,Be,vf,zt,yf,wf,Hr,Xt,Lr,ee,Ae,No,Ht,bf,Po,gf,xr,Lt,qr,te,Ne,Vo,xt,_f,Ro,Mf,Dr,Kl,Qo,Ef,Or,qt,Kr,Dt,en,le,Pe,Fo,Ot,Uf,Yo,Tf,tn,Ve,Kt,Jf,zo,$f,kf,jf,ea,If,Xo,Cf,ln,el,an,ae,Re,Ho,tl,Gf,Lo,Sf,on,ta,Zf,sn,ll,Wf,al,Bf,rn,I,Af,xo,Nf,Pf,qo,Vf,Rf,nn,la,Qf,hn,J,ol,Ff,Do,Yf,zf,Xf,sl,Hf,Oo,Lf,xf,qf,rl,Df,Ko,Of,Kf,ec,Qe,tc,es,lc,ac,nl,oc,pn,aa,sc,fn,il,rc,hl,nc,cn,oa,ic,un,sa,Fe,hc,pl,pc,fc,ts,cc,mn,fl,Gu,dn,Ye,ls,uc,mc,as,dc,vn,cl,yn,ul,vc,os,yc,wn,oe,wc,ss,bc,gc,rs,_c,bn,ml,gn,se,ze,ns,dl,Mc,is,Ec,_n,C,Uc,hs,Tc,Jc,ps,$c,kc,Mn,G,jc,fs,Ic,Cc,cs,Gc,Sc,En,vl,Zc,yl,Wc,Un,wl,Bc,bl,Ac,Tn,ra,Nc,Jn,g,us,Pc,Vc,ms,Rc,Qc,na,Fc,gl,Yc,zc,ds,Xc,Hc,vs,Lc,xc,ys,qc,Dc,ws,Oc,Kc,ia,eu,bs,tu,$n,re,Xe,gs,_l,lu,_s,au,kn,Ml,jn,El,In,ha,ou,Cn,ne,He,Ms,Ul,su,Es,ru,Gn;return qe=new b({}),De=new iy({props:{classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/deep-rl-class/blob/main/notebooks/unit6/unit6.ipynb"}],askForHelpUrl:"http://hf.co/join/discord"}}),ot=new b({}),st=new b({}),nt=new b({}),pt=new b({}),dt=new b({}),yt=new b({}),wt=new b({}),_t=new b({}),Mt=new _({props:{code:"JTI1JTI1Y2FwdHVyZSUwQSFhcHQlMjBpbnN0YWxsJTIwcHl0aG9uLW9wZW5nbCUwQSFhcHQlMjBpbnN0YWxsJTIwZmZtcGVnJTBBIWFwdCUyMGluc3RhbGwlMjB4dmZiJTBBIXBpcDMlMjBpbnN0YWxsJTIwcHl2aXJ0dWFsZGlzcGxheQ==",highlighted:`%%capture
!apt install python-opengl
!apt install ffmpeg
!apt install xvfb
!pip3 install pyvirtualdisplay`}}),Et=new _({props:{code:"JTIzJTIwVmlydHVhbCUyMGRpc3BsYXklMEFmcm9tJTIwcHl2aXJ0dWFsZGlzcGxheSUyMGltcG9ydCUyMERpc3BsYXklMEElMEF2aXJ0dWFsX2Rpc3BsYXklMjAlM0QlMjBEaXNwbGF5KHZpc2libGUlM0QwJTJDJTIwc2l6ZSUzRCgxNDAwJTJDJTIwOTAwKSklMEF2aXJ0dWFsX2Rpc3BsYXkuc3RhcnQoKQ==",highlighted:`<span class="hljs-comment"># Virtual display</span>
<span class="hljs-keyword">from</span> pyvirtualdisplay <span class="hljs-keyword">import</span> Display

virtual_display = Display(visible=<span class="hljs-number">0</span>, size=(<span class="hljs-number">1400</span>, <span class="hljs-number">900</span>))
virtual_display.start()`}}),Ut=new b({}),Tt=new _({props:{code:"IXBpcCUyMGluc3RhbGwlMjBzdGFibGUtYmFzZWxpbmVzMyU1QmV4dHJhJTVEJTBBIXBpcCUyMGluc3RhbGwlMjBneW1uYXNpdW0lMEEhcGlwJTIwaW5zdGFsbCUyMGdpdCUyQmh0dHBzJTNBJTJGJTJGZ2l0aHViLmNvbSUyRmh1Z2dpbmdmYWNlJTJGaHVnZ2luZ2ZhY2Vfc2IzJTQwZ3ltbmFzaXVtLXYyJTBBIXBpcCUyMGluc3RhbGwlMjBodWdnaW5nZmFjZV9odWIlMEEhcGlwJTIwaW5zdGFsbCUyMHBhbmRhX2d5bQ==",highlighted:`!pip install stable-baselines3[extra]
!pip install gymnasium
!pip install git+https://github.com/huggingface/huggingface_sb3@gymnasium-v2
!pip install huggingface_hub
!pip install panda_gym`}}),Jt=new b({}),$t=new _({props:{code:"aW1wb3J0JTIwb3MlMEElMEFpbXBvcnQlMjBneW1uYXNpdW0lMjBhcyUyMGd5bSUwQWltcG9ydCUyMHBhbmRhX2d5bSUwQSUwQWZyb20lMjBodWdnaW5nZmFjZV9zYjMlMjBpbXBvcnQlMjBsb2FkX2Zyb21faHViJTJDJTIwcGFja2FnZV90b19odWIlMEElMEFmcm9tJTIwc3RhYmxlX2Jhc2VsaW5lczMlMjBpbXBvcnQlMjBBMkMlMEFmcm9tJTIwc3RhYmxlX2Jhc2VsaW5lczMuY29tbW9uLmV2YWx1YXRpb24lMjBpbXBvcnQlMjBldmFsdWF0ZV9wb2xpY3klMEFmcm9tJTIwc3RhYmxlX2Jhc2VsaW5lczMuY29tbW9uLnZlY19lbnYlMjBpbXBvcnQlMjBEdW1teVZlY0VudiUyQyUyMFZlY05vcm1hbGl6ZSUwQWZyb20lMjBzdGFibGVfYmFzZWxpbmVzMy5jb21tb24uZW52X3V0aWwlMjBpbXBvcnQlMjBtYWtlX3ZlY19lbnYlMEElMEFmcm9tJTIwaHVnZ2luZ2ZhY2VfaHViJTIwaW1wb3J0JTIwbm90ZWJvb2tfbG9naW4=",highlighted:`<span class="hljs-keyword">import</span> os

<span class="hljs-keyword">import</span> gymnasium <span class="hljs-keyword">as</span> gym
<span class="hljs-keyword">import</span> panda_gym

<span class="hljs-keyword">from</span> huggingface_sb3 <span class="hljs-keyword">import</span> load_from_hub, package_to_hub

<span class="hljs-keyword">from</span> stable_baselines3 <span class="hljs-keyword">import</span> A2C
<span class="hljs-keyword">from</span> stable_baselines3.common.evaluation <span class="hljs-keyword">import</span> evaluate_policy
<span class="hljs-keyword">from</span> stable_baselines3.common.vec_env <span class="hljs-keyword">import</span> DummyVecEnv, VecNormalize
<span class="hljs-keyword">from</span> stable_baselines3.common.env_util <span class="hljs-keyword">import</span> make_vec_env

<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login`}}),kt=new b({}),It=new b({}),Ct=new b({}),Gt=new _({props:{code:"ZW52X2lkJTIwJTNEJTIwJTIyUGFuZGFSZWFjaERlbnNlLXYzJTIyJTBBJTBBJTIzJTIwQ3JlYXRlJTIwdGhlJTIwZW52JTBBZW52JTIwJTNEJTIwZ3ltLm1ha2UoZW52X2lkKSUwQSUwQSUyMyUyMEdldCUyMHRoZSUyMHN0YXRlJTIwc3BhY2UlMjBhbmQlMjBhY3Rpb24lMjBzcGFjZSUwQXNfc2l6ZSUyMCUzRCUyMGVudi5vYnNlcnZhdGlvbl9zcGFjZS5zaGFwZSUwQWFfc2l6ZSUyMCUzRCUyMGVudi5hY3Rpb25fc3BhY2U=",highlighted:`env_id = <span class="hljs-string">&quot;PandaReachDense-v3&quot;</span>

<span class="hljs-comment"># Create the env</span>
env = gym.make(env_id)

<span class="hljs-comment"># Get the state space and action space</span>
s_size = env.observation_space.shape
a_size = env.action_space`}}),St=new _({props:{code:"cHJpbnQoJTIyX19fX19PQlNFUlZBVElPTiUyMFNQQUNFX19fX18lMjAlNUNuJTIyKSUwQXByaW50KCUyMlRoZSUyMFN0YXRlJTIwU3BhY2UlMjBpcyUzQSUyMCUyMiUyQyUyMHNfc2l6ZSklMEFwcmludCglMjJTYW1wbGUlMjBvYnNlcnZhdGlvbiUyMiUyQyUyMGVudi5vYnNlcnZhdGlvbl9zcGFjZS5zYW1wbGUoKSklMjAlMjMlMjBHZXQlMjBhJTIwcmFuZG9tJTIwb2JzZXJ2YXRpb24=",highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;_____OBSERVATION SPACE_____ \\n&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;The State Space is: &quot;</span>, s_size)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Sample observation&quot;</span>, env.observation_space.sample()) <span class="hljs-comment"># Get a random observation</span>`}}),Zt=new _({props:{code:"cHJpbnQoJTIyJTVDbiUyMF9fX19fQUNUSU9OJTIwU1BBQ0VfX19fXyUyMCU1Q24lMjIpJTBBcHJpbnQoJTIyVGhlJTIwQWN0aW9uJTIwU3BhY2UlMjBpcyUzQSUyMCUyMiUyQyUyMGFfc2l6ZSklMEFwcmludCglMjJBY3Rpb24lMjBTcGFjZSUyMFNhbXBsZSUyMiUyQyUyMGVudi5hY3Rpb25fc3BhY2Uuc2FtcGxlKCkpJTIwJTIzJTIwVGFrZSUyMGElMjByYW5kb20lMjBhY3Rpb24=",highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;\\n _____ACTION SPACE_____ \\n&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;The Action Space is: &quot;</span>, a_size)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Action Space Sample&quot;</span>, env.action_space.sample()) <span class="hljs-comment"># Take a random action</span>`}}),Wt=new b({}),Pt=new _({props:{code:"ZW52JTIwJTNEJTIwbWFrZV92ZWNfZW52KGVudl9pZCUyQyUyMG5fZW52cyUzRDQpJTBBJTBBJTIzJTIwQWRkaW5nJTIwdGhpcyUyMHdyYXBwZXIlMjB0byUyMG5vcm1hbGl6ZSUyMHRoZSUyMG9ic2VydmF0aW9uJTIwYW5kJTIwdGhlJTIwcmV3YXJkJTBBZW52JTIwJTNEJTIwJTIzJTIwVE9ETyUzQSUyMEFkZCUyMHRoZSUyMHdyYXBwZXI=",highlighted:`env = make_vec_env(env_id, n_envs=<span class="hljs-number">4</span>)

<span class="hljs-comment"># Adding this wrapper to normalize the observation and the reward</span>
env = <span class="hljs-comment"># <span class="hljs-doctag">TODO:</span> Add the wrapper</span>`}}),Vt=new b({}),Rt=new _({props:{code:"ZW52JTIwJTNEJTIwbWFrZV92ZWNfZW52KGVudl9pZCUyQyUyMG5fZW52cyUzRDQpJTBBJTBBZW52JTIwJTNEJTIwVmVjTm9ybWFsaXplKGVudiUyQyUyMG5vcm1fb2JzJTNEVHJ1ZSUyQyUyMG5vcm1fcmV3YXJkJTNEVHJ1ZSUyQyUyMGNsaXBfb2JzJTNEMTAuKQ==",highlighted:`env = make_vec_env(env_id, n_envs=<span class="hljs-number">4</span>)

env = VecNormalize(env, norm_obs=<span class="hljs-literal">True</span>, norm_reward=<span class="hljs-literal">True</span>, clip_obs=<span class="hljs-number">10.</span>)`}}),Qt=new b({}),Xt=new _({props:{code:"bW9kZWwlMjAlM0QlMjAlMjMlMjBDcmVhdGUlMjB0aGUlMjBBMkMlMjBtb2RlbCUyMGFuZCUyMHRyeSUyMHRvJTIwZmluZCUyMHRoZSUyMGJlc3QlMjBwYXJhbWV0ZXJz",highlighted:'model = <span class="hljs-comment"># Create the A2C model and try to find the best parameters</span>'}}),Ht=new b({}),Lt=new _({props:{code:"bW9kZWwlMjAlM0QlMjBBMkMocG9saWN5JTIwJTNEJTIwJTIyTXVsdGlJbnB1dFBvbGljeSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVudiUyMCUzRCUyMGVudiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHZlcmJvc2UlM0QxKQ==",highlighted:`model = A2C(policy = <span class="hljs-string">&quot;MultiInputPolicy&quot;</span>,
            env = env,
            verbose=<span class="hljs-number">1</span>)`}}),xt=new b({}),qt=new _({props:{code:"bW9kZWwubGVhcm4oMV8wMDBfMDAwKQ==",highlighted:'model.learn(<span class="hljs-number">1_000_000</span>)'}}),Dt=new _({props:{code:"JTIzJTIwU2F2ZSUyMHRoZSUyMG1vZGVsJTIwYW5kJTIwJTIwVmVjTm9ybWFsaXplJTIwc3RhdGlzdGljcyUyMHdoZW4lMjBzYXZpbmclMjB0aGUlMjBhZ2VudCUwQW1vZGVsLnNhdmUoJTIyYTJjLVBhbmRhUmVhY2hEZW5zZS12MyUyMiklMEFlbnYuc2F2ZSglMjJ2ZWNfbm9ybWFsaXplLnBrbCUyMik=",highlighted:`<span class="hljs-comment"># Save the model and  VecNormalize statistics when saving the agent</span>
model.save(<span class="hljs-string">&quot;a2c-PandaReachDense-v3&quot;</span>)
env.save(<span class="hljs-string">&quot;vec_normalize.pkl&quot;</span>)`}}),Ot=new b({}),el=new _({props:{code:"ZnJvbSUyMHN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi52ZWNfZW52JTIwaW1wb3J0JTIwRHVtbXlWZWNFbnYlMkMlMjBWZWNOb3JtYWxpemUlMEElMEElMjMlMjBMb2FkJTIwdGhlJTIwc2F2ZWQlMjBzdGF0aXN0aWNzJTBBZXZhbF9lbnYlMjAlM0QlMjBEdW1teVZlY0VudiglNUJsYW1iZGElM0ElMjBneW0ubWFrZSglMjJQYW5kYVJlYWNoRGVuc2UtdjMlMjIpJTVEKSUwQWV2YWxfZW52JTIwJTNEJTIwVmVjTm9ybWFsaXplLmxvYWQoJTIydmVjX25vcm1hbGl6ZS5wa2wlMjIlMkMlMjBldmFsX2VudiklMEElMEElMjMlMjBXZSUyMG5lZWQlMjB0byUyMG92ZXJyaWRlJTIwdGhlJTIwcmVuZGVyX21vZGUlMEFldmFsX2Vudi5yZW5kZXJfbW9kZSUyMCUzRCUyMCUyMnJnYl9hcnJheSUyMiUwQSUwQSUyMyUyMCUyMGRvJTIwbm90JTIwdXBkYXRlJTIwdGhlbSUyMGF0JTIwdGVzdCUyMHRpbWUlMEFldmFsX2Vudi50cmFpbmluZyUyMCUzRCUyMEZhbHNlJTBBJTIzJTIwcmV3YXJkJTIwbm9ybWFsaXphdGlvbiUyMGlzJTIwbm90JTIwbmVlZGVkJTIwYXQlMjB0ZXN0JTIwdGltZSUwQWV2YWxfZW52Lm5vcm1fcmV3YXJkJTIwJTNEJTIwRmFsc2UlMEElMEElMjMlMjBMb2FkJTIwdGhlJTIwYWdlbnQlMEFtb2RlbCUyMCUzRCUyMEEyQy5sb2FkKCUyMmEyYy1QYW5kYVJlYWNoRGVuc2UtdjMlMjIpJTBBJTBBbWVhbl9yZXdhcmQlMkMlMjBzdGRfcmV3YXJkJTIwJTNEJTIwZXZhbHVhdGVfcG9saWN5KG1vZGVsJTJDJTIwZXZhbF9lbnYpJTBBJTBBcHJpbnQoZiUyMk1lYW4lMjByZXdhcmQlMjAlM0QlMjAlN0JtZWFuX3Jld2FyZCUzQS4yZiU3RCUyMCUyQiUyRi0lMjAlN0JzdGRfcmV3YXJkJTNBLjJmJTdEJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> stable_baselines3.common.vec_env <span class="hljs-keyword">import</span> DummyVecEnv, VecNormalize

<span class="hljs-comment"># Load the saved statistics</span>
eval_env = DummyVecEnv([<span class="hljs-keyword">lambda</span>: gym.make(<span class="hljs-string">&quot;PandaReachDense-v3&quot;</span>)])
eval_env = VecNormalize.load(<span class="hljs-string">&quot;vec_normalize.pkl&quot;</span>, eval_env)

<span class="hljs-comment"># We need to override the render_mode</span>
eval_env.render_mode = <span class="hljs-string">&quot;rgb_array&quot;</span>

<span class="hljs-comment">#  do not update them at test time</span>
eval_env.training = <span class="hljs-literal">False</span>
<span class="hljs-comment"># reward normalization is not needed at test time</span>
eval_env.norm_reward = <span class="hljs-literal">False</span>

<span class="hljs-comment"># Load the agent</span>
model = A2C.load(<span class="hljs-string">&quot;a2c-PandaReachDense-v3&quot;</span>)

mean_reward, std_reward = evaluate_policy(model, eval_env)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Mean reward = <span class="hljs-subst">{mean_reward:<span class="hljs-number">.2</span>f}</span> +/- <span class="hljs-subst">{std_reward:<span class="hljs-number">.2</span>f}</span>&quot;</span>)`}}),tl=new b({}),cl=new _({props:{code:"bm90ZWJvb2tfbG9naW4oKSUwQSFnaXQlMjBjb25maWclMjAtLWdsb2JhbCUyMGNyZWRlbnRpYWwuaGVscGVyJTIwc3RvcmU=",highlighted:`notebook_login()
!git config --<span class="hljs-keyword">global</span> credential.helper store`}}),ml=new _({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX3NiMyUyMGltcG9ydCUyMHBhY2thZ2VfdG9faHViJTBBJTBBcGFja2FnZV90b19odWIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0Rtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0RmJTIyYTJjLSU3QmVudl9pZCU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX2FyY2hpdGVjdHVyZSUzRCUyMkEyQyUyMiUyQyUwQSUyMCUyMCUyMCUyMGVudl9pZCUzRGVudl9pZCUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfZW52JTNEZXZhbF9lbnYlMkMlMEElMjAlMjAlMjAlMjByZXBvX2lkJTNEZiUyMlRob21hc1NpbW9uaW5pJTJGYTJjLSU3QmVudl9pZCU3RCUyMiUyQyUyMCUyMyUyMENoYW5nZSUyMHRoZSUyMHVzZXJuYW1lJTBBJTIwJTIwJTIwJTIwY29tbWl0X21lc3NhZ2UlM0QlMjJJbml0aWFsJTIwY29tbWl0JTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_sb3 <span class="hljs-keyword">import</span> package_to_hub

package_to_hub(
    model=model,
    model_name=<span class="hljs-string">f&quot;a2c-<span class="hljs-subst">{env_id}</span>&quot;</span>,
    model_architecture=<span class="hljs-string">&quot;A2C&quot;</span>,
    env_id=env_id,
    eval_env=eval_env,
    repo_id=<span class="hljs-string">f&quot;ThomasSimonini/a2c-<span class="hljs-subst">{env_id}</span>&quot;</span>, <span class="hljs-comment"># Change the username</span>
    commit_message=<span class="hljs-string">&quot;Initial commit&quot;</span>,
)`}}),dl=new b({}),_l=new b({}),Ml=new _({props:{code:"JTIzJTIwMSUyMC0lMjAyJTBBZW52X2lkJTIwJTNEJTIwJTIyUGFuZGFQaWNrQW5kUGxhY2UtdjMlMjIlMEFlbnYlMjAlM0QlMjBtYWtlX3ZlY19lbnYoZW52X2lkJTJDJTIwbl9lbnZzJTNENCklMEElMEElMjMlMjAzJTBBZW52JTIwJTNEJTIwVmVjTm9ybWFsaXplKGVudiUyQyUyMG5vcm1fb2JzJTNEVHJ1ZSUyQyUyMG5vcm1fcmV3YXJkJTNEVHJ1ZSUyQyUyMGNsaXBfb2JzJTNEMTAuKSUwQSUwQSUyMyUyMDQlMEFtb2RlbCUyMCUzRCUyMEEyQyhwb2xpY3klMjAlM0QlMjAlMjJNdWx0aUlucHV0UG9saWN5JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW52JTIwJTNEJTIwZW52JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdmVyYm9zZSUzRDEpJTBBJTIzJTIwNSUwQW1vZGVsLmxlYXJuKDFfMDAwXzAwMCk=",highlighted:`<span class="hljs-comment"># 1 - 2</span>
env_id = <span class="hljs-string">&quot;PandaPickAndPlace-v3&quot;</span>
env = make_vec_env(env_id, n_envs=<span class="hljs-number">4</span>)

<span class="hljs-comment"># 3</span>
env = VecNormalize(env, norm_obs=<span class="hljs-literal">True</span>, norm_reward=<span class="hljs-literal">True</span>, clip_obs=<span class="hljs-number">10.</span>)

<span class="hljs-comment"># 4</span>
model = A2C(policy = <span class="hljs-string">&quot;MultiInputPolicy&quot;</span>,
            env = env,
            verbose=<span class="hljs-number">1</span>)
<span class="hljs-comment"># 5</span>
model.learn(<span class="hljs-number">1_000_000</span>)`}}),El=new _({props:{code:"JTIzJTIwNiUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJhMmMtUGFuZGFQaWNrQW5kUGxhY2UtdjMlMjIlM0IlMEFtb2RlbC5zYXZlKG1vZGVsX25hbWUpJTBBZW52LnNhdmUoJTIydmVjX25vcm1hbGl6ZS5wa2wlMjIpJTBBJTBBJTIzJTIwNyUwQWZyb20lMjBzdGFibGVfYmFzZWxpbmVzMy5jb21tb24udmVjX2VudiUyMGltcG9ydCUyMER1bW15VmVjRW52JTJDJTIwVmVjTm9ybWFsaXplJTBBJTBBJTIzJTIwTG9hZCUyMHRoZSUyMHNhdmVkJTIwc3RhdGlzdGljcyUwQWV2YWxfZW52JTIwJTNEJTIwRHVtbXlWZWNFbnYoJTVCbGFtYmRhJTNBJTIwZ3ltLm1ha2UoJTIyUGFuZGFQaWNrQW5kUGxhY2UtdjMlMjIpJTVEKSUwQWV2YWxfZW52JTIwJTNEJTIwVmVjTm9ybWFsaXplLmxvYWQoJTIydmVjX25vcm1hbGl6ZS5wa2wlMjIlMkMlMjBldmFsX2VudiklMEElMEElMjMlMjAlMjBkbyUyMG5vdCUyMHVwZGF0ZSUyMHRoZW0lMjBhdCUyMHRlc3QlMjB0aW1lJTBBZXZhbF9lbnYudHJhaW5pbmclMjAlM0QlMjBGYWxzZSUwQSUyMyUyMHJld2FyZCUyMG5vcm1hbGl6YXRpb24lMjBpcyUyMG5vdCUyMG5lZWRlZCUyMGF0JTIwdGVzdCUyMHRpbWUlMEFldmFsX2Vudi5ub3JtX3Jld2FyZCUyMCUzRCUyMEZhbHNlJTBBJTBBJTIzJTIwTG9hZCUyMHRoZSUyMGFnZW50JTBBbW9kZWwlMjAlM0QlMjBBMkMubG9hZChtb2RlbF9uYW1lKSUwQSUwQW1lYW5fcmV3YXJkJTJDJTIwc3RkX3Jld2FyZCUyMCUzRCUyMGV2YWx1YXRlX3BvbGljeShtb2RlbCUyQyUyMGV2YWxfZW52KSUwQSUwQXByaW50KGYlMjJNZWFuJTIwcmV3YXJkJTIwJTNEJTIwJTdCbWVhbl9yZXdhcmQlM0EuMmYlN0QlMjAlMkIlMkYtJTIwJTdCc3RkX3Jld2FyZCUzQS4yZiU3RCUyMiklMEElMEElMjMlMjA4JTBBcGFja2FnZV90b19odWIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0Rtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0RmJTIyYTJjLSU3QmVudl9pZCU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX2FyY2hpdGVjdHVyZSUzRCUyMkEyQyUyMiUyQyUwQSUyMCUyMCUyMCUyMGVudl9pZCUzRGVudl9pZCUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfZW52JTNEZXZhbF9lbnYlMkMlMEElMjAlMjAlMjAlMjByZXBvX2lkJTNEZiUyMlRob21hc1NpbW9uaW5pJTJGYTJjLSU3QmVudl9pZCU3RCUyMiUyQyUyMCUyMyUyMFRPRE8lM0ElMjBDaGFuZ2UlMjB0aGUlMjB1c2VybmFtZSUwQSUyMCUyMCUyMCUyMGNvbW1pdF9tZXNzYWdlJTNEJTIySW5pdGlhbCUyMGNvbW1pdCUyMiUyQyUwQSk=",highlighted:`<span class="hljs-comment"># 6</span>
model_name = <span class="hljs-string">&quot;a2c-PandaPickAndPlace-v3&quot;</span>;
model.save(model_name)
env.save(<span class="hljs-string">&quot;vec_normalize.pkl&quot;</span>)

<span class="hljs-comment"># 7</span>
<span class="hljs-keyword">from</span> stable_baselines3.common.vec_env <span class="hljs-keyword">import</span> DummyVecEnv, VecNormalize

<span class="hljs-comment"># Load the saved statistics</span>
eval_env = DummyVecEnv([<span class="hljs-keyword">lambda</span>: gym.make(<span class="hljs-string">&quot;PandaPickAndPlace-v3&quot;</span>)])
eval_env = VecNormalize.load(<span class="hljs-string">&quot;vec_normalize.pkl&quot;</span>, eval_env)

<span class="hljs-comment">#  do not update them at test time</span>
eval_env.training = <span class="hljs-literal">False</span>
<span class="hljs-comment"># reward normalization is not needed at test time</span>
eval_env.norm_reward = <span class="hljs-literal">False</span>

<span class="hljs-comment"># Load the agent</span>
model = A2C.load(model_name)

mean_reward, std_reward = evaluate_policy(model, eval_env)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Mean reward = <span class="hljs-subst">{mean_reward:<span class="hljs-number">.2</span>f}</span> +/- <span class="hljs-subst">{std_reward:<span class="hljs-number">.2</span>f}</span>&quot;</span>)

<span class="hljs-comment"># 8</span>
package_to_hub(
    model=model,
    model_name=<span class="hljs-string">f&quot;a2c-<span class="hljs-subst">{env_id}</span>&quot;</span>,
    model_architecture=<span class="hljs-string">&quot;A2C&quot;</span>,
    env_id=env_id,
    eval_env=eval_env,
    repo_id=<span class="hljs-string">f&quot;ThomasSimonini/a2c-<span class="hljs-subst">{env_id}</span>&quot;</span>, <span class="hljs-comment"># <span class="hljs-doctag">TODO:</span> Change the username</span>
    commit_message=<span class="hljs-string">&quot;Initial commit&quot;</span>,
)`}}),Ul=new b({}),{c(){W=a("meta"),Is=p(),B=a("h1"),ie=a("a"),va=a("span"),u(qe.$$.fragment),Ui=p(),ya=a("span"),Ti=n("Advantage Actor Critic (A2C) using Robotics Simulations with Panda-Gym \u{1F916}"),Cs=p(),u(De.$$.fragment),Gs=p(),he=a("p"),Ji=n("Now that you\u2019ve studied the theory behind Advantage Actor Critic (A2C), "),wa=a("strong"),$i=n("you\u2019re ready to train your A2C agent"),ki=n(" using Stable-Baselines3 in a robotic environment. And train a:"),Ss=p(),Tl=a("ul"),ba=a("li"),ji=n("A robotic arm \u{1F9BE} to move to the correct position."),Zs=p(),Jl=a("p"),Ii=n("We\u2019re going to use"),Ws=p(),$l=a("ul"),ga=a("li"),Oe=a("a"),Ci=n("panda-gym"),Bs=p(),kl=a("p"),Gi=n("To validate this hands-on for the certification process, you need to push your two trained models to the Hub and get the following results:"),As=p(),jl=a("ul"),Il=a("li"),_a=a("code"),Si=n("PandaReachDense-v3"),Zi=n(" get a result of >= -3.5."),Ns=p(),A=a("p"),Wi=n("To find your result, "),Ke=a("a"),Bi=n("go to the leaderboard"),Ai=n(" and find your model, "),Ma=a("strong"),Ni=n("the result = mean_reward - std of reward"),Ps=p(),et=a("p"),Pi=n("For more information about the certification process, check this section \u{1F449} "),tt=a("a"),Vi=n("https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),Vs=p(),lt=a("p"),Ea=a("strong"),Ri=n("To start the hands-on click on Open In Colab button"),Qi=n(" \u{1F447} :"),Rs=p(),Cl=a("p"),at=a("a"),Gl=a("img"),Qs=p(),N=a("h1"),pe=a("a"),Ua=a("span"),u(ot.$$.fragment),Fi=p(),Ta=a("span"),Yi=n("Unit 6: Advantage Actor Critic (A2C) using Robotics Simulations with Panda-Gym \u{1F916}"),Fs=p(),P=a("h3"),fe=a("a"),Ja=a("span"),u(st.$$.fragment),zi=p(),$a=a("span"),Xi=n("\u{1F3AE} Environments:"),Ys=p(),Sl=a("ul"),ka=a("li"),rt=a("a"),Hi=n("Panda-Gym"),zs=p(),V=a("h3"),ce=a("a"),ja=a("span"),u(nt.$$.fragment),Li=p(),Ia=a("span"),xi=n("\u{1F4DA} RL-Library:"),Xs=p(),Zl=a("ul"),Ca=a("li"),it=a("a"),qi=n("Stable-Baselines3"),Hs=p(),$=a("p"),Di=n("We\u2019re constantly trying to improve our tutorials, so "),Ga=a("strong"),Oi=n("if you find some issues in this notebook"),Ki=n(", please "),ht=a("a"),eh=n("open an issue on the GitHub Repo"),th=n("."),Ls=p(),R=a("h2"),ue=a("a"),Sa=a("span"),u(pt.$$.fragment),lh=p(),Za=a("span"),ah=n("Objectives of this notebook \u{1F3C6}"),xs=p(),Wl=a("p"),oh=n("At the end of the notebook, you will:"),qs=p(),T=a("ul"),ft=a("li"),sh=n("Be able to use "),Wa=a("strong"),rh=n("Panda-Gym"),nh=n(", the environment library."),ih=p(),ct=a("li"),hh=n("Be able to "),Ba=a("strong"),ph=n("train robots using A2C"),fh=n("."),ch=p(),ut=a("li"),uh=n("Understand why "),Aa=a("strong"),mh=n("we need to normalize the input"),dh=n("."),vh=p(),mt=a("li"),yh=n("Be able to "),Na=a("strong"),wh=n("push your trained agent and the code to the Hub"),bh=n(" with a nice video replay and an evaluation score \u{1F525}."),Ds=p(),Q=a("h2"),me=a("a"),Pa=a("span"),u(dt.$$.fragment),gh=p(),Va=a("span"),_h=n("Prerequisites \u{1F3D7}\uFE0F"),Os=p(),Bl=a("p"),Mh=n("Before diving into the notebook, you need to:"),Ks=p(),de=a("p"),Eh=n("\u{1F532} \u{1F4DA} Study "),vt=a("a"),Uh=n("Actor-Critic methods by reading Unit 6"),Th=n(" \u{1F917}"),er=p(),F=a("h1"),ve=a("a"),Ra=a("span"),u(yt.$$.fragment),Jh=p(),Qa=a("span"),$h=n("Let's train our first robots \u{1F916}"),tr=p(),Y=a("h2"),ye=a("a"),Fa=a("span"),u(wt.$$.fragment),kh=p(),Ya=a("span"),jh=n("Set the GPU \u{1F4AA}"),lr=p(),Al=a("ul"),we=a("li"),Ih=n("To "),za=a("strong"),Ch=n("accelerate the agent\u2019s training, we\u2019ll use a GPU"),Gh=n(". To do that, go to "),Xa=a("code"),Sh=n("Runtime > Change Runtime type"),ar=p(),bt=a("img"),or=p(),Nl=a("ul"),Ha=a("li"),La=a("code"),Zh=n("Hardware Accelerator > GPU"),sr=p(),gt=a("img"),rr=p(),z=a("h2"),be=a("a"),xa=a("span"),u(_t.$$.fragment),Wh=p(),qa=a("span"),Bh=n("Create a virtual display \u{1F53D}"),nr=p(),ge=a("p"),Ah=n("During the notebook, we\u2019ll need to generate a replay video. To do so, with colab, "),Da=a("strong"),Nh=n("we need to have a virtual screen to be able to render the environment"),Ph=n(" (and thus record the frames)."),ir=p(),Pl=a("p"),Vh=n("The following cell will install the librairies and create and run a virtual screen \u{1F5A5}"),hr=p(),u(Mt.$$.fragment),pr=p(),u(Et.$$.fragment),fr=p(),X=a("h3"),_e=a("a"),Oa=a("span"),u(Ut.$$.fragment),Rh=p(),Ka=a("span"),Qh=n("Install dependencies \u{1F53D}"),cr=p(),Vl=a("p"),Fh=n("We\u2019ll install multiple ones:"),ur=p(),E=a("ul"),eo=a("li"),to=a("code"),Yh=n("gymnasium"),zh=p(),Rl=a("li"),lo=a("code"),Xh=n("panda-gym"),Hh=n(": Contains the robotics arm environments."),Lh=p(),Ql=a("li"),ao=a("code"),xh=n("stable-baselines3"),qh=n(": The SB3 deep reinforcement learning library."),Dh=p(),Fl=a("li"),oo=a("code"),Oh=n("huggingface_sb3"),Kh=n(": Additional code for Stable-baselines3 to load and upload models from the Hugging Face \u{1F917} Hub."),ep=p(),Yl=a("li"),so=a("code"),tp=n("huggingface_hub"),lp=n(": Library allowing anyone to work with the Hub repositories."),mr=p(),u(Tt.$$.fragment),dr=p(),H=a("h2"),Me=a("a"),ro=a("span"),u(Jt.$$.fragment),ap=p(),no=a("span"),op=n("Import the packages \u{1F4E6}"),vr=p(),u($t.$$.fragment),yr=p(),L=a("h2"),Ee=a("a"),io=a("span"),u(kt.$$.fragment),sp=p(),ho=a("span"),rp=n("PandaReachDense-v3 \u{1F9BE}"),wr=p(),zl=a("p"),np=n("The agent we\u2019re going to train is a robotic arm that needs to do controls (moving the arm and using the end-effector)."),br=p(),Ue=a("p"),ip=n("In robotics, the "),po=a("em"),hp=n("end-effector"),pp=n(" is the device at the end of a robotic arm designed to interact with the environment."),gr=p(),Te=a("p"),fp=n("In "),fo=a("code"),cp=n("PandaReach"),up=n(", the robot must place its end-effector at a target position (green ball)."),_r=p(),U=a("p"),mp=n("We\u2019re going to use the dense version of this environment. It means we\u2019ll get a "),co=a("em"),dp=n("dense reward function"),vp=n(" that "),uo=a("strong"),yp=n("will provide a reward at each timestep"),wp=n(" (the closer the agent is to completing the task, the higher the reward). Contrary to a "),mo=a("em"),bp=n("sparse reward function"),gp=n(" where the environment "),vo=a("strong"),_p=n("return a reward if and only if the task is completed"),Mp=n("."),Mr=p(),k=a("p"),Ep=n("Also, we\u2019re going to use the "),yo=a("em"),Up=n("End-effector displacement control"),Tp=n(", it means the "),wo=a("strong"),Jp=n("action corresponds to the displacement of the end-effector"),$p=n(". We don\u2019t control the individual motion of each joint (joint control)."),Er=p(),jt=a("img"),Ur=p(),Je=a("p"),kp=n("This way "),bo=a("strong"),jp=n("the training will be easier"),Ip=n("."),Tr=p(),x=a("h3"),$e=a("a"),go=a("span"),u(It.$$.fragment),Cp=p(),_o=a("span"),Gp=n("Create the environment"),Jr=p(),q=a("h4"),ke=a("a"),Mo=a("span"),u(Ct.$$.fragment),Sp=p(),Eo=a("span"),Zp=n("The environment \u{1F3AE}"),$r=p(),je=a("p"),Wp=n("In "),Uo=a("code"),Bp=n("PandaReachDense-v3"),Ap=n(" the robotic arm must place its end-effector at a target position (green ball)."),kr=p(),u(Gt.$$.fragment),jr=p(),u(St.$$.fragment),Ir=p(),Ie=a("p"),Np=n("The observation space "),To=a("strong"),Pp=n("is a dictionary with 3 different elements"),Vp=n(":"),Cr=p(),j=a("ul"),Xl=a("li"),Jo=a("code"),Rp=n("achieved_goal"),Qp=n(": (x,y,z) position of the goal."),Fp=p(),Hl=a("li"),$o=a("code"),Yp=n("desired_goal"),zp=n(": (x,y,z) distance between the goal position and the current object position."),Xp=p(),Ll=a("li"),ko=a("code"),Hp=n("observation"),Lp=n(": position (x,y,z) and velocity of the end-effector (vx, vy, vz)."),Gr=p(),Ce=a("p"),xp=n("Given it\u2019s a dictionary as observation, "),jo=a("strong"),qp=n("we will need to use a MultiInputPolicy policy instead of MlpPolicy"),Dp=n("."),Sr=p(),u(Zt.$$.fragment),Zr=p(),xl=a("p"),Op=n("The action space is a vector with 3 values:"),Wr=p(),ql=a("ul"),Io=a("li"),Kp=n("Control x, y, z movement"),Br=p(),D=a("h3"),Ge=a("a"),Co=a("span"),u(Wt.$$.fragment),ef=p(),Go=a("span"),tf=n("Normalize observation and rewards"),Ar=p(),Se=a("p"),lf=n("A good practice in reinforcement learning is to "),Bt=a("a"),af=n("normalize input features"),of=n("."),Nr=p(),Dl=a("p"),sf=n("For that purpose, there is a wrapper that will compute a running average and standard deviation of input features."),Pr=p(),At=a("p"),rf=n("We also normalize rewards with this same wrapper by adding "),So=a("code"),nf=n("norm_reward = True"),Vr=p(),Ol=a("p"),Nt=a("a"),hf=n("You should check the documentation to fill this cell"),Rr=p(),u(Pt.$$.fragment),Qr=p(),O=a("h4"),Ze=a("a"),Zo=a("span"),u(Vt.$$.fragment),pf=p(),Wo=a("span"),ff=n("Solution"),Fr=p(),u(Rt.$$.fragment),Yr=p(),K=a("h3"),We=a("a"),Bo=a("span"),u(Qt.$$.fragment),cf=p(),Ao=a("span"),uf=n("Create the A2C Model \u{1F916}"),zr=p(),Ft=a("p"),mf=n("For more information about A2C implementation with StableBaselines3 check: "),Yt=a("a"),df=n("https://stable-baselines3.readthedocs.io/en/master/modules/a2c.html#notes"),Xr=p(),Be=a("p"),vf=n("To find the best parameters I checked the "),zt=a("a"),yf=n("official trained agents by Stable-Baselines3 team"),wf=n("."),Hr=p(),u(Xt.$$.fragment),Lr=p(),ee=a("h4"),Ae=a("a"),No=a("span"),u(Ht.$$.fragment),bf=p(),Po=a("span"),gf=n("Solution"),xr=p(),u(Lt.$$.fragment),qr=p(),te=a("h3"),Ne=a("a"),Vo=a("span"),u(xt.$$.fragment),_f=p(),Ro=a("span"),Mf=n("Train the A2C agent \u{1F3C3}"),Dr=p(),Kl=a("ul"),Qo=a("li"),Ef=n("Let\u2019s train our agent for 1,000,000 timesteps, don\u2019t forget to use GPU on Colab. It will take approximately ~25-40min"),Or=p(),u(qt.$$.fragment),Kr=p(),u(Dt.$$.fragment),en=p(),le=a("h3"),Pe=a("a"),Fo=a("span"),u(Ot.$$.fragment),Uf=p(),Yo=a("span"),Tf=n("Evaluate the agent \u{1F4C8}"),tn=p(),Ve=a("ul"),Kt=a("li"),Jf=n("Now that\u2019s our  agent is trained, we need to "),zo=a("strong"),$f=n("check its performance"),kf=n("."),jf=p(),ea=a("li"),If=n("Stable-Baselines3 provides a method to do that: "),Xo=a("code"),Cf=n("evaluate_policy"),ln=p(),u(el.$$.fragment),an=p(),ae=a("h3"),Re=a("a"),Ho=a("span"),u(tl.$$.fragment),Gf=p(),Lo=a("span"),Sf=n("Publish your trained model on the Hub \u{1F525}"),on=p(),ta=a("p"),Zf=n("Now that we saw we got good results after the training, we can publish our trained model on the Hub with one line of code."),sn=p(),ll=a("p"),Wf=n("\u{1F4DA} The libraries documentation \u{1F449} "),al=a("a"),Bf=n("https://github.com/huggingface/huggingface_sb3/tree/main#hugging-face\u2014x-stable-baselines3-v20"),rn=p(),I=a("p"),Af=n("By using "),xo=a("code"),Nf=n("package_to_hub"),Pf=n(", as we already mentionned in the former units, "),qo=a("strong"),Vf=n("you evaluate, record a replay, generate a model card of your agent and push it to the hub"),Rf=n("."),nn=p(),la=a("p"),Qf=n("This way:"),hn=p(),J=a("ul"),ol=a("li"),Ff=n("You can "),Do=a("strong"),Yf=n("showcase our work"),zf=n(" \u{1F525}"),Xf=p(),sl=a("li"),Hf=n("You can "),Oo=a("strong"),Lf=n("visualize your agent playing"),xf=n(" \u{1F440}"),qf=p(),rl=a("li"),Df=n("You can "),Ko=a("strong"),Of=n("share with the community an agent that others can use"),Kf=n(" \u{1F4BE}"),ec=p(),Qe=a("li"),tc=n("You can "),es=a("strong"),lc=n("access a leaderboard \u{1F3C6} to see how well your agent is performing compared to your classmates"),ac=n(" \u{1F449} "),nl=a("a"),oc=n("https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),pn=p(),aa=a("p"),sc=n("To be able to share your model with the community there are three more steps to follow:"),fn=p(),il=a("p"),rc=n("1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),hl=a("a"),nc=n("https://huggingface.co/join"),cn=p(),oa=a("p"),ic=n("2\uFE0F\u20E3 Sign in and then, you need to store your authentication token from the Hugging Face website."),un=p(),sa=a("ul"),Fe=a("li"),hc=n("Create a new token ("),pl=a("a"),pc=n("https://huggingface.co/settings/tokens"),fc=n(") "),ts=a("strong"),cc=n("with write role"),mn=p(),fl=a("img"),dn=p(),Ye=a("ul"),ls=a("li"),uc=n("Copy the token"),mc=p(),as=a("li"),dc=n("Run the cell below and paste the token"),vn=p(),u(cl.$$.fragment),yn=p(),ul=a("p"),vc=n("If you don\u2019t want to use a Google Colab or a Jupyter Notebook, you need to use this command instead: "),os=a("code"),yc=n("huggingface-cli login"),wn=p(),oe=a("p"),wc=n("3\uFE0F\u20E3 We\u2019re now ready to push our trained agent to the \u{1F917} Hub \u{1F525} using "),ss=a("code"),bc=n("package_to_hub()"),gc=n(` function.
For this environment, `),rs=a("strong"),_c=n("running this cell can take approximately 10min"),bn=p(),u(ml.$$.fragment),gn=p(),se=a("h2"),ze=a("a"),ns=a("span"),u(dl.$$.fragment),Mc=p(),is=a("span"),Ec=n("Some additional challenges \u{1F3C6}"),_n=p(),C=a("p"),Uc=n("The best way to learn "),hs=a("strong"),Tc=n("is to try things by your own"),Jc=n("! Why not trying  "),ps=a("code"),$c=n("PandaPickAndPlace-v3"),kc=n("?"),Mn=p(),G=a("p"),jc=n("If you want to try more advanced tasks for panda-gym, you need to check what was done using "),fs=a("strong"),Ic=n("TQC or SAC"),Cc=n(" (a more sample-efficient algorithm suited for robotics tasks). In real robotics, you\u2019ll use a more sample-efficient algorithm for a simple reason: contrary to a simulation "),cs=a("strong"),Gc=n("if you move your robotic arm too much, you have a risk of breaking it"),Sc=n("."),En=p(),vl=a("p"),Zc=n("PandaPickAndPlace-v1 (this model uses the v1 version of the environment): "),yl=a("a"),Wc=n("https://huggingface.co/sb3/tqc-PandaPickAndPlace-v1"),Un=p(),wl=a("p"),Bc=n("And don\u2019t hesitate to check panda-gym documentation here: "),bl=a("a"),Ac=n("https://panda-gym.readthedocs.io/en/latest/usage/train_with_sb3.html"),Tn=p(),ra=a("p"),Nc=n("We provide you the steps to train another agent (optional):"),Jn=p(),g=a("ol"),us=a("li"),Pc=n("Define the environment called \u201CPandaPickAndPlace-v3\u201D"),Vc=p(),ms=a("li"),Rc=n("Make a vectorized environment"),Qc=p(),na=a("li"),Fc=n("Add a wrapper to normalize the observations and rewards. "),gl=a("a"),Yc=n("Check the documentation"),zc=p(),ds=a("li"),Xc=n("Create the A2C Model (don\u2019t forget verbose=1 to print the training logs)."),Hc=p(),vs=a("li"),Lc=n("Train it for 1M Timesteps"),xc=p(),ys=a("li"),qc=n("Save the model and  VecNormalize statistics when saving the agent"),Dc=p(),ws=a("li"),Oc=n("Evaluate your agent"),Kc=p(),ia=a("li"),eu=n("Publish your trained model on the Hub \u{1F525} with "),bs=a("code"),tu=n("package_to_hub"),$n=p(),re=a("h3"),Xe=a("a"),gs=a("span"),u(_l.$$.fragment),lu=p(),_s=a("span"),au=n("Solution (optional)"),kn=p(),u(Ml.$$.fragment),jn=p(),u(El.$$.fragment),In=p(),ha=a("p"),ou=n("See you on Unit 7! \u{1F525}"),Cn=p(),ne=a("h2"),He=a("a"),Ms=a("span"),u(Ul.$$.fragment),su=p(),Es=a("span"),ru=n("Keep learning, stay awesome \u{1F917}"),this.h()},l(e){const r=sy('[data-svelte="svelte-1phssyn"]',document.head);W=o(r,"META",{name:!0,content:!0}),r.forEach(t),Is=f(e),B=o(e,"H1",{class:!0});var Sn=s(B);ie=o(Sn,"A",{id:!0,class:!0,href:!0});var Su=s(ie);va=o(Su,"SPAN",{});var Zu=s(va);m(qe.$$.fragment,Zu),Zu.forEach(t),Su.forEach(t),Ui=f(Sn),ya=o(Sn,"SPAN",{});var Wu=s(ya);Ti=i(Wu,"Advantage Actor Critic (A2C) using Robotics Simulations with Panda-Gym \u{1F916}"),Wu.forEach(t),Sn.forEach(t),Cs=f(e),m(De.$$.fragment,e),Gs=f(e),he=o(e,"P",{});var Zn=s(he);Ji=i(Zn,"Now that you\u2019ve studied the theory behind Advantage Actor Critic (A2C), "),wa=o(Zn,"STRONG",{});var Bu=s(wa);$i=i(Bu,"you\u2019re ready to train your A2C agent"),Bu.forEach(t),ki=i(Zn," using Stable-Baselines3 in a robotic environment. And train a:"),Zn.forEach(t),Ss=f(e),Tl=o(e,"UL",{});var Au=s(Tl);ba=o(Au,"LI",{});var Nu=s(ba);ji=i(Nu,"A robotic arm \u{1F9BE} to move to the correct position."),Nu.forEach(t),Au.forEach(t),Zs=f(e),Jl=o(e,"P",{});var Pu=s(Jl);Ii=i(Pu,"We\u2019re going to use"),Pu.forEach(t),Ws=f(e),$l=o(e,"UL",{});var Vu=s($l);ga=o(Vu,"LI",{});var Ru=s(ga);Oe=o(Ru,"A",{href:!0,rel:!0});var Qu=s(Oe);Ci=i(Qu,"panda-gym"),Qu.forEach(t),Ru.forEach(t),Vu.forEach(t),Bs=f(e),kl=o(e,"P",{});var Fu=s(kl);Gi=i(Fu,"To validate this hands-on for the certification process, you need to push your two trained models to the Hub and get the following results:"),Fu.forEach(t),As=f(e),jl=o(e,"UL",{});var Yu=s(jl);Il=o(Yu,"LI",{});var nu=s(Il);_a=o(nu,"CODE",{});var zu=s(_a);Si=i(zu,"PandaReachDense-v3"),zu.forEach(t),Zi=i(nu," get a result of >= -3.5."),nu.forEach(t),Yu.forEach(t),Ns=f(e),A=o(e,"P",{});var Us=s(A);Wi=i(Us,"To find your result, "),Ke=o(Us,"A",{href:!0,rel:!0});var Xu=s(Ke);Bi=i(Xu,"go to the leaderboard"),Xu.forEach(t),Ai=i(Us," and find your model, "),Ma=o(Us,"STRONG",{});var Hu=s(Ma);Ni=i(Hu,"the result = mean_reward - std of reward"),Hu.forEach(t),Us.forEach(t),Ps=f(e),et=o(e,"P",{});var iu=s(et);Pi=i(iu,"For more information about the certification process, check this section \u{1F449} "),tt=o(iu,"A",{href:!0,rel:!0});var Lu=s(tt);Vi=i(Lu,"https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),Lu.forEach(t),iu.forEach(t),Vs=f(e),lt=o(e,"P",{});var hu=s(lt);Ea=o(hu,"STRONG",{});var xu=s(Ea);Ri=i(xu,"To start the hands-on click on Open In Colab button"),xu.forEach(t),Qi=i(hu," \u{1F447} :"),hu.forEach(t),Rs=f(e),Cl=o(e,"P",{});var qu=s(Cl);at=o(qu,"A",{href:!0,rel:!0});var Du=s(at);Gl=o(Du,"IMG",{src:!0,alt:!0}),Du.forEach(t),qu.forEach(t),Qs=f(e),N=o(e,"H1",{class:!0});var Wn=s(N);pe=o(Wn,"A",{id:!0,class:!0,href:!0});var Ou=s(pe);Ua=o(Ou,"SPAN",{});var Ku=s(Ua);m(ot.$$.fragment,Ku),Ku.forEach(t),Ou.forEach(t),Fi=f(Wn),Ta=o(Wn,"SPAN",{});var em=s(Ta);Yi=i(em,"Unit 6: Advantage Actor Critic (A2C) using Robotics Simulations with Panda-Gym \u{1F916}"),em.forEach(t),Wn.forEach(t),Fs=f(e),P=o(e,"H3",{class:!0});var Bn=s(P);fe=o(Bn,"A",{id:!0,class:!0,href:!0});var tm=s(fe);Ja=o(tm,"SPAN",{});var lm=s(Ja);m(st.$$.fragment,lm),lm.forEach(t),tm.forEach(t),zi=f(Bn),$a=o(Bn,"SPAN",{});var am=s($a);Xi=i(am,"\u{1F3AE} Environments:"),am.forEach(t),Bn.forEach(t),Ys=f(e),Sl=o(e,"UL",{});var om=s(Sl);ka=o(om,"LI",{});var sm=s(ka);rt=o(sm,"A",{href:!0,rel:!0});var rm=s(rt);Hi=i(rm,"Panda-Gym"),rm.forEach(t),sm.forEach(t),om.forEach(t),zs=f(e),V=o(e,"H3",{class:!0});var An=s(V);ce=o(An,"A",{id:!0,class:!0,href:!0});var nm=s(ce);ja=o(nm,"SPAN",{});var im=s(ja);m(nt.$$.fragment,im),im.forEach(t),nm.forEach(t),Li=f(An),Ia=o(An,"SPAN",{});var hm=s(Ia);xi=i(hm,"\u{1F4DA} RL-Library:"),hm.forEach(t),An.forEach(t),Xs=f(e),Zl=o(e,"UL",{});var pm=s(Zl);Ca=o(pm,"LI",{});var fm=s(Ca);it=o(fm,"A",{href:!0,rel:!0});var cm=s(it);qi=i(cm,"Stable-Baselines3"),cm.forEach(t),fm.forEach(t),pm.forEach(t),Hs=f(e),$=o(e,"P",{});var pa=s($);Di=i(pa,"We\u2019re constantly trying to improve our tutorials, so "),Ga=o(pa,"STRONG",{});var um=s(Ga);Oi=i(um,"if you find some issues in this notebook"),um.forEach(t),Ki=i(pa,", please "),ht=o(pa,"A",{href:!0,rel:!0});var mm=s(ht);eh=i(mm,"open an issue on the GitHub Repo"),mm.forEach(t),th=i(pa,"."),pa.forEach(t),Ls=f(e),R=o(e,"H2",{class:!0});var Nn=s(R);ue=o(Nn,"A",{id:!0,class:!0,href:!0});var dm=s(ue);Sa=o(dm,"SPAN",{});var vm=s(Sa);m(pt.$$.fragment,vm),vm.forEach(t),dm.forEach(t),lh=f(Nn),Za=o(Nn,"SPAN",{});var ym=s(Za);ah=i(ym,"Objectives of this notebook \u{1F3C6}"),ym.forEach(t),Nn.forEach(t),xs=f(e),Wl=o(e,"P",{});var wm=s(Wl);oh=i(wm,"At the end of the notebook, you will:"),wm.forEach(t),qs=f(e),T=o(e,"UL",{});var Le=s(T);ft=o(Le,"LI",{});var Pn=s(ft);sh=i(Pn,"Be able to use "),Wa=o(Pn,"STRONG",{});var bm=s(Wa);rh=i(bm,"Panda-Gym"),bm.forEach(t),nh=i(Pn,", the environment library."),Pn.forEach(t),ih=f(Le),ct=o(Le,"LI",{});var Vn=s(ct);hh=i(Vn,"Be able to "),Ba=o(Vn,"STRONG",{});var gm=s(Ba);ph=i(gm,"train robots using A2C"),gm.forEach(t),fh=i(Vn,"."),Vn.forEach(t),ch=f(Le),ut=o(Le,"LI",{});var Rn=s(ut);uh=i(Rn,"Understand why "),Aa=o(Rn,"STRONG",{});var _m=s(Aa);mh=i(_m,"we need to normalize the input"),_m.forEach(t),dh=i(Rn,"."),Rn.forEach(t),vh=f(Le),mt=o(Le,"LI",{});var Qn=s(mt);yh=i(Qn,"Be able to "),Na=o(Qn,"STRONG",{});var Mm=s(Na);wh=i(Mm,"push your trained agent and the code to the Hub"),Mm.forEach(t),bh=i(Qn," with a nice video replay and an evaluation score \u{1F525}."),Qn.forEach(t),Le.forEach(t),Ds=f(e),Q=o(e,"H2",{class:!0});var Fn=s(Q);me=o(Fn,"A",{id:!0,class:!0,href:!0});var Em=s(me);Pa=o(Em,"SPAN",{});var Um=s(Pa);m(dt.$$.fragment,Um),Um.forEach(t),Em.forEach(t),gh=f(Fn),Va=o(Fn,"SPAN",{});var Tm=s(Va);_h=i(Tm,"Prerequisites \u{1F3D7}\uFE0F"),Tm.forEach(t),Fn.forEach(t),Os=f(e),Bl=o(e,"P",{});var Jm=s(Bl);Mh=i(Jm,"Before diving into the notebook, you need to:"),Jm.forEach(t),Ks=f(e),de=o(e,"P",{});var Yn=s(de);Eh=i(Yn,"\u{1F532} \u{1F4DA} Study "),vt=o(Yn,"A",{href:!0,rel:!0});var $m=s(vt);Uh=i($m,"Actor-Critic methods by reading Unit 6"),$m.forEach(t),Th=i(Yn," \u{1F917}"),Yn.forEach(t),er=f(e),F=o(e,"H1",{class:!0});var zn=s(F);ve=o(zn,"A",{id:!0,class:!0,href:!0});var km=s(ve);Ra=o(km,"SPAN",{});var jm=s(Ra);m(yt.$$.fragment,jm),jm.forEach(t),km.forEach(t),Jh=f(zn),Qa=o(zn,"SPAN",{});var Im=s(Qa);$h=i(Im,"Let's train our first robots \u{1F916}"),Im.forEach(t),zn.forEach(t),tr=f(e),Y=o(e,"H2",{class:!0});var Xn=s(Y);ye=o(Xn,"A",{id:!0,class:!0,href:!0});var Cm=s(ye);Fa=o(Cm,"SPAN",{});var Gm=s(Fa);m(wt.$$.fragment,Gm),Gm.forEach(t),Cm.forEach(t),kh=f(Xn),Ya=o(Xn,"SPAN",{});var Sm=s(Ya);jh=i(Sm,"Set the GPU \u{1F4AA}"),Sm.forEach(t),Xn.forEach(t),lr=f(e),Al=o(e,"UL",{});var Zm=s(Al);we=o(Zm,"LI",{});var Ts=s(we);Ih=i(Ts,"To "),za=o(Ts,"STRONG",{});var Wm=s(za);Ch=i(Wm,"accelerate the agent\u2019s training, we\u2019ll use a GPU"),Wm.forEach(t),Gh=i(Ts,". To do that, go to "),Xa=o(Ts,"CODE",{});var Bm=s(Xa);Sh=i(Bm,"Runtime > Change Runtime type"),Bm.forEach(t),Ts.forEach(t),Zm.forEach(t),ar=f(e),bt=o(e,"IMG",{src:!0,alt:!0}),or=f(e),Nl=o(e,"UL",{});var Am=s(Nl);Ha=o(Am,"LI",{});var Nm=s(Ha);La=o(Nm,"CODE",{});var Pm=s(La);Zh=i(Pm,"Hardware Accelerator > GPU"),Pm.forEach(t),Nm.forEach(t),Am.forEach(t),sr=f(e),gt=o(e,"IMG",{src:!0,alt:!0}),rr=f(e),z=o(e,"H2",{class:!0});var Hn=s(z);be=o(Hn,"A",{id:!0,class:!0,href:!0});var Vm=s(be);xa=o(Vm,"SPAN",{});var Rm=s(xa);m(_t.$$.fragment,Rm),Rm.forEach(t),Vm.forEach(t),Wh=f(Hn),qa=o(Hn,"SPAN",{});var Qm=s(qa);Bh=i(Qm,"Create a virtual display \u{1F53D}"),Qm.forEach(t),Hn.forEach(t),nr=f(e),ge=o(e,"P",{});var Ln=s(ge);Ah=i(Ln,"During the notebook, we\u2019ll need to generate a replay video. To do so, with colab, "),Da=o(Ln,"STRONG",{});var Fm=s(Da);Nh=i(Fm,"we need to have a virtual screen to be able to render the environment"),Fm.forEach(t),Ph=i(Ln," (and thus record the frames)."),Ln.forEach(t),ir=f(e),Pl=o(e,"P",{});var Ym=s(Pl);Vh=i(Ym,"The following cell will install the librairies and create and run a virtual screen \u{1F5A5}"),Ym.forEach(t),hr=f(e),m(Mt.$$.fragment,e),pr=f(e),m(Et.$$.fragment,e),fr=f(e),X=o(e,"H3",{class:!0});var xn=s(X);_e=o(xn,"A",{id:!0,class:!0,href:!0});var zm=s(_e);Oa=o(zm,"SPAN",{});var Xm=s(Oa);m(Ut.$$.fragment,Xm),Xm.forEach(t),zm.forEach(t),Rh=f(xn),Ka=o(xn,"SPAN",{});var Hm=s(Ka);Qh=i(Hm,"Install dependencies \u{1F53D}"),Hm.forEach(t),xn.forEach(t),cr=f(e),Vl=o(e,"P",{});var Lm=s(Vl);Fh=i(Lm,"We\u2019ll install multiple ones:"),Lm.forEach(t),ur=f(e),E=o(e,"UL",{});var S=s(E);eo=o(S,"LI",{});var xm=s(eo);to=o(xm,"CODE",{});var qm=s(to);Yh=i(qm,"gymnasium"),qm.forEach(t),xm.forEach(t),zh=f(S),Rl=o(S,"LI",{});var pu=s(Rl);lo=o(pu,"CODE",{});var Dm=s(lo);Xh=i(Dm,"panda-gym"),Dm.forEach(t),Hh=i(pu,": Contains the robotics arm environments."),pu.forEach(t),Lh=f(S),Ql=o(S,"LI",{});var fu=s(Ql);ao=o(fu,"CODE",{});var Om=s(ao);xh=i(Om,"stable-baselines3"),Om.forEach(t),qh=i(fu,": The SB3 deep reinforcement learning library."),fu.forEach(t),Dh=f(S),Fl=o(S,"LI",{});var cu=s(Fl);oo=o(cu,"CODE",{});var Km=s(oo);Oh=i(Km,"huggingface_sb3"),Km.forEach(t),Kh=i(cu,": Additional code for Stable-baselines3 to load and upload models from the Hugging Face \u{1F917} Hub."),cu.forEach(t),ep=f(S),Yl=o(S,"LI",{});var uu=s(Yl);so=o(uu,"CODE",{});var ed=s(so);tp=i(ed,"huggingface_hub"),ed.forEach(t),lp=i(uu,": Library allowing anyone to work with the Hub repositories."),uu.forEach(t),S.forEach(t),mr=f(e),m(Tt.$$.fragment,e),dr=f(e),H=o(e,"H2",{class:!0});var qn=s(H);Me=o(qn,"A",{id:!0,class:!0,href:!0});var td=s(Me);ro=o(td,"SPAN",{});var ld=s(ro);m(Jt.$$.fragment,ld),ld.forEach(t),td.forEach(t),ap=f(qn),no=o(qn,"SPAN",{});var ad=s(no);op=i(ad,"Import the packages \u{1F4E6}"),ad.forEach(t),qn.forEach(t),vr=f(e),m($t.$$.fragment,e),yr=f(e),L=o(e,"H2",{class:!0});var Dn=s(L);Ee=o(Dn,"A",{id:!0,class:!0,href:!0});var od=s(Ee);io=o(od,"SPAN",{});var sd=s(io);m(kt.$$.fragment,sd),sd.forEach(t),od.forEach(t),sp=f(Dn),ho=o(Dn,"SPAN",{});var rd=s(ho);rp=i(rd,"PandaReachDense-v3 \u{1F9BE}"),rd.forEach(t),Dn.forEach(t),wr=f(e),zl=o(e,"P",{});var nd=s(zl);np=i(nd,"The agent we\u2019re going to train is a robotic arm that needs to do controls (moving the arm and using the end-effector)."),nd.forEach(t),br=f(e),Ue=o(e,"P",{});var On=s(Ue);ip=i(On,"In robotics, the "),po=o(On,"EM",{});var id=s(po);hp=i(id,"end-effector"),id.forEach(t),pp=i(On," is the device at the end of a robotic arm designed to interact with the environment."),On.forEach(t),gr=f(e),Te=o(e,"P",{});var Kn=s(Te);fp=i(Kn,"In "),fo=o(Kn,"CODE",{});var hd=s(fo);cp=i(hd,"PandaReach"),hd.forEach(t),up=i(Kn,", the robot must place its end-effector at a target position (green ball)."),Kn.forEach(t),_r=f(e),U=o(e,"P",{});var Z=s(U);mp=i(Z,"We\u2019re going to use the dense version of this environment. It means we\u2019ll get a "),co=o(Z,"EM",{});var pd=s(co);dp=i(pd,"dense reward function"),pd.forEach(t),vp=i(Z," that "),uo=o(Z,"STRONG",{});var fd=s(uo);yp=i(fd,"will provide a reward at each timestep"),fd.forEach(t),wp=i(Z," (the closer the agent is to completing the task, the higher the reward). Contrary to a "),mo=o(Z,"EM",{});var cd=s(mo);bp=i(cd,"sparse reward function"),cd.forEach(t),gp=i(Z," where the environment "),vo=o(Z,"STRONG",{});var ud=s(vo);_p=i(ud,"return a reward if and only if the task is completed"),ud.forEach(t),Mp=i(Z,"."),Z.forEach(t),Mr=f(e),k=o(e,"P",{});var fa=s(k);Ep=i(fa,"Also, we\u2019re going to use the "),yo=o(fa,"EM",{});var md=s(yo);Up=i(md,"End-effector displacement control"),md.forEach(t),Tp=i(fa,", it means the "),wo=o(fa,"STRONG",{});var dd=s(wo);Jp=i(dd,"action corresponds to the displacement of the end-effector"),dd.forEach(t),$p=i(fa,". We don\u2019t control the individual motion of each joint (joint control)."),fa.forEach(t),Er=f(e),jt=o(e,"IMG",{src:!0,alt:!0}),Ur=f(e),Je=o(e,"P",{});var ei=s(Je);kp=i(ei,"This way "),bo=o(ei,"STRONG",{});var vd=s(bo);jp=i(vd,"the training will be easier"),vd.forEach(t),Ip=i(ei,"."),ei.forEach(t),Tr=f(e),x=o(e,"H3",{class:!0});var ti=s(x);$e=o(ti,"A",{id:!0,class:!0,href:!0});var yd=s($e);go=o(yd,"SPAN",{});var wd=s(go);m(It.$$.fragment,wd),wd.forEach(t),yd.forEach(t),Cp=f(ti),_o=o(ti,"SPAN",{});var bd=s(_o);Gp=i(bd,"Create the environment"),bd.forEach(t),ti.forEach(t),Jr=f(e),q=o(e,"H4",{class:!0});var li=s(q);ke=o(li,"A",{id:!0,class:!0,href:!0});var gd=s(ke);Mo=o(gd,"SPAN",{});var _d=s(Mo);m(Ct.$$.fragment,_d),_d.forEach(t),gd.forEach(t),Sp=f(li),Eo=o(li,"SPAN",{});var Md=s(Eo);Zp=i(Md,"The environment \u{1F3AE}"),Md.forEach(t),li.forEach(t),$r=f(e),je=o(e,"P",{});var ai=s(je);Wp=i(ai,"In "),Uo=o(ai,"CODE",{});var Ed=s(Uo);Bp=i(Ed,"PandaReachDense-v3"),Ed.forEach(t),Ap=i(ai," the robotic arm must place its end-effector at a target position (green ball)."),ai.forEach(t),kr=f(e),m(Gt.$$.fragment,e),jr=f(e),m(St.$$.fragment,e),Ir=f(e),Ie=o(e,"P",{});var oi=s(Ie);Np=i(oi,"The observation space "),To=o(oi,"STRONG",{});var Ud=s(To);Pp=i(Ud,"is a dictionary with 3 different elements"),Ud.forEach(t),Vp=i(oi,":"),oi.forEach(t),Cr=f(e),j=o(e,"UL",{});var ca=s(j);Xl=o(ca,"LI",{});var mu=s(Xl);Jo=o(mu,"CODE",{});var Td=s(Jo);Rp=i(Td,"achieved_goal"),Td.forEach(t),Qp=i(mu,": (x,y,z) position of the goal."),mu.forEach(t),Fp=f(ca),Hl=o(ca,"LI",{});var du=s(Hl);$o=o(du,"CODE",{});var Jd=s($o);Yp=i(Jd,"desired_goal"),Jd.forEach(t),zp=i(du,": (x,y,z) distance between the goal position and the current object position."),du.forEach(t),Xp=f(ca),Ll=o(ca,"LI",{});var vu=s(Ll);ko=o(vu,"CODE",{});var $d=s(ko);Hp=i($d,"observation"),$d.forEach(t),Lp=i(vu,": position (x,y,z) and velocity of the end-effector (vx, vy, vz)."),vu.forEach(t),ca.forEach(t),Gr=f(e),Ce=o(e,"P",{});var si=s(Ce);xp=i(si,"Given it\u2019s a dictionary as observation, "),jo=o(si,"STRONG",{});var kd=s(jo);qp=i(kd,"we will need to use a MultiInputPolicy policy instead of MlpPolicy"),kd.forEach(t),Dp=i(si,"."),si.forEach(t),Sr=f(e),m(Zt.$$.fragment,e),Zr=f(e),xl=o(e,"P",{});var jd=s(xl);Op=i(jd,"The action space is a vector with 3 values:"),jd.forEach(t),Wr=f(e),ql=o(e,"UL",{});var Id=s(ql);Io=o(Id,"LI",{});var Cd=s(Io);Kp=i(Cd,"Control x, y, z movement"),Cd.forEach(t),Id.forEach(t),Br=f(e),D=o(e,"H3",{class:!0});var ri=s(D);Ge=o(ri,"A",{id:!0,class:!0,href:!0});var Gd=s(Ge);Co=o(Gd,"SPAN",{});var Sd=s(Co);m(Wt.$$.fragment,Sd),Sd.forEach(t),Gd.forEach(t),ef=f(ri),Go=o(ri,"SPAN",{});var Zd=s(Go);tf=i(Zd,"Normalize observation and rewards"),Zd.forEach(t),ri.forEach(t),Ar=f(e),Se=o(e,"P",{});var ni=s(Se);lf=i(ni,"A good practice in reinforcement learning is to "),Bt=o(ni,"A",{href:!0,rel:!0});var Wd=s(Bt);af=i(Wd,"normalize input features"),Wd.forEach(t),of=i(ni,"."),ni.forEach(t),Nr=f(e),Dl=o(e,"P",{});var Bd=s(Dl);sf=i(Bd,"For that purpose, there is a wrapper that will compute a running average and standard deviation of input features."),Bd.forEach(t),Pr=f(e),At=o(e,"P",{});var yu=s(At);rf=i(yu,"We also normalize rewards with this same wrapper by adding "),So=o(yu,"CODE",{});var Ad=s(So);nf=i(Ad,"norm_reward = True"),Ad.forEach(t),yu.forEach(t),Vr=f(e),Ol=o(e,"P",{});var Nd=s(Ol);Nt=o(Nd,"A",{href:!0,rel:!0});var Pd=s(Nt);hf=i(Pd,"You should check the documentation to fill this cell"),Pd.forEach(t),Nd.forEach(t),Rr=f(e),m(Pt.$$.fragment,e),Qr=f(e),O=o(e,"H4",{class:!0});var ii=s(O);Ze=o(ii,"A",{id:!0,class:!0,href:!0});var Vd=s(Ze);Zo=o(Vd,"SPAN",{});var Rd=s(Zo);m(Vt.$$.fragment,Rd),Rd.forEach(t),Vd.forEach(t),pf=f(ii),Wo=o(ii,"SPAN",{});var Qd=s(Wo);ff=i(Qd,"Solution"),Qd.forEach(t),ii.forEach(t),Fr=f(e),m(Rt.$$.fragment,e),Yr=f(e),K=o(e,"H3",{class:!0});var hi=s(K);We=o(hi,"A",{id:!0,class:!0,href:!0});var Fd=s(We);Bo=o(Fd,"SPAN",{});var Yd=s(Bo);m(Qt.$$.fragment,Yd),Yd.forEach(t),Fd.forEach(t),cf=f(hi),Ao=o(hi,"SPAN",{});var zd=s(Ao);uf=i(zd,"Create the A2C Model \u{1F916}"),zd.forEach(t),hi.forEach(t),zr=f(e),Ft=o(e,"P",{});var wu=s(Ft);mf=i(wu,"For more information about A2C implementation with StableBaselines3 check: "),Yt=o(wu,"A",{href:!0,rel:!0});var Xd=s(Yt);df=i(Xd,"https://stable-baselines3.readthedocs.io/en/master/modules/a2c.html#notes"),Xd.forEach(t),wu.forEach(t),Xr=f(e),Be=o(e,"P",{});var pi=s(Be);vf=i(pi,"To find the best parameters I checked the "),zt=o(pi,"A",{href:!0,rel:!0});var Hd=s(zt);yf=i(Hd,"official trained agents by Stable-Baselines3 team"),Hd.forEach(t),wf=i(pi,"."),pi.forEach(t),Hr=f(e),m(Xt.$$.fragment,e),Lr=f(e),ee=o(e,"H4",{class:!0});var fi=s(ee);Ae=o(fi,"A",{id:!0,class:!0,href:!0});var Ld=s(Ae);No=o(Ld,"SPAN",{});var xd=s(No);m(Ht.$$.fragment,xd),xd.forEach(t),Ld.forEach(t),bf=f(fi),Po=o(fi,"SPAN",{});var qd=s(Po);gf=i(qd,"Solution"),qd.forEach(t),fi.forEach(t),xr=f(e),m(Lt.$$.fragment,e),qr=f(e),te=o(e,"H3",{class:!0});var ci=s(te);Ne=o(ci,"A",{id:!0,class:!0,href:!0});var Dd=s(Ne);Vo=o(Dd,"SPAN",{});var Od=s(Vo);m(xt.$$.fragment,Od),Od.forEach(t),Dd.forEach(t),_f=f(ci),Ro=o(ci,"SPAN",{});var Kd=s(Ro);Mf=i(Kd,"Train the A2C agent \u{1F3C3}"),Kd.forEach(t),ci.forEach(t),Dr=f(e),Kl=o(e,"UL",{});var ev=s(Kl);Qo=o(ev,"LI",{});var tv=s(Qo);Ef=i(tv,"Let\u2019s train our agent for 1,000,000 timesteps, don\u2019t forget to use GPU on Colab. It will take approximately ~25-40min"),tv.forEach(t),ev.forEach(t),Or=f(e),m(qt.$$.fragment,e),Kr=f(e),m(Dt.$$.fragment,e),en=f(e),le=o(e,"H3",{class:!0});var ui=s(le);Pe=o(ui,"A",{id:!0,class:!0,href:!0});var lv=s(Pe);Fo=o(lv,"SPAN",{});var av=s(Fo);m(Ot.$$.fragment,av),av.forEach(t),lv.forEach(t),Uf=f(ui),Yo=o(ui,"SPAN",{});var ov=s(Yo);Tf=i(ov,"Evaluate the agent \u{1F4C8}"),ov.forEach(t),ui.forEach(t),tn=f(e),Ve=o(e,"UL",{});var mi=s(Ve);Kt=o(mi,"LI",{});var di=s(Kt);Jf=i(di,"Now that\u2019s our  agent is trained, we need to "),zo=o(di,"STRONG",{});var sv=s(zo);$f=i(sv,"check its performance"),sv.forEach(t),kf=i(di,"."),di.forEach(t),jf=f(mi),ea=o(mi,"LI",{});var bu=s(ea);If=i(bu,"Stable-Baselines3 provides a method to do that: "),Xo=o(bu,"CODE",{});var rv=s(Xo);Cf=i(rv,"evaluate_policy"),rv.forEach(t),bu.forEach(t),mi.forEach(t),ln=f(e),m(el.$$.fragment,e),an=f(e),ae=o(e,"H3",{class:!0});var vi=s(ae);Re=o(vi,"A",{id:!0,class:!0,href:!0});var nv=s(Re);Ho=o(nv,"SPAN",{});var iv=s(Ho);m(tl.$$.fragment,iv),iv.forEach(t),nv.forEach(t),Gf=f(vi),Lo=o(vi,"SPAN",{});var hv=s(Lo);Sf=i(hv,"Publish your trained model on the Hub \u{1F525}"),hv.forEach(t),vi.forEach(t),on=f(e),ta=o(e,"P",{});var pv=s(ta);Zf=i(pv,"Now that we saw we got good results after the training, we can publish our trained model on the Hub with one line of code."),pv.forEach(t),sn=f(e),ll=o(e,"P",{});var gu=s(ll);Wf=i(gu,"\u{1F4DA} The libraries documentation \u{1F449} "),al=o(gu,"A",{href:!0,rel:!0});var fv=s(al);Bf=i(fv,"https://github.com/huggingface/huggingface_sb3/tree/main#hugging-face\u2014x-stable-baselines3-v20"),fv.forEach(t),gu.forEach(t),rn=f(e),I=o(e,"P",{});var ua=s(I);Af=i(ua,"By using "),xo=o(ua,"CODE",{});var cv=s(xo);Nf=i(cv,"package_to_hub"),cv.forEach(t),Pf=i(ua,", as we already mentionned in the former units, "),qo=o(ua,"STRONG",{});var uv=s(qo);Vf=i(uv,"you evaluate, record a replay, generate a model card of your agent and push it to the hub"),uv.forEach(t),Rf=i(ua,"."),ua.forEach(t),nn=f(e),la=o(e,"P",{});var mv=s(la);Qf=i(mv,"This way:"),mv.forEach(t),hn=f(e),J=o(e,"UL",{});var xe=s(J);ol=o(xe,"LI",{});var yi=s(ol);Ff=i(yi,"You can "),Do=o(yi,"STRONG",{});var dv=s(Do);Yf=i(dv,"showcase our work"),dv.forEach(t),zf=i(yi," \u{1F525}"),yi.forEach(t),Xf=f(xe),sl=o(xe,"LI",{});var wi=s(sl);Hf=i(wi,"You can "),Oo=o(wi,"STRONG",{});var vv=s(Oo);Lf=i(vv,"visualize your agent playing"),vv.forEach(t),xf=i(wi," \u{1F440}"),wi.forEach(t),qf=f(xe),rl=o(xe,"LI",{});var bi=s(rl);Df=i(bi,"You can "),Ko=o(bi,"STRONG",{});var yv=s(Ko);Of=i(yv,"share with the community an agent that others can use"),yv.forEach(t),Kf=i(bi," \u{1F4BE}"),bi.forEach(t),ec=f(xe),Qe=o(xe,"LI",{});var Js=s(Qe);tc=i(Js,"You can "),es=o(Js,"STRONG",{});var wv=s(es);lc=i(wv,"access a leaderboard \u{1F3C6} to see how well your agent is performing compared to your classmates"),wv.forEach(t),ac=i(Js," \u{1F449} "),nl=o(Js,"A",{href:!0,rel:!0});var bv=s(nl);oc=i(bv,"https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),bv.forEach(t),Js.forEach(t),xe.forEach(t),pn=f(e),aa=o(e,"P",{});var gv=s(aa);sc=i(gv,"To be able to share your model with the community there are three more steps to follow:"),gv.forEach(t),fn=f(e),il=o(e,"P",{});var _u=s(il);rc=i(_u,"1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),hl=o(_u,"A",{href:!0,rel:!0});var _v=s(hl);nc=i(_v,"https://huggingface.co/join"),_v.forEach(t),_u.forEach(t),cn=f(e),oa=o(e,"P",{});var Mv=s(oa);ic=i(Mv,"2\uFE0F\u20E3 Sign in and then, you need to store your authentication token from the Hugging Face website."),Mv.forEach(t),un=f(e),sa=o(e,"UL",{});var Ev=s(sa);Fe=o(Ev,"LI",{});var $s=s(Fe);hc=i($s,"Create a new token ("),pl=o($s,"A",{href:!0,rel:!0});var Uv=s(pl);pc=i(Uv,"https://huggingface.co/settings/tokens"),Uv.forEach(t),fc=i($s,") "),ts=o($s,"STRONG",{});var Tv=s(ts);cc=i(Tv,"with write role"),Tv.forEach(t),$s.forEach(t),Ev.forEach(t),mn=f(e),fl=o(e,"IMG",{src:!0,alt:!0}),dn=f(e),Ye=o(e,"UL",{});var gi=s(Ye);ls=o(gi,"LI",{});var Jv=s(ls);uc=i(Jv,"Copy the token"),Jv.forEach(t),mc=f(gi),as=o(gi,"LI",{});var $v=s(as);dc=i($v,"Run the cell below and paste the token"),$v.forEach(t),gi.forEach(t),vn=f(e),m(cl.$$.fragment,e),yn=f(e),ul=o(e,"P",{});var Mu=s(ul);vc=i(Mu,"If you don\u2019t want to use a Google Colab or a Jupyter Notebook, you need to use this command instead: "),os=o(Mu,"CODE",{});var kv=s(os);yc=i(kv,"huggingface-cli login"),kv.forEach(t),Mu.forEach(t),wn=f(e),oe=o(e,"P",{});var ks=s(oe);wc=i(ks,"3\uFE0F\u20E3 We\u2019re now ready to push our trained agent to the \u{1F917} Hub \u{1F525} using "),ss=o(ks,"CODE",{});var jv=s(ss);bc=i(jv,"package_to_hub()"),jv.forEach(t),gc=i(ks,` function.
For this environment, `),rs=o(ks,"STRONG",{});var Iv=s(rs);_c=i(Iv,"running this cell can take approximately 10min"),Iv.forEach(t),ks.forEach(t),bn=f(e),m(ml.$$.fragment,e),gn=f(e),se=o(e,"H2",{class:!0});var _i=s(se);ze=o(_i,"A",{id:!0,class:!0,href:!0});var Cv=s(ze);ns=o(Cv,"SPAN",{});var Gv=s(ns);m(dl.$$.fragment,Gv),Gv.forEach(t),Cv.forEach(t),Mc=f(_i),is=o(_i,"SPAN",{});var Sv=s(is);Ec=i(Sv,"Some additional challenges \u{1F3C6}"),Sv.forEach(t),_i.forEach(t),_n=f(e),C=o(e,"P",{});var ma=s(C);Uc=i(ma,"The best way to learn "),hs=o(ma,"STRONG",{});var Zv=s(hs);Tc=i(Zv,"is to try things by your own"),Zv.forEach(t),Jc=i(ma,"! Why not trying  "),ps=o(ma,"CODE",{});var Wv=s(ps);$c=i(Wv,"PandaPickAndPlace-v3"),Wv.forEach(t),kc=i(ma,"?"),ma.forEach(t),Mn=f(e),G=o(e,"P",{});var da=s(G);jc=i(da,"If you want to try more advanced tasks for panda-gym, you need to check what was done using "),fs=o(da,"STRONG",{});var Bv=s(fs);Ic=i(Bv,"TQC or SAC"),Bv.forEach(t),Cc=i(da," (a more sample-efficient algorithm suited for robotics tasks). In real robotics, you\u2019ll use a more sample-efficient algorithm for a simple reason: contrary to a simulation "),cs=o(da,"STRONG",{});var Av=s(cs);Gc=i(Av,"if you move your robotic arm too much, you have a risk of breaking it"),Av.forEach(t),Sc=i(da,"."),da.forEach(t),En=f(e),vl=o(e,"P",{});var Eu=s(vl);Zc=i(Eu,"PandaPickAndPlace-v1 (this model uses the v1 version of the environment): "),yl=o(Eu,"A",{href:!0,rel:!0});var Nv=s(yl);Wc=i(Nv,"https://huggingface.co/sb3/tqc-PandaPickAndPlace-v1"),Nv.forEach(t),Eu.forEach(t),Un=f(e),wl=o(e,"P",{});var Uu=s(wl);Bc=i(Uu,"And don\u2019t hesitate to check panda-gym documentation here: "),bl=o(Uu,"A",{href:!0,rel:!0});var Pv=s(bl);Ac=i(Pv,"https://panda-gym.readthedocs.io/en/latest/usage/train_with_sb3.html"),Pv.forEach(t),Uu.forEach(t),Tn=f(e),ra=o(e,"P",{});var Vv=s(ra);Nc=i(Vv,"We provide you the steps to train another agent (optional):"),Vv.forEach(t),Jn=f(e),g=o(e,"OL",{});var M=s(g);us=o(M,"LI",{});var Rv=s(us);Pc=i(Rv,"Define the environment called \u201CPandaPickAndPlace-v3\u201D"),Rv.forEach(t),Vc=f(M),ms=o(M,"LI",{});var Qv=s(ms);Rc=i(Qv,"Make a vectorized environment"),Qv.forEach(t),Qc=f(M),na=o(M,"LI",{});var Tu=s(na);Fc=i(Tu,"Add a wrapper to normalize the observations and rewards. "),gl=o(Tu,"A",{href:!0,rel:!0});var Fv=s(gl);Yc=i(Fv,"Check the documentation"),Fv.forEach(t),Tu.forEach(t),zc=f(M),ds=o(M,"LI",{});var Yv=s(ds);Xc=i(Yv,"Create the A2C Model (don\u2019t forget verbose=1 to print the training logs)."),Yv.forEach(t),Hc=f(M),vs=o(M,"LI",{});var zv=s(vs);Lc=i(zv,"Train it for 1M Timesteps"),zv.forEach(t),xc=f(M),ys=o(M,"LI",{});var Xv=s(ys);qc=i(Xv,"Save the model and  VecNormalize statistics when saving the agent"),Xv.forEach(t),Dc=f(M),ws=o(M,"LI",{});var Hv=s(ws);Oc=i(Hv,"Evaluate your agent"),Hv.forEach(t),Kc=f(M),ia=o(M,"LI",{});var Ju=s(ia);eu=i(Ju,"Publish your trained model on the Hub \u{1F525} with "),bs=o(Ju,"CODE",{});var Lv=s(bs);tu=i(Lv,"package_to_hub"),Lv.forEach(t),Ju.forEach(t),M.forEach(t),$n=f(e),re=o(e,"H3",{class:!0});var Mi=s(re);Xe=o(Mi,"A",{id:!0,class:!0,href:!0});var xv=s(Xe);gs=o(xv,"SPAN",{});var qv=s(gs);m(_l.$$.fragment,qv),qv.forEach(t),xv.forEach(t),lu=f(Mi),_s=o(Mi,"SPAN",{});var Dv=s(_s);au=i(Dv,"Solution (optional)"),Dv.forEach(t),Mi.forEach(t),kn=f(e),m(Ml.$$.fragment,e),jn=f(e),m(El.$$.fragment,e),In=f(e),ha=o(e,"P",{});var Ov=s(ha);ou=i(Ov,"See you on Unit 7! \u{1F525}"),Ov.forEach(t),Cn=f(e),ne=o(e,"H2",{class:!0});var Ei=s(ne);He=o(Ei,"A",{id:!0,class:!0,href:!0});var Kv=s(He);Ms=o(Kv,"SPAN",{});var ey=s(Ms);m(Ul.$$.fragment,ey),ey.forEach(t),Kv.forEach(t),su=f(Ei),Es=o(Ei,"SPAN",{});var ty=s(Es);ru=i(ty,"Keep learning, stay awesome \u{1F917}"),ty.forEach(t),Ei.forEach(t),this.h()},h(){c(W,"name","hf:doc:metadata"),c(W,"content",JSON.stringify(py)),c(ie,"id","hands-on"),c(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ie,"href","#hands-on"),c(B,"class","relative group"),c(Oe,"href","https://github.com/qgallouedec/panda-gym"),c(Oe,"rel","nofollow"),c(Ke,"href","https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),c(Ke,"rel","nofollow"),c(tt,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),c(tt,"rel","nofollow"),js(Gl.src,ku="https://colab.research.google.com/assets/colab-badge.svg")||c(Gl,"src",ku),c(Gl,"alt","Open In Colab"),c(at,"href","https://colab.research.google.com/github/huggingface/deep-rl-class/blob/master/notebooks/unit6/unit6.ipynb"),c(at,"rel","nofollow"),c(pe,"id","unit-6-advantage-actor-critic-a2c-using-robotics-simulations-with-pandagym"),c(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(pe,"href","#unit-6-advantage-actor-critic-a2c-using-robotics-simulations-with-pandagym"),c(N,"class","relative group"),c(fe,"id","environments"),c(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(fe,"href","#environments"),c(P,"class","relative group"),c(rt,"href","https://github.com/qgallouedec/panda-gym"),c(rt,"rel","nofollow"),c(ce,"id","rllibrary"),c(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ce,"href","#rllibrary"),c(V,"class","relative group"),c(it,"href","https://stable-baselines3.readthedocs.io/"),c(it,"rel","nofollow"),c(ht,"href","https://github.com/huggingface/deep-rl-class/issues"),c(ht,"rel","nofollow"),c(ue,"id","objectives-of-this-notebook"),c(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ue,"href","#objectives-of-this-notebook"),c(R,"class","relative group"),c(me,"id","prerequisites"),c(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(me,"href","#prerequisites"),c(Q,"class","relative group"),c(vt,"href","https://huggingface.co/deep-rl-course/unit6/introduction"),c(vt,"rel","nofollow"),c(ve,"id","lets-train-our-first-robots"),c(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ve,"href","#lets-train-our-first-robots"),c(F,"class","relative group"),c(ye,"id","set-the-gpu"),c(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ye,"href","#set-the-gpu"),c(Y,"class","relative group"),js(bt.src,ju="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step1.jpg")||c(bt,"src",ju),c(bt,"alt","GPU Step 1"),js(gt.src,Iu="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step2.jpg")||c(gt,"src",Iu),c(gt,"alt","GPU Step 2"),c(be,"id","create-a-virtual-display"),c(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(be,"href","#create-a-virtual-display"),c(z,"class","relative group"),c(_e,"id","install-dependencies"),c(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(_e,"href","#install-dependencies"),c(X,"class","relative group"),c(Me,"id","import-the-packages"),c(Me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Me,"href","#import-the-packages"),c(H,"class","relative group"),c(Ee,"id","pandareachdensev3"),c(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ee,"href","#pandareachdensev3"),c(L,"class","relative group"),js(jt.src,Cu="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/robotics.jpg")||c(jt,"src",Cu),c(jt,"alt","Robotics"),c($e,"id","create-the-environment"),c($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c($e,"href","#create-the-environment"),c(x,"class","relative group"),c(ke,"id","the-environment"),c(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ke,"href","#the-environment"),c(q,"class","relative group"),c(Ge,"id","normalize-observation-and-rewards"),c(Ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ge,"href","#normalize-observation-and-rewards"),c(D,"class","relative group"),c(Bt,"href","https://stable-baselines3.readthedocs.io/en/master/guide/rl_tips.html"),c(Bt,"rel","nofollow"),c(Nt,"href","https://stable-baselines3.readthedocs.io/en/master/guide/vec_envs.html#vecnormalize"),c(Nt,"rel","nofollow"),c(Ze,"id","solution"),c(Ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ze,"href","#solution"),c(O,"class","relative group"),c(We,"id","create-the-a2c-model"),c(We,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(We,"href","#create-the-a2c-model"),c(K,"class","relative group"),c(Yt,"href","https://stable-baselines3.readthedocs.io/en/master/modules/a2c.html#notes"),c(Yt,"rel","nofollow"),c(zt,"href","https://huggingface.co/sb3"),c(zt,"rel","nofollow"),c(Ae,"id","solution"),c(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ae,"href","#solution"),c(ee,"class","relative group"),c(Ne,"id","train-the-a2c-agent"),c(Ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ne,"href","#train-the-a2c-agent"),c(te,"class","relative group"),c(Pe,"id","evaluate-the-agent"),c(Pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Pe,"href","#evaluate-the-agent"),c(le,"class","relative group"),c(Re,"id","publish-your-trained-model-on-the-hub"),c(Re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Re,"href","#publish-your-trained-model-on-the-hub"),c(ae,"class","relative group"),c(al,"href","https://github.com/huggingface/huggingface_sb3/tree/main#hugging-face--x-stable-baselines3-v20"),c(al,"rel","nofollow"),c(nl,"href","https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),c(nl,"rel","nofollow"),c(hl,"href","https://huggingface.co/join"),c(hl,"rel","nofollow"),c(pl,"href","https://huggingface.co/settings/tokens"),c(pl,"rel","nofollow"),js(fl.src,Gu="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/create-token.jpg")||c(fl,"src",Gu),c(fl,"alt","Create HF Token"),c(ze,"id","some-additional-challenges"),c(ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ze,"href","#some-additional-challenges"),c(se,"class","relative group"),c(yl,"href","https://huggingface.co/sb3/tqc-PandaPickAndPlace-v1"),c(yl,"rel","nofollow"),c(bl,"href","https://panda-gym.readthedocs.io/en/latest/usage/train_with_sb3.html"),c(bl,"rel","nofollow"),c(gl,"href","https://stable-baselines3.readthedocs.io/en/master/guide/vec_envs.html#vecnormalize"),c(gl,"rel","nofollow"),c(Xe,"id","solution-optional"),c(Xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Xe,"href","#solution-optional"),c(re,"class","relative group"),c(He,"id","keep-learning-stay-awesome"),c(He,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(He,"href","#keep-learning-stay-awesome"),c(ne,"class","relative group")},m(e,r){l(document.head,W),h(e,Is,r),h(e,B,r),l(B,ie),l(ie,va),d(qe,va,null),l(B,Ui),l(B,ya),l(ya,Ti),h(e,Cs,r),d(De,e,r),h(e,Gs,r),h(e,he,r),l(he,Ji),l(he,wa),l(wa,$i),l(he,ki),h(e,Ss,r),h(e,Tl,r),l(Tl,ba),l(ba,ji),h(e,Zs,r),h(e,Jl,r),l(Jl,Ii),h(e,Ws,r),h(e,$l,r),l($l,ga),l(ga,Oe),l(Oe,Ci),h(e,Bs,r),h(e,kl,r),l(kl,Gi),h(e,As,r),h(e,jl,r),l(jl,Il),l(Il,_a),l(_a,Si),l(Il,Zi),h(e,Ns,r),h(e,A,r),l(A,Wi),l(A,Ke),l(Ke,Bi),l(A,Ai),l(A,Ma),l(Ma,Ni),h(e,Ps,r),h(e,et,r),l(et,Pi),l(et,tt),l(tt,Vi),h(e,Vs,r),h(e,lt,r),l(lt,Ea),l(Ea,Ri),l(lt,Qi),h(e,Rs,r),h(e,Cl,r),l(Cl,at),l(at,Gl),h(e,Qs,r),h(e,N,r),l(N,pe),l(pe,Ua),d(ot,Ua,null),l(N,Fi),l(N,Ta),l(Ta,Yi),h(e,Fs,r),h(e,P,r),l(P,fe),l(fe,Ja),d(st,Ja,null),l(P,zi),l(P,$a),l($a,Xi),h(e,Ys,r),h(e,Sl,r),l(Sl,ka),l(ka,rt),l(rt,Hi),h(e,zs,r),h(e,V,r),l(V,ce),l(ce,ja),d(nt,ja,null),l(V,Li),l(V,Ia),l(Ia,xi),h(e,Xs,r),h(e,Zl,r),l(Zl,Ca),l(Ca,it),l(it,qi),h(e,Hs,r),h(e,$,r),l($,Di),l($,Ga),l(Ga,Oi),l($,Ki),l($,ht),l(ht,eh),l($,th),h(e,Ls,r),h(e,R,r),l(R,ue),l(ue,Sa),d(pt,Sa,null),l(R,lh),l(R,Za),l(Za,ah),h(e,xs,r),h(e,Wl,r),l(Wl,oh),h(e,qs,r),h(e,T,r),l(T,ft),l(ft,sh),l(ft,Wa),l(Wa,rh),l(ft,nh),l(T,ih),l(T,ct),l(ct,hh),l(ct,Ba),l(Ba,ph),l(ct,fh),l(T,ch),l(T,ut),l(ut,uh),l(ut,Aa),l(Aa,mh),l(ut,dh),l(T,vh),l(T,mt),l(mt,yh),l(mt,Na),l(Na,wh),l(mt,bh),h(e,Ds,r),h(e,Q,r),l(Q,me),l(me,Pa),d(dt,Pa,null),l(Q,gh),l(Q,Va),l(Va,_h),h(e,Os,r),h(e,Bl,r),l(Bl,Mh),h(e,Ks,r),h(e,de,r),l(de,Eh),l(de,vt),l(vt,Uh),l(de,Th),h(e,er,r),h(e,F,r),l(F,ve),l(ve,Ra),d(yt,Ra,null),l(F,Jh),l(F,Qa),l(Qa,$h),h(e,tr,r),h(e,Y,r),l(Y,ye),l(ye,Fa),d(wt,Fa,null),l(Y,kh),l(Y,Ya),l(Ya,jh),h(e,lr,r),h(e,Al,r),l(Al,we),l(we,Ih),l(we,za),l(za,Ch),l(we,Gh),l(we,Xa),l(Xa,Sh),h(e,ar,r),h(e,bt,r),h(e,or,r),h(e,Nl,r),l(Nl,Ha),l(Ha,La),l(La,Zh),h(e,sr,r),h(e,gt,r),h(e,rr,r),h(e,z,r),l(z,be),l(be,xa),d(_t,xa,null),l(z,Wh),l(z,qa),l(qa,Bh),h(e,nr,r),h(e,ge,r),l(ge,Ah),l(ge,Da),l(Da,Nh),l(ge,Ph),h(e,ir,r),h(e,Pl,r),l(Pl,Vh),h(e,hr,r),d(Mt,e,r),h(e,pr,r),d(Et,e,r),h(e,fr,r),h(e,X,r),l(X,_e),l(_e,Oa),d(Ut,Oa,null),l(X,Rh),l(X,Ka),l(Ka,Qh),h(e,cr,r),h(e,Vl,r),l(Vl,Fh),h(e,ur,r),h(e,E,r),l(E,eo),l(eo,to),l(to,Yh),l(E,zh),l(E,Rl),l(Rl,lo),l(lo,Xh),l(Rl,Hh),l(E,Lh),l(E,Ql),l(Ql,ao),l(ao,xh),l(Ql,qh),l(E,Dh),l(E,Fl),l(Fl,oo),l(oo,Oh),l(Fl,Kh),l(E,ep),l(E,Yl),l(Yl,so),l(so,tp),l(Yl,lp),h(e,mr,r),d(Tt,e,r),h(e,dr,r),h(e,H,r),l(H,Me),l(Me,ro),d(Jt,ro,null),l(H,ap),l(H,no),l(no,op),h(e,vr,r),d($t,e,r),h(e,yr,r),h(e,L,r),l(L,Ee),l(Ee,io),d(kt,io,null),l(L,sp),l(L,ho),l(ho,rp),h(e,wr,r),h(e,zl,r),l(zl,np),h(e,br,r),h(e,Ue,r),l(Ue,ip),l(Ue,po),l(po,hp),l(Ue,pp),h(e,gr,r),h(e,Te,r),l(Te,fp),l(Te,fo),l(fo,cp),l(Te,up),h(e,_r,r),h(e,U,r),l(U,mp),l(U,co),l(co,dp),l(U,vp),l(U,uo),l(uo,yp),l(U,wp),l(U,mo),l(mo,bp),l(U,gp),l(U,vo),l(vo,_p),l(U,Mp),h(e,Mr,r),h(e,k,r),l(k,Ep),l(k,yo),l(yo,Up),l(k,Tp),l(k,wo),l(wo,Jp),l(k,$p),h(e,Er,r),h(e,jt,r),h(e,Ur,r),h(e,Je,r),l(Je,kp),l(Je,bo),l(bo,jp),l(Je,Ip),h(e,Tr,r),h(e,x,r),l(x,$e),l($e,go),d(It,go,null),l(x,Cp),l(x,_o),l(_o,Gp),h(e,Jr,r),h(e,q,r),l(q,ke),l(ke,Mo),d(Ct,Mo,null),l(q,Sp),l(q,Eo),l(Eo,Zp),h(e,$r,r),h(e,je,r),l(je,Wp),l(je,Uo),l(Uo,Bp),l(je,Ap),h(e,kr,r),d(Gt,e,r),h(e,jr,r),d(St,e,r),h(e,Ir,r),h(e,Ie,r),l(Ie,Np),l(Ie,To),l(To,Pp),l(Ie,Vp),h(e,Cr,r),h(e,j,r),l(j,Xl),l(Xl,Jo),l(Jo,Rp),l(Xl,Qp),l(j,Fp),l(j,Hl),l(Hl,$o),l($o,Yp),l(Hl,zp),l(j,Xp),l(j,Ll),l(Ll,ko),l(ko,Hp),l(Ll,Lp),h(e,Gr,r),h(e,Ce,r),l(Ce,xp),l(Ce,jo),l(jo,qp),l(Ce,Dp),h(e,Sr,r),d(Zt,e,r),h(e,Zr,r),h(e,xl,r),l(xl,Op),h(e,Wr,r),h(e,ql,r),l(ql,Io),l(Io,Kp),h(e,Br,r),h(e,D,r),l(D,Ge),l(Ge,Co),d(Wt,Co,null),l(D,ef),l(D,Go),l(Go,tf),h(e,Ar,r),h(e,Se,r),l(Se,lf),l(Se,Bt),l(Bt,af),l(Se,of),h(e,Nr,r),h(e,Dl,r),l(Dl,sf),h(e,Pr,r),h(e,At,r),l(At,rf),l(At,So),l(So,nf),h(e,Vr,r),h(e,Ol,r),l(Ol,Nt),l(Nt,hf),h(e,Rr,r),d(Pt,e,r),h(e,Qr,r),h(e,O,r),l(O,Ze),l(Ze,Zo),d(Vt,Zo,null),l(O,pf),l(O,Wo),l(Wo,ff),h(e,Fr,r),d(Rt,e,r),h(e,Yr,r),h(e,K,r),l(K,We),l(We,Bo),d(Qt,Bo,null),l(K,cf),l(K,Ao),l(Ao,uf),h(e,zr,r),h(e,Ft,r),l(Ft,mf),l(Ft,Yt),l(Yt,df),h(e,Xr,r),h(e,Be,r),l(Be,vf),l(Be,zt),l(zt,yf),l(Be,wf),h(e,Hr,r),d(Xt,e,r),h(e,Lr,r),h(e,ee,r),l(ee,Ae),l(Ae,No),d(Ht,No,null),l(ee,bf),l(ee,Po),l(Po,gf),h(e,xr,r),d(Lt,e,r),h(e,qr,r),h(e,te,r),l(te,Ne),l(Ne,Vo),d(xt,Vo,null),l(te,_f),l(te,Ro),l(Ro,Mf),h(e,Dr,r),h(e,Kl,r),l(Kl,Qo),l(Qo,Ef),h(e,Or,r),d(qt,e,r),h(e,Kr,r),d(Dt,e,r),h(e,en,r),h(e,le,r),l(le,Pe),l(Pe,Fo),d(Ot,Fo,null),l(le,Uf),l(le,Yo),l(Yo,Tf),h(e,tn,r),h(e,Ve,r),l(Ve,Kt),l(Kt,Jf),l(Kt,zo),l(zo,$f),l(Kt,kf),l(Ve,jf),l(Ve,ea),l(ea,If),l(ea,Xo),l(Xo,Cf),h(e,ln,r),d(el,e,r),h(e,an,r),h(e,ae,r),l(ae,Re),l(Re,Ho),d(tl,Ho,null),l(ae,Gf),l(ae,Lo),l(Lo,Sf),h(e,on,r),h(e,ta,r),l(ta,Zf),h(e,sn,r),h(e,ll,r),l(ll,Wf),l(ll,al),l(al,Bf),h(e,rn,r),h(e,I,r),l(I,Af),l(I,xo),l(xo,Nf),l(I,Pf),l(I,qo),l(qo,Vf),l(I,Rf),h(e,nn,r),h(e,la,r),l(la,Qf),h(e,hn,r),h(e,J,r),l(J,ol),l(ol,Ff),l(ol,Do),l(Do,Yf),l(ol,zf),l(J,Xf),l(J,sl),l(sl,Hf),l(sl,Oo),l(Oo,Lf),l(sl,xf),l(J,qf),l(J,rl),l(rl,Df),l(rl,Ko),l(Ko,Of),l(rl,Kf),l(J,ec),l(J,Qe),l(Qe,tc),l(Qe,es),l(es,lc),l(Qe,ac),l(Qe,nl),l(nl,oc),h(e,pn,r),h(e,aa,r),l(aa,sc),h(e,fn,r),h(e,il,r),l(il,rc),l(il,hl),l(hl,nc),h(e,cn,r),h(e,oa,r),l(oa,ic),h(e,un,r),h(e,sa,r),l(sa,Fe),l(Fe,hc),l(Fe,pl),l(pl,pc),l(Fe,fc),l(Fe,ts),l(ts,cc),h(e,mn,r),h(e,fl,r),h(e,dn,r),h(e,Ye,r),l(Ye,ls),l(ls,uc),l(Ye,mc),l(Ye,as),l(as,dc),h(e,vn,r),d(cl,e,r),h(e,yn,r),h(e,ul,r),l(ul,vc),l(ul,os),l(os,yc),h(e,wn,r),h(e,oe,r),l(oe,wc),l(oe,ss),l(ss,bc),l(oe,gc),l(oe,rs),l(rs,_c),h(e,bn,r),d(ml,e,r),h(e,gn,r),h(e,se,r),l(se,ze),l(ze,ns),d(dl,ns,null),l(se,Mc),l(se,is),l(is,Ec),h(e,_n,r),h(e,C,r),l(C,Uc),l(C,hs),l(hs,Tc),l(C,Jc),l(C,ps),l(ps,$c),l(C,kc),h(e,Mn,r),h(e,G,r),l(G,jc),l(G,fs),l(fs,Ic),l(G,Cc),l(G,cs),l(cs,Gc),l(G,Sc),h(e,En,r),h(e,vl,r),l(vl,Zc),l(vl,yl),l(yl,Wc),h(e,Un,r),h(e,wl,r),l(wl,Bc),l(wl,bl),l(bl,Ac),h(e,Tn,r),h(e,ra,r),l(ra,Nc),h(e,Jn,r),h(e,g,r),l(g,us),l(us,Pc),l(g,Vc),l(g,ms),l(ms,Rc),l(g,Qc),l(g,na),l(na,Fc),l(na,gl),l(gl,Yc),l(g,zc),l(g,ds),l(ds,Xc),l(g,Hc),l(g,vs),l(vs,Lc),l(g,xc),l(g,ys),l(ys,qc),l(g,Dc),l(g,ws),l(ws,Oc),l(g,Kc),l(g,ia),l(ia,eu),l(ia,bs),l(bs,tu),h(e,$n,r),h(e,re,r),l(re,Xe),l(Xe,gs),d(_l,gs,null),l(re,lu),l(re,_s),l(_s,au),h(e,kn,r),d(Ml,e,r),h(e,jn,r),d(El,e,r),h(e,In,r),h(e,ha,r),l(ha,ou),h(e,Cn,r),h(e,ne,r),l(ne,He),l(He,Ms),d(Ul,Ms,null),l(ne,su),l(ne,Es),l(Es,ru),Gn=!0},p:ry,i(e){Gn||(v(qe.$$.fragment,e),v(De.$$.fragment,e),v(ot.$$.fragment,e),v(st.$$.fragment,e),v(nt.$$.fragment,e),v(pt.$$.fragment,e),v(dt.$$.fragment,e),v(yt.$$.fragment,e),v(wt.$$.fragment,e),v(_t.$$.fragment,e),v(Mt.$$.fragment,e),v(Et.$$.fragment,e),v(Ut.$$.fragment,e),v(Tt.$$.fragment,e),v(Jt.$$.fragment,e),v($t.$$.fragment,e),v(kt.$$.fragment,e),v(It.$$.fragment,e),v(Ct.$$.fragment,e),v(Gt.$$.fragment,e),v(St.$$.fragment,e),v(Zt.$$.fragment,e),v(Wt.$$.fragment,e),v(Pt.$$.fragment,e),v(Vt.$$.fragment,e),v(Rt.$$.fragment,e),v(Qt.$$.fragment,e),v(Xt.$$.fragment,e),v(Ht.$$.fragment,e),v(Lt.$$.fragment,e),v(xt.$$.fragment,e),v(qt.$$.fragment,e),v(Dt.$$.fragment,e),v(Ot.$$.fragment,e),v(el.$$.fragment,e),v(tl.$$.fragment,e),v(cl.$$.fragment,e),v(ml.$$.fragment,e),v(dl.$$.fragment,e),v(_l.$$.fragment,e),v(Ml.$$.fragment,e),v(El.$$.fragment,e),v(Ul.$$.fragment,e),Gn=!0)},o(e){y(qe.$$.fragment,e),y(De.$$.fragment,e),y(ot.$$.fragment,e),y(st.$$.fragment,e),y(nt.$$.fragment,e),y(pt.$$.fragment,e),y(dt.$$.fragment,e),y(yt.$$.fragment,e),y(wt.$$.fragment,e),y(_t.$$.fragment,e),y(Mt.$$.fragment,e),y(Et.$$.fragment,e),y(Ut.$$.fragment,e),y(Tt.$$.fragment,e),y(Jt.$$.fragment,e),y($t.$$.fragment,e),y(kt.$$.fragment,e),y(It.$$.fragment,e),y(Ct.$$.fragment,e),y(Gt.$$.fragment,e),y(St.$$.fragment,e),y(Zt.$$.fragment,e),y(Wt.$$.fragment,e),y(Pt.$$.fragment,e),y(Vt.$$.fragment,e),y(Rt.$$.fragment,e),y(Qt.$$.fragment,e),y(Xt.$$.fragment,e),y(Ht.$$.fragment,e),y(Lt.$$.fragment,e),y(xt.$$.fragment,e),y(qt.$$.fragment,e),y(Dt.$$.fragment,e),y(Ot.$$.fragment,e),y(el.$$.fragment,e),y(tl.$$.fragment,e),y(cl.$$.fragment,e),y(ml.$$.fragment,e),y(dl.$$.fragment,e),y(_l.$$.fragment,e),y(Ml.$$.fragment,e),y(El.$$.fragment,e),y(Ul.$$.fragment,e),Gn=!1},d(e){t(W),e&&t(Is),e&&t(B),w(qe),e&&t(Cs),w(De,e),e&&t(Gs),e&&t(he),e&&t(Ss),e&&t(Tl),e&&t(Zs),e&&t(Jl),e&&t(Ws),e&&t($l),e&&t(Bs),e&&t(kl),e&&t(As),e&&t(jl),e&&t(Ns),e&&t(A),e&&t(Ps),e&&t(et),e&&t(Vs),e&&t(lt),e&&t(Rs),e&&t(Cl),e&&t(Qs),e&&t(N),w(ot),e&&t(Fs),e&&t(P),w(st),e&&t(Ys),e&&t(Sl),e&&t(zs),e&&t(V),w(nt),e&&t(Xs),e&&t(Zl),e&&t(Hs),e&&t($),e&&t(Ls),e&&t(R),w(pt),e&&t(xs),e&&t(Wl),e&&t(qs),e&&t(T),e&&t(Ds),e&&t(Q),w(dt),e&&t(Os),e&&t(Bl),e&&t(Ks),e&&t(de),e&&t(er),e&&t(F),w(yt),e&&t(tr),e&&t(Y),w(wt),e&&t(lr),e&&t(Al),e&&t(ar),e&&t(bt),e&&t(or),e&&t(Nl),e&&t(sr),e&&t(gt),e&&t(rr),e&&t(z),w(_t),e&&t(nr),e&&t(ge),e&&t(ir),e&&t(Pl),e&&t(hr),w(Mt,e),e&&t(pr),w(Et,e),e&&t(fr),e&&t(X),w(Ut),e&&t(cr),e&&t(Vl),e&&t(ur),e&&t(E),e&&t(mr),w(Tt,e),e&&t(dr),e&&t(H),w(Jt),e&&t(vr),w($t,e),e&&t(yr),e&&t(L),w(kt),e&&t(wr),e&&t(zl),e&&t(br),e&&t(Ue),e&&t(gr),e&&t(Te),e&&t(_r),e&&t(U),e&&t(Mr),e&&t(k),e&&t(Er),e&&t(jt),e&&t(Ur),e&&t(Je),e&&t(Tr),e&&t(x),w(It),e&&t(Jr),e&&t(q),w(Ct),e&&t($r),e&&t(je),e&&t(kr),w(Gt,e),e&&t(jr),w(St,e),e&&t(Ir),e&&t(Ie),e&&t(Cr),e&&t(j),e&&t(Gr),e&&t(Ce),e&&t(Sr),w(Zt,e),e&&t(Zr),e&&t(xl),e&&t(Wr),e&&t(ql),e&&t(Br),e&&t(D),w(Wt),e&&t(Ar),e&&t(Se),e&&t(Nr),e&&t(Dl),e&&t(Pr),e&&t(At),e&&t(Vr),e&&t(Ol),e&&t(Rr),w(Pt,e),e&&t(Qr),e&&t(O),w(Vt),e&&t(Fr),w(Rt,e),e&&t(Yr),e&&t(K),w(Qt),e&&t(zr),e&&t(Ft),e&&t(Xr),e&&t(Be),e&&t(Hr),w(Xt,e),e&&t(Lr),e&&t(ee),w(Ht),e&&t(xr),w(Lt,e),e&&t(qr),e&&t(te),w(xt),e&&t(Dr),e&&t(Kl),e&&t(Or),w(qt,e),e&&t(Kr),w(Dt,e),e&&t(en),e&&t(le),w(Ot),e&&t(tn),e&&t(Ve),e&&t(ln),w(el,e),e&&t(an),e&&t(ae),w(tl),e&&t(on),e&&t(ta),e&&t(sn),e&&t(ll),e&&t(rn),e&&t(I),e&&t(nn),e&&t(la),e&&t(hn),e&&t(J),e&&t(pn),e&&t(aa),e&&t(fn),e&&t(il),e&&t(cn),e&&t(oa),e&&t(un),e&&t(sa),e&&t(mn),e&&t(fl),e&&t(dn),e&&t(Ye),e&&t(vn),w(cl,e),e&&t(yn),e&&t(ul),e&&t(wn),e&&t(oe),e&&t(bn),w(ml,e),e&&t(gn),e&&t(se),w(dl),e&&t(_n),e&&t(C),e&&t(Mn),e&&t(G),e&&t(En),e&&t(vl),e&&t(Un),e&&t(wl),e&&t(Tn),e&&t(ra),e&&t(Jn),e&&t(g),e&&t($n),e&&t(re),w(_l),e&&t(kn),w(Ml,e),e&&t(jn),w(El,e),e&&t(In),e&&t(ha),e&&t(Cn),e&&t(ne),w(Ul)}}}const py={local:"lets-train-our-first-robots",sections:[{local:"set-the-gpu",title:"Set the GPU \u{1F4AA}"},{local:"create-a-virtual-display",sections:[{local:"install-dependencies",title:"Install dependencies \u{1F53D}"}],title:"Create a virtual display \u{1F53D}"},{local:"import-the-packages",title:"Import the packages \u{1F4E6}"},{local:"pandareachdensev3",sections:[{local:"create-the-environment",sections:[{local:"the-environment",title:"The environment \u{1F3AE}"}],title:"Create the environment"},{local:"normalize-observation-and-rewards",sections:[{local:"solution",title:"Solution"}],title:"Normalize observation and rewards"},{local:"create-the-a2c-model",sections:[{local:"solution",title:"Solution"}],title:"Create the A2C Model \u{1F916}"},{local:"train-the-a2c-agent",title:"Train the A2C agent \u{1F3C3}"},{local:"evaluate-the-agent",title:"Evaluate the agent \u{1F4C8}"},{local:"publish-your-trained-model-on-the-hub",title:"Publish your trained model on the Hub \u{1F525}"}],title:"PandaReachDense-v3 \u{1F9BE}"},{local:"some-additional-challenges",sections:[{local:"solution-optional",title:"Solution (optional)"}],title:"Some additional challenges \u{1F3C6}"},{local:"keep-learning-stay-awesome",title:"Keep learning, stay awesome \u{1F917}"}],title:"Let's train our first robots \u{1F916}"};function fy($u){return ny(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class vy extends ly{constructor(W){super();ay(this,W,fy,hy,oy,{})}}export{vy as default,py as metadata};
