import{S as gd,i as md,s as dd,e as o,k as u,w as c,t as n,M as vd,c as s,d as t,m as p,a as r,x as g,h as i,b as f,N as _,G as l,g as h,y as m,L as yd,q as d,o as v,B as y,v as wd}from"../../chunks/vendor-hf-doc-builder.js";import{I as w}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as U}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as bd}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function _d(Of){let G,dr,C,K,Dl,qe,Qn,Yl,qn,vr,Ke,yr,ee,Kn,Ql,ei,ti,wr,te,li,ql,oi,si,br,S,le,Kl,et,ri,eo,ai,_r,tt,to,ni,ii,Er,nl,lt,il,zf,Mr,ot,Vf,Tr,st,hi,lo,ui,$r,rt,Xf,kr,A,oe,oo,at,pi,so,fi,Ur,hl,se,ci,nt,gi,mi,it,di,jr,N,re,ro,ht,vi,ao,yi,Jr,ul,no,ut,wi,Ir,j,bi,io,_i,Ei,pt,Mi,Ti,Hr,Z,ae,ho,ft,$i,uo,ki,Gr,pl,Ui,Cr,J,ct,ji,po,Ji,Ii,Hi,fl,fo,Gi,Ci,Si,gt,Ai,co,Ni,Zi,Sr,W,ne,go,mt,Wi,mo,Li,Ar,cl,Pi,Nr,ie,xi,vo,Ri,Bi,Zr,he,Oi,yo,zi,Vi,Wr,L,ue,wo,dt,Xi,bo,Fi,Lr,vt,Ff,Pr,gl,_o,Eo,Di,xr,yt,Df,Rr,P,pe,Mo,wt,Yi,To,Qi,Br,ml,$o,qi,Or,bt,zr,_t,Vr,x,fe,ko,Et,Ki,dl,eh,Uo,th,Xr,ce,jo,lh,oh,vl,sh,Jo,rh,Fr,Mt,Dr,Tt,Yr,k,ah,$t,nh,ih,Io,hh,uh,kt,ph,Qr,Ut,qr,yl,fh,Kr,jt,ea,R,ge,Ho,Jt,ch,Go,gh,ta,B,me,Co,It,mh,So,dh,la,wl,vh,oa,I,Ao,yh,wh,No,bh,_h,Zo,Eh,sa,de,Mh,Wo,Th,$h,ra,ve,Yf,aa,O,ye,Lo,Ht,kh,Po,Uh,na,we,Qf,ia,z,xo,jh,Jh,Ro,Ih,Hh,ha,V,be,Bo,Gt,Gh,Oo,Ch,ua,_e,Sh,zo,Ah,Nh,pa,H,Zh,Vo,Wh,Lh,Xo,Ph,xh,fa,Ee,Rh,Fo,Bh,Oh,ca,bl,zh,ga,Me,qf,ma,E,Te,Do,Vh,Xh,Yo,Fh,Dh,Yh,$e,Qo,Qh,qh,qo,Kh,eu,tu,ke,Ko,lu,ou,es,su,ru,au,Ue,ts,nu,iu,ls,hu,uu,da,X,je,os,Ct,pu,ss,fu,va,b,rs,_l,cu,as,gu,mu,ns,St,du,At,vu,yu,wu,is,hs,bu,_u,us,El,Eu,ps,Mu,Tu,fs,Ml,$u,cs,ku,Uu,gs,ms,ju,ya,Nt,wa,Je,ds,vs,Ju,Iu,ys,Zt,ws,Hu,Gu,bs,Cu,ba,Tl,Su,_a,F,Ie,_s,Wt,Au,Es,Nu,Ea,Lt,Zu,Ms,Wu,Ma,He,Kf,Ta,$l,Lu,$a,M,kl,Ts,Pu,xu,Ru,Ul,$s,Bu,Ou,zu,jl,ks,Vu,Xu,Fu,Jl,Us,Du,Yu,ka,Ge,Qu,js,qu,Ku,Ua,Il,Pt,ep,Js,tp,lp,ja,Ce,op,Is,sp,rp,Ja,xt,Ia,D,Se,Hs,Rt,ap,Gs,np,Ha,Hl,Gl,ip,Cs,hp,Ga,Cl,up,Ca,Bt,pp,Ot,fp,Sa,Sl,cp,Aa,Al,Ae,gp,zt,mp,dp,Ss,vp,Na,Vt,ec,Za,Ne,As,yp,wp,Ns,bp,Wa,Xt,La,Ft,_p,Zs,Ep,Pa,Ze,Mp,Ws,Tp,$p,xa,We,tc,Ra,Nl,kp,Ba,T,Zl,Ls,Up,jp,Jp,Dt,Ps,Ip,Hp,Wl,Gp,Cp,Yt,xs,Sp,Ap,Le,Np,Pe,Zp,Rs,Wp,Lp,Ll,Bs,Pp,xp,Oa,Qt,za,Pl,Rp,Va,qt,Xa,Kt,Bp,Os,Op,Fa,xe,lc,Da,el,zp,zs,Vp,Ya,Y,Re,Vs,tl,Xp,Xs,Fp,Qa,xl,Dp,qa,Be,Fs,Rl,Yp,ll,Qp,qp,Ds,Ys,Kp,Ka,Oe,oc,en,ze,Qs,qs,ef,tf,Ks,ol,lf,er,of,sf,tn,Ve,tr,rf,af,Bl,nf,lr,hf,ln,sl,uf,or,pf,on,Ol,ff,sn,Q,cf,sr,gf,mf,rr,df,rn,Xe,sc,an,q,Fe,ar,rl,vf,nr,yf,nn;return qe=new w({}),Ke=new bd({props:{classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/deep-rl-class/blob/master/notebooks/bonus-unit1/bonus-unit1.ipynb"}],askForHelpUrl:"http://hf.co/join/discord"}}),et=new w({}),at=new w({}),ht=new w({}),ft=new w({}),mt=new w({}),dt=new w({}),wt=new w({}),bt=new U({props:{code:"JTIzJTIwQ2xvbmUlMjB0aGUlMjByZXBvc2l0b3J5JTIwKGNhbiUyMHRha2UlMjAzbWluKSUwQWdpdCUyMGNsb25lJTIwLS1kZXB0aCUyMDElMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZVbml0eS1UZWNobm9sb2dpZXMlMkZtbC1hZ2VudHM=",highlighted:`<span class="hljs-comment"># Clone the repository (can take 3min)</span>
git <span class="hljs-built_in">clone</span> --depth 1 https://github.com/Unity-Technologies/ml-agents`}}),_t=new U({props:{code:"JTIzJTIwR28lMjBpbnNpZGUlMjB0aGUlMjByZXBvc2l0b3J5JTIwYW5kJTIwaW5zdGFsbCUyMHRoZSUyMHBhY2thZ2UlMjAoY2FuJTIwdGFrZSUyMDNtaW4pJTBBJTI1Y2QlMjBtbC1hZ2VudHMlMEFwaXAzJTIwaW5zdGFsbCUyMC1lJTIwLiUyRm1sLWFnZW50cy1lbnZzJTBBcGlwMyUyMGluc3RhbGwlMjAtZSUyMC4lMkZtbC1hZ2VudHM=",highlighted:`<span class="hljs-comment"># Go inside the repository and install the package (can take 3min)</span>
%<span class="hljs-built_in">cd</span> ml-agents
pip3 install -e ./ml-agents-envs
pip3 install -e ./ml-agents`}}),Et=new w({}),Mt=new U({props:{code:"bWtkaXIlMjAuJTJGdHJhaW5lZC1lbnZzLWV4ZWN1dGFibGVzJTBBbWtkaXIlMjAuJTJGdHJhaW5lZC1lbnZzLWV4ZWN1dGFibGVzJTJGbGludXg=",highlighted:`<span class="hljs-built_in">mkdir</span> ./trained-envs-executables
<span class="hljs-built_in">mkdir</span> ./trained-envs-executables/linux`}}),Tt=new U({props:{code:"d2dldCUyMC0tbG9hZC1jb29raWVzJTIwJTJGdG1wJTJGY29va2llcy50eHQlMjAlMjJodHRwcyUzQSUyRiUyRmRvY3MuZ29vZ2xlLmNvbSUyRnVjJTNGZXhwb3J0JTNEZG93bmxvYWQlMjZjb25maXJtJTNEJTI0KHdnZXQlMjAtLXF1aWV0JTIwLS1zYXZlLWNvb2tpZXMlMjAlMkZ0bXAlMkZjb29raWVzLnR4dCUyMC0ta2VlcC1zZXNzaW9uLWNvb2tpZXMlMjAtLW5vLWNoZWNrLWNlcnRpZmljYXRlJTIwJ2h0dHBzJTNBJTJGJTJGZG9jcy5nb29nbGUuY29tJTJGdWMlM0ZleHBvcnQlM0Rkb3dubG9hZCUyNmlkJTNEMXp2M005NVpKVFdIVVZPV1Q2Y2txX2NtOThuZnQ4Z2RGJyUyMC1PLSUyMCU3QyUyMHNlZCUyMC1ybiUyMCdzJTJGLipjb25maXJtJTNEKCU1QjAtOUEtWmEtel8lNUQlMkIpLiolMkYlNUMxJTVDbiUyRnAnKSUyNmlkJTNEMXp2M005NVpKVFdIVVZPV1Q2Y2txX2NtOThuZnQ4Z2RGJTIyJTIwLU8lMjAuJTJGdHJhaW5lZC1lbnZzLWV4ZWN1dGFibGVzJTJGbGludXglMkZIdWdneS56aXAlMjAlMjYlMjYlMjBybSUyMC1yZiUyMCUyRnRtcCUyRmNvb2tpZXMudHh0",highlighted:'wget --load-cookies /tmp/cookies.txt <span class="hljs-string">&quot;https://docs.google.com/uc?export=download&amp;confirm=<span class="hljs-subst">$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate &#x27;https://docs.google.com/uc?export=download&amp;id=1zv3M95ZJTWHUVOWT6ckq_cm98nft8gdF&#x27; -O- | sed -rn &#x27;s/.*confirm=([0-9A-Za-z_]+)</span>.*/\\1\\n/p&#x27;)&amp;id=1zv3M95ZJTWHUVOWT6ckq_cm98nft8gdF&quot;</span> -O ./trained-envs-executables/linux/Huggy.zip &amp;&amp; <span class="hljs-built_in">rm</span> -rf /tmp/cookies.txt'}}),Ut=new U({props:{code:"JTI1JTI1Y2FwdHVyZSUwQXVuemlwJTIwLWQlMjAuJTJGdHJhaW5lZC1lbnZzLWV4ZWN1dGFibGVzJTJGbGludXglMkYlMjAuJTJGdHJhaW5lZC1lbnZzLWV4ZWN1dGFibGVzJTJGbGludXglMkZIdWdneS56aXA=",highlighted:`%%capture
unzip -d ./trained-envs-executables/linux/ ./trained-envs-executables/linux/Huggy.zip`}}),jt=new U({props:{code:"Y2htb2QlMjAtUiUyMDc1NSUyMC4lMkZ0cmFpbmVkLWVudnMtZXhlY3V0YWJsZXMlMkZsaW51eCUyRkh1Z2d5",highlighted:'<span class="hljs-built_in">chmod</span> -R 755 ./trained-envs-executables/linux/Huggy'}}),Jt=new w({}),It=new w({}),Ht=new w({}),Gt=new w({}),Ct=new w({}),Nt=new U({props:{code:"YmVoYXZpb3JzJTNBJTBBJTIwJTIwSHVnZ3klM0ElMEElMjAlMjAlMjAlMjB0cmFpbmVyX3R5cGUlM0ElMjBwcG8lMEElMjAlMjAlMjAlMjBoeXBlcnBhcmFtZXRlcnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNBJTIwMjA0OCUwQSUyMCUyMCUyMCUyMCUyMCUyMGJ1ZmZlcl9zaXplJTNBJTIwMjA0ODAlMEElMjAlMjAlMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNBJTIwMC4wMDAzJTBBJTIwJTIwJTIwJTIwJTIwJTIwYmV0YSUzQSUyMDAuMDA1JTBBJTIwJTIwJTIwJTIwJTIwJTIwZXBzaWxvbiUzQSUyMDAuMiUwQSUyMCUyMCUyMCUyMCUyMCUyMGxhbWJkJTNBJTIwMC45NSUwQSUyMCUyMCUyMCUyMCUyMCUyMG51bV9lcG9jaCUzQSUyMDMlMEElMjAlMjAlMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlX3NjaGVkdWxlJTNBJTIwbGluZWFyJTBBJTIwJTIwJTIwJTIwbmV0d29ya19zZXR0aW5ncyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMG5vcm1hbGl6ZSUzQSUyMHRydWUlMEElMjAlMjAlMjAlMjAlMjAlMjBoaWRkZW5fdW5pdHMlM0ElMjA1MTIlMEElMjAlMjAlMjAlMjAlMjAlMjBudW1fbGF5ZXJzJTNBJTIwMyUwQSUyMCUyMCUyMCUyMCUyMCUyMHZpc19lbmNvZGVfdHlwZSUzQSUyMHNpbXBsZSUwQSUyMCUyMCUyMCUyMHJld2FyZF9zaWduYWxzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwZXh0cmluc2ljJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ2FtbWElM0ElMjAwLjk5NSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0cmVuZ3RoJTNBJTIwMS4wJTBBJTIwJTIwJTIwJTIwY2hlY2twb2ludF9pbnRlcnZhbCUzQSUyMDIwMDAwMCUwQSUyMCUyMCUyMCUyMGtlZXBfY2hlY2twb2ludHMlM0ElMjAxNSUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzQSUyMDJlNiUwQSUyMCUyMCUyMCUyMHRpbWVfaG9yaXpvbiUzQSUyMDEwMDAlMEElMjAlMjAlMjAlMjBzdW1tYXJ5X2ZyZXElM0ElMjA1MDAwMA==",highlighted:`<span class="hljs-attr">behaviors:</span>
  <span class="hljs-attr">Huggy:</span>
    <span class="hljs-attr">trainer_type:</span> <span class="hljs-string">ppo</span>
    <span class="hljs-attr">hyperparameters:</span>
      <span class="hljs-attr">batch_size:</span> <span class="hljs-number">2048</span>
      <span class="hljs-attr">buffer_size:</span> <span class="hljs-number">20480</span>
      <span class="hljs-attr">learning_rate:</span> <span class="hljs-number">0.0003</span>
      <span class="hljs-attr">beta:</span> <span class="hljs-number">0.005</span>
      <span class="hljs-attr">epsilon:</span> <span class="hljs-number">0.2</span>
      <span class="hljs-attr">lambd:</span> <span class="hljs-number">0.95</span>
      <span class="hljs-attr">num_epoch:</span> <span class="hljs-number">3</span>
      <span class="hljs-attr">learning_rate_schedule:</span> <span class="hljs-string">linear</span>
    <span class="hljs-attr">network_settings:</span>
      <span class="hljs-attr">normalize:</span> <span class="hljs-literal">true</span>
      <span class="hljs-attr">hidden_units:</span> <span class="hljs-number">512</span>
      <span class="hljs-attr">num_layers:</span> <span class="hljs-number">3</span>
      <span class="hljs-attr">vis_encode_type:</span> <span class="hljs-string">simple</span>
    <span class="hljs-attr">reward_signals:</span>
      <span class="hljs-attr">extrinsic:</span>
        <span class="hljs-attr">gamma:</span> <span class="hljs-number">0.995</span>
        <span class="hljs-attr">strength:</span> <span class="hljs-number">1.0</span>
    <span class="hljs-attr">checkpoint_interval:</span> <span class="hljs-number">200000</span>
    <span class="hljs-attr">keep_checkpoints:</span> <span class="hljs-number">15</span>
    <span class="hljs-attr">max_steps:</span> <span class="hljs-number">2e6</span>
    <span class="hljs-attr">time_horizon:</span> <span class="hljs-number">1000</span>
    <span class="hljs-attr">summary_freq:</span> <span class="hljs-number">50000</span>`}}),Wt=new w({}),xt=new U({props:{code:"bWxhZ2VudHMtbGVhcm4lMjAuJTJGY29uZmlnJTJGcHBvJTJGSHVnZ3kueWFtbCUyMC0tZW52JTNELiUyRnRyYWluZWQtZW52cy1leGVjdXRhYmxlcyUyRmxpbnV4JTJGSHVnZ3klMkZIdWdneSUyMC0tcnVuLWlkJTNEJTIySHVnZ3klMjIlMjAtLW5vLWdyYXBoaWNz",highlighted:'mlagents-learn ./config/ppo/Huggy.yaml --<span class="hljs-built_in">env</span>=./trained-envs-executables/linux/Huggy/Huggy --run-id=<span class="hljs-string">&quot;Huggy&quot;</span> --no-graphics'}}),Rt=new w({}),Xt=new U({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),Qt=new U({props:{code:"bWxhZ2VudHMtcHVzaC10by1oZiUyMC0tcnVuLWlkJTNEJTIySHVnZ3lUcmFpbmluZyUyMiUyMC0tbG9jYWwtZGlyJTNEJTIyLiUyRnJlc3VsdHMlMkZIdWdneSUyMiUyMC0tcmVwby1pZCUzRCUyMlRob21hc1NpbW9uaW5pJTJGcHBvLUh1Z2d5JTIyJTIwLS1jb21taXQtbWVzc2FnZSUzRCUyMkh1Z2d5JTIy",highlighted:'mlagents-push-to-hf --run-id=<span class="hljs-string">&quot;HuggyTraining&quot;</span> --local-dir=<span class="hljs-string">&quot;./results/Huggy&quot;</span> --repo-id=<span class="hljs-string">&quot;ThomasSimonini/ppo-Huggy&quot;</span> --commit-message=<span class="hljs-string">&quot;Huggy&quot;</span>'}}),qt=new U({props:{code:"WW91ciUyMG1vZGVsJTIwaXMlMjBwdXNoZWQlMjB0byUyMHRoZSUyMGh1Yi4lMjBZb3UlMjBjYW4lMjB2aWV3JTIweW91ciUyMG1vZGVsJTIwaGVyZSUzQSUyMGh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZUaG9tYXNTaW1vbmluaSUyRnBwby1IdWdneQ==",highlighted:'Your model is pushed to the hub. You can view your model here: https:<span class="hljs-regexp">//</span>huggingface.co<span class="hljs-regexp">/ThomasSimonini/</span>ppo-Huggy'}}),tl=new w({}),rl=new w({}),{c(){G=o("meta"),dr=u(),C=o("h1"),K=o("a"),Dl=o("span"),c(qe.$$.fragment),Qn=u(),Yl=o("span"),qn=n("Let's train and play with Huggy \u{1F436}"),vr=u(),c(Ke.$$.fragment),yr=u(),ee=o("p"),Kn=n("We strongly "),Ql=o("strong"),ei=n("recommend students use Google Colab for the hands-on exercises"),ti=n(" instead of running them on their personal computers."),wr=u(),te=o("p"),li=n("By using Google Colab, "),ql=o("strong"),oi=n("you can focus on learning and experimenting without worrying about the technical aspects"),si=n(" of setting up your environments."),br=u(),S=o("h2"),le=o("a"),Kl=o("span"),c(et.$$.fragment),ri=u(),eo=o("span"),ai=n("Let's train Huggy \u{1F436}"),_r=u(),tt=o("p"),to=o("strong"),ni=n("To start to train Huggy, click on Open In Colab button"),ii=n(" \u{1F447} :"),Er=u(),nl=o("p"),lt=o("a"),il=o("img"),Mr=u(),ot=o("img"),Tr=u(),st=o("p"),hi=n("In this notebook, we\u2019ll reinforce what we learned in the first Unit by "),lo=o("strong"),ui=n("teaching Huggy the Dog to fetch the stick and then play with it directly in your browser"),$r=u(),rt=o("img"),kr=u(),A=o("h3"),oe=o("a"),oo=o("span"),c(at.$$.fragment),pi=u(),so=o("span"),fi=n("The environment \u{1F3AE}"),Ur=u(),hl=o("ul"),se=o("li"),ci=n("Huggy the Dog, an environment created by "),nt=o("a"),gi=n("Thomas Simonini"),mi=n(" based on "),it=o("a"),di=n("Puppo The Corgi"),jr=u(),N=o("h3"),re=o("a"),ro=o("span"),c(ht.$$.fragment),vi=u(),ao=o("span"),yi=n("The library used \u{1F4DA}"),Jr=u(),ul=o("ul"),no=o("li"),ut=o("a"),wi=n("MLAgents"),Ir=u(),j=o("p"),bi=n("We\u2019re constantly trying to improve our tutorials, so "),io=o("strong"),_i=n("if you find some issues in this notebook"),Ei=n(", please "),pt=o("a"),Mi=n("open an issue on the Github Repo"),Ti=n("."),Hr=u(),Z=o("h2"),ae=o("a"),ho=o("span"),c(ft.$$.fragment),$i=u(),uo=o("span"),ki=n("Objectives of this notebook \u{1F3C6}"),Gr=u(),pl=o("p"),Ui=n("At the end of the notebook, you will:"),Cr=u(),J=o("ul"),ct=o("li"),ji=n("Understand "),po=o("strong"),Ji=n("the state space, action space, and reward function used to train Huggy"),Ii=n("."),Hi=u(),fl=o("li"),fo=o("strong"),Gi=n("Train your own Huggy"),Ci=n(" to fetch the stick."),Si=u(),gt=o("li"),Ai=n("Be able to play "),co=o("strong"),Ni=n("with your trained Huggy directly in your browser"),Zi=n("."),Sr=u(),W=o("h2"),ne=o("a"),go=o("span"),c(mt.$$.fragment),Wi=u(),mo=o("span"),Li=n("Prerequisites \u{1F3D7}\uFE0F"),Ar=u(),cl=o("p"),Pi=n("Before diving into the notebook, you need to:"),Nr=u(),ie=o("p"),xi=n("\u{1F532} \u{1F4DA} "),vo=o("strong"),Ri=n("Develop an understanding of the foundations of Reinforcement learning"),Bi=n(" (MC, TD, Rewards hypothesis\u2026) by doing Unit 1"),Zr=u(),he=o("p"),Oi=n("\u{1F532} \u{1F4DA} "),yo=o("strong"),zi=n("Read the introduction to Huggy"),Vi=n(" by doing Bonus Unit 1"),Wr=u(),L=o("h2"),ue=o("a"),wo=o("span"),c(dt.$$.fragment),Xi=u(),bo=o("span"),Fi=n("Set the GPU \u{1F4AA}"),Lr=n(`

- To **accelerate the agent's training, we'll use a GPU**. To do that, go to \`Runtime > Change Runtime type\`
`),vt=o("img"),Pr=u(),gl=o("ul"),_o=o("li"),Eo=o("code"),Di=n("Hardware Accelerator > GPU"),xr=u(),yt=o("img"),Rr=u(),P=o("h2"),pe=o("a"),Mo=o("span"),c(wt.$$.fragment),Yi=u(),To=o("span"),Qi=n("Clone the repository and install the dependencies \u{1F53D}"),Br=u(),ml=o("ul"),$o=o("li"),qi=n("We need to clone the repository, that contains ML-Agents."),Or=u(),c(bt.$$.fragment),zr=u(),c(_t.$$.fragment),Vr=u(),x=o("h2"),fe=o("a"),ko=o("span"),c(Et.$$.fragment),Ki=u(),dl=o("span"),eh=n("Download and move the environment zip file in "),Uo=o("code"),th=n("./trained-envs-executables/linux/"),Xr=u(),ce=o("ul"),jo=o("li"),lh=n("Our environment executable is in a zip file."),oh=u(),vl=o("li"),sh=n("We need to download it and place it to "),Jo=o("code"),rh=n("./trained-envs-executables/linux/"),Fr=u(),c(Mt.$$.fragment),Dr=u(),c(Tt.$$.fragment),Yr=u(),k=o("p"),ah=n("Download the file Huggy.zip from "),$t=o("a"),nh=n("https://drive.google.com/uc?export=download&id=1zv3M95ZJTWHUVOWT6ckq_cm98nft8gdF"),ih=n(" using "),Io=o("code"),hh=n("wget"),uh=n(". Check out the full solution to download large files from GDrive "),kt=o("a"),ph=n("here"),Qr=u(),c(Ut.$$.fragment),qr=u(),yl=o("p"),fh=n("Make sure your file is accessible"),Kr=u(),c(jt.$$.fragment),ea=u(),R=o("h2"),ge=o("a"),Ho=o("span"),c(Jt.$$.fragment),ch=u(),Go=o("span"),gh=n("Let's recap how this environment works"),ta=u(),B=o("h3"),me=o("a"),Co=o("span"),c(It.$$.fragment),mh=u(),So=o("span"),dh=n("The State Space: what Huggy perceives."),la=u(),wl=o("p"),vh=n("Huggy doesn\u2019t \u201Csee\u201D his environment. Instead, we provide him information about the environment:"),oa=u(),I=o("ul"),Ao=o("li"),yh=n("The target (stick) position"),wh=u(),No=o("li"),bh=n("The relative position between himself and the target"),_h=u(),Zo=o("li"),Eh=n("The orientation of his legs."),sa=u(),de=o("p"),Mh=n("Given all this information, Huggy "),Wo=o("strong"),Th=n("can decide which action to take next to fulfill his goal"),$h=n("."),ra=u(),ve=o("img"),aa=u(),O=o("h3"),ye=o("a"),Lo=o("span"),c(Ht.$$.fragment),kh=u(),Po=o("span"),Uh=n("The Action Space: what moves Huggy can do"),na=u(),we=o("img"),ia=u(),z=o("p"),xo=o("strong"),jh=n("Joint motors drive huggy legs"),Jh=n(". This means that to get the target, Huggy needs to "),Ro=o("strong"),Ih=n("learn to rotate the joint motors of each of his legs correctly so he can move"),Hh=n("."),ha=u(),V=o("h3"),be=o("a"),Bo=o("span"),c(Gt.$$.fragment),Gh=u(),Oo=o("span"),Ch=n("The Reward Function"),ua=u(),_e=o("p"),Sh=n("The reward function is designed so that "),zo=o("strong"),Ah=n("Huggy will fulfill his goal"),Nh=n(" : fetch the stick."),pa=u(),H=o("p"),Zh=n("Remember that one of the foundations of Reinforcement Learning is the "),Vo=o("em"),Wh=n("reward hypothesis"),Lh=n(": a goal can be described as the "),Xo=o("strong"),Ph=n("maximization of the expected cumulative reward"),xh=n("."),fa=u(),Ee=o("p"),Rh=n("Here, our goal is that Huggy "),Fo=o("strong"),Bh=n("goes towards the stick but without spinning too much"),Oh=n(". Hence, our reward function must translate this goal."),ca=u(),bl=o("p"),zh=n("Our reward function:"),ga=u(),Me=o("img"),ma=u(),E=o("ul"),Te=o("li"),Do=o("em"),Vh=n("Orientation bonus"),Xh=n(": we "),Yo=o("strong"),Fh=n("reward him for getting close to the target"),Dh=n("."),Yh=u(),$e=o("li"),Qo=o("em"),Qh=n("Time penalty"),qh=n(": a fixed-time penalty given at every action to "),qo=o("strong"),Kh=n("force him to get to the stick as fast as possible"),eu=n("."),tu=u(),ke=o("li"),Ko=o("em"),lu=n("Rotation penalty"),ou=n(": we penalize Huggy if "),es=o("strong"),su=n("he spins too much and turns too quickly"),ru=n("."),au=u(),Ue=o("li"),ts=o("em"),nu=n("Getting to the target reward"),iu=n(": we reward Huggy for "),ls=o("strong"),hu=n("reaching the target"),uu=n("."),da=u(),X=o("h2"),je=o("a"),os=o("span"),c(Ct.$$.fragment),pu=u(),ss=o("span"),fu=n("Check the Huggy config file"),va=u(),b=o("ul"),rs=o("li"),_l=o("p"),cu=n("In ML-Agents, you define the "),as=o("strong"),gu=n("training hyperparameters in config.yaml files."),mu=u(),ns=o("li"),St=o("p"),du=n("For the scope of this notebook, we\u2019re not going to modify the hyperparameters, but if you want to try as an experiment, Unity provides very "),At=o("a"),vu=n("good documentation explaining each of them here"),yu=n("."),wu=u(),is=o("li"),hs=o("p"),bu=n("We need to create a config file for Huggy."),_u=u(),us=o("li"),El=o("p"),Eu=n("Go to "),ps=o("code"),Mu=n("/content/ml-agents/config/ppo"),Tu=u(),fs=o("li"),Ml=o("p"),$u=n("Create a new file called "),cs=o("code"),ku=n("Huggy.yaml"),Uu=u(),gs=o("li"),ms=o("p"),ju=n("Copy and paste the content below \u{1F53D}"),ya=u(),c(Nt.$$.fragment),wa=u(),Je=o("ul"),ds=o("li"),vs=o("p"),Ju=n("Don\u2019t forget to save the file!"),Iu=u(),ys=o("li"),Zt=o("p"),ws=o("strong"),Hu=n("In the case you want to modify the hyperparameters"),Gu=n(", in Google Colab notebook, you can click here to open the config.yaml: "),bs=o("code"),Cu=n("/content/ml-agents/config/ppo/Huggy.yaml"),ba=u(),Tl=o("p"),Su=n("We\u2019re now ready to train our agent \u{1F525}."),_a=u(),F=o("h2"),Ie=o("a"),_s=o("span"),c(Wt.$$.fragment),Au=u(),Es=o("span"),Nu=n("Train our agent"),Ea=u(),Lt=o("p"),Zu=n("To train our agent, we just need to "),Ms=o("strong"),Wu=n("launch mlagents-learn and select the executable containing the environment."),Ma=u(),He=o("img"),Ta=u(),$l=o("p"),Lu=n("With ML Agents, we run a training script. We define four parameters:"),$a=u(),M=o("ol"),kl=o("li"),Ts=o("code"),Pu=n("mlagents-learn <config>"),xu=n(": the path where the hyperparameter config file is."),Ru=u(),Ul=o("li"),$s=o("code"),Bu=n("--env"),Ou=n(": where the environment executable is."),zu=u(),jl=o("li"),ks=o("code"),Vu=n("--run_id"),Xu=n(": the name you want to give to your training run id."),Fu=u(),Jl=o("li"),Us=o("code"),Du=n("--no-graphics"),Yu=n(": to not launch the visualization during the training."),ka=u(),Ge=o("p"),Qu=n("Train the model and use the "),js=o("code"),qu=n("--resume"),Ku=n(" flag to continue training in case of interruption."),Ua=u(),Il=o("blockquote"),Pt=o("p"),ep=n("It will fail first time when you use "),Js=o("code"),tp=n("--resume"),lp=n(", try running the block again to bypass the error."),ja=u(),Ce=o("p"),op=n("The training will take 30 to 45min depending on your machine (don\u2019t forget to "),Is=o("strong"),sp=n("set up a GPU"),rp=n("), go take a \u2615\uFE0F you deserve it \u{1F917}."),Ja=u(),c(xt.$$.fragment),Ia=u(),D=o("h2"),Se=o("a"),Hs=o("span"),c(Rt.$$.fragment),ap=u(),Gs=o("span"),np=n("Push the agent to the \u{1F917} Hub"),Ha=u(),Hl=o("ul"),Gl=o("li"),ip=n("Now that we trained our agent, we\u2019re "),Cs=o("strong"),hp=n("ready to push it to the Hub to be able to play with Huggy on your browser\u{1F525}."),Ga=u(),Cl=o("p"),up=n("To be able to share your model with the community there are three more steps to follow:"),Ca=u(),Bt=o("p"),pp=n("1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),Ot=o("a"),fp=n("https://huggingface.co/join"),Sa=u(),Sl=o("p"),cp=n("2\uFE0F\u20E3 Sign in and then get your token from the Hugging Face website."),Aa=u(),Al=o("ul"),Ae=o("li"),gp=n("Create a new token ("),zt=o("a"),mp=n("https://huggingface.co/settings/tokens"),dp=n(") "),Ss=o("strong"),vp=n("with write role"),Na=u(),Vt=o("img"),Za=u(),Ne=o("ul"),As=o("li"),yp=n("Copy the token"),wp=u(),Ns=o("li"),bp=n("Run the cell below and paste the token"),Wa=u(),c(Xt.$$.fragment),La=u(),Ft=o("p"),_p=n("If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),Zs=o("code"),Ep=n("huggingface-cli login"),Pa=u(),Ze=o("p"),Mp=n("Then, we simply need to run "),Ws=o("code"),Tp=n("mlagents-push-to-hf"),$p=n("."),xa=u(),We=o("img"),Ra=u(),Nl=o("p"),kp=n("And we define 4 parameters:"),Ba=u(),T=o("ol"),Zl=o("li"),Ls=o("code"),Up=n("--run-id"),jp=n(": the name of the training run id."),Jp=u(),Dt=o("li"),Ps=o("code"),Ip=n("--local-dir"),Hp=n(": where the agent was saved, it\u2019s results/"),Wl=o("run_id"),Gp=n(", so in my case results/First Training."),Cp=u(),Yt=o("li"),xs=o("code"),Sp=n("--repo-id"),Ap=n(": the name of the Hugging Face repo you want to create or update. It\u2019s always "),Le=o("your"),Np=n("/"),Pe=o("the"),Zp=n("If the repo does not exist "),Rs=o("strong"),Wp=n("it will be created automatically"),Lp=u(),Ll=o("li"),Bs=o("code"),Pp=n("--commit-message"),xp=n(": since HF repos are git repositories you need to give a commit message."),Oa=u(),c(Qt.$$.fragment),za=u(),Pl=o("p"),Rp=n("If everything worked you should see this at the end of the process (but with a different url \u{1F606}) :"),Va=u(),c(qt.$$.fragment),Xa=u(),Kt=o("p"),Bp=n("It\u2019s the link to your model repository. The repository contains a model card that explains how to use the model, your Tensorboard logs and your config file. "),Os=o("strong"),Op=n("What\u2019s awesome is that it\u2019s a git repository, which means you can have different commits, update your repository with a new push, open Pull Requests, etc."),Fa=u(),xe=o("img"),Da=u(),el=o("p"),zp=n("But now comes the best part: "),zs=o("strong"),Vp=n("being able to play with Huggy online \u{1F440}."),Ya=u(),Y=o("h2"),Re=o("a"),Vs=o("span"),c(tl.$$.fragment),Xp=u(),Xs=o("span"),Fp=n("Play with your Huggy \u{1F415}"),Qa=u(),xl=o("p"),Dp=n("This step is the simplest:"),qa=u(),Be=o("ul"),Fs=o("li"),Rl=o("p"),Yp=n("Open the Huggy game in your browser: "),ll=o("a"),Qp=n("https://huggingface.co/spaces/ThomasSimonini/Huggy"),qp=u(),Ds=o("li"),Ys=o("p"),Kp=n("Click on Play with my Huggy model"),Ka=u(),Oe=o("img"),en=u(),ze=o("ol"),Qs=o("li"),qs=o("p"),ef=n("In step 1, choose your model repository, which is the model id (in my case ThomasSimonini/ppo-Huggy)."),tf=u(),Ks=o("li"),ol=o("p"),lf=n("In step 2, "),er=o("strong"),of=n("choose which model you want to replay"),sf=n(":"),tn=u(),Ve=o("ul"),tr=o("li"),rf=n("I have multiple ones, since we saved a model every 500000 timesteps."),af=u(),Bl=o("li"),nf=n("But since I want the most recent one, I choose "),lr=o("code"),hf=n("Huggy.onnx"),ln=u(),sl=o("p"),uf=n("\u{1F449} It\u2019s good "),or=o("strong"),pf=n("to try with different models steps to see the improvement of the agent."),on=u(),Ol=o("p"),ff=n("Congrats on finishing this bonus unit!"),sn=u(),Q=o("p"),cf=n("You can now sit and enjoy playing with your Huggy \u{1F436}. And don\u2019t "),sr=o("strong"),gf=n("forget to spread the love by sharing Huggy with your friends \u{1F917}"),mf=n(". And if you share about it on social media, "),rr=o("strong"),df=n("please tag us @huggingface and me @simoninithomas"),rn=u(),Xe=o("img"),an=u(),q=o("h2"),Fe=o("a"),ar=o("span"),c(rl.$$.fragment),vf=u(),nr=o("span"),yf=n("Keep Learning, Stay awesome \u{1F917}"),this.h()},l(e){const a=vd('[data-svelte="svelte-1phssyn"]',document.head);G=s(a,"META",{name:!0,content:!0}),a.forEach(t),dr=p(e),C=s(e,"H1",{class:!0});var hn=r(C);K=s(hn,"A",{id:!0,class:!0,href:!0});var rc=r(K);Dl=s(rc,"SPAN",{});var ac=r(Dl);g(qe.$$.fragment,ac),ac.forEach(t),rc.forEach(t),Qn=p(hn),Yl=s(hn,"SPAN",{});var nc=r(Yl);qn=i(nc,"Let's train and play with Huggy \u{1F436}"),nc.forEach(t),hn.forEach(t),vr=p(e),g(Ke.$$.fragment,e),yr=p(e),ee=s(e,"P",{});var un=r(ee);Kn=i(un,"We strongly "),Ql=s(un,"STRONG",{});var ic=r(Ql);ei=i(ic,"recommend students use Google Colab for the hands-on exercises"),ic.forEach(t),ti=i(un," instead of running them on their personal computers."),un.forEach(t),wr=p(e),te=s(e,"P",{});var pn=r(te);li=i(pn,"By using Google Colab, "),ql=s(pn,"STRONG",{});var hc=r(ql);oi=i(hc,"you can focus on learning and experimenting without worrying about the technical aspects"),hc.forEach(t),si=i(pn," of setting up your environments."),pn.forEach(t),br=p(e),S=s(e,"H2",{class:!0});var fn=r(S);le=s(fn,"A",{id:!0,class:!0,href:!0});var uc=r(le);Kl=s(uc,"SPAN",{});var pc=r(Kl);g(et.$$.fragment,pc),pc.forEach(t),uc.forEach(t),ri=p(fn),eo=s(fn,"SPAN",{});var fc=r(eo);ai=i(fc,"Let's train Huggy \u{1F436}"),fc.forEach(t),fn.forEach(t),_r=p(e),tt=s(e,"P",{});var wf=r(tt);to=s(wf,"STRONG",{});var cc=r(to);ni=i(cc,"To start to train Huggy, click on Open In Colab button"),cc.forEach(t),ii=i(wf," \u{1F447} :"),wf.forEach(t),Er=p(e),nl=s(e,"P",{});var gc=r(nl);lt=s(gc,"A",{href:!0,rel:!0});var mc=r(lt);il=s(mc,"IMG",{src:!0,alt:!0}),mc.forEach(t),gc.forEach(t),Mr=p(e),ot=s(e,"IMG",{src:!0,alt:!0}),Tr=p(e),st=s(e,"P",{});var bf=r(st);hi=i(bf,"In this notebook, we\u2019ll reinforce what we learned in the first Unit by "),lo=s(bf,"STRONG",{});var dc=r(lo);ui=i(dc,"teaching Huggy the Dog to fetch the stick and then play with it directly in your browser"),dc.forEach(t),bf.forEach(t),$r=p(e),rt=s(e,"IMG",{src:!0,alt:!0}),kr=p(e),A=s(e,"H3",{class:!0});var cn=r(A);oe=s(cn,"A",{id:!0,class:!0,href:!0});var vc=r(oe);oo=s(vc,"SPAN",{});var yc=r(oo);g(at.$$.fragment,yc),yc.forEach(t),vc.forEach(t),pi=p(cn),so=s(cn,"SPAN",{});var wc=r(so);fi=i(wc,"The environment \u{1F3AE}"),wc.forEach(t),cn.forEach(t),Ur=p(e),hl=s(e,"UL",{});var bc=r(hl);se=s(bc,"LI",{});var ir=r(se);ci=i(ir,"Huggy the Dog, an environment created by "),nt=s(ir,"A",{href:!0,rel:!0});var _c=r(nt);gi=i(_c,"Thomas Simonini"),_c.forEach(t),mi=i(ir," based on "),it=s(ir,"A",{href:!0,rel:!0});var Ec=r(it);di=i(Ec,"Puppo The Corgi"),Ec.forEach(t),ir.forEach(t),bc.forEach(t),jr=p(e),N=s(e,"H3",{class:!0});var gn=r(N);re=s(gn,"A",{id:!0,class:!0,href:!0});var Mc=r(re);ro=s(Mc,"SPAN",{});var Tc=r(ro);g(ht.$$.fragment,Tc),Tc.forEach(t),Mc.forEach(t),vi=p(gn),ao=s(gn,"SPAN",{});var $c=r(ao);yi=i($c,"The library used \u{1F4DA}"),$c.forEach(t),gn.forEach(t),Jr=p(e),ul=s(e,"UL",{});var kc=r(ul);no=s(kc,"LI",{});var Uc=r(no);ut=s(Uc,"A",{href:!0,rel:!0});var jc=r(ut);wi=i(jc,"MLAgents"),jc.forEach(t),Uc.forEach(t),kc.forEach(t),Ir=p(e),j=s(e,"P",{});var zl=r(j);bi=i(zl,"We\u2019re constantly trying to improve our tutorials, so "),io=s(zl,"STRONG",{});var Jc=r(io);_i=i(Jc,"if you find some issues in this notebook"),Jc.forEach(t),Ei=i(zl,", please "),pt=s(zl,"A",{href:!0,rel:!0});var Ic=r(pt);Mi=i(Ic,"open an issue on the Github Repo"),Ic.forEach(t),Ti=i(zl,"."),zl.forEach(t),Hr=p(e),Z=s(e,"H2",{class:!0});var mn=r(Z);ae=s(mn,"A",{id:!0,class:!0,href:!0});var Hc=r(ae);ho=s(Hc,"SPAN",{});var Gc=r(ho);g(ft.$$.fragment,Gc),Gc.forEach(t),Hc.forEach(t),$i=p(mn),uo=s(mn,"SPAN",{});var Cc=r(uo);ki=i(Cc,"Objectives of this notebook \u{1F3C6}"),Cc.forEach(t),mn.forEach(t),Gr=p(e),pl=s(e,"P",{});var Sc=r(pl);Ui=i(Sc,"At the end of the notebook, you will:"),Sc.forEach(t),Cr=p(e),J=s(e,"UL",{});var Vl=r(J);ct=s(Vl,"LI",{});var dn=r(ct);ji=i(dn,"Understand "),po=s(dn,"STRONG",{});var Ac=r(po);Ji=i(Ac,"the state space, action space, and reward function used to train Huggy"),Ac.forEach(t),Ii=i(dn,"."),dn.forEach(t),Hi=p(Vl),fl=s(Vl,"LI",{});var _f=r(fl);fo=s(_f,"STRONG",{});var Nc=r(fo);Gi=i(Nc,"Train your own Huggy"),Nc.forEach(t),Ci=i(_f," to fetch the stick."),_f.forEach(t),Si=p(Vl),gt=s(Vl,"LI",{});var vn=r(gt);Ai=i(vn,"Be able to play "),co=s(vn,"STRONG",{});var Zc=r(co);Ni=i(Zc,"with your trained Huggy directly in your browser"),Zc.forEach(t),Zi=i(vn,"."),vn.forEach(t),Vl.forEach(t),Sr=p(e),W=s(e,"H2",{class:!0});var yn=r(W);ne=s(yn,"A",{id:!0,class:!0,href:!0});var Wc=r(ne);go=s(Wc,"SPAN",{});var Lc=r(go);g(mt.$$.fragment,Lc),Lc.forEach(t),Wc.forEach(t),Wi=p(yn),mo=s(yn,"SPAN",{});var Pc=r(mo);Li=i(Pc,"Prerequisites \u{1F3D7}\uFE0F"),Pc.forEach(t),yn.forEach(t),Ar=p(e),cl=s(e,"P",{});var xc=r(cl);Pi=i(xc,"Before diving into the notebook, you need to:"),xc.forEach(t),Nr=p(e),ie=s(e,"P",{});var wn=r(ie);xi=i(wn,"\u{1F532} \u{1F4DA} "),vo=s(wn,"STRONG",{});var Rc=r(vo);Ri=i(Rc,"Develop an understanding of the foundations of Reinforcement learning"),Rc.forEach(t),Bi=i(wn," (MC, TD, Rewards hypothesis\u2026) by doing Unit 1"),wn.forEach(t),Zr=p(e),he=s(e,"P",{});var bn=r(he);Oi=i(bn,"\u{1F532} \u{1F4DA} "),yo=s(bn,"STRONG",{});var Bc=r(yo);zi=i(Bc,"Read the introduction to Huggy"),Bc.forEach(t),Vi=i(bn," by doing Bonus Unit 1"),bn.forEach(t),Wr=p(e),L=s(e,"H2",{class:!0});var _n=r(L);ue=s(_n,"A",{id:!0,class:!0,href:!0});var Oc=r(ue);wo=s(Oc,"SPAN",{});var zc=r(wo);g(dt.$$.fragment,zc),zc.forEach(t),Oc.forEach(t),Xi=p(_n),bo=s(_n,"SPAN",{});var Vc=r(bo);Fi=i(Vc,"Set the GPU \u{1F4AA}"),Vc.forEach(t),_n.forEach(t),Lr=i(e,`

- To **accelerate the agent's training, we'll use a GPU**. To do that, go to \`Runtime > Change Runtime type\`
`),vt=s(e,"IMG",{src:!0,alt:!0}),Pr=p(e),gl=s(e,"UL",{});var Xc=r(gl);_o=s(Xc,"LI",{});var Fc=r(_o);Eo=s(Fc,"CODE",{});var Dc=r(Eo);Di=i(Dc,"Hardware Accelerator > GPU"),Dc.forEach(t),Fc.forEach(t),Xc.forEach(t),xr=p(e),yt=s(e,"IMG",{src:!0,alt:!0}),Rr=p(e),P=s(e,"H2",{class:!0});var En=r(P);pe=s(En,"A",{id:!0,class:!0,href:!0});var Yc=r(pe);Mo=s(Yc,"SPAN",{});var Qc=r(Mo);g(wt.$$.fragment,Qc),Qc.forEach(t),Yc.forEach(t),Yi=p(En),To=s(En,"SPAN",{});var qc=r(To);Qi=i(qc,"Clone the repository and install the dependencies \u{1F53D}"),qc.forEach(t),En.forEach(t),Br=p(e),ml=s(e,"UL",{});var Kc=r(ml);$o=s(Kc,"LI",{});var eg=r($o);qi=i(eg,"We need to clone the repository, that contains ML-Agents."),eg.forEach(t),Kc.forEach(t),Or=p(e),g(bt.$$.fragment,e),zr=p(e),g(_t.$$.fragment,e),Vr=p(e),x=s(e,"H2",{class:!0});var Mn=r(x);fe=s(Mn,"A",{id:!0,class:!0,href:!0});var tg=r(fe);ko=s(tg,"SPAN",{});var lg=r(ko);g(Et.$$.fragment,lg),lg.forEach(t),tg.forEach(t),Ki=p(Mn),dl=s(Mn,"SPAN",{});var Ef=r(dl);eh=i(Ef,"Download and move the environment zip file in "),Uo=s(Ef,"CODE",{});var og=r(Uo);th=i(og,"./trained-envs-executables/linux/"),og.forEach(t),Ef.forEach(t),Mn.forEach(t),Xr=p(e),ce=s(e,"UL",{});var Tn=r(ce);jo=s(Tn,"LI",{});var sg=r(jo);lh=i(sg,"Our environment executable is in a zip file."),sg.forEach(t),oh=p(Tn),vl=s(Tn,"LI",{});var Mf=r(vl);sh=i(Mf,"We need to download it and place it to "),Jo=s(Mf,"CODE",{});var rg=r(Jo);rh=i(rg,"./trained-envs-executables/linux/"),rg.forEach(t),Mf.forEach(t),Tn.forEach(t),Fr=p(e),g(Mt.$$.fragment,e),Dr=p(e),g(Tt.$$.fragment,e),Yr=p(e),k=s(e,"P",{});var al=r(k);ah=i(al,"Download the file Huggy.zip from "),$t=s(al,"A",{href:!0,rel:!0});var ag=r($t);nh=i(ag,"https://drive.google.com/uc?export=download&id=1zv3M95ZJTWHUVOWT6ckq_cm98nft8gdF"),ag.forEach(t),ih=i(al," using "),Io=s(al,"CODE",{});var ng=r(Io);hh=i(ng,"wget"),ng.forEach(t),uh=i(al,". Check out the full solution to download large files from GDrive "),kt=s(al,"A",{href:!0,rel:!0});var ig=r(kt);ph=i(ig,"here"),ig.forEach(t),al.forEach(t),Qr=p(e),g(Ut.$$.fragment,e),qr=p(e),yl=s(e,"P",{});var hg=r(yl);fh=i(hg,"Make sure your file is accessible"),hg.forEach(t),Kr=p(e),g(jt.$$.fragment,e),ea=p(e),R=s(e,"H2",{class:!0});var $n=r(R);ge=s($n,"A",{id:!0,class:!0,href:!0});var ug=r(ge);Ho=s(ug,"SPAN",{});var pg=r(Ho);g(Jt.$$.fragment,pg),pg.forEach(t),ug.forEach(t),ch=p($n),Go=s($n,"SPAN",{});var fg=r(Go);gh=i(fg,"Let's recap how this environment works"),fg.forEach(t),$n.forEach(t),ta=p(e),B=s(e,"H3",{class:!0});var kn=r(B);me=s(kn,"A",{id:!0,class:!0,href:!0});var cg=r(me);Co=s(cg,"SPAN",{});var gg=r(Co);g(It.$$.fragment,gg),gg.forEach(t),cg.forEach(t),mh=p(kn),So=s(kn,"SPAN",{});var mg=r(So);dh=i(mg,"The State Space: what Huggy perceives."),mg.forEach(t),kn.forEach(t),la=p(e),wl=s(e,"P",{});var dg=r(wl);vh=i(dg,"Huggy doesn\u2019t \u201Csee\u201D his environment. Instead, we provide him information about the environment:"),dg.forEach(t),oa=p(e),I=s(e,"UL",{});var Xl=r(I);Ao=s(Xl,"LI",{});var vg=r(Ao);yh=i(vg,"The target (stick) position"),vg.forEach(t),wh=p(Xl),No=s(Xl,"LI",{});var yg=r(No);bh=i(yg,"The relative position between himself and the target"),yg.forEach(t),_h=p(Xl),Zo=s(Xl,"LI",{});var wg=r(Zo);Eh=i(wg,"The orientation of his legs."),wg.forEach(t),Xl.forEach(t),sa=p(e),de=s(e,"P",{});var Un=r(de);Mh=i(Un,"Given all this information, Huggy "),Wo=s(Un,"STRONG",{});var bg=r(Wo);Th=i(bg,"can decide which action to take next to fulfill his goal"),bg.forEach(t),$h=i(Un,"."),Un.forEach(t),ra=p(e),ve=s(e,"IMG",{src:!0,alt:!0,width:!0}),aa=p(e),O=s(e,"H3",{class:!0});var jn=r(O);ye=s(jn,"A",{id:!0,class:!0,href:!0});var _g=r(ye);Lo=s(_g,"SPAN",{});var Eg=r(Lo);g(Ht.$$.fragment,Eg),Eg.forEach(t),_g.forEach(t),kh=p(jn),Po=s(jn,"SPAN",{});var Mg=r(Po);Uh=i(Mg,"The Action Space: what moves Huggy can do"),Mg.forEach(t),jn.forEach(t),na=p(e),we=s(e,"IMG",{src:!0,alt:!0,width:!0}),ia=p(e),z=s(e,"P",{});var hr=r(z);xo=s(hr,"STRONG",{});var Tg=r(xo);jh=i(Tg,"Joint motors drive huggy legs"),Tg.forEach(t),Jh=i(hr,". This means that to get the target, Huggy needs to "),Ro=s(hr,"STRONG",{});var $g=r(Ro);Ih=i($g,"learn to rotate the joint motors of each of his legs correctly so he can move"),$g.forEach(t),Hh=i(hr,"."),hr.forEach(t),ha=p(e),V=s(e,"H3",{class:!0});var Jn=r(V);be=s(Jn,"A",{id:!0,class:!0,href:!0});var kg=r(be);Bo=s(kg,"SPAN",{});var Ug=r(Bo);g(Gt.$$.fragment,Ug),Ug.forEach(t),kg.forEach(t),Gh=p(Jn),Oo=s(Jn,"SPAN",{});var jg=r(Oo);Ch=i(jg,"The Reward Function"),jg.forEach(t),Jn.forEach(t),ua=p(e),_e=s(e,"P",{});var In=r(_e);Sh=i(In,"The reward function is designed so that "),zo=s(In,"STRONG",{});var Jg=r(zo);Ah=i(Jg,"Huggy will fulfill his goal"),Jg.forEach(t),Nh=i(In," : fetch the stick."),In.forEach(t),pa=p(e),H=s(e,"P",{});var Fl=r(H);Zh=i(Fl,"Remember that one of the foundations of Reinforcement Learning is the "),Vo=s(Fl,"EM",{});var Ig=r(Vo);Wh=i(Ig,"reward hypothesis"),Ig.forEach(t),Lh=i(Fl,": a goal can be described as the "),Xo=s(Fl,"STRONG",{});var Hg=r(Xo);Ph=i(Hg,"maximization of the expected cumulative reward"),Hg.forEach(t),xh=i(Fl,"."),Fl.forEach(t),fa=p(e),Ee=s(e,"P",{});var Hn=r(Ee);Rh=i(Hn,"Here, our goal is that Huggy "),Fo=s(Hn,"STRONG",{});var Gg=r(Fo);Bh=i(Gg,"goes towards the stick but without spinning too much"),Gg.forEach(t),Oh=i(Hn,". Hence, our reward function must translate this goal."),Hn.forEach(t),ca=p(e),bl=s(e,"P",{});var Cg=r(bl);zh=i(Cg,"Our reward function:"),Cg.forEach(t),ga=p(e),Me=s(e,"IMG",{src:!0,alt:!0,width:!0}),ma=p(e),E=s(e,"UL",{});var De=r(E);Te=s(De,"LI",{});var ur=r(Te);Do=s(ur,"EM",{});var Sg=r(Do);Vh=i(Sg,"Orientation bonus"),Sg.forEach(t),Xh=i(ur,": we "),Yo=s(ur,"STRONG",{});var Ag=r(Yo);Fh=i(Ag,"reward him for getting close to the target"),Ag.forEach(t),Dh=i(ur,"."),ur.forEach(t),Yh=p(De),$e=s(De,"LI",{});var pr=r($e);Qo=s(pr,"EM",{});var Ng=r(Qo);Qh=i(Ng,"Time penalty"),Ng.forEach(t),qh=i(pr,": a fixed-time penalty given at every action to "),qo=s(pr,"STRONG",{});var Zg=r(qo);Kh=i(Zg,"force him to get to the stick as fast as possible"),Zg.forEach(t),eu=i(pr,"."),pr.forEach(t),tu=p(De),ke=s(De,"LI",{});var fr=r(ke);Ko=s(fr,"EM",{});var Wg=r(Ko);lu=i(Wg,"Rotation penalty"),Wg.forEach(t),ou=i(fr,": we penalize Huggy if "),es=s(fr,"STRONG",{});var Lg=r(es);su=i(Lg,"he spins too much and turns too quickly"),Lg.forEach(t),ru=i(fr,"."),fr.forEach(t),au=p(De),Ue=s(De,"LI",{});var cr=r(Ue);ts=s(cr,"EM",{});var Pg=r(ts);nu=i(Pg,"Getting to the target reward"),Pg.forEach(t),iu=i(cr,": we reward Huggy for "),ls=s(cr,"STRONG",{});var xg=r(ls);hu=i(xg,"reaching the target"),xg.forEach(t),uu=i(cr,"."),cr.forEach(t),De.forEach(t),da=p(e),X=s(e,"H2",{class:!0});var Gn=r(X);je=s(Gn,"A",{id:!0,class:!0,href:!0});var Rg=r(je);os=s(Rg,"SPAN",{});var Bg=r(os);g(Ct.$$.fragment,Bg),Bg.forEach(t),Rg.forEach(t),pu=p(Gn),ss=s(Gn,"SPAN",{});var Og=r(ss);fu=i(Og,"Check the Huggy config file"),Og.forEach(t),Gn.forEach(t),va=p(e),b=s(e,"UL",{});var $=r(b);rs=s($,"LI",{});var zg=r(rs);_l=s(zg,"P",{});var Tf=r(_l);cu=i(Tf,"In ML-Agents, you define the "),as=s(Tf,"STRONG",{});var Vg=r(as);gu=i(Vg,"training hyperparameters in config.yaml files."),Vg.forEach(t),Tf.forEach(t),zg.forEach(t),mu=p($),ns=s($,"LI",{});var Xg=r(ns);St=s(Xg,"P",{});var Cn=r(St);du=i(Cn,"For the scope of this notebook, we\u2019re not going to modify the hyperparameters, but if you want to try as an experiment, Unity provides very "),At=s(Cn,"A",{href:!0,rel:!0});var Fg=r(At);vu=i(Fg,"good documentation explaining each of them here"),Fg.forEach(t),yu=i(Cn,"."),Cn.forEach(t),Xg.forEach(t),wu=p($),is=s($,"LI",{});var Dg=r(is);hs=s(Dg,"P",{});var Yg=r(hs);bu=i(Yg,"We need to create a config file for Huggy."),Yg.forEach(t),Dg.forEach(t),_u=p($),us=s($,"LI",{});var Qg=r(us);El=s(Qg,"P",{});var $f=r(El);Eu=i($f,"Go to "),ps=s($f,"CODE",{});var qg=r(ps);Mu=i(qg,"/content/ml-agents/config/ppo"),qg.forEach(t),$f.forEach(t),Qg.forEach(t),Tu=p($),fs=s($,"LI",{});var Kg=r(fs);Ml=s(Kg,"P",{});var kf=r(Ml);$u=i(kf,"Create a new file called "),cs=s(kf,"CODE",{});var em=r(cs);ku=i(em,"Huggy.yaml"),em.forEach(t),kf.forEach(t),Kg.forEach(t),Uu=p($),gs=s($,"LI",{});var tm=r(gs);ms=s(tm,"P",{});var lm=r(ms);ju=i(lm,"Copy and paste the content below \u{1F53D}"),lm.forEach(t),tm.forEach(t),$.forEach(t),ya=p(e),g(Nt.$$.fragment,e),wa=p(e),Je=s(e,"UL",{});var Sn=r(Je);ds=s(Sn,"LI",{});var om=r(ds);vs=s(om,"P",{});var sm=r(vs);Ju=i(sm,"Don\u2019t forget to save the file!"),sm.forEach(t),om.forEach(t),Iu=p(Sn),ys=s(Sn,"LI",{});var rm=r(ys);Zt=s(rm,"P",{});var An=r(Zt);ws=s(An,"STRONG",{});var am=r(ws);Hu=i(am,"In the case you want to modify the hyperparameters"),am.forEach(t),Gu=i(An,", in Google Colab notebook, you can click here to open the config.yaml: "),bs=s(An,"CODE",{});var nm=r(bs);Cu=i(nm,"/content/ml-agents/config/ppo/Huggy.yaml"),nm.forEach(t),An.forEach(t),rm.forEach(t),Sn.forEach(t),ba=p(e),Tl=s(e,"P",{});var im=r(Tl);Su=i(im,"We\u2019re now ready to train our agent \u{1F525}."),im.forEach(t),_a=p(e),F=s(e,"H2",{class:!0});var Nn=r(F);Ie=s(Nn,"A",{id:!0,class:!0,href:!0});var hm=r(Ie);_s=s(hm,"SPAN",{});var um=r(_s);g(Wt.$$.fragment,um),um.forEach(t),hm.forEach(t),Au=p(Nn),Es=s(Nn,"SPAN",{});var pm=r(Es);Nu=i(pm,"Train our agent"),pm.forEach(t),Nn.forEach(t),Ea=p(e),Lt=s(e,"P",{});var Uf=r(Lt);Zu=i(Uf,"To train our agent, we just need to "),Ms=s(Uf,"STRONG",{});var fm=r(Ms);Wu=i(fm,"launch mlagents-learn and select the executable containing the environment."),fm.forEach(t),Uf.forEach(t),Ma=p(e),He=s(e,"IMG",{src:!0,alt:!0,width:!0}),Ta=p(e),$l=s(e,"P",{});var cm=r($l);Lu=i(cm,"With ML Agents, we run a training script. We define four parameters:"),cm.forEach(t),$a=p(e),M=s(e,"OL",{});var Ye=r(M);kl=s(Ye,"LI",{});var jf=r(kl);Ts=s(jf,"CODE",{});var gm=r(Ts);Pu=i(gm,"mlagents-learn <config>"),gm.forEach(t),xu=i(jf,": the path where the hyperparameter config file is."),jf.forEach(t),Ru=p(Ye),Ul=s(Ye,"LI",{});var Jf=r(Ul);$s=s(Jf,"CODE",{});var mm=r($s);Bu=i(mm,"--env"),mm.forEach(t),Ou=i(Jf,": where the environment executable is."),Jf.forEach(t),zu=p(Ye),jl=s(Ye,"LI",{});var If=r(jl);ks=s(If,"CODE",{});var dm=r(ks);Vu=i(dm,"--run_id"),dm.forEach(t),Xu=i(If,": the name you want to give to your training run id."),If.forEach(t),Fu=p(Ye),Jl=s(Ye,"LI",{});var Hf=r(Jl);Us=s(Hf,"CODE",{});var vm=r(Us);Du=i(vm,"--no-graphics"),vm.forEach(t),Yu=i(Hf,": to not launch the visualization during the training."),Hf.forEach(t),Ye.forEach(t),ka=p(e),Ge=s(e,"P",{});var Zn=r(Ge);Qu=i(Zn,"Train the model and use the "),js=s(Zn,"CODE",{});var ym=r(js);qu=i(ym,"--resume"),ym.forEach(t),Ku=i(Zn," flag to continue training in case of interruption."),Zn.forEach(t),Ua=p(e),Il=s(e,"BLOCKQUOTE",{});var wm=r(Il);Pt=s(wm,"P",{});var Wn=r(Pt);ep=i(Wn,"It will fail first time when you use "),Js=s(Wn,"CODE",{});var bm=r(Js);tp=i(bm,"--resume"),bm.forEach(t),lp=i(Wn,", try running the block again to bypass the error."),Wn.forEach(t),wm.forEach(t),ja=p(e),Ce=s(e,"P",{});var Ln=r(Ce);op=i(Ln,"The training will take 30 to 45min depending on your machine (don\u2019t forget to "),Is=s(Ln,"STRONG",{});var _m=r(Is);sp=i(_m,"set up a GPU"),_m.forEach(t),rp=i(Ln,"), go take a \u2615\uFE0F you deserve it \u{1F917}."),Ln.forEach(t),Ja=p(e),g(xt.$$.fragment,e),Ia=p(e),D=s(e,"H2",{class:!0});var Pn=r(D);Se=s(Pn,"A",{id:!0,class:!0,href:!0});var Em=r(Se);Hs=s(Em,"SPAN",{});var Mm=r(Hs);g(Rt.$$.fragment,Mm),Mm.forEach(t),Em.forEach(t),ap=p(Pn),Gs=s(Pn,"SPAN",{});var Tm=r(Gs);np=i(Tm,"Push the agent to the \u{1F917} Hub"),Tm.forEach(t),Pn.forEach(t),Ha=p(e),Hl=s(e,"UL",{});var $m=r(Hl);Gl=s($m,"LI",{});var Gf=r(Gl);ip=i(Gf,"Now that we trained our agent, we\u2019re "),Cs=s(Gf,"STRONG",{});var km=r(Cs);hp=i(km,"ready to push it to the Hub to be able to play with Huggy on your browser\u{1F525}."),km.forEach(t),Gf.forEach(t),$m.forEach(t),Ga=p(e),Cl=s(e,"P",{});var Um=r(Cl);up=i(Um,"To be able to share your model with the community there are three more steps to follow:"),Um.forEach(t),Ca=p(e),Bt=s(e,"P",{});var Cf=r(Bt);pp=i(Cf,"1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),Ot=s(Cf,"A",{href:!0,rel:!0});var jm=r(Ot);fp=i(jm,"https://huggingface.co/join"),jm.forEach(t),Cf.forEach(t),Sa=p(e),Sl=s(e,"P",{});var Jm=r(Sl);cp=i(Jm,"2\uFE0F\u20E3 Sign in and then get your token from the Hugging Face website."),Jm.forEach(t),Aa=p(e),Al=s(e,"UL",{});var Im=r(Al);Ae=s(Im,"LI",{});var gr=r(Ae);gp=i(gr,"Create a new token ("),zt=s(gr,"A",{href:!0,rel:!0});var Hm=r(zt);mp=i(Hm,"https://huggingface.co/settings/tokens"),Hm.forEach(t),dp=i(gr,") "),Ss=s(gr,"STRONG",{});var Gm=r(Ss);vp=i(Gm,"with write role"),Gm.forEach(t),gr.forEach(t),Im.forEach(t),Na=p(e),Vt=s(e,"IMG",{src:!0,alt:!0}),Za=p(e),Ne=s(e,"UL",{});var xn=r(Ne);As=s(xn,"LI",{});var Cm=r(As);yp=i(Cm,"Copy the token"),Cm.forEach(t),wp=p(xn),Ns=s(xn,"LI",{});var Sm=r(Ns);bp=i(Sm,"Run the cell below and paste the token"),Sm.forEach(t),xn.forEach(t),Wa=p(e),g(Xt.$$.fragment,e),La=p(e),Ft=s(e,"P",{});var Sf=r(Ft);_p=i(Sf,"If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),Zs=s(Sf,"CODE",{});var Am=r(Zs);Ep=i(Am,"huggingface-cli login"),Am.forEach(t),Sf.forEach(t),Pa=p(e),Ze=s(e,"P",{});var Rn=r(Ze);Mp=i(Rn,"Then, we simply need to run "),Ws=s(Rn,"CODE",{});var Nm=r(Ws);Tp=i(Nm,"mlagents-push-to-hf"),Nm.forEach(t),$p=i(Rn,"."),Rn.forEach(t),xa=p(e),We=s(e,"IMG",{src:!0,alt:!0,width:!0}),Ra=p(e),Nl=s(e,"P",{});var Zm=r(Nl);kp=i(Zm,"And we define 4 parameters:"),Zm.forEach(t),Ba=p(e),T=s(e,"OL",{});var Qe=r(T);Zl=s(Qe,"LI",{});var Af=r(Zl);Ls=s(Af,"CODE",{});var Wm=r(Ls);Up=i(Wm,"--run-id"),Wm.forEach(t),jp=i(Af,": the name of the training run id."),Af.forEach(t),Jp=p(Qe),Dt=s(Qe,"LI",{});var Bn=r(Dt);Ps=s(Bn,"CODE",{});var Lm=r(Ps);Ip=i(Lm,"--local-dir"),Lm.forEach(t),Hp=i(Bn,": where the agent was saved, it\u2019s results/"),Wl=s(Bn,"RUN_ID",{name:!0});var Pm=r(Wl);Gp=i(Pm,", so in my case results/First Training."),Pm.forEach(t),Bn.forEach(t),Cp=p(Qe),Yt=s(Qe,"LI",{});var On=r(Yt);xs=s(On,"CODE",{});var xm=r(xs);Sp=i(xm,"--repo-id"),xm.forEach(t),Ap=i(On,": the name of the Hugging Face repo you want to create or update. It\u2019s always "),Le=s(On,"YOUR",{huggingface:!0,username:!0});var Nf=r(Le);Np=i(Nf,"/"),Pe=s(Nf,"THE",{repo:!0,name:!0});var Zf=r(Pe);Zp=i(Zf,"If the repo does not exist "),Rs=s(Zf,"STRONG",{});var Rm=r(Rs);Wp=i(Rm,"it will be created automatically"),Rm.forEach(t),Zf.forEach(t),Nf.forEach(t),On.forEach(t),Lp=p(Qe),Ll=s(Qe,"LI",{});var Wf=r(Ll);Bs=s(Wf,"CODE",{});var Bm=r(Bs);Pp=i(Bm,"--commit-message"),Bm.forEach(t),xp=i(Wf,": since HF repos are git repositories you need to give a commit message."),Wf.forEach(t),Qe.forEach(t),Oa=p(e),g(Qt.$$.fragment,e),za=p(e),Pl=s(e,"P",{});var Om=r(Pl);Rp=i(Om,"If everything worked you should see this at the end of the process (but with a different url \u{1F606}) :"),Om.forEach(t),Va=p(e),g(qt.$$.fragment,e),Xa=p(e),Kt=s(e,"P",{});var Lf=r(Kt);Bp=i(Lf,"It\u2019s the link to your model repository. The repository contains a model card that explains how to use the model, your Tensorboard logs and your config file. "),Os=s(Lf,"STRONG",{});var zm=r(Os);Op=i(zm,"What\u2019s awesome is that it\u2019s a git repository, which means you can have different commits, update your repository with a new push, open Pull Requests, etc."),zm.forEach(t),Lf.forEach(t),Fa=p(e),xe=s(e,"IMG",{src:!0,alt:!0,width:!0}),Da=p(e),el=s(e,"P",{});var Pf=r(el);zp=i(Pf,"But now comes the best part: "),zs=s(Pf,"STRONG",{});var Vm=r(zs);Vp=i(Vm,"being able to play with Huggy online \u{1F440}."),Vm.forEach(t),Pf.forEach(t),Ya=p(e),Y=s(e,"H2",{class:!0});var zn=r(Y);Re=s(zn,"A",{id:!0,class:!0,href:!0});var Xm=r(Re);Vs=s(Xm,"SPAN",{});var Fm=r(Vs);g(tl.$$.fragment,Fm),Fm.forEach(t),Xm.forEach(t),Xp=p(zn),Xs=s(zn,"SPAN",{});var Dm=r(Xs);Fp=i(Dm,"Play with your Huggy \u{1F415}"),Dm.forEach(t),zn.forEach(t),Qa=p(e),xl=s(e,"P",{});var Ym=r(xl);Dp=i(Ym,"This step is the simplest:"),Ym.forEach(t),qa=p(e),Be=s(e,"UL",{});var Vn=r(Be);Fs=s(Vn,"LI",{});var Qm=r(Fs);Rl=s(Qm,"P",{});var xf=r(Rl);Yp=i(xf,"Open the Huggy game in your browser: "),ll=s(xf,"A",{href:!0,rel:!0});var qm=r(ll);Qp=i(qm,"https://huggingface.co/spaces/ThomasSimonini/Huggy"),qm.forEach(t),xf.forEach(t),Qm.forEach(t),qp=p(Vn),Ds=s(Vn,"LI",{});var Km=r(Ds);Ys=s(Km,"P",{});var ed=r(Ys);Kp=i(ed,"Click on Play with my Huggy model"),ed.forEach(t),Km.forEach(t),Vn.forEach(t),Ka=p(e),Oe=s(e,"IMG",{src:!0,alt:!0,width:!0}),en=p(e),ze=s(e,"OL",{});var Xn=r(ze);Qs=s(Xn,"LI",{});var td=r(Qs);qs=s(td,"P",{});var ld=r(qs);ef=i(ld,"In step 1, choose your model repository, which is the model id (in my case ThomasSimonini/ppo-Huggy)."),ld.forEach(t),td.forEach(t),tf=p(Xn),Ks=s(Xn,"LI",{});var od=r(Ks);ol=s(od,"P",{});var Fn=r(ol);lf=i(Fn,"In step 2, "),er=s(Fn,"STRONG",{});var sd=r(er);of=i(sd,"choose which model you want to replay"),sd.forEach(t),sf=i(Fn,":"),Fn.forEach(t),od.forEach(t),Xn.forEach(t),tn=p(e),Ve=s(e,"UL",{});var Dn=r(Ve);tr=s(Dn,"LI",{});var rd=r(tr);rf=i(rd,"I have multiple ones, since we saved a model every 500000 timesteps."),rd.forEach(t),af=p(Dn),Bl=s(Dn,"LI",{});var Rf=r(Bl);nf=i(Rf,"But since I want the most recent one, I choose "),lr=s(Rf,"CODE",{});var ad=r(lr);hf=i(ad,"Huggy.onnx"),ad.forEach(t),Rf.forEach(t),Dn.forEach(t),ln=p(e),sl=s(e,"P",{});var Bf=r(sl);uf=i(Bf,"\u{1F449} It\u2019s good "),or=s(Bf,"STRONG",{});var nd=r(or);pf=i(nd,"to try with different models steps to see the improvement of the agent."),nd.forEach(t),Bf.forEach(t),on=p(e),Ol=s(e,"P",{});var id=r(Ol);ff=i(id,"Congrats on finishing this bonus unit!"),id.forEach(t),sn=p(e),Q=s(e,"P",{});var mr=r(Q);cf=i(mr,"You can now sit and enjoy playing with your Huggy \u{1F436}. And don\u2019t "),sr=s(mr,"STRONG",{});var hd=r(sr);gf=i(hd,"forget to spread the love by sharing Huggy with your friends \u{1F917}"),hd.forEach(t),mf=i(mr,". And if you share about it on social media, "),rr=s(mr,"STRONG",{});var ud=r(rr);df=i(ud,"please tag us @huggingface and me @simoninithomas"),ud.forEach(t),mr.forEach(t),rn=p(e),Xe=s(e,"IMG",{src:!0,alt:!0,width:!0}),an=p(e),q=s(e,"H2",{class:!0});var Yn=r(q);Fe=s(Yn,"A",{id:!0,class:!0,href:!0});var pd=r(Fe);ar=s(pd,"SPAN",{});var fd=r(ar);g(rl.$$.fragment,fd),fd.forEach(t),pd.forEach(t),vf=p(Yn),nr=s(Yn,"SPAN",{});var cd=r(nr);yf=i(cd,"Keep Learning, Stay awesome \u{1F917}"),cd.forEach(t),Yn.forEach(t),this.h()},h(){f(G,"name","hf:doc:metadata"),f(G,"content",JSON.stringify(Ed)),f(K,"id","train"),f(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(K,"href","#train"),f(C,"class","relative group"),f(le,"id","lets-train-huggy"),f(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(le,"href","#lets-train-huggy"),f(S,"class","relative group"),_(il.src,zf="https://colab.research.google.com/assets/colab-badge.svg")||f(il,"src",zf),f(il,"alt","Open In Colab"),f(lt,"href","https://colab.research.google.com/github/huggingface/deep-rl-class/blob/master/notebooks/bonus-unit1/bonus-unit1.ipynb"),f(lt,"rel","nofollow"),_(ot.src,Vf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit2/thumbnail.png")||f(ot,"src",Vf),f(ot,"alt","Bonus Unit 1Thumbnail"),_(rt.src,Xf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/huggy.jpg")||f(rt,"src",Xf),f(rt,"alt","Huggy"),f(oe,"id","the-environment"),f(oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(oe,"href","#the-environment"),f(A,"class","relative group"),f(nt,"href","https://twitter.com/ThomasSimonini"),f(nt,"rel","nofollow"),f(it,"href","https://blog.unity.com/technology/puppo-the-corgi-cuteness-overload-with-the-unity-ml-agents-toolkit"),f(it,"rel","nofollow"),f(re,"id","the-library-used"),f(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(re,"href","#the-library-used"),f(N,"class","relative group"),f(ut,"href","https://github.com/Unity-Technologies/ml-agents"),f(ut,"rel","nofollow"),f(pt,"href","https://github.com/huggingface/deep-rl-class/issues"),f(pt,"rel","nofollow"),f(ae,"id","objectives-of-this-notebook"),f(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ae,"href","#objectives-of-this-notebook"),f(Z,"class","relative group"),f(ne,"id","prerequisites"),f(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ne,"href","#prerequisites"),f(W,"class","relative group"),f(ue,"id","set-the-gpu"),f(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ue,"href","#set-the-gpu"),f(L,"class","relative group"),_(vt.src,Ff="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step1.jpg")||f(vt,"src",Ff),f(vt,"alt","GPU Step 1"),_(yt.src,Df="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step2.jpg")||f(yt,"src",Df),f(yt,"alt","GPU Step 2"),f(pe,"id","clone-the-repository-and-install-the-dependencies"),f(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(pe,"href","#clone-the-repository-and-install-the-dependencies"),f(P,"class","relative group"),f(fe,"id","download-and-move-the-environment-zip-file-in-trainedenvsexecutableslinux"),f(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(fe,"href","#download-and-move-the-environment-zip-file-in-trainedenvsexecutableslinux"),f(x,"class","relative group"),f($t,"href","https://drive.google.com/uc?export=download&id=1zv3M95ZJTWHUVOWT6ckq_cm98nft8gdF"),f($t,"rel","nofollow"),f(kt,"href","https://bcrf.biochem.wisc.edu/2021/02/05/download-google-drive-files-using-wget/"),f(kt,"rel","nofollow"),f(ge,"id","lets-recap-how-this-environment-works"),f(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ge,"href","#lets-recap-how-this-environment-works"),f(R,"class","relative group"),f(me,"id","the-state-space-what-huggy-perceives"),f(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(me,"href","#the-state-space-what-huggy-perceives"),f(B,"class","relative group"),_(ve.src,Yf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/huggy.jpg")||f(ve,"src",Yf),f(ve,"alt","Huggy"),f(ve,"width","100%"),f(ye,"id","the-action-space-what-moves-huggy-can-do"),f(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ye,"href","#the-action-space-what-moves-huggy-can-do"),f(O,"class","relative group"),_(we.src,Qf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/huggy-action.jpg")||f(we,"src",Qf),f(we,"alt","Huggy action"),f(we,"width","100%"),f(be,"id","the-reward-function"),f(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(be,"href","#the-reward-function"),f(V,"class","relative group"),_(Me.src,qf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/reward.jpg")||f(Me,"src",qf),f(Me,"alt","Huggy reward function"),f(Me,"width","100%"),f(je,"id","check-the-huggy-config-file"),f(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(je,"href","#check-the-huggy-config-file"),f(X,"class","relative group"),f(At,"href","https://github.com/Unity-Technologies/ml-agents/blob/main/docs/Training-Configuration-File.md"),f(At,"rel","nofollow"),f(Ie,"id","train-our-agent"),f(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ie,"href","#train-our-agent"),f(F,"class","relative group"),_(He.src,Kf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/mllearn.png")||f(He,"src",Kf),f(He,"alt","ml learn function"),f(He,"width","100%"),f(Se,"id","push-the-agent-to-the-hub"),f(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Se,"href","#push-the-agent-to-the-hub"),f(D,"class","relative group"),f(Ot,"href","https://huggingface.co/join"),f(Ot,"rel","nofollow"),f(zt,"href","https://huggingface.co/settings/tokens"),f(zt,"rel","nofollow"),_(Vt.src,ec="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/create-token.jpg")||f(Vt,"src",ec),f(Vt,"alt","Create HF Token"),_(We.src,tc="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/mlpush.png")||f(We,"src",tc),f(We,"alt","ml learn function"),f(We,"width","100%"),f(Wl,"name",""),f(Pe,"repo",""),f(Pe,"name",""),f(Le,"huggingface",""),f(Le,"username",""),_(xe.src,lc="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/modelcard.png")||f(xe,"src",lc),f(xe,"alt","ml learn function"),f(xe,"width","100%"),f(Re,"id","play-with-your-huggy"),f(Re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Re,"href","#play-with-your-huggy"),f(Y,"class","relative group"),f(ll,"href","https://huggingface.co/spaces/ThomasSimonini/Huggy"),f(ll,"rel","nofollow"),_(Oe.src,oc="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/load-huggy.jpg")||f(Oe,"src",oc),f(Oe,"alt","load-huggy"),f(Oe,"width","100%"),_(Xe.src,sc="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/huggy-cover.jpeg")||f(Xe,"src",sc),f(Xe,"alt","Huggy cover"),f(Xe,"width","100%"),f(Fe,"id","keep-learning-stay-awesome"),f(Fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Fe,"href","#keep-learning-stay-awesome"),f(q,"class","relative group")},m(e,a){l(document.head,G),h(e,dr,a),h(e,C,a),l(C,K),l(K,Dl),m(qe,Dl,null),l(C,Qn),l(C,Yl),l(Yl,qn),h(e,vr,a),m(Ke,e,a),h(e,yr,a),h(e,ee,a),l(ee,Kn),l(ee,Ql),l(Ql,ei),l(ee,ti),h(e,wr,a),h(e,te,a),l(te,li),l(te,ql),l(ql,oi),l(te,si),h(e,br,a),h(e,S,a),l(S,le),l(le,Kl),m(et,Kl,null),l(S,ri),l(S,eo),l(eo,ai),h(e,_r,a),h(e,tt,a),l(tt,to),l(to,ni),l(tt,ii),h(e,Er,a),h(e,nl,a),l(nl,lt),l(lt,il),h(e,Mr,a),h(e,ot,a),h(e,Tr,a),h(e,st,a),l(st,hi),l(st,lo),l(lo,ui),h(e,$r,a),h(e,rt,a),h(e,kr,a),h(e,A,a),l(A,oe),l(oe,oo),m(at,oo,null),l(A,pi),l(A,so),l(so,fi),h(e,Ur,a),h(e,hl,a),l(hl,se),l(se,ci),l(se,nt),l(nt,gi),l(se,mi),l(se,it),l(it,di),h(e,jr,a),h(e,N,a),l(N,re),l(re,ro),m(ht,ro,null),l(N,vi),l(N,ao),l(ao,yi),h(e,Jr,a),h(e,ul,a),l(ul,no),l(no,ut),l(ut,wi),h(e,Ir,a),h(e,j,a),l(j,bi),l(j,io),l(io,_i),l(j,Ei),l(j,pt),l(pt,Mi),l(j,Ti),h(e,Hr,a),h(e,Z,a),l(Z,ae),l(ae,ho),m(ft,ho,null),l(Z,$i),l(Z,uo),l(uo,ki),h(e,Gr,a),h(e,pl,a),l(pl,Ui),h(e,Cr,a),h(e,J,a),l(J,ct),l(ct,ji),l(ct,po),l(po,Ji),l(ct,Ii),l(J,Hi),l(J,fl),l(fl,fo),l(fo,Gi),l(fl,Ci),l(J,Si),l(J,gt),l(gt,Ai),l(gt,co),l(co,Ni),l(gt,Zi),h(e,Sr,a),h(e,W,a),l(W,ne),l(ne,go),m(mt,go,null),l(W,Wi),l(W,mo),l(mo,Li),h(e,Ar,a),h(e,cl,a),l(cl,Pi),h(e,Nr,a),h(e,ie,a),l(ie,xi),l(ie,vo),l(vo,Ri),l(ie,Bi),h(e,Zr,a),h(e,he,a),l(he,Oi),l(he,yo),l(yo,zi),l(he,Vi),h(e,Wr,a),h(e,L,a),l(L,ue),l(ue,wo),m(dt,wo,null),l(L,Xi),l(L,bo),l(bo,Fi),h(e,Lr,a),h(e,vt,a),h(e,Pr,a),h(e,gl,a),l(gl,_o),l(_o,Eo),l(Eo,Di),h(e,xr,a),h(e,yt,a),h(e,Rr,a),h(e,P,a),l(P,pe),l(pe,Mo),m(wt,Mo,null),l(P,Yi),l(P,To),l(To,Qi),h(e,Br,a),h(e,ml,a),l(ml,$o),l($o,qi),h(e,Or,a),m(bt,e,a),h(e,zr,a),m(_t,e,a),h(e,Vr,a),h(e,x,a),l(x,fe),l(fe,ko),m(Et,ko,null),l(x,Ki),l(x,dl),l(dl,eh),l(dl,Uo),l(Uo,th),h(e,Xr,a),h(e,ce,a),l(ce,jo),l(jo,lh),l(ce,oh),l(ce,vl),l(vl,sh),l(vl,Jo),l(Jo,rh),h(e,Fr,a),m(Mt,e,a),h(e,Dr,a),m(Tt,e,a),h(e,Yr,a),h(e,k,a),l(k,ah),l(k,$t),l($t,nh),l(k,ih),l(k,Io),l(Io,hh),l(k,uh),l(k,kt),l(kt,ph),h(e,Qr,a),m(Ut,e,a),h(e,qr,a),h(e,yl,a),l(yl,fh),h(e,Kr,a),m(jt,e,a),h(e,ea,a),h(e,R,a),l(R,ge),l(ge,Ho),m(Jt,Ho,null),l(R,ch),l(R,Go),l(Go,gh),h(e,ta,a),h(e,B,a),l(B,me),l(me,Co),m(It,Co,null),l(B,mh),l(B,So),l(So,dh),h(e,la,a),h(e,wl,a),l(wl,vh),h(e,oa,a),h(e,I,a),l(I,Ao),l(Ao,yh),l(I,wh),l(I,No),l(No,bh),l(I,_h),l(I,Zo),l(Zo,Eh),h(e,sa,a),h(e,de,a),l(de,Mh),l(de,Wo),l(Wo,Th),l(de,$h),h(e,ra,a),h(e,ve,a),h(e,aa,a),h(e,O,a),l(O,ye),l(ye,Lo),m(Ht,Lo,null),l(O,kh),l(O,Po),l(Po,Uh),h(e,na,a),h(e,we,a),h(e,ia,a),h(e,z,a),l(z,xo),l(xo,jh),l(z,Jh),l(z,Ro),l(Ro,Ih),l(z,Hh),h(e,ha,a),h(e,V,a),l(V,be),l(be,Bo),m(Gt,Bo,null),l(V,Gh),l(V,Oo),l(Oo,Ch),h(e,ua,a),h(e,_e,a),l(_e,Sh),l(_e,zo),l(zo,Ah),l(_e,Nh),h(e,pa,a),h(e,H,a),l(H,Zh),l(H,Vo),l(Vo,Wh),l(H,Lh),l(H,Xo),l(Xo,Ph),l(H,xh),h(e,fa,a),h(e,Ee,a),l(Ee,Rh),l(Ee,Fo),l(Fo,Bh),l(Ee,Oh),h(e,ca,a),h(e,bl,a),l(bl,zh),h(e,ga,a),h(e,Me,a),h(e,ma,a),h(e,E,a),l(E,Te),l(Te,Do),l(Do,Vh),l(Te,Xh),l(Te,Yo),l(Yo,Fh),l(Te,Dh),l(E,Yh),l(E,$e),l($e,Qo),l(Qo,Qh),l($e,qh),l($e,qo),l(qo,Kh),l($e,eu),l(E,tu),l(E,ke),l(ke,Ko),l(Ko,lu),l(ke,ou),l(ke,es),l(es,su),l(ke,ru),l(E,au),l(E,Ue),l(Ue,ts),l(ts,nu),l(Ue,iu),l(Ue,ls),l(ls,hu),l(Ue,uu),h(e,da,a),h(e,X,a),l(X,je),l(je,os),m(Ct,os,null),l(X,pu),l(X,ss),l(ss,fu),h(e,va,a),h(e,b,a),l(b,rs),l(rs,_l),l(_l,cu),l(_l,as),l(as,gu),l(b,mu),l(b,ns),l(ns,St),l(St,du),l(St,At),l(At,vu),l(St,yu),l(b,wu),l(b,is),l(is,hs),l(hs,bu),l(b,_u),l(b,us),l(us,El),l(El,Eu),l(El,ps),l(ps,Mu),l(b,Tu),l(b,fs),l(fs,Ml),l(Ml,$u),l(Ml,cs),l(cs,ku),l(b,Uu),l(b,gs),l(gs,ms),l(ms,ju),h(e,ya,a),m(Nt,e,a),h(e,wa,a),h(e,Je,a),l(Je,ds),l(ds,vs),l(vs,Ju),l(Je,Iu),l(Je,ys),l(ys,Zt),l(Zt,ws),l(ws,Hu),l(Zt,Gu),l(Zt,bs),l(bs,Cu),h(e,ba,a),h(e,Tl,a),l(Tl,Su),h(e,_a,a),h(e,F,a),l(F,Ie),l(Ie,_s),m(Wt,_s,null),l(F,Au),l(F,Es),l(Es,Nu),h(e,Ea,a),h(e,Lt,a),l(Lt,Zu),l(Lt,Ms),l(Ms,Wu),h(e,Ma,a),h(e,He,a),h(e,Ta,a),h(e,$l,a),l($l,Lu),h(e,$a,a),h(e,M,a),l(M,kl),l(kl,Ts),l(Ts,Pu),l(kl,xu),l(M,Ru),l(M,Ul),l(Ul,$s),l($s,Bu),l(Ul,Ou),l(M,zu),l(M,jl),l(jl,ks),l(ks,Vu),l(jl,Xu),l(M,Fu),l(M,Jl),l(Jl,Us),l(Us,Du),l(Jl,Yu),h(e,ka,a),h(e,Ge,a),l(Ge,Qu),l(Ge,js),l(js,qu),l(Ge,Ku),h(e,Ua,a),h(e,Il,a),l(Il,Pt),l(Pt,ep),l(Pt,Js),l(Js,tp),l(Pt,lp),h(e,ja,a),h(e,Ce,a),l(Ce,op),l(Ce,Is),l(Is,sp),l(Ce,rp),h(e,Ja,a),m(xt,e,a),h(e,Ia,a),h(e,D,a),l(D,Se),l(Se,Hs),m(Rt,Hs,null),l(D,ap),l(D,Gs),l(Gs,np),h(e,Ha,a),h(e,Hl,a),l(Hl,Gl),l(Gl,ip),l(Gl,Cs),l(Cs,hp),h(e,Ga,a),h(e,Cl,a),l(Cl,up),h(e,Ca,a),h(e,Bt,a),l(Bt,pp),l(Bt,Ot),l(Ot,fp),h(e,Sa,a),h(e,Sl,a),l(Sl,cp),h(e,Aa,a),h(e,Al,a),l(Al,Ae),l(Ae,gp),l(Ae,zt),l(zt,mp),l(Ae,dp),l(Ae,Ss),l(Ss,vp),h(e,Na,a),h(e,Vt,a),h(e,Za,a),h(e,Ne,a),l(Ne,As),l(As,yp),l(Ne,wp),l(Ne,Ns),l(Ns,bp),h(e,Wa,a),m(Xt,e,a),h(e,La,a),h(e,Ft,a),l(Ft,_p),l(Ft,Zs),l(Zs,Ep),h(e,Pa,a),h(e,Ze,a),l(Ze,Mp),l(Ze,Ws),l(Ws,Tp),l(Ze,$p),h(e,xa,a),h(e,We,a),h(e,Ra,a),h(e,Nl,a),l(Nl,kp),h(e,Ba,a),h(e,T,a),l(T,Zl),l(Zl,Ls),l(Ls,Up),l(Zl,jp),l(T,Jp),l(T,Dt),l(Dt,Ps),l(Ps,Ip),l(Dt,Hp),l(Dt,Wl),l(Wl,Gp),l(T,Cp),l(T,Yt),l(Yt,xs),l(xs,Sp),l(Yt,Ap),l(Yt,Le),l(Le,Np),l(Le,Pe),l(Pe,Zp),l(Pe,Rs),l(Rs,Wp),l(T,Lp),l(T,Ll),l(Ll,Bs),l(Bs,Pp),l(Ll,xp),h(e,Oa,a),m(Qt,e,a),h(e,za,a),h(e,Pl,a),l(Pl,Rp),h(e,Va,a),m(qt,e,a),h(e,Xa,a),h(e,Kt,a),l(Kt,Bp),l(Kt,Os),l(Os,Op),h(e,Fa,a),h(e,xe,a),h(e,Da,a),h(e,el,a),l(el,zp),l(el,zs),l(zs,Vp),h(e,Ya,a),h(e,Y,a),l(Y,Re),l(Re,Vs),m(tl,Vs,null),l(Y,Xp),l(Y,Xs),l(Xs,Fp),h(e,Qa,a),h(e,xl,a),l(xl,Dp),h(e,qa,a),h(e,Be,a),l(Be,Fs),l(Fs,Rl),l(Rl,Yp),l(Rl,ll),l(ll,Qp),l(Be,qp),l(Be,Ds),l(Ds,Ys),l(Ys,Kp),h(e,Ka,a),h(e,Oe,a),h(e,en,a),h(e,ze,a),l(ze,Qs),l(Qs,qs),l(qs,ef),l(ze,tf),l(ze,Ks),l(Ks,ol),l(ol,lf),l(ol,er),l(er,of),l(ol,sf),h(e,tn,a),h(e,Ve,a),l(Ve,tr),l(tr,rf),l(Ve,af),l(Ve,Bl),l(Bl,nf),l(Bl,lr),l(lr,hf),h(e,ln,a),h(e,sl,a),l(sl,uf),l(sl,or),l(or,pf),h(e,on,a),h(e,Ol,a),l(Ol,ff),h(e,sn,a),h(e,Q,a),l(Q,cf),l(Q,sr),l(sr,gf),l(Q,mf),l(Q,rr),l(rr,df),h(e,rn,a),h(e,Xe,a),h(e,an,a),h(e,q,a),l(q,Fe),l(Fe,ar),m(rl,ar,null),l(q,vf),l(q,nr),l(nr,yf),nn=!0},p:yd,i(e){nn||(d(qe.$$.fragment,e),d(Ke.$$.fragment,e),d(et.$$.fragment,e),d(at.$$.fragment,e),d(ht.$$.fragment,e),d(ft.$$.fragment,e),d(mt.$$.fragment,e),d(dt.$$.fragment,e),d(wt.$$.fragment,e),d(bt.$$.fragment,e),d(_t.$$.fragment,e),d(Et.$$.fragment,e),d(Mt.$$.fragment,e),d(Tt.$$.fragment,e),d(Ut.$$.fragment,e),d(jt.$$.fragment,e),d(Jt.$$.fragment,e),d(It.$$.fragment,e),d(Ht.$$.fragment,e),d(Gt.$$.fragment,e),d(Ct.$$.fragment,e),d(Nt.$$.fragment,e),d(Wt.$$.fragment,e),d(xt.$$.fragment,e),d(Rt.$$.fragment,e),d(Xt.$$.fragment,e),d(Qt.$$.fragment,e),d(qt.$$.fragment,e),d(tl.$$.fragment,e),d(rl.$$.fragment,e),nn=!0)},o(e){v(qe.$$.fragment,e),v(Ke.$$.fragment,e),v(et.$$.fragment,e),v(at.$$.fragment,e),v(ht.$$.fragment,e),v(ft.$$.fragment,e),v(mt.$$.fragment,e),v(dt.$$.fragment,e),v(wt.$$.fragment,e),v(bt.$$.fragment,e),v(_t.$$.fragment,e),v(Et.$$.fragment,e),v(Mt.$$.fragment,e),v(Tt.$$.fragment,e),v(Ut.$$.fragment,e),v(jt.$$.fragment,e),v(Jt.$$.fragment,e),v(It.$$.fragment,e),v(Ht.$$.fragment,e),v(Gt.$$.fragment,e),v(Ct.$$.fragment,e),v(Nt.$$.fragment,e),v(Wt.$$.fragment,e),v(xt.$$.fragment,e),v(Rt.$$.fragment,e),v(Xt.$$.fragment,e),v(Qt.$$.fragment,e),v(qt.$$.fragment,e),v(tl.$$.fragment,e),v(rl.$$.fragment,e),nn=!1},d(e){t(G),e&&t(dr),e&&t(C),y(qe),e&&t(vr),y(Ke,e),e&&t(yr),e&&t(ee),e&&t(wr),e&&t(te),e&&t(br),e&&t(S),y(et),e&&t(_r),e&&t(tt),e&&t(Er),e&&t(nl),e&&t(Mr),e&&t(ot),e&&t(Tr),e&&t(st),e&&t($r),e&&t(rt),e&&t(kr),e&&t(A),y(at),e&&t(Ur),e&&t(hl),e&&t(jr),e&&t(N),y(ht),e&&t(Jr),e&&t(ul),e&&t(Ir),e&&t(j),e&&t(Hr),e&&t(Z),y(ft),e&&t(Gr),e&&t(pl),e&&t(Cr),e&&t(J),e&&t(Sr),e&&t(W),y(mt),e&&t(Ar),e&&t(cl),e&&t(Nr),e&&t(ie),e&&t(Zr),e&&t(he),e&&t(Wr),e&&t(L),y(dt),e&&t(Lr),e&&t(vt),e&&t(Pr),e&&t(gl),e&&t(xr),e&&t(yt),e&&t(Rr),e&&t(P),y(wt),e&&t(Br),e&&t(ml),e&&t(Or),y(bt,e),e&&t(zr),y(_t,e),e&&t(Vr),e&&t(x),y(Et),e&&t(Xr),e&&t(ce),e&&t(Fr),y(Mt,e),e&&t(Dr),y(Tt,e),e&&t(Yr),e&&t(k),e&&t(Qr),y(Ut,e),e&&t(qr),e&&t(yl),e&&t(Kr),y(jt,e),e&&t(ea),e&&t(R),y(Jt),e&&t(ta),e&&t(B),y(It),e&&t(la),e&&t(wl),e&&t(oa),e&&t(I),e&&t(sa),e&&t(de),e&&t(ra),e&&t(ve),e&&t(aa),e&&t(O),y(Ht),e&&t(na),e&&t(we),e&&t(ia),e&&t(z),e&&t(ha),e&&t(V),y(Gt),e&&t(ua),e&&t(_e),e&&t(pa),e&&t(H),e&&t(fa),e&&t(Ee),e&&t(ca),e&&t(bl),e&&t(ga),e&&t(Me),e&&t(ma),e&&t(E),e&&t(da),e&&t(X),y(Ct),e&&t(va),e&&t(b),e&&t(ya),y(Nt,e),e&&t(wa),e&&t(Je),e&&t(ba),e&&t(Tl),e&&t(_a),e&&t(F),y(Wt),e&&t(Ea),e&&t(Lt),e&&t(Ma),e&&t(He),e&&t(Ta),e&&t($l),e&&t($a),e&&t(M),e&&t(ka),e&&t(Ge),e&&t(Ua),e&&t(Il),e&&t(ja),e&&t(Ce),e&&t(Ja),y(xt,e),e&&t(Ia),e&&t(D),y(Rt),e&&t(Ha),e&&t(Hl),e&&t(Ga),e&&t(Cl),e&&t(Ca),e&&t(Bt),e&&t(Sa),e&&t(Sl),e&&t(Aa),e&&t(Al),e&&t(Na),e&&t(Vt),e&&t(Za),e&&t(Ne),e&&t(Wa),y(Xt,e),e&&t(La),e&&t(Ft),e&&t(Pa),e&&t(Ze),e&&t(xa),e&&t(We),e&&t(Ra),e&&t(Nl),e&&t(Ba),e&&t(T),e&&t(Oa),y(Qt,e),e&&t(za),e&&t(Pl),e&&t(Va),y(qt,e),e&&t(Xa),e&&t(Kt),e&&t(Fa),e&&t(xe),e&&t(Da),e&&t(el),e&&t(Ya),e&&t(Y),y(tl),e&&t(Qa),e&&t(xl),e&&t(qa),e&&t(Be),e&&t(Ka),e&&t(Oe),e&&t(en),e&&t(ze),e&&t(tn),e&&t(Ve),e&&t(ln),e&&t(sl),e&&t(on),e&&t(Ol),e&&t(sn),e&&t(Q),e&&t(rn),e&&t(Xe),e&&t(an),e&&t(q),y(rl)}}}const Ed={local:"train",sections:[{local:"lets-train-huggy",sections:[{local:"the-environment",title:"The environment \u{1F3AE}"},{local:"the-library-used",title:"The library used \u{1F4DA}"}],title:"Let's train Huggy \u{1F436}"},{local:"objectives-of-this-notebook",title:"Objectives of this notebook \u{1F3C6}"},{local:"prerequisites",title:"Prerequisites \u{1F3D7}\uFE0F"},{local:"set-the-gpu",title:"Set the GPU \u{1F4AA}"},{local:"clone-the-repository-and-install-the-dependencies",title:"Clone the repository and install the dependencies \u{1F53D}"},{local:"download-and-move-the-environment-zip-file-in-trainedenvsexecutableslinux",title:"Download and move the environment zip file in `./trained-envs-executables/linux/`"},{local:"lets-recap-how-this-environment-works",sections:[{local:"the-state-space-what-huggy-perceives",title:"The State Space: what Huggy perceives."},{local:"the-action-space-what-moves-huggy-can-do",title:"The Action Space: what moves Huggy can do"},{local:"the-reward-function",title:"The Reward Function"}],title:"Let's recap how this environment works"},{local:"check-the-huggy-config-file",title:"Check the Huggy config file"},{local:"train-our-agent",title:"Train our agent"},{local:"push-the-agent-to-the-hub",title:"Push the agent to the \u{1F917} Hub"},{local:"play-with-your-huggy",title:"Play with your Huggy \u{1F415}"},{local:"keep-learning-stay-awesome",title:"Keep Learning, Stay awesome \u{1F917}"}],title:"Let's train and play with Huggy \u{1F436} "};function Md(Of){return wd(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class jd extends gd{constructor(G){super();md(this,G,Md,_d,dd,{})}}export{jd as default,Ed as metadata};
