import{S as kw,i as jw,s as Jw,e as l,k as u,w as p,t as r,M as Uw,c as a,d as t,m as f,a as s,x as d,h as i,b as c,N as Oe,G as o,g as h,y as m,L as Pw,q as v,o as w,B as y,v as Nw}from"../../chunks/vendor-hf-doc-builder.js";import{I as b}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as P}from"../../chunks/CodeBlock-hf-doc-builder.js";function Cw(cd){let N,ys,C,te,Ml,He,mn,Al,vn,gs,oe,wn,Il,yn,gn,bs,Re,bn,$l,En,Es,ze,_n,Sl,Tn,_s,Ze,Mn,Ye,An,Ts,le,In,kl,$n,Sn,Ms,wo,kn,As,x,ae,jl,De,jn,Jl,Jn,Is,yo,Un,$s,Fe,Pn,Ul,Nn,Ss,go,Cn,ks,I,Ve,xn,Pl,Ln,Wn,Gn,se,Bn,Nl,On,Hn,Xe,Rn,zn,re,Zn,Cl,Yn,Dn,qe,Fn,js,Qe,Vn,Ke,Xn,Js,ie,qn,et,Qn,Kn,Us,E,bo,eh,xl,th,oh,Eo,lh,Ll,ah,sh,tt,rh,Wl,ih,nh,hh,_o,uh,Gl,fh,Ps,L,ne,Bl,ot,ch,Ol,ph,Ns,_,dh,Hl,mh,vh,Rl,wh,yh,zl,gh,bh,Cs,To,Eh,xs,$,Mo,Zl,_h,Th,Mh,lt,Ah,Yl,Ih,$h,Sh,at,kh,Dl,jh,Jh,Ls,he,Uh,Fl,Ph,Nh,Ws,S,Ch,Vl,xh,Lh,st,Wh,Gh,Gs,W,ue,Xl,rt,Bh,ql,Oh,Bs,fe,it,Hh,Ql,Rh,zh,Zh,Ao,Yh,nt,Dh,Os,G,ce,Kl,ht,Fh,ea,Vh,Hs,pe,Xh,ut,qh,Qh,Rs,de,Kh,ta,eu,tu,zs,ft,Zs,Io,ou,Ys,ct,Ds,$o,lu,Fs,pt,Vs,So,au,Xs,dt,qs,ko,su,Qs,mt,Ks,vt,ru,wt,iu,er,B,nu,oa,hu,uu,la,fu,tr,yt,cu,aa,pu,or,gt,du,bt,mu,lr,Et,vu,_t,wu,ar,k,yu,Tt,gu,bu,sa,Eu,_u,sr,O,me,ra,Mt,Tu,ia,Mu,rr,H,Au,na,Iu,$u,At,Su,ir,It,ku,ha,ju,nr,ve,jo,pd,Ju,Jo,Uu,Uo,Pu,hr,R,we,ua,$t,Nu,fa,Cu,ur,Po,xu,fr,St,dd,cr,z,ye,ca,kt,Lu,pa,Wu,pr,No,Gu,dr,j,da,Bu,Ou,ma,Hu,Ru,Co,zu,g,va,Zu,Yu,wa,Du,Fu,ya,Vu,Xu,ga,qu,Qu,ba,Ku,ef,Ea,tf,mr,Z,ge,_a,jt,of,Ta,lf,vr,xo,af,wr,Jt,md,yr,Y,be,Ma,Ut,sf,Aa,rf,gr,Ee,nf,Ia,hf,uf,br,Pt,ff,$a,cf,Er,J,pf,Nt,df,mf,Sa,vf,wf,_r,_e,yf,ka,gf,bf,Tr,Te,Ef,ja,_f,Tf,Mr,U,Mf,Ja,Af,If,Ua,$f,Sf,Ar,Me,Lo,vd,kf,Wo,jf,Go,Jf,Ir,Bo,Uf,$r,Ae,Pf,Ct,Nf,Cf,Sr,D,Ie,Pa,xt,xf,Na,Lf,kr,F,Wf,Lt,Gf,Bf,Wt,Of,Ca,Hf,Rf,jr,Gt,zf,xa,Bt,Zf,Jr,$e,Yf,La,Df,Ff,Ur,Ot,Pr,Oo,Vf,Nr,Ht,Xf,Ho,Rt,qf,Qf,Cr,Ro,Kf,xr,V,Se,Wa,zt,ec,Ga,tc,Lr,Zt,oc,Ba,lc,Wr,zo,ac,Gr,T,Zo,Oa,sc,rc,ic,Yo,Ha,nc,hc,uc,Do,Ra,fc,cc,pc,Fo,za,dc,mc,Br,Vo,vc,Or,Xo,wc,Hr,Yt,Rr,qo,yc,zr,Qo,gc,Zr,Ko,bc,Yr,X,ke,Za,Dt,Ec,Ya,_c,Dr,Ft,Tc,Da,Mc,Fr,el,Ac,Vr,Vt,Ic,Xt,$c,Xr,tl,Sc,qr,q,kc,qt,jc,Jc,Fa,Uc,Qr,Qt,wd,Kr,ol,Pc,ei,Kt,ti,je,Nc,Va,Cc,xc,oi,ll,Lc,li,M,al,Xa,Wc,Gc,Bc,eo,qa,Oc,Hc,sl,Rc,zc,to,Qa,Zc,Yc,Je,Dc,Ue,Fc,Ka,Vc,Xc,rl,es,qc,Qc,ai,il,Kc,si,oo,ri,lo,ii,nl,ep,ni,ao,tp,so,op,hi,ro,lp,ts,ap,ui,Q,Pe,os,io,sp,ls,rp,fi,Ne,ip,as,np,hp,ci,hl,up,pi,ul,ss,fp,di,no,yd,mi,fl,cp,vi,ho,gd,wi,uo,fo,pp,rs,dp,mp,yi,co,bd,gi,cl,vp,bi,K,Ce,is,po,wp,ns,yp,Ei,ee,gp,hs,bp,Ep,mo,_p,_i,pl,Tp,Ti,dl,ml,Mp,vo,Ap,Mi,xe,Ip,us,$p,Sp,Ai,vl,kp,Ii;return He=new b({}),De=new b({}),ot=new b({}),rt=new b({}),ht=new b({}),ft=new P({props:{code:"Y29uZGElMjBjcmVhdGUlMjAtLW5hbWUlMjBybCUyMHB5dGhvbiUzRDMuOSUwQWNvbmRhJTIwYWN0aXZhdGUlMjBybA==",highlighted:`conda create --name rl python=3.9
conda activate rl`}}),ct=new P({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZVbml0eS1UZWNobm9sb2dpZXMlMkZtbC1hZ2VudHM=",highlighted:'git <span class="hljs-built_in">clone</span> https://github.com/Unity-Technologies/ml-agents'}}),pt=new P({props:{code:"Y2QlMjBtbC1hZ2VudHMlMEFwaXAlMjBpbnN0YWxsJTIwLWUlMjAuJTJGbWwtYWdlbnRzLWVudnMlMEFwaXAlMjBpbnN0YWxsJTIwLWUlMjAuJTJGbWwtYWdlbnRz",highlighted:`<span class="hljs-built_in">cd</span> ml-agents
pip install -e ./ml-agents-envs
pip install -e ./ml-agents`}}),dt=new P({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRvcmNo",highlighted:"pip install torch"}}),mt=new P({props:{code:"cGlwJTIwaW5zdGFsbCUyMG9ubnglM0QlM0QxLjEyLjA=",highlighted:"pip install onnx==1.12.0"}}),Mt=new b({}),$t=new b({}),kt=new b({}),jt=new b({}),Ut=new b({}),xt=new b({}),Ot=new P({props:{code:"YmVoYXZpb3JzJTNBJTBBJTIwJTIwU29jY2VyVHdvcyUzQSUwQSUyMCUyMCUyMCUyMHRyYWluZXJfdHlwZSUzQSUyMHBvY2ElMEElMjAlMjAlMjAlMjBoeXBlcnBhcmFtZXRlcnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNBJTIwMjA0OCUwQSUyMCUyMCUyMCUyMCUyMCUyMGJ1ZmZlcl9zaXplJTNBJTIwMjA0ODAlMEElMjAlMjAlMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNBJTIwMC4wMDAzJTBBJTIwJTIwJTIwJTIwJTIwJTIwYmV0YSUzQSUyMDAuMDA1JTBBJTIwJTIwJTIwJTIwJTIwJTIwZXBzaWxvbiUzQSUyMDAuMiUwQSUyMCUyMCUyMCUyMCUyMCUyMGxhbWJkJTNBJTIwMC45NSUwQSUyMCUyMCUyMCUyMCUyMCUyMG51bV9lcG9jaCUzQSUyMDMlMEElMjAlMjAlMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlX3NjaGVkdWxlJTNBJTIwY29uc3RhbnQlMEElMjAlMjAlMjAlMjBuZXR3b3JrX3NldHRpbmdzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwbm9ybWFsaXplJTNBJTIwZmFsc2UlMEElMjAlMjAlMjAlMjAlMjAlMjBoaWRkZW5fdW5pdHMlM0ElMjA1MTIlMEElMjAlMjAlMjAlMjAlMjAlMjBudW1fbGF5ZXJzJTNBJTIwMiUwQSUyMCUyMCUyMCUyMCUyMCUyMHZpc19lbmNvZGVfdHlwZSUzQSUyMHNpbXBsZSUwQSUyMCUyMCUyMCUyMHJld2FyZF9zaWduYWxzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwZXh0cmluc2ljJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ2FtbWElM0ElMjAwLjk5JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RyZW5ndGglM0ElMjAxLjAlMEElMjAlMjAlMjAlMjBrZWVwX2NoZWNrcG9pbnRzJTNBJTIwNSUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzQSUyMDUwMDAwMDAlMEElMjAlMjAlMjAlMjB0aW1lX2hvcml6b24lM0ElMjAxMDAwJTBBJTIwJTIwJTIwJTIwc3VtbWFyeV9mcmVxJTNBJTIwMTAwMDAlMEElMjAlMjAlMjAlMjBzZWxmX3BsYXklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjBzYXZlX3N0ZXBzJTNBJTIwNTAwMDAlMEElMjAlMjAlMjAlMjAlMjAlMjB0ZWFtX2NoYW5nZSUzQSUyMDIwMDAwMCUwQSUyMCUyMCUyMCUyMCUyMCUyMHN3YXBfc3RlcHMlM0ElMjAyMDAwJTBBJTIwJTIwJTIwJTIwJTIwJTIwd2luZG93JTNBJTIwMTAlMEElMjAlMjAlMjAlMjAlMjAlMjBwbGF5X2FnYWluc3RfbGF0ZXN0X21vZGVsX3JhdGlvJTNBJTIwMC41JTBBJTIwJTIwJTIwJTIwJTIwJTIwaW5pdGlhbF9lbG8lM0ElMjAxMjAwLjA=",highlighted:`behaviors:
  SoccerTwos:
    trainer_type: poca
    hyperparameters:
      batch_size: <span class="hljs-number">2048</span>
      buffer_size: <span class="hljs-number">20480</span>
      learning_rate: <span class="hljs-number">0.0003</span>
      beta: <span class="hljs-number">0.005</span>
      epsilon: <span class="hljs-number">0.2</span>
      lambd: <span class="hljs-number">0.95</span>
      num_epoch: <span class="hljs-number">3</span>
      learning_rate_schedule: constant
    network_settings:
      normalize: <span class="hljs-literal">false</span>
      hidden_units: <span class="hljs-number">512</span>
      num_layers: <span class="hljs-number">2</span>
      vis_encode_type: simple
    reward_signals:
      extrinsic:
        gamma: <span class="hljs-number">0.99</span>
        strength: <span class="hljs-number">1.0</span>
    keep_checkpoints: <span class="hljs-number">5</span>
    max_steps: <span class="hljs-number">5000000</span>
    time_horizon: <span class="hljs-number">1000</span>
    summary_freq: <span class="hljs-number">10000</span>
    self_play:
      save_steps: <span class="hljs-number">50000</span>
      team_change: <span class="hljs-number">200000</span>
      swap_steps: <span class="hljs-number">2000</span>
      window: <span class="hljs-number">10</span>
      play_against_latest_model_ratio: <span class="hljs-number">0.5</span>
      initial_elo: <span class="hljs-number">1200.0</span>`}}),zt=new b({}),Yt=new P({props:{code:"bWxhZ2VudHMtbGVhcm4lMjAuJTJGY29uZmlnJTJGcG9jYSUyRlNvY2NlclR3b3MueWFtbCUyMC0tZW52JTNELiUyRnRyYWluaW5nLWVudnMtZXhlY3V0YWJsZXMlMkZTb2NjZXJUd29zLmV4ZSUyMC0tcnVuLWlkJTNEJTIyU29jY2VyVHdvcyUyMiUyMC0tbm8tZ3JhcGhpY3M=",highlighted:'mlagents-learn ./config/poca/SoccerTwos.yaml --<span class="hljs-built_in">env</span>=./training-envs-executables/SoccerTwos.exe --run-id=<span class="hljs-string">&quot;SoccerTwos&quot;</span> --no-graphics'}}),Dt=new b({}),Kt=new P({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:"huggingface-cli login"}}),oo=new P({props:{code:"bWxhZ2VudHMtcHVzaC10by1oZiUyMCUyMC0tcnVuLWlkJTNEJTIyU29jY2VyVHdvcyUyMiUyMC0tbG9jYWwtZGlyJTNEJTIyLiUyRnJlc3VsdHMlMkZTb2NjZXJUd29zJTIyJTIwLS1yZXBvLWlkJTNEJTIyVGhvbWFzU2ltb25pbmklMkZwb2NhLVNvY2NlclR3b3MlMjIlMjAtLWNvbW1pdC1tZXNzYWdlJTNEJTIyRmlyc3QlMjBQdXNoJTIyJTYw",highlighted:'mlagents-push-to-hf  --run-id=<span class="hljs-string">&quot;SoccerTwos&quot;</span> --local-dir=<span class="hljs-string">&quot;./results/SoccerTwos&quot;</span> --repo-id=<span class="hljs-string">&quot;ThomasSimonini/poca-SoccerTwos&quot;</span> --commit-message=<span class="hljs-string">&quot;First Push&quot;</span>`'}}),lo=new P({props:{code:"bWxhZ2VudHMtcHVzaC10by1oZiUyMCUyMC0tcnVuLWlkJTNEJTIwJTIzJTIwQWRkJTIweW91ciUyMHJ1biUyMGlkJTIwJTIwLS1sb2NhbC1kaXIlM0QlMjAlMjMlMjBZb3VyJTIwbG9jYWwlMjBkaXIlMjAlMjAtLXJlcG8taWQlM0QlMjAlMjMlMjBZb3VyJTIwcmVwbyUyMGlkJTIwLS1jb21taXQtbWVzc2FnZSUzRCUyMkZpcnN0JTIwUHVzaCUyMg==",highlighted:'mlagents-push-to-hf  --run-id= <span class="hljs-comment"># Add your run id  --local-dir= # Your local dir  --repo-id= # Your repo id --commit-message=&quot;First Push&quot;</span>'}}),io=new b({}),po=new b({}),{c(){N=l("meta"),ys=u(),C=l("h1"),te=l("a"),Ml=l("span"),p(He.$$.fragment),mn=u(),Al=l("span"),vn=r("Hands-on"),gs=u(),oe=l("p"),wn=r("Now that you learned the basics of multi-agents, you\u2019re ready to train your first agents in a multi-agent system: "),Il=l("strong"),yn=r("a 2vs2 soccer team that needs to beat the opponent team"),gn=r("."),bs=u(),Re=l("p"),bn=r("And you\u2019re going to participate in AI vs. AI challenges where your trained agent will compete against other classmates\u2019 "),$l=l("strong"),En=r("agents every day and be ranked on a new leaderboard."),Es=u(),ze=l("p"),_n=r("To validate this hands-on for the certification process, you just need to push a trained model. There "),Sl=l("strong"),Tn=r("are no minimal results to attain to validate it."),_s=u(),Ze=l("p"),Mn=r("For more information about the certification process, check this section \u{1F449} "),Ye=l("a"),An=r("https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),Ts=u(),le=l("p"),In=r("This hands-on will be different since to get correct results "),kl=l("strong"),$n=r("you need to train your agents from 4 hours to 8 hours"),Sn=r(". And given the risk of timeout in Colab, we advise you to train on your computer. You don\u2019t need a supercomputer: a simple laptop is good enough for this exercise."),Ms=u(),wo=l("p"),kn=r("Let\u2019s get started! \u{1F525}"),As=u(),x=l("h2"),ae=l("a"),jl=l("span"),p(De.$$.fragment),jn=u(),Jl=l("span"),Jn=r("What is AI vs. AI?"),Is=u(),yo=l("p"),Un=r("AI vs. AI is an open-source tool we developed at Hugging Face to compete agents on the Hub against one another in a multi-agent setting. These models are then ranked in a leaderboard."),$s=u(),Fe=l("p"),Pn=r("The idea of this tool is to have a robust evaluation tool: "),Ul=l("strong"),Nn=r("by evaluating your agent with a lot of others, you\u2019ll get a good idea of the quality of your policy."),Ss=u(),go=l("p"),Cn=r("More precisely, AI vs. AI is three tools:"),ks=u(),I=l("ul"),Ve=l("li"),xn=r("A "),Pl=l("em"),Ln=r("matchmaking process"),Wn=r(" defining the matches (which model against which) and running the model fights using a background task in the Space."),Gn=u(),se=l("li"),Bn=r("A "),Nl=l("em"),On=r("leaderboard"),Hn=r(" getting the match history results and displaying the models\u2019 ELO ratings: "),Xe=l("a"),Rn=r("https://huggingface.co/spaces/huggingface-projects/AIvsAI-SoccerTwos"),zn=u(),re=l("li"),Zn=r("A "),Cl=l("em"),Yn=r("Space demo"),Dn=r(" to visualize your agents playing against others: "),qe=l("a"),Fn=r("https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),js=u(),Qe=l("p"),Vn=r("In addition to these three tools, your classmate cyllum created a \u{1F917} SoccerTwos Challenge Analytics where you can check the detailed match results of a model: "),Ke=l("a"),Xn=r("https://huggingface.co/spaces/cyllum/soccertwos-analytics"),Js=u(),ie=l("p"),qn=r("We\u2019re "),et=l("a"),Qn=r("wrote a blog post to explain this AI vs. AI tool in detail"),Kn=r(", but to give you the big picture it works this way:"),Us=u(),E=l("ul"),bo=l("li"),eh=r("Every four hours, our algorithm "),xl=l("strong"),th=r("fetches all the available models for a given environment (in our case ML-Agents-SoccerTwos)."),oh=u(),Eo=l("li"),lh=r("It creates a "),Ll=l("strong"),ah=r("queue of matches with the matchmaking algorithm."),sh=u(),tt=l("li"),rh=r("We simulate the match in a Unity headless process and "),Wl=l("strong"),ih=r("gather the match result"),nh=r(" (1 if the first model won, 0.5 if it\u2019s a draw, 0 if the second model won) in a Dataset."),hh=u(),_o=l("li"),uh=r("Then, when all matches from the matches queue are done, "),Gl=l("strong"),fh=r("we update the ELO score for each model and update the leaderboard."),Ps=u(),L=l("h3"),ne=l("a"),Bl=l("span"),p(ot.$$.fragment),ch=u(),Ol=l("span"),ph=r("Competition Rules"),Ns=u(),_=l("p"),dh=r("This first AI vs. AI competition "),Hl=l("strong"),mh=r("is an experiment"),vh=r(": the goal is to improve the tool in the future with your feedback. So some "),Rl=l("strong"),wh=r("breakups can happen during the challenge"),yh=r(`. But don\u2019t worry
`),zl=l("strong"),gh=r("all the results are saved in a dataset so we can always restart the calculation correctly without losing information"),bh=r("."),Cs=u(),To=l("p"),Eh=r("In order for your model to get correctly evaluated against others you need to follow these rules:"),xs=u(),$=l("ol"),Mo=l("li"),Zl=l("strong"),_h=r("You can\u2019t change the observation space or action space of the agent."),Th=r(" By doing that your model will not work during evaluation."),Mh=u(),lt=l("li"),Ah=r("You "),Yl=l("strong"),Ih=r("can\u2019t use a custom trainer for now,"),$h=r(" you need to use the Unity MLAgents ones."),Sh=u(),at=l("li"),kh=r("We provide executables to train your agents. You can also use the Unity Editor if you prefer "),Dl=l("strong"),jh=r(", but to avoid bugs, we advise that you use our executables"),Jh=r("."),Ls=u(),he=l("p"),Uh=r("What will make the difference during this challenge are "),Fl=l("strong"),Ph=r("the hyperparameters you choose"),Nh=r("."),Ws=u(),S=l("p"),Ch=r("We\u2019re constantly trying to improve our tutorials, so\xA0"),Vl=l("strong"),xh=r("if you find some issues in this notebook"),Lh=r(", please\xA0"),st=l("a"),Wh=r("open an issue on the GitHub Repo"),Gh=r("."),Gs=u(),W=l("h3"),ue=l("a"),Xl=l("span"),p(rt.$$.fragment),Bh=u(),ql=l("span"),Oh=r("Chat with your classmates, share advice and ask questions on Discord"),Bs=u(),fe=l("ul"),it=l("li"),Hh=r("We created a new channel called "),Ql=l("code"),Rh=r("ai-vs-ai-challenge"),zh=r(" to exchange advice and ask questions."),Zh=u(),Ao=l("li"),Yh=r("If you didn\u2019t join the discord server yet, you can "),nt=l("a"),Dh=r("join here"),Os=u(),G=l("h2"),ce=l("a"),Kl=l("span"),p(ht.$$.fragment),Fh=u(),ea=l("span"),Vh=r("Step 0: Install MLAgents and download the correct executable"),Hs=u(),pe=l("p"),Xh=r("We advise you to use "),ut=l("a"),qh=r("conda"),Qh=r(" as a package manager and create a new environment."),Rs=u(),de=l("p"),Kh=r("With conda, we create a new environment called rl with "),ta=l("strong"),eu=r("Python 3.9"),tu=r(":"),zs=u(),p(ft.$$.fragment),Zs=u(),Io=l("p"),ou=r("To be able to train our agents correctly and push to the Hub, we need to install ML-Agents"),Ys=u(),p(ct.$$.fragment),Ds=u(),$o=l("p"),lu=r("When the cloning is done (it takes 2.63 GB), we go inside the repository and install the package"),Fs=u(),p(pt.$$.fragment),Vs=u(),So=l("p"),au=r("We also need to install pytorch with:"),Xs=u(),p(dt.$$.fragment),qs=u(),ko=l("p"),su=r("As well as the following dependency:"),Qs=u(),p(mt.$$.fragment),Ks=u(),vt=l("p"),ru=r("Finally, you need to install git-lfs: "),wt=l("a"),iu=r("https://git-lfs.com/"),er=u(),B=l("p"),nu=r("Now that it\u2019s installed, we need to add the environment training executable. Based on your operating system you need to download one of them, unzip it and place it in a new folder inside "),oa=l("code"),hu=r("ml-agents"),uu=r(" that you call "),la=l("code"),fu=r("training-envs-executables"),tr=u(),yt=l("p"),cu=r("At the end your executable should be in "),aa=l("code"),pu=r("ml-agents/training-envs-executables/SoccerTwos"),or=u(),gt=l("p"),du=r("Windows: Download "),bt=l("a"),mu=r("this executable"),lr=u(),Et=l("p"),vu=r("Linux (Ubuntu): Download "),_t=l("a"),wu=r("this executable"),ar=u(),k=l("p"),yu=r("Mac: Download "),Tt=l("a"),gu=r("this executable"),bu=r(`
\u26A0 For Mac you need also to call this `),sa=l("code"),Eu=r("xattr -cr training-envs-executables/SoccerTwos/SoccerTwos.app"),_u=r(" to be able to run SoccerTwos"),sr=u(),O=l("h2"),me=l("a"),ra=l("span"),p(Mt.$$.fragment),Tu=u(),ia=l("span"),Mu=r("Step 1: Understand the environment"),rr=u(),H=l("p"),Au=r("The environment is called "),na=l("code"),Iu=r("SoccerTwos"),$u=r(". The Unity MLAgents Team made it. You can find its documentation "),At=l("a"),Su=r("here"),ir=u(),It=l("p"),ku=r("The goal in this environment "),ha=l("strong"),ju=r("is to get the ball into the opponent\u2019s goal while preventing the ball from entering your own goal."),nr=u(),ve=l("figure"),jo=l("img"),Ju=u(),Jo=l("figcaption"),Uu=r("This environment was made by the "),Uo=l("a"),Pu=r("Unity MLAgents Team"),hr=u(),R=l("h3"),we=l("a"),ua=l("span"),p($t.$$.fragment),Nu=u(),fa=l("span"),Cu=r("The reward function"),ur=u(),Po=l("p"),xu=r("The reward function is:"),fr=u(),St=l("img"),cr=u(),z=l("h3"),ye=l("a"),ca=l("span"),p(kt.$$.fragment),Lu=u(),pa=l("span"),Wu=r("The observation space"),pr=u(),No=l("p"),Gu=r("The observation space is composed of vectors of size 336:"),dr=u(),j=l("ul"),da=l("li"),Bu=r("11 ray-casts forward distributed over 120 degrees (264 state dimensions)"),Ou=u(),ma=l("li"),Hu=r("3 ray-casts backward distributed over 90 degrees (72 state dimensions)"),Ru=u(),Co=l("li"),zu=r("Both of these ray-casts can detect 6 objects:"),g=l("ul"),va=l("li"),Zu=r("Ball"),Yu=u(),wa=l("li"),Du=r("Blue Goal"),Fu=u(),ya=l("li"),Vu=r("Purple Goal"),Xu=u(),ga=l("li"),qu=r("Wall"),Qu=u(),ba=l("li"),Ku=r("Blue Agent"),ef=u(),Ea=l("li"),tf=r("Purple Agent"),mr=u(),Z=l("h3"),ge=l("a"),_a=l("span"),p(jt.$$.fragment),of=u(),Ta=l("span"),lf=r("The action space"),vr=u(),xo=l("p"),af=r("The action space is three discrete branches:"),wr=u(),Jt=l("img"),yr=u(),Y=l("h2"),be=l("a"),Ma=l("span"),p(Ut.$$.fragment),sf=u(),Aa=l("span"),rf=r("Step 2: Understand MA-POCA"),gr=u(),Ee=l("p"),nf=r("We know how to train agents to play against others: "),Ia=l("strong"),hf=r("we can use self-play."),uf=r(" This is a perfect technique for a 1vs1."),br=u(),Pt=l("p"),ff=r("But in our case we\u2019re 2vs2, and each team has 2 agents. How then can we "),$a=l("strong"),cf=r("train cooperative behavior for groups of agents?"),Er=u(),J=l("p"),pf=r("As explained in the "),Nt=l("a"),df=r("Unity Blog"),mf=r(", agents typically receive a reward as a group (+1 - penalty) when the team scores a goal. This implies that "),Sa=l("strong"),vf=r("every agent on the team is rewarded even if each agent didn\u2019t contribute the same to the win"),wf=r(", which makes it difficult to learn what to do independently."),_r=u(),_e=l("p"),yf=r("The Unity MLAgents team developed the solution in a new multi-agent trainer called "),ka=l("em"),gf=r("MA-POCA (Multi-Agent POsthumous Credit Assignment)"),bf=r("."),Tr=u(),Te=l("p"),Ef=r("The idea is simple but powerful: a centralized critic "),ja=l("strong"),_f=r("processes the states of all agents in the team to estimate how well each agent is doing"),Tf=r(". Think of this critic as a coach."),Mr=u(),U=l("p"),Mf=r("This allows each agent to "),Ja=l("strong"),Af=r("make decisions based only on what it perceives locally"),If=r(", and "),Ua=l("strong"),$f=r("simultaneously evaluate how good its behavior is in the context of the whole group"),Sf=r("."),Ar=u(),Me=l("figure"),Lo=l("img"),kf=u(),Wo=l("figcaption"),jf=r("This illustrates MA-POCA\u2019s centralized learning and decentralized execution. Source: "),Go=l("a"),Jf=r("MLAgents Plays Dodgeball"),Ir=u(),Bo=l("p"),Uf=r("The solution then is to use Self-Play with an MA-POCA trainer (called poca). The poca trainer will help us to train cooperative behavior and self-play to win against an opponent team."),$r=u(),Ae=l("p"),Pf=r("If you want to dive deeper into this MA-POCA algorithm, you need to read the paper they published "),Ct=l("a"),Nf=r("here"),Cf=r(" and the sources we put on the additional readings section."),Sr=u(),D=l("h2"),Ie=l("a"),Pa=l("span"),p(xt.$$.fragment),xf=u(),Na=l("span"),Lf=r("Step 3: Define the config file"),kr=u(),F=l("p"),Wf=r("We already learned in "),Lt=l("a"),Gf=r("Unit 5"),Bf=r(" that in ML-Agents, you define "),Wt=l("strong"),Of=r("the training hyperparameters in "),Ca=l("code"),Hf=r("config.yaml"),Rf=r(" files."),jr=u(),Gt=l("p"),zf=r("There are multiple hyperparameters. To understand them better, you should read the explanations for each of them in\xA0"),xa=l("strong"),Bt=l("a"),Zf=r("the documentation"),Jr=u(),$e=l("p"),Yf=r("The config file we\u2019re going to use here is in  "),La=l("code"),Df=r("./config/poca/SoccerTwos.yaml"),Ff=r(". It looks like this:"),Ur=u(),p(Ot.$$.fragment),Pr=u(),Oo=l("p"),Vf=r("Compared to Pyramids or SnowballTarget, we have new hyperparameters with a self-play part. How you modify them can be critical in getting good results."),Nr=u(),Ht=l("p"),Xf=r("The advice I can give you here is to check the explanation and recommended value for each parameters (especially self-play ones) against\xA0"),Ho=l("strong"),Rt=l("a"),qf=r("the documentation"),Qf=r("."),Cr=u(),Ro=l("p"),Kf=r("Now that you\u2019ve modified our config file, you\u2019re ready to train your agents."),xr=u(),V=l("h2"),Se=l("a"),Wa=l("span"),p(zt.$$.fragment),ec=u(),Ga=l("span"),tc=r("Step 4: Start the training"),Lr=u(),Zt=l("p"),oc=r("To train the agents, we need to\xA0"),Ba=l("strong"),lc=r("launch mlagents-learn and select the executable containing the environment."),Wr=u(),zo=l("p"),ac=r("We define four parameters:"),Gr=u(),T=l("ol"),Zo=l("li"),Oa=l("code"),sc=r("mlagents-learn <config>"),rc=r(": the path where the hyperparameter config file is."),ic=u(),Yo=l("li"),Ha=l("code"),nc=r("-env"),hc=r(": where the environment executable is."),uc=u(),Do=l("li"),Ra=l("code"),fc=r("-run_id"),cc=r(": the name you want to give to your training run id."),pc=u(),Fo=l("li"),za=l("code"),dc=r("-no-graphics"),mc=r(": to not launch the visualization during the training."),Br=u(),Vo=l("p"),vc=r("Depending on your hardware, 5M timesteps (the recommended value, but you can also try 10M) will take 5 to 8 hours of training. You can continue using your computer in the meantime, but I advise deactivating the computer standby mode to prevent the training from being stopped."),Or=u(),Xo=l("p"),wc=r("Depending on the executable you use (windows, ubuntu, mac) the training command will look like this (your executable path can be different so don\u2019t hesitate to check before running)."),Hr=u(),p(Yt.$$.fragment),Rr=u(),qo=l("p"),yc=r("The executable contains 8 copies of SoccerTwos."),zr=u(),Qo=l("p"),gc=r("\u26A0\uFE0F It\u2019s normal if you don\u2019t see a big increase of ELO score (and even a decrease below 1200) before 2M timesteps, since your agents will spend most of their time moving randomly on the field before being able to goal."),Zr=u(),Ko=l("p"),bc=r("\u26A0\uFE0F You can stop the training with Ctrl + C but beware of typing this command only once to stop the training since MLAgents needs to generate a final .onnx file before closing the run."),Yr=u(),X=l("h2"),ke=l("a"),Za=l("span"),p(Dt.$$.fragment),Ec=u(),Ya=l("span"),_c=r("Step 5: **Push the agent to the Hugging Face Hub**"),Dr=u(),Ft=l("p"),Tc=r("Now that we trained our agents, we\u2019re\xA0"),Da=l("strong"),Mc=r("ready to push them to the Hub to be able to participate in the AI vs. AI challenge and visualize them playing on your browser\u{1F525}."),Fr=u(),el=l("p"),Ac=r("To be able to share your model with the community, there are three more steps to follow:"),Vr=u(),Vt=l("p"),Ic=r("1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1\xA0"),Xt=l("a"),$c=r("https://huggingface.co/join"),Xr=u(),tl=l("p"),Sc=r("2\uFE0F\u20E3 Sign in and store your authentication token from the Hugging Face website."),qr=u(),q=l("p"),kc=r("Create a new token ("),qt=l("a"),jc=r("https://huggingface.co/settings/tokens"),Jc=r(")\xA0"),Fa=l("strong"),Uc=r("with write role"),Qr=u(),Qt=l("img"),Kr=u(),ol=l("p"),Pc=r("Copy the token, run this, and paste the token"),ei=u(),p(Kt.$$.fragment),ti=u(),je=l("p"),Nc=r("Then, we need to run "),Va=l("code"),Cc=r("mlagents-push-to-hf"),xc=r("."),oi=u(),ll=l("p"),Lc=r("And we define four parameters:"),li=u(),M=l("ol"),al=l("li"),Xa=l("code"),Wc=r("-run-id"),Gc=r(": the name of the training run id."),Bc=u(),eo=l("li"),qa=l("code"),Oc=r("-local-dir"),Hc=r(": where the agent was saved, it\u2019s results/"),sl=l("run_id"),Rc=r(", so in my case results/First Training."),zc=u(),to=l("li"),Qa=l("code"),Zc=r("-repo-id"),Yc=r(": the name of the Hugging Face repo you want to create or update. It\u2019s always "),Je=l("your"),Dc=r("/"),Ue=l("the"),Fc=r("If the repo does not exist "),Ka=l("strong"),Vc=r("it will be created automatically"),Xc=u(),rl=l("li"),es=l("code"),qc=r("--commit-message"),Qc=r(": since HF repos are git repositories you need to give a commit message."),ai=u(),il=l("p"),Kc=r("In my case"),si=u(),p(oo.$$.fragment),ri=u(),p(lo.$$.fragment),ii=u(),nl=l("p"),ep=r("If everything worked you should see this at the end of the process (but with a different url \u{1F606}) :"),ni=u(),ao=l("p"),tp=r("Your model is pushed to the Hub. You can view your model here: "),so=l("a"),op=r("https://huggingface.co/ThomasSimonini/poca-SoccerTwos"),hi=u(),ro=l("p"),lp=r("It\u2019s the link to your model. It contains a model card that explains how to use it, your Tensorboard, and your config file. "),ts=l("strong"),ap=r("What\u2019s awesome is that it\u2019s a git repository, which means you can have different commits, update your repository with a new push, etc."),ui=u(),Q=l("h2"),Pe=l("a"),os=l("span"),p(io.$$.fragment),sp=u(),ls=l("span"),rp=r("Step 6: Verify that your model is ready for AI vs AI Challenge"),fi=u(),Ne=l("p"),ip=r("Now that your model is pushed to the Hub, "),as=l("strong"),np=r("it\u2019s going to be added automatically to the AI vs AI Challenge model pool."),hp=r(" It can take a little bit of time before your model is added to the leaderboard given we do a run of matches every 4h."),ci=u(),hl=l("p"),up=r("But to ensure that everything works perfectly you need to check:"),pi=u(),ul=l("ol"),ss=l("li"),fp=r("That you have this tag in your model: ML-Agents-SoccerTwos. This is the tag we use to select models to be added to the challenge pool. To do that go to your model and check the tags"),di=u(),no=l("img"),mi=u(),fl=l("p"),cp=r("If it\u2019s not the case you just need to modify the readme and add it"),vi=u(),ho=l("img"),wi=u(),uo=l("ol"),fo=l("li"),pp=r("That you have a "),rs=l("code"),dp=r("SoccerTwos.onnx"),mp=r(" file"),yi=u(),co=l("img"),gi=u(),cl=l("p"),vp=r("We strongly suggest that you create a new model when you push to the Hub if you want to train it again or train a new version."),bi=u(),K=l("h2"),Ce=l("a"),is=l("span"),p(po.$$.fragment),wp=u(),ns=l("span"),yp=r("Step 7: Visualize some match in our demo"),Ei=u(),ee=l("p"),gp=r("Now that your model is part of AI vs AI Challenge, "),hs=l("strong"),bp=r("you can visualize how good it is compared to others"),Ep=r(": "),mo=l("a"),_p=r("https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),_i=u(),pl=l("p"),Tp=r("In order to do that, you just need to go to this demo:"),Ti=u(),dl=l("ul"),ml=l("li"),Mp=r("Select your model as team blue (or team purple if you prefer) and another model to compete against. The best opponents to compare your model to are either whoever is on top of the leaderboard or the "),vo=l("a"),Ap=r("baseline model"),Mi=u(),xe=l("p"),Ip=r("The matches you see live are not used in the calculation of your result "),us=l("strong"),$p=r("but they are a good way to visualize how good your agent is"),Sp=r("."),Ai=u(),vl=l("p"),kp=r("And don\u2019t hesitate to share the best score your agent gets on discord in the #rl-i-made-this channel \u{1F525}"),this.h()},l(e){const n=Uw('[data-svelte="svelte-1phssyn"]',document.head);N=a(n,"META",{name:!0,content:!0}),n.forEach(t),ys=f(e),C=a(e,"H1",{class:!0});var $i=s(C);te=a($i,"A",{id:!0,class:!0,href:!0});var Ed=s(te);Ml=a(Ed,"SPAN",{});var _d=s(Ml);d(He.$$.fragment,_d),_d.forEach(t),Ed.forEach(t),mn=f($i),Al=a($i,"SPAN",{});var Td=s(Al);vn=i(Td,"Hands-on"),Td.forEach(t),$i.forEach(t),gs=f(e),oe=a(e,"P",{});var Si=s(oe);wn=i(Si,"Now that you learned the basics of multi-agents, you\u2019re ready to train your first agents in a multi-agent system: "),Il=a(Si,"STRONG",{});var Md=s(Il);yn=i(Md,"a 2vs2 soccer team that needs to beat the opponent team"),Md.forEach(t),gn=i(Si,"."),Si.forEach(t),bs=f(e),Re=a(e,"P",{});var jp=s(Re);bn=i(jp,"And you\u2019re going to participate in AI vs. AI challenges where your trained agent will compete against other classmates\u2019 "),$l=a(jp,"STRONG",{});var Ad=s($l);En=i(Ad,"agents every day and be ranked on a new leaderboard."),Ad.forEach(t),jp.forEach(t),Es=f(e),ze=a(e,"P",{});var Jp=s(ze);_n=i(Jp,"To validate this hands-on for the certification process, you just need to push a trained model. There "),Sl=a(Jp,"STRONG",{});var Id=s(Sl);Tn=i(Id,"are no minimal results to attain to validate it."),Id.forEach(t),Jp.forEach(t),_s=f(e),Ze=a(e,"P",{});var Up=s(Ze);Mn=i(Up,"For more information about the certification process, check this section \u{1F449} "),Ye=a(Up,"A",{href:!0,rel:!0});var $d=s(Ye);An=i($d,"https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),$d.forEach(t),Up.forEach(t),Ts=f(e),le=a(e,"P",{});var ki=s(le);In=i(ki,"This hands-on will be different since to get correct results "),kl=a(ki,"STRONG",{});var Sd=s(kl);$n=i(Sd,"you need to train your agents from 4 hours to 8 hours"),Sd.forEach(t),Sn=i(ki,". And given the risk of timeout in Colab, we advise you to train on your computer. You don\u2019t need a supercomputer: a simple laptop is good enough for this exercise."),ki.forEach(t),Ms=f(e),wo=a(e,"P",{});var kd=s(wo);kn=i(kd,"Let\u2019s get started! \u{1F525}"),kd.forEach(t),As=f(e),x=a(e,"H2",{class:!0});var ji=s(x);ae=a(ji,"A",{id:!0,class:!0,href:!0});var jd=s(ae);jl=a(jd,"SPAN",{});var Jd=s(jl);d(De.$$.fragment,Jd),Jd.forEach(t),jd.forEach(t),jn=f(ji),Jl=a(ji,"SPAN",{});var Ud=s(Jl);Jn=i(Ud,"What is AI vs. AI?"),Ud.forEach(t),ji.forEach(t),Is=f(e),yo=a(e,"P",{});var Pd=s(yo);Un=i(Pd,"AI vs. AI is an open-source tool we developed at Hugging Face to compete agents on the Hub against one another in a multi-agent setting. These models are then ranked in a leaderboard."),Pd.forEach(t),$s=f(e),Fe=a(e,"P",{});var Pp=s(Fe);Pn=i(Pp,"The idea of this tool is to have a robust evaluation tool: "),Ul=a(Pp,"STRONG",{});var Nd=s(Ul);Nn=i(Nd,"by evaluating your agent with a lot of others, you\u2019ll get a good idea of the quality of your policy."),Nd.forEach(t),Pp.forEach(t),Ss=f(e),go=a(e,"P",{});var Cd=s(go);Cn=i(Cd,"More precisely, AI vs. AI is three tools:"),Cd.forEach(t),ks=f(e),I=a(e,"UL",{});var wl=s(I);Ve=a(wl,"LI",{});var Ji=s(Ve);xn=i(Ji,"A "),Pl=a(Ji,"EM",{});var xd=s(Pl);Ln=i(xd,"matchmaking process"),xd.forEach(t),Wn=i(Ji," defining the matches (which model against which) and running the model fights using a background task in the Space."),Ji.forEach(t),Gn=f(wl),se=a(wl,"LI",{});var fs=s(se);Bn=i(fs,"A "),Nl=a(fs,"EM",{});var Ld=s(Nl);On=i(Ld,"leaderboard"),Ld.forEach(t),Hn=i(fs," getting the match history results and displaying the models\u2019 ELO ratings: "),Xe=a(fs,"A",{href:!0,rel:!0});var Wd=s(Xe);Rn=i(Wd,"https://huggingface.co/spaces/huggingface-projects/AIvsAI-SoccerTwos"),Wd.forEach(t),fs.forEach(t),zn=f(wl),re=a(wl,"LI",{});var cs=s(re);Zn=i(cs,"A "),Cl=a(cs,"EM",{});var Gd=s(Cl);Yn=i(Gd,"Space demo"),Gd.forEach(t),Dn=i(cs," to visualize your agents playing against others: "),qe=a(cs,"A",{href:!0,rel:!0});var Bd=s(qe);Fn=i(Bd,"https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),Bd.forEach(t),cs.forEach(t),wl.forEach(t),js=f(e),Qe=a(e,"P",{});var Np=s(Qe);Vn=i(Np,"In addition to these three tools, your classmate cyllum created a \u{1F917} SoccerTwos Challenge Analytics where you can check the detailed match results of a model: "),Ke=a(Np,"A",{href:!0,rel:!0});var Od=s(Ke);Xn=i(Od,"https://huggingface.co/spaces/cyllum/soccertwos-analytics"),Od.forEach(t),Np.forEach(t),Js=f(e),ie=a(e,"P",{});var Ui=s(ie);qn=i(Ui,"We\u2019re "),et=a(Ui,"A",{href:!0,rel:!0});var Hd=s(et);Qn=i(Hd,"wrote a blog post to explain this AI vs. AI tool in detail"),Hd.forEach(t),Kn=i(Ui,", but to give you the big picture it works this way:"),Ui.forEach(t),Us=f(e),E=a(e,"UL",{});var Le=s(E);bo=a(Le,"LI",{});var Cp=s(bo);eh=i(Cp,"Every four hours, our algorithm "),xl=a(Cp,"STRONG",{});var Rd=s(xl);th=i(Rd,"fetches all the available models for a given environment (in our case ML-Agents-SoccerTwos)."),Rd.forEach(t),Cp.forEach(t),oh=f(Le),Eo=a(Le,"LI",{});var xp=s(Eo);lh=i(xp,"It creates a "),Ll=a(xp,"STRONG",{});var zd=s(Ll);ah=i(zd,"queue of matches with the matchmaking algorithm."),zd.forEach(t),xp.forEach(t),sh=f(Le),tt=a(Le,"LI",{});var Pi=s(tt);rh=i(Pi,"We simulate the match in a Unity headless process and "),Wl=a(Pi,"STRONG",{});var Zd=s(Wl);ih=i(Zd,"gather the match result"),Zd.forEach(t),nh=i(Pi," (1 if the first model won, 0.5 if it\u2019s a draw, 0 if the second model won) in a Dataset."),Pi.forEach(t),hh=f(Le),_o=a(Le,"LI",{});var Lp=s(_o);uh=i(Lp,"Then, when all matches from the matches queue are done, "),Gl=a(Lp,"STRONG",{});var Yd=s(Gl);fh=i(Yd,"we update the ELO score for each model and update the leaderboard."),Yd.forEach(t),Lp.forEach(t),Le.forEach(t),Ps=f(e),L=a(e,"H3",{class:!0});var Ni=s(L);ne=a(Ni,"A",{id:!0,class:!0,href:!0});var Dd=s(ne);Bl=a(Dd,"SPAN",{});var Fd=s(Bl);d(ot.$$.fragment,Fd),Fd.forEach(t),Dd.forEach(t),ch=f(Ni),Ol=a(Ni,"SPAN",{});var Vd=s(Ol);ph=i(Vd,"Competition Rules"),Vd.forEach(t),Ni.forEach(t),Ns=f(e),_=a(e,"P",{});var We=s(_);dh=i(We,"This first AI vs. AI competition "),Hl=a(We,"STRONG",{});var Xd=s(Hl);mh=i(Xd,"is an experiment"),Xd.forEach(t),vh=i(We,": the goal is to improve the tool in the future with your feedback. So some "),Rl=a(We,"STRONG",{});var qd=s(Rl);wh=i(qd,"breakups can happen during the challenge"),qd.forEach(t),yh=i(We,`. But don\u2019t worry
`),zl=a(We,"STRONG",{});var Qd=s(zl);gh=i(Qd,"all the results are saved in a dataset so we can always restart the calculation correctly without losing information"),Qd.forEach(t),bh=i(We,"."),We.forEach(t),Cs=f(e),To=a(e,"P",{});var Kd=s(To);Eh=i(Kd,"In order for your model to get correctly evaluated against others you need to follow these rules:"),Kd.forEach(t),xs=f(e),$=a(e,"OL",{});var yl=s($);Mo=a(yl,"LI",{});var Wp=s(Mo);Zl=a(Wp,"STRONG",{});var em=s(Zl);_h=i(em,"You can\u2019t change the observation space or action space of the agent."),em.forEach(t),Th=i(Wp," By doing that your model will not work during evaluation."),Wp.forEach(t),Mh=f(yl),lt=a(yl,"LI",{});var Ci=s(lt);Ah=i(Ci,"You "),Yl=a(Ci,"STRONG",{});var tm=s(Yl);Ih=i(tm,"can\u2019t use a custom trainer for now,"),tm.forEach(t),$h=i(Ci," you need to use the Unity MLAgents ones."),Ci.forEach(t),Sh=f(yl),at=a(yl,"LI",{});var xi=s(at);kh=i(xi,"We provide executables to train your agents. You can also use the Unity Editor if you prefer "),Dl=a(xi,"STRONG",{});var om=s(Dl);jh=i(om,", but to avoid bugs, we advise that you use our executables"),om.forEach(t),Jh=i(xi,"."),xi.forEach(t),yl.forEach(t),Ls=f(e),he=a(e,"P",{});var Li=s(he);Uh=i(Li,"What will make the difference during this challenge are "),Fl=a(Li,"STRONG",{});var lm=s(Fl);Ph=i(lm,"the hyperparameters you choose"),lm.forEach(t),Nh=i(Li,"."),Li.forEach(t),Ws=f(e),S=a(e,"P",{});var gl=s(S);Ch=i(gl,"We\u2019re constantly trying to improve our tutorials, so\xA0"),Vl=a(gl,"STRONG",{});var am=s(Vl);xh=i(am,"if you find some issues in this notebook"),am.forEach(t),Lh=i(gl,", please\xA0"),st=a(gl,"A",{href:!0,rel:!0});var sm=s(st);Wh=i(sm,"open an issue on the GitHub Repo"),sm.forEach(t),Gh=i(gl,"."),gl.forEach(t),Gs=f(e),W=a(e,"H3",{class:!0});var Wi=s(W);ue=a(Wi,"A",{id:!0,class:!0,href:!0});var rm=s(ue);Xl=a(rm,"SPAN",{});var im=s(Xl);d(rt.$$.fragment,im),im.forEach(t),rm.forEach(t),Bh=f(Wi),ql=a(Wi,"SPAN",{});var nm=s(ql);Oh=i(nm,"Chat with your classmates, share advice and ask questions on Discord"),nm.forEach(t),Wi.forEach(t),Bs=f(e),fe=a(e,"UL",{});var Gi=s(fe);it=a(Gi,"LI",{});var Bi=s(it);Hh=i(Bi,"We created a new channel called "),Ql=a(Bi,"CODE",{});var hm=s(Ql);Rh=i(hm,"ai-vs-ai-challenge"),hm.forEach(t),zh=i(Bi," to exchange advice and ask questions."),Bi.forEach(t),Zh=f(Gi),Ao=a(Gi,"LI",{});var Gp=s(Ao);Yh=i(Gp,"If you didn\u2019t join the discord server yet, you can "),nt=a(Gp,"A",{href:!0,rel:!0});var um=s(nt);Dh=i(um,"join here"),um.forEach(t),Gp.forEach(t),Gi.forEach(t),Os=f(e),G=a(e,"H2",{class:!0});var Oi=s(G);ce=a(Oi,"A",{id:!0,class:!0,href:!0});var fm=s(ce);Kl=a(fm,"SPAN",{});var cm=s(Kl);d(ht.$$.fragment,cm),cm.forEach(t),fm.forEach(t),Fh=f(Oi),ea=a(Oi,"SPAN",{});var pm=s(ea);Vh=i(pm,"Step 0: Install MLAgents and download the correct executable"),pm.forEach(t),Oi.forEach(t),Hs=f(e),pe=a(e,"P",{});var Hi=s(pe);Xh=i(Hi,"We advise you to use "),ut=a(Hi,"A",{href:!0,rel:!0});var dm=s(ut);qh=i(dm,"conda"),dm.forEach(t),Qh=i(Hi," as a package manager and create a new environment."),Hi.forEach(t),Rs=f(e),de=a(e,"P",{});var Ri=s(de);Kh=i(Ri,"With conda, we create a new environment called rl with "),ta=a(Ri,"STRONG",{});var mm=s(ta);eu=i(mm,"Python 3.9"),mm.forEach(t),tu=i(Ri,":"),Ri.forEach(t),zs=f(e),d(ft.$$.fragment,e),Zs=f(e),Io=a(e,"P",{});var vm=s(Io);ou=i(vm,"To be able to train our agents correctly and push to the Hub, we need to install ML-Agents"),vm.forEach(t),Ys=f(e),d(ct.$$.fragment,e),Ds=f(e),$o=a(e,"P",{});var wm=s($o);lu=i(wm,"When the cloning is done (it takes 2.63 GB), we go inside the repository and install the package"),wm.forEach(t),Fs=f(e),d(pt.$$.fragment,e),Vs=f(e),So=a(e,"P",{});var ym=s(So);au=i(ym,"We also need to install pytorch with:"),ym.forEach(t),Xs=f(e),d(dt.$$.fragment,e),qs=f(e),ko=a(e,"P",{});var gm=s(ko);su=i(gm,"As well as the following dependency:"),gm.forEach(t),Qs=f(e),d(mt.$$.fragment,e),Ks=f(e),vt=a(e,"P",{});var Bp=s(vt);ru=i(Bp,"Finally, you need to install git-lfs: "),wt=a(Bp,"A",{href:!0,rel:!0});var bm=s(wt);iu=i(bm,"https://git-lfs.com/"),bm.forEach(t),Bp.forEach(t),er=f(e),B=a(e,"P",{});var ps=s(B);nu=i(ps,"Now that it\u2019s installed, we need to add the environment training executable. Based on your operating system you need to download one of them, unzip it and place it in a new folder inside "),oa=a(ps,"CODE",{});var Em=s(oa);hu=i(Em,"ml-agents"),Em.forEach(t),uu=i(ps," that you call "),la=a(ps,"CODE",{});var _m=s(la);fu=i(_m,"training-envs-executables"),_m.forEach(t),ps.forEach(t),tr=f(e),yt=a(e,"P",{});var Op=s(yt);cu=i(Op,"At the end your executable should be in "),aa=a(Op,"CODE",{});var Tm=s(aa);pu=i(Tm,"ml-agents/training-envs-executables/SoccerTwos"),Tm.forEach(t),Op.forEach(t),or=f(e),gt=a(e,"P",{});var Hp=s(gt);du=i(Hp,"Windows: Download "),bt=a(Hp,"A",{href:!0,rel:!0});var Mm=s(bt);mu=i(Mm,"this executable"),Mm.forEach(t),Hp.forEach(t),lr=f(e),Et=a(e,"P",{});var Rp=s(Et);vu=i(Rp,"Linux (Ubuntu): Download "),_t=a(Rp,"A",{href:!0,rel:!0});var Am=s(_t);wu=i(Am,"this executable"),Am.forEach(t),Rp.forEach(t),ar=f(e),k=a(e,"P",{});var bl=s(k);yu=i(bl,"Mac: Download "),Tt=a(bl,"A",{href:!0,rel:!0});var Im=s(Tt);gu=i(Im,"this executable"),Im.forEach(t),bu=i(bl,`
\u26A0 For Mac you need also to call this `),sa=a(bl,"CODE",{});var $m=s(sa);Eu=i($m,"xattr -cr training-envs-executables/SoccerTwos/SoccerTwos.app"),$m.forEach(t),_u=i(bl," to be able to run SoccerTwos"),bl.forEach(t),sr=f(e),O=a(e,"H2",{class:!0});var zi=s(O);me=a(zi,"A",{id:!0,class:!0,href:!0});var Sm=s(me);ra=a(Sm,"SPAN",{});var km=s(ra);d(Mt.$$.fragment,km),km.forEach(t),Sm.forEach(t),Tu=f(zi),ia=a(zi,"SPAN",{});var jm=s(ia);Mu=i(jm,"Step 1: Understand the environment"),jm.forEach(t),zi.forEach(t),rr=f(e),H=a(e,"P",{});var ds=s(H);Au=i(ds,"The environment is called "),na=a(ds,"CODE",{});var Jm=s(na);Iu=i(Jm,"SoccerTwos"),Jm.forEach(t),$u=i(ds,". The Unity MLAgents Team made it. You can find its documentation "),At=a(ds,"A",{href:!0,rel:!0});var Um=s(At);Su=i(Um,"here"),Um.forEach(t),ds.forEach(t),ir=f(e),It=a(e,"P",{});var zp=s(It);ku=i(zp,"The goal in this environment "),ha=a(zp,"STRONG",{});var Pm=s(ha);ju=i(Pm,"is to get the ball into the opponent\u2019s goal while preventing the ball from entering your own goal."),Pm.forEach(t),zp.forEach(t),nr=f(e),ve=a(e,"FIGURE",{});var Zi=s(ve);jo=a(Zi,"IMG",{src:!0,alt:!0}),Ju=f(Zi),Jo=a(Zi,"FIGCAPTION",{});var Zp=s(Jo);Uu=i(Zp,"This environment was made by the "),Uo=a(Zp,"A",{href:!0});var Nm=s(Uo);Pu=i(Nm,"Unity MLAgents Team"),Nm.forEach(t),Zp.forEach(t),Zi.forEach(t),hr=f(e),R=a(e,"H3",{class:!0});var Yi=s(R);we=a(Yi,"A",{id:!0,class:!0,href:!0});var Cm=s(we);ua=a(Cm,"SPAN",{});var xm=s(ua);d($t.$$.fragment,xm),xm.forEach(t),Cm.forEach(t),Nu=f(Yi),fa=a(Yi,"SPAN",{});var Lm=s(fa);Cu=i(Lm,"The reward function"),Lm.forEach(t),Yi.forEach(t),ur=f(e),Po=a(e,"P",{});var Wm=s(Po);xu=i(Wm,"The reward function is:"),Wm.forEach(t),fr=f(e),St=a(e,"IMG",{src:!0,alt:!0}),cr=f(e),z=a(e,"H3",{class:!0});var Di=s(z);ye=a(Di,"A",{id:!0,class:!0,href:!0});var Gm=s(ye);ca=a(Gm,"SPAN",{});var Bm=s(ca);d(kt.$$.fragment,Bm),Bm.forEach(t),Gm.forEach(t),Lu=f(Di),pa=a(Di,"SPAN",{});var Om=s(pa);Wu=i(Om,"The observation space"),Om.forEach(t),Di.forEach(t),pr=f(e),No=a(e,"P",{});var Hm=s(No);Gu=i(Hm,"The observation space is composed of vectors of size 336:"),Hm.forEach(t),dr=f(e),j=a(e,"UL",{});var El=s(j);da=a(El,"LI",{});var Rm=s(da);Bu=i(Rm,"11 ray-casts forward distributed over 120 degrees (264 state dimensions)"),Rm.forEach(t),Ou=f(El),ma=a(El,"LI",{});var zm=s(ma);Hu=i(zm,"3 ray-casts backward distributed over 90 degrees (72 state dimensions)"),zm.forEach(t),Ru=f(El),Co=a(El,"LI",{});var Yp=s(Co);zu=i(Yp,"Both of these ray-casts can detect 6 objects:"),g=a(Yp,"UL",{});var A=s(g);va=a(A,"LI",{});var Zm=s(va);Zu=i(Zm,"Ball"),Zm.forEach(t),Yu=f(A),wa=a(A,"LI",{});var Ym=s(wa);Du=i(Ym,"Blue Goal"),Ym.forEach(t),Fu=f(A),ya=a(A,"LI",{});var Dm=s(ya);Vu=i(Dm,"Purple Goal"),Dm.forEach(t),Xu=f(A),ga=a(A,"LI",{});var Fm=s(ga);qu=i(Fm,"Wall"),Fm.forEach(t),Qu=f(A),ba=a(A,"LI",{});var Vm=s(ba);Ku=i(Vm,"Blue Agent"),Vm.forEach(t),ef=f(A),Ea=a(A,"LI",{});var Xm=s(Ea);tf=i(Xm,"Purple Agent"),Xm.forEach(t),A.forEach(t),Yp.forEach(t),El.forEach(t),mr=f(e),Z=a(e,"H3",{class:!0});var Fi=s(Z);ge=a(Fi,"A",{id:!0,class:!0,href:!0});var qm=s(ge);_a=a(qm,"SPAN",{});var Qm=s(_a);d(jt.$$.fragment,Qm),Qm.forEach(t),qm.forEach(t),of=f(Fi),Ta=a(Fi,"SPAN",{});var Km=s(Ta);lf=i(Km,"The action space"),Km.forEach(t),Fi.forEach(t),vr=f(e),xo=a(e,"P",{});var ev=s(xo);af=i(ev,"The action space is three discrete branches:"),ev.forEach(t),wr=f(e),Jt=a(e,"IMG",{src:!0,alt:!0}),yr=f(e),Y=a(e,"H2",{class:!0});var Vi=s(Y);be=a(Vi,"A",{id:!0,class:!0,href:!0});var tv=s(be);Ma=a(tv,"SPAN",{});var ov=s(Ma);d(Ut.$$.fragment,ov),ov.forEach(t),tv.forEach(t),sf=f(Vi),Aa=a(Vi,"SPAN",{});var lv=s(Aa);rf=i(lv,"Step 2: Understand MA-POCA"),lv.forEach(t),Vi.forEach(t),gr=f(e),Ee=a(e,"P",{});var Xi=s(Ee);nf=i(Xi,"We know how to train agents to play against others: "),Ia=a(Xi,"STRONG",{});var av=s(Ia);hf=i(av,"we can use self-play."),av.forEach(t),uf=i(Xi," This is a perfect technique for a 1vs1."),Xi.forEach(t),br=f(e),Pt=a(e,"P",{});var Dp=s(Pt);ff=i(Dp,"But in our case we\u2019re 2vs2, and each team has 2 agents. How then can we "),$a=a(Dp,"STRONG",{});var sv=s($a);cf=i(sv,"train cooperative behavior for groups of agents?"),sv.forEach(t),Dp.forEach(t),Er=f(e),J=a(e,"P",{});var _l=s(J);pf=i(_l,"As explained in the "),Nt=a(_l,"A",{href:!0,rel:!0});var rv=s(Nt);df=i(rv,"Unity Blog"),rv.forEach(t),mf=i(_l,", agents typically receive a reward as a group (+1 - penalty) when the team scores a goal. This implies that "),Sa=a(_l,"STRONG",{});var iv=s(Sa);vf=i(iv,"every agent on the team is rewarded even if each agent didn\u2019t contribute the same to the win"),iv.forEach(t),wf=i(_l,", which makes it difficult to learn what to do independently."),_l.forEach(t),_r=f(e),_e=a(e,"P",{});var qi=s(_e);yf=i(qi,"The Unity MLAgents team developed the solution in a new multi-agent trainer called "),ka=a(qi,"EM",{});var nv=s(ka);gf=i(nv,"MA-POCA (Multi-Agent POsthumous Credit Assignment)"),nv.forEach(t),bf=i(qi,"."),qi.forEach(t),Tr=f(e),Te=a(e,"P",{});var Qi=s(Te);Ef=i(Qi,"The idea is simple but powerful: a centralized critic "),ja=a(Qi,"STRONG",{});var hv=s(ja);_f=i(hv,"processes the states of all agents in the team to estimate how well each agent is doing"),hv.forEach(t),Tf=i(Qi,". Think of this critic as a coach."),Qi.forEach(t),Mr=f(e),U=a(e,"P",{});var Tl=s(U);Mf=i(Tl,"This allows each agent to "),Ja=a(Tl,"STRONG",{});var uv=s(Ja);Af=i(uv,"make decisions based only on what it perceives locally"),uv.forEach(t),If=i(Tl,", and "),Ua=a(Tl,"STRONG",{});var fv=s(Ua);$f=i(fv,"simultaneously evaluate how good its behavior is in the context of the whole group"),fv.forEach(t),Sf=i(Tl,"."),Tl.forEach(t),Ar=f(e),Me=a(e,"FIGURE",{});var Ki=s(Me);Lo=a(Ki,"IMG",{src:!0,alt:!0}),kf=f(Ki),Wo=a(Ki,"FIGCAPTION",{});var Fp=s(Wo);jf=i(Fp,"This illustrates MA-POCA\u2019s centralized learning and decentralized execution. Source: "),Go=a(Fp,"A",{href:!0});var cv=s(Go);Jf=i(cv,"MLAgents Plays Dodgeball"),cv.forEach(t),Fp.forEach(t),Ki.forEach(t),Ir=f(e),Bo=a(e,"P",{});var pv=s(Bo);Uf=i(pv,"The solution then is to use Self-Play with an MA-POCA trainer (called poca). The poca trainer will help us to train cooperative behavior and self-play to win against an opponent team."),pv.forEach(t),$r=f(e),Ae=a(e,"P",{});var en=s(Ae);Pf=i(en,"If you want to dive deeper into this MA-POCA algorithm, you need to read the paper they published "),Ct=a(en,"A",{href:!0,rel:!0});var dv=s(Ct);Nf=i(dv,"here"),dv.forEach(t),Cf=i(en," and the sources we put on the additional readings section."),en.forEach(t),Sr=f(e),D=a(e,"H2",{class:!0});var tn=s(D);Ie=a(tn,"A",{id:!0,class:!0,href:!0});var mv=s(Ie);Pa=a(mv,"SPAN",{});var vv=s(Pa);d(xt.$$.fragment,vv),vv.forEach(t),mv.forEach(t),xf=f(tn),Na=a(tn,"SPAN",{});var wv=s(Na);Lf=i(wv,"Step 3: Define the config file"),wv.forEach(t),tn.forEach(t),kr=f(e),F=a(e,"P",{});var ms=s(F);Wf=i(ms,"We already learned in "),Lt=a(ms,"A",{href:!0,rel:!0});var yv=s(Lt);Gf=i(yv,"Unit 5"),yv.forEach(t),Bf=i(ms," that in ML-Agents, you define "),Wt=a(ms,"STRONG",{});var on=s(Wt);Of=i(on,"the training hyperparameters in "),Ca=a(on,"CODE",{});var gv=s(Ca);Hf=i(gv,"config.yaml"),gv.forEach(t),Rf=i(on," files."),on.forEach(t),ms.forEach(t),jr=f(e),Gt=a(e,"P",{});var Vp=s(Gt);zf=i(Vp,"There are multiple hyperparameters. To understand them better, you should read the explanations for each of them in\xA0"),xa=a(Vp,"STRONG",{});var bv=s(xa);Bt=a(bv,"A",{href:!0,rel:!0});var Ev=s(Bt);Zf=i(Ev,"the documentation"),Ev.forEach(t),bv.forEach(t),Vp.forEach(t),Jr=f(e),$e=a(e,"P",{});var ln=s($e);Yf=i(ln,"The config file we\u2019re going to use here is in  "),La=a(ln,"CODE",{});var _v=s(La);Df=i(_v,"./config/poca/SoccerTwos.yaml"),_v.forEach(t),Ff=i(ln,". It looks like this:"),ln.forEach(t),Ur=f(e),d(Ot.$$.fragment,e),Pr=f(e),Oo=a(e,"P",{});var Tv=s(Oo);Vf=i(Tv,"Compared to Pyramids or SnowballTarget, we have new hyperparameters with a self-play part. How you modify them can be critical in getting good results."),Tv.forEach(t),Nr=f(e),Ht=a(e,"P",{});var Xp=s(Ht);Xf=i(Xp,"The advice I can give you here is to check the explanation and recommended value for each parameters (especially self-play ones) against\xA0"),Ho=a(Xp,"STRONG",{});var qp=s(Ho);Rt=a(qp,"A",{href:!0,rel:!0});var Mv=s(Rt);qf=i(Mv,"the documentation"),Mv.forEach(t),Qf=i(qp,"."),qp.forEach(t),Xp.forEach(t),Cr=f(e),Ro=a(e,"P",{});var Av=s(Ro);Kf=i(Av,"Now that you\u2019ve modified our config file, you\u2019re ready to train your agents."),Av.forEach(t),xr=f(e),V=a(e,"H2",{class:!0});var an=s(V);Se=a(an,"A",{id:!0,class:!0,href:!0});var Iv=s(Se);Wa=a(Iv,"SPAN",{});var $v=s(Wa);d(zt.$$.fragment,$v),$v.forEach(t),Iv.forEach(t),ec=f(an),Ga=a(an,"SPAN",{});var Sv=s(Ga);tc=i(Sv,"Step 4: Start the training"),Sv.forEach(t),an.forEach(t),Lr=f(e),Zt=a(e,"P",{});var Qp=s(Zt);oc=i(Qp,"To train the agents, we need to\xA0"),Ba=a(Qp,"STRONG",{});var kv=s(Ba);lc=i(kv,"launch mlagents-learn and select the executable containing the environment."),kv.forEach(t),Qp.forEach(t),Wr=f(e),zo=a(e,"P",{});var jv=s(zo);ac=i(jv,"We define four parameters:"),jv.forEach(t),Gr=f(e),T=a(e,"OL",{});var Ge=s(T);Zo=a(Ge,"LI",{});var Kp=s(Zo);Oa=a(Kp,"CODE",{});var Jv=s(Oa);sc=i(Jv,"mlagents-learn <config>"),Jv.forEach(t),rc=i(Kp,": the path where the hyperparameter config file is."),Kp.forEach(t),ic=f(Ge),Yo=a(Ge,"LI",{});var ed=s(Yo);Ha=a(ed,"CODE",{});var Uv=s(Ha);nc=i(Uv,"-env"),Uv.forEach(t),hc=i(ed,": where the environment executable is."),ed.forEach(t),uc=f(Ge),Do=a(Ge,"LI",{});var td=s(Do);Ra=a(td,"CODE",{});var Pv=s(Ra);fc=i(Pv,"-run_id"),Pv.forEach(t),cc=i(td,": the name you want to give to your training run id."),td.forEach(t),pc=f(Ge),Fo=a(Ge,"LI",{});var od=s(Fo);za=a(od,"CODE",{});var Nv=s(za);dc=i(Nv,"-no-graphics"),Nv.forEach(t),mc=i(od,": to not launch the visualization during the training."),od.forEach(t),Ge.forEach(t),Br=f(e),Vo=a(e,"P",{});var Cv=s(Vo);vc=i(Cv,"Depending on your hardware, 5M timesteps (the recommended value, but you can also try 10M) will take 5 to 8 hours of training. You can continue using your computer in the meantime, but I advise deactivating the computer standby mode to prevent the training from being stopped."),Cv.forEach(t),Or=f(e),Xo=a(e,"P",{});var xv=s(Xo);wc=i(xv,"Depending on the executable you use (windows, ubuntu, mac) the training command will look like this (your executable path can be different so don\u2019t hesitate to check before running)."),xv.forEach(t),Hr=f(e),d(Yt.$$.fragment,e),Rr=f(e),qo=a(e,"P",{});var Lv=s(qo);yc=i(Lv,"The executable contains 8 copies of SoccerTwos."),Lv.forEach(t),zr=f(e),Qo=a(e,"P",{});var Wv=s(Qo);gc=i(Wv,"\u26A0\uFE0F It\u2019s normal if you don\u2019t see a big increase of ELO score (and even a decrease below 1200) before 2M timesteps, since your agents will spend most of their time moving randomly on the field before being able to goal."),Wv.forEach(t),Zr=f(e),Ko=a(e,"P",{});var Gv=s(Ko);bc=i(Gv,"\u26A0\uFE0F You can stop the training with Ctrl + C but beware of typing this command only once to stop the training since MLAgents needs to generate a final .onnx file before closing the run."),Gv.forEach(t),Yr=f(e),X=a(e,"H2",{class:!0});var sn=s(X);ke=a(sn,"A",{id:!0,class:!0,href:!0});var Bv=s(ke);Za=a(Bv,"SPAN",{});var Ov=s(Za);d(Dt.$$.fragment,Ov),Ov.forEach(t),Bv.forEach(t),Ec=f(sn),Ya=a(sn,"SPAN",{});var Hv=s(Ya);_c=i(Hv,"Step 5: **Push the agent to the Hugging Face Hub**"),Hv.forEach(t),sn.forEach(t),Dr=f(e),Ft=a(e,"P",{});var ld=s(Ft);Tc=i(ld,"Now that we trained our agents, we\u2019re\xA0"),Da=a(ld,"STRONG",{});var Rv=s(Da);Mc=i(Rv,"ready to push them to the Hub to be able to participate in the AI vs. AI challenge and visualize them playing on your browser\u{1F525}."),Rv.forEach(t),ld.forEach(t),Fr=f(e),el=a(e,"P",{});var zv=s(el);Ac=i(zv,"To be able to share your model with the community, there are three more steps to follow:"),zv.forEach(t),Vr=f(e),Vt=a(e,"P",{});var ad=s(Vt);Ic=i(ad,"1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1\xA0"),Xt=a(ad,"A",{href:!0,rel:!0});var Zv=s(Xt);$c=i(Zv,"https://huggingface.co/join"),Zv.forEach(t),ad.forEach(t),Xr=f(e),tl=a(e,"P",{});var Yv=s(tl);Sc=i(Yv,"2\uFE0F\u20E3 Sign in and store your authentication token from the Hugging Face website."),Yv.forEach(t),qr=f(e),q=a(e,"P",{});var vs=s(q);kc=i(vs,"Create a new token ("),qt=a(vs,"A",{href:!0,rel:!0});var Dv=s(qt);jc=i(Dv,"https://huggingface.co/settings/tokens"),Dv.forEach(t),Jc=i(vs,")\xA0"),Fa=a(vs,"STRONG",{});var Fv=s(Fa);Uc=i(Fv,"with write role"),Fv.forEach(t),vs.forEach(t),Qr=f(e),Qt=a(e,"IMG",{src:!0,alt:!0}),Kr=f(e),ol=a(e,"P",{});var Vv=s(ol);Pc=i(Vv,"Copy the token, run this, and paste the token"),Vv.forEach(t),ei=f(e),d(Kt.$$.fragment,e),ti=f(e),je=a(e,"P",{});var rn=s(je);Nc=i(rn,"Then, we need to run "),Va=a(rn,"CODE",{});var Xv=s(Va);Cc=i(Xv,"mlagents-push-to-hf"),Xv.forEach(t),xc=i(rn,"."),rn.forEach(t),oi=f(e),ll=a(e,"P",{});var qv=s(ll);Lc=i(qv,"And we define four parameters:"),qv.forEach(t),li=f(e),M=a(e,"OL",{});var Be=s(M);al=a(Be,"LI",{});var sd=s(al);Xa=a(sd,"CODE",{});var Qv=s(Xa);Wc=i(Qv,"-run-id"),Qv.forEach(t),Gc=i(sd,": the name of the training run id."),sd.forEach(t),Bc=f(Be),eo=a(Be,"LI",{});var nn=s(eo);qa=a(nn,"CODE",{});var Kv=s(qa);Oc=i(Kv,"-local-dir"),Kv.forEach(t),Hc=i(nn,": where the agent was saved, it\u2019s results/"),sl=a(nn,"RUN_ID",{name:!0});var ew=s(sl);Rc=i(ew,", so in my case results/First Training."),ew.forEach(t),nn.forEach(t),zc=f(Be),to=a(Be,"LI",{});var hn=s(to);Qa=a(hn,"CODE",{});var tw=s(Qa);Zc=i(tw,"-repo-id"),tw.forEach(t),Yc=i(hn,": the name of the Hugging Face repo you want to create or update. It\u2019s always "),Je=a(hn,"YOUR",{huggingface:!0,username:!0});var rd=s(Je);Dc=i(rd,"/"),Ue=a(rd,"THE",{repo:!0,name:!0});var id=s(Ue);Fc=i(id,"If the repo does not exist "),Ka=a(id,"STRONG",{});var ow=s(Ka);Vc=i(ow,"it will be created automatically"),ow.forEach(t),id.forEach(t),rd.forEach(t),hn.forEach(t),Xc=f(Be),rl=a(Be,"LI",{});var nd=s(rl);es=a(nd,"CODE",{});var lw=s(es);qc=i(lw,"--commit-message"),lw.forEach(t),Qc=i(nd,": since HF repos are git repositories you need to give a commit message."),nd.forEach(t),Be.forEach(t),ai=f(e),il=a(e,"P",{});var aw=s(il);Kc=i(aw,"In my case"),aw.forEach(t),si=f(e),d(oo.$$.fragment,e),ri=f(e),d(lo.$$.fragment,e),ii=f(e),nl=a(e,"P",{});var sw=s(nl);ep=i(sw,"If everything worked you should see this at the end of the process (but with a different url \u{1F606}) :"),sw.forEach(t),ni=f(e),ao=a(e,"P",{});var hd=s(ao);tp=i(hd,"Your model is pushed to the Hub. You can view your model here: "),so=a(hd,"A",{href:!0,rel:!0});var rw=s(so);op=i(rw,"https://huggingface.co/ThomasSimonini/poca-SoccerTwos"),rw.forEach(t),hd.forEach(t),hi=f(e),ro=a(e,"P",{});var ud=s(ro);lp=i(ud,"It\u2019s the link to your model. It contains a model card that explains how to use it, your Tensorboard, and your config file. "),ts=a(ud,"STRONG",{});var iw=s(ts);ap=i(iw,"What\u2019s awesome is that it\u2019s a git repository, which means you can have different commits, update your repository with a new push, etc."),iw.forEach(t),ud.forEach(t),ui=f(e),Q=a(e,"H2",{class:!0});var un=s(Q);Pe=a(un,"A",{id:!0,class:!0,href:!0});var nw=s(Pe);os=a(nw,"SPAN",{});var hw=s(os);d(io.$$.fragment,hw),hw.forEach(t),nw.forEach(t),sp=f(un),ls=a(un,"SPAN",{});var uw=s(ls);rp=i(uw,"Step 6: Verify that your model is ready for AI vs AI Challenge"),uw.forEach(t),un.forEach(t),fi=f(e),Ne=a(e,"P",{});var fn=s(Ne);ip=i(fn,"Now that your model is pushed to the Hub, "),as=a(fn,"STRONG",{});var fw=s(as);np=i(fw,"it\u2019s going to be added automatically to the AI vs AI Challenge model pool."),fw.forEach(t),hp=i(fn," It can take a little bit of time before your model is added to the leaderboard given we do a run of matches every 4h."),fn.forEach(t),ci=f(e),hl=a(e,"P",{});var cw=s(hl);up=i(cw,"But to ensure that everything works perfectly you need to check:"),cw.forEach(t),pi=f(e),ul=a(e,"OL",{});var pw=s(ul);ss=a(pw,"LI",{});var dw=s(ss);fp=i(dw,"That you have this tag in your model: ML-Agents-SoccerTwos. This is the tag we use to select models to be added to the challenge pool. To do that go to your model and check the tags"),dw.forEach(t),pw.forEach(t),di=f(e),no=a(e,"IMG",{src:!0,alt:!0}),mi=f(e),fl=a(e,"P",{});var mw=s(fl);cp=i(mw,"If it\u2019s not the case you just need to modify the readme and add it"),mw.forEach(t),vi=f(e),ho=a(e,"IMG",{src:!0,alt:!0}),wi=f(e),uo=a(e,"OL",{start:!0});var vw=s(uo);fo=a(vw,"LI",{});var cn=s(fo);pp=i(cn,"That you have a "),rs=a(cn,"CODE",{});var ww=s(rs);dp=i(ww,"SoccerTwos.onnx"),ww.forEach(t),mp=i(cn," file"),cn.forEach(t),vw.forEach(t),yi=f(e),co=a(e,"IMG",{src:!0,alt:!0}),gi=f(e),cl=a(e,"P",{});var yw=s(cl);vp=i(yw,"We strongly suggest that you create a new model when you push to the Hub if you want to train it again or train a new version."),yw.forEach(t),bi=f(e),K=a(e,"H2",{class:!0});var pn=s(K);Ce=a(pn,"A",{id:!0,class:!0,href:!0});var gw=s(Ce);is=a(gw,"SPAN",{});var bw=s(is);d(po.$$.fragment,bw),bw.forEach(t),gw.forEach(t),wp=f(pn),ns=a(pn,"SPAN",{});var Ew=s(ns);yp=i(Ew,"Step 7: Visualize some match in our demo"),Ew.forEach(t),pn.forEach(t),Ei=f(e),ee=a(e,"P",{});var ws=s(ee);gp=i(ws,"Now that your model is part of AI vs AI Challenge, "),hs=a(ws,"STRONG",{});var _w=s(hs);bp=i(_w,"you can visualize how good it is compared to others"),_w.forEach(t),Ep=i(ws,": "),mo=a(ws,"A",{href:!0,rel:!0});var Tw=s(mo);_p=i(Tw,"https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),Tw.forEach(t),ws.forEach(t),_i=f(e),pl=a(e,"P",{});var Mw=s(pl);Tp=i(Mw,"In order to do that, you just need to go to this demo:"),Mw.forEach(t),Ti=f(e),dl=a(e,"UL",{});var Aw=s(dl);ml=a(Aw,"LI",{});var fd=s(ml);Mp=i(fd,"Select your model as team blue (or team purple if you prefer) and another model to compete against. The best opponents to compare your model to are either whoever is on top of the leaderboard or the "),vo=a(fd,"A",{href:!0,rel:!0});var Iw=s(vo);Ap=i(Iw,"baseline model"),Iw.forEach(t),fd.forEach(t),Aw.forEach(t),Mi=f(e),xe=a(e,"P",{});var dn=s(xe);Ip=i(dn,"The matches you see live are not used in the calculation of your result "),us=a(dn,"STRONG",{});var $w=s(us);$p=i($w,"but they are a good way to visualize how good your agent is"),$w.forEach(t),Sp=i(dn,"."),dn.forEach(t),Ai=f(e),vl=a(e,"P",{});var Sw=s(vl);kp=i(Sw,"And don\u2019t hesitate to share the best score your agent gets on discord in the #rl-i-made-this channel \u{1F525}"),Sw.forEach(t),this.h()},h(){c(N,"name","hf:doc:metadata"),c(N,"content",JSON.stringify(xw)),c(te,"id","handson"),c(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(te,"href","#handson"),c(C,"class","relative group"),c(Ye,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),c(Ye,"rel","nofollow"),c(ae,"id","what-is-ai-vs-ai"),c(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ae,"href","#what-is-ai-vs-ai"),c(x,"class","relative group"),c(Xe,"href","https://huggingface.co/spaces/huggingface-projects/AIvsAI-SoccerTwos"),c(Xe,"rel","nofollow"),c(qe,"href","https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),c(qe,"rel","nofollow"),c(Ke,"href","https://huggingface.co/spaces/cyllum/soccertwos-analytics"),c(Ke,"rel","nofollow"),c(et,"href","https://huggingface.co/blog/aivsai"),c(et,"rel","nofollow"),c(ne,"id","competition-rules"),c(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ne,"href","#competition-rules"),c(L,"class","relative group"),c(st,"href","https://github.com/huggingface/deep-rl-class/issues"),c(st,"rel","nofollow"),c(ue,"id","chat-with-your-classmates-share-advice-and-ask-questions-on-discord"),c(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ue,"href","#chat-with-your-classmates-share-advice-and-ask-questions-on-discord"),c(W,"class","relative group"),c(nt,"href","https://discord.gg/ydHrjt3WP5"),c(nt,"rel","nofollow"),c(ce,"id","step-0-install-mlagents-and-download-the-correct-executable"),c(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ce,"href","#step-0-install-mlagents-and-download-the-correct-executable"),c(G,"class","relative group"),c(ut,"href","https://docs.conda.io/en/latest/"),c(ut,"rel","nofollow"),c(wt,"href","https://git-lfs.com/"),c(wt,"rel","nofollow"),c(bt,"href","https://drive.google.com/file/d/1sqFxbEdTMubjVktnV4C6ICjp89wLhUcP/view?usp=sharing"),c(bt,"rel","nofollow"),c(_t,"href","https://drive.google.com/file/d/1KuqBKYiXiIcU4kNMqEzhgypuFP5_45CL/view?usp=sharing"),c(_t,"rel","nofollow"),c(Tt,"href","https://drive.google.com/drive/folders/1h7YB0qwjoxxghApQdEUQmk95ZwIDxrPG?usp=share_link"),c(Tt,"rel","nofollow"),c(me,"id","step-1-understand-the-environment"),c(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(me,"href","#step-1-understand-the-environment"),c(O,"class","relative group"),c(At,"href","https://github.com/Unity-Technologies/ml-agents/blob/develop/docs/Learning-Environment-Examples.md#soccer-twos"),c(At,"rel","nofollow"),Oe(jo.src,pd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/soccertwos.gif")||c(jo,"src",pd),c(jo,"alt","SoccerTwos"),c(Uo,"href","https://github.com/Unity-Technologies/ml-agents"),c(we,"id","the-reward-function"),c(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(we,"href","#the-reward-function"),c(R,"class","relative group"),Oe(St.src,dd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/soccerreward.png")||c(St,"src",dd),c(St,"alt","SoccerTwos Reward"),c(ye,"id","the-observation-space"),c(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ye,"href","#the-observation-space"),c(z,"class","relative group"),c(ge,"id","the-action-space"),c(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ge,"href","#the-action-space"),c(Z,"class","relative group"),Oe(Jt.src,md="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/socceraction.png")||c(Jt,"src",md),c(Jt,"alt","SoccerTwos Action"),c(be,"id","step-2-understand-mapoca"),c(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(be,"href","#step-2-understand-mapoca"),c(Y,"class","relative group"),c(Nt,"href","https://blog.unity.com/technology/ml-agents-v20-release-now-supports-training-complex-cooperative-behaviors"),c(Nt,"rel","nofollow"),Oe(Lo.src,vd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/mapoca.png")||c(Lo,"src",vd),c(Lo,"alt","MA POCA"),c(Go,"href","https://blog.unity.com/technology/ml-agents-plays-dodgeball"),c(Ct,"href","https://arxiv.org/pdf/2111.05992.pdf"),c(Ct,"rel","nofollow"),c(Ie,"id","step-3-define-the-config-file"),c(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ie,"href","#step-3-define-the-config-file"),c(D,"class","relative group"),c(Lt,"href","https://huggingface.co/deep-rl-course/unit5/introduction"),c(Lt,"rel","nofollow"),c(Bt,"href","https://github.com/Unity-Technologies/ml-agents/blob/release_20_docs/docs/Training-Configuration-File.md"),c(Bt,"rel","nofollow"),c(Rt,"href","https://github.com/Unity-Technologies/ml-agents/blob/release_20_docs/docs/Training-Configuration-File.md"),c(Rt,"rel","nofollow"),c(Se,"id","step-4-start-the-training"),c(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Se,"href","#step-4-start-the-training"),c(V,"class","relative group"),c(ke,"id","step-5-push-the-agent-to-the-hugging-face-hub"),c(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ke,"href","#step-5-push-the-agent-to-the-hugging-face-hub"),c(X,"class","relative group"),c(Xt,"href","https://huggingface.co/join"),c(Xt,"rel","nofollow"),c(qt,"href","https://huggingface.co/settings/tokens"),c(qt,"rel","nofollow"),Oe(Qt.src,wd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/create-token.jpg")||c(Qt,"src",wd),c(Qt,"alt","Create HF Token"),c(sl,"name",""),c(Ue,"repo",""),c(Ue,"name",""),c(Je,"huggingface",""),c(Je,"username",""),c(so,"href","https://huggingface.co/ThomasSimonini/poca-SoccerTwos"),c(so,"rel","nofollow"),c(Pe,"id","step-6-verify-that-your-model-is-ready-for-ai-vs-ai-challenge"),c(Pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Pe,"href","#step-6-verify-that-your-model-is-ready-for-ai-vs-ai-challenge"),c(Q,"class","relative group"),Oe(no.src,yd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/verify1.png")||c(no,"src",yd),c(no,"alt","Verify"),Oe(ho.src,gd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/verify2.png")||c(ho,"src",gd),c(ho,"alt","Verify"),c(uo,"start","2"),Oe(co.src,bd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/verify3.png")||c(co,"src",bd),c(co,"alt","Verify"),c(Ce,"id","step-7-visualize-some-match-in-our-demo"),c(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ce,"href","#step-7-visualize-some-match-in-our-demo"),c(K,"class","relative group"),c(mo,"href","https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),c(mo,"rel","nofollow"),c(vo,"href","https://huggingface.co/unity/MLAgents-SoccerTwos"),c(vo,"rel","nofollow")},m(e,n){o(document.head,N),h(e,ys,n),h(e,C,n),o(C,te),o(te,Ml),m(He,Ml,null),o(C,mn),o(C,Al),o(Al,vn),h(e,gs,n),h(e,oe,n),o(oe,wn),o(oe,Il),o(Il,yn),o(oe,gn),h(e,bs,n),h(e,Re,n),o(Re,bn),o(Re,$l),o($l,En),h(e,Es,n),h(e,ze,n),o(ze,_n),o(ze,Sl),o(Sl,Tn),h(e,_s,n),h(e,Ze,n),o(Ze,Mn),o(Ze,Ye),o(Ye,An),h(e,Ts,n),h(e,le,n),o(le,In),o(le,kl),o(kl,$n),o(le,Sn),h(e,Ms,n),h(e,wo,n),o(wo,kn),h(e,As,n),h(e,x,n),o(x,ae),o(ae,jl),m(De,jl,null),o(x,jn),o(x,Jl),o(Jl,Jn),h(e,Is,n),h(e,yo,n),o(yo,Un),h(e,$s,n),h(e,Fe,n),o(Fe,Pn),o(Fe,Ul),o(Ul,Nn),h(e,Ss,n),h(e,go,n),o(go,Cn),h(e,ks,n),h(e,I,n),o(I,Ve),o(Ve,xn),o(Ve,Pl),o(Pl,Ln),o(Ve,Wn),o(I,Gn),o(I,se),o(se,Bn),o(se,Nl),o(Nl,On),o(se,Hn),o(se,Xe),o(Xe,Rn),o(I,zn),o(I,re),o(re,Zn),o(re,Cl),o(Cl,Yn),o(re,Dn),o(re,qe),o(qe,Fn),h(e,js,n),h(e,Qe,n),o(Qe,Vn),o(Qe,Ke),o(Ke,Xn),h(e,Js,n),h(e,ie,n),o(ie,qn),o(ie,et),o(et,Qn),o(ie,Kn),h(e,Us,n),h(e,E,n),o(E,bo),o(bo,eh),o(bo,xl),o(xl,th),o(E,oh),o(E,Eo),o(Eo,lh),o(Eo,Ll),o(Ll,ah),o(E,sh),o(E,tt),o(tt,rh),o(tt,Wl),o(Wl,ih),o(tt,nh),o(E,hh),o(E,_o),o(_o,uh),o(_o,Gl),o(Gl,fh),h(e,Ps,n),h(e,L,n),o(L,ne),o(ne,Bl),m(ot,Bl,null),o(L,ch),o(L,Ol),o(Ol,ph),h(e,Ns,n),h(e,_,n),o(_,dh),o(_,Hl),o(Hl,mh),o(_,vh),o(_,Rl),o(Rl,wh),o(_,yh),o(_,zl),o(zl,gh),o(_,bh),h(e,Cs,n),h(e,To,n),o(To,Eh),h(e,xs,n),h(e,$,n),o($,Mo),o(Mo,Zl),o(Zl,_h),o(Mo,Th),o($,Mh),o($,lt),o(lt,Ah),o(lt,Yl),o(Yl,Ih),o(lt,$h),o($,Sh),o($,at),o(at,kh),o(at,Dl),o(Dl,jh),o(at,Jh),h(e,Ls,n),h(e,he,n),o(he,Uh),o(he,Fl),o(Fl,Ph),o(he,Nh),h(e,Ws,n),h(e,S,n),o(S,Ch),o(S,Vl),o(Vl,xh),o(S,Lh),o(S,st),o(st,Wh),o(S,Gh),h(e,Gs,n),h(e,W,n),o(W,ue),o(ue,Xl),m(rt,Xl,null),o(W,Bh),o(W,ql),o(ql,Oh),h(e,Bs,n),h(e,fe,n),o(fe,it),o(it,Hh),o(it,Ql),o(Ql,Rh),o(it,zh),o(fe,Zh),o(fe,Ao),o(Ao,Yh),o(Ao,nt),o(nt,Dh),h(e,Os,n),h(e,G,n),o(G,ce),o(ce,Kl),m(ht,Kl,null),o(G,Fh),o(G,ea),o(ea,Vh),h(e,Hs,n),h(e,pe,n),o(pe,Xh),o(pe,ut),o(ut,qh),o(pe,Qh),h(e,Rs,n),h(e,de,n),o(de,Kh),o(de,ta),o(ta,eu),o(de,tu),h(e,zs,n),m(ft,e,n),h(e,Zs,n),h(e,Io,n),o(Io,ou),h(e,Ys,n),m(ct,e,n),h(e,Ds,n),h(e,$o,n),o($o,lu),h(e,Fs,n),m(pt,e,n),h(e,Vs,n),h(e,So,n),o(So,au),h(e,Xs,n),m(dt,e,n),h(e,qs,n),h(e,ko,n),o(ko,su),h(e,Qs,n),m(mt,e,n),h(e,Ks,n),h(e,vt,n),o(vt,ru),o(vt,wt),o(wt,iu),h(e,er,n),h(e,B,n),o(B,nu),o(B,oa),o(oa,hu),o(B,uu),o(B,la),o(la,fu),h(e,tr,n),h(e,yt,n),o(yt,cu),o(yt,aa),o(aa,pu),h(e,or,n),h(e,gt,n),o(gt,du),o(gt,bt),o(bt,mu),h(e,lr,n),h(e,Et,n),o(Et,vu),o(Et,_t),o(_t,wu),h(e,ar,n),h(e,k,n),o(k,yu),o(k,Tt),o(Tt,gu),o(k,bu),o(k,sa),o(sa,Eu),o(k,_u),h(e,sr,n),h(e,O,n),o(O,me),o(me,ra),m(Mt,ra,null),o(O,Tu),o(O,ia),o(ia,Mu),h(e,rr,n),h(e,H,n),o(H,Au),o(H,na),o(na,Iu),o(H,$u),o(H,At),o(At,Su),h(e,ir,n),h(e,It,n),o(It,ku),o(It,ha),o(ha,ju),h(e,nr,n),h(e,ve,n),o(ve,jo),o(ve,Ju),o(ve,Jo),o(Jo,Uu),o(Jo,Uo),o(Uo,Pu),h(e,hr,n),h(e,R,n),o(R,we),o(we,ua),m($t,ua,null),o(R,Nu),o(R,fa),o(fa,Cu),h(e,ur,n),h(e,Po,n),o(Po,xu),h(e,fr,n),h(e,St,n),h(e,cr,n),h(e,z,n),o(z,ye),o(ye,ca),m(kt,ca,null),o(z,Lu),o(z,pa),o(pa,Wu),h(e,pr,n),h(e,No,n),o(No,Gu),h(e,dr,n),h(e,j,n),o(j,da),o(da,Bu),o(j,Ou),o(j,ma),o(ma,Hu),o(j,Ru),o(j,Co),o(Co,zu),o(Co,g),o(g,va),o(va,Zu),o(g,Yu),o(g,wa),o(wa,Du),o(g,Fu),o(g,ya),o(ya,Vu),o(g,Xu),o(g,ga),o(ga,qu),o(g,Qu),o(g,ba),o(ba,Ku),o(g,ef),o(g,Ea),o(Ea,tf),h(e,mr,n),h(e,Z,n),o(Z,ge),o(ge,_a),m(jt,_a,null),o(Z,of),o(Z,Ta),o(Ta,lf),h(e,vr,n),h(e,xo,n),o(xo,af),h(e,wr,n),h(e,Jt,n),h(e,yr,n),h(e,Y,n),o(Y,be),o(be,Ma),m(Ut,Ma,null),o(Y,sf),o(Y,Aa),o(Aa,rf),h(e,gr,n),h(e,Ee,n),o(Ee,nf),o(Ee,Ia),o(Ia,hf),o(Ee,uf),h(e,br,n),h(e,Pt,n),o(Pt,ff),o(Pt,$a),o($a,cf),h(e,Er,n),h(e,J,n),o(J,pf),o(J,Nt),o(Nt,df),o(J,mf),o(J,Sa),o(Sa,vf),o(J,wf),h(e,_r,n),h(e,_e,n),o(_e,yf),o(_e,ka),o(ka,gf),o(_e,bf),h(e,Tr,n),h(e,Te,n),o(Te,Ef),o(Te,ja),o(ja,_f),o(Te,Tf),h(e,Mr,n),h(e,U,n),o(U,Mf),o(U,Ja),o(Ja,Af),o(U,If),o(U,Ua),o(Ua,$f),o(U,Sf),h(e,Ar,n),h(e,Me,n),o(Me,Lo),o(Me,kf),o(Me,Wo),o(Wo,jf),o(Wo,Go),o(Go,Jf),h(e,Ir,n),h(e,Bo,n),o(Bo,Uf),h(e,$r,n),h(e,Ae,n),o(Ae,Pf),o(Ae,Ct),o(Ct,Nf),o(Ae,Cf),h(e,Sr,n),h(e,D,n),o(D,Ie),o(Ie,Pa),m(xt,Pa,null),o(D,xf),o(D,Na),o(Na,Lf),h(e,kr,n),h(e,F,n),o(F,Wf),o(F,Lt),o(Lt,Gf),o(F,Bf),o(F,Wt),o(Wt,Of),o(Wt,Ca),o(Ca,Hf),o(Wt,Rf),h(e,jr,n),h(e,Gt,n),o(Gt,zf),o(Gt,xa),o(xa,Bt),o(Bt,Zf),h(e,Jr,n),h(e,$e,n),o($e,Yf),o($e,La),o(La,Df),o($e,Ff),h(e,Ur,n),m(Ot,e,n),h(e,Pr,n),h(e,Oo,n),o(Oo,Vf),h(e,Nr,n),h(e,Ht,n),o(Ht,Xf),o(Ht,Ho),o(Ho,Rt),o(Rt,qf),o(Ho,Qf),h(e,Cr,n),h(e,Ro,n),o(Ro,Kf),h(e,xr,n),h(e,V,n),o(V,Se),o(Se,Wa),m(zt,Wa,null),o(V,ec),o(V,Ga),o(Ga,tc),h(e,Lr,n),h(e,Zt,n),o(Zt,oc),o(Zt,Ba),o(Ba,lc),h(e,Wr,n),h(e,zo,n),o(zo,ac),h(e,Gr,n),h(e,T,n),o(T,Zo),o(Zo,Oa),o(Oa,sc),o(Zo,rc),o(T,ic),o(T,Yo),o(Yo,Ha),o(Ha,nc),o(Yo,hc),o(T,uc),o(T,Do),o(Do,Ra),o(Ra,fc),o(Do,cc),o(T,pc),o(T,Fo),o(Fo,za),o(za,dc),o(Fo,mc),h(e,Br,n),h(e,Vo,n),o(Vo,vc),h(e,Or,n),h(e,Xo,n),o(Xo,wc),h(e,Hr,n),m(Yt,e,n),h(e,Rr,n),h(e,qo,n),o(qo,yc),h(e,zr,n),h(e,Qo,n),o(Qo,gc),h(e,Zr,n),h(e,Ko,n),o(Ko,bc),h(e,Yr,n),h(e,X,n),o(X,ke),o(ke,Za),m(Dt,Za,null),o(X,Ec),o(X,Ya),o(Ya,_c),h(e,Dr,n),h(e,Ft,n),o(Ft,Tc),o(Ft,Da),o(Da,Mc),h(e,Fr,n),h(e,el,n),o(el,Ac),h(e,Vr,n),h(e,Vt,n),o(Vt,Ic),o(Vt,Xt),o(Xt,$c),h(e,Xr,n),h(e,tl,n),o(tl,Sc),h(e,qr,n),h(e,q,n),o(q,kc),o(q,qt),o(qt,jc),o(q,Jc),o(q,Fa),o(Fa,Uc),h(e,Qr,n),h(e,Qt,n),h(e,Kr,n),h(e,ol,n),o(ol,Pc),h(e,ei,n),m(Kt,e,n),h(e,ti,n),h(e,je,n),o(je,Nc),o(je,Va),o(Va,Cc),o(je,xc),h(e,oi,n),h(e,ll,n),o(ll,Lc),h(e,li,n),h(e,M,n),o(M,al),o(al,Xa),o(Xa,Wc),o(al,Gc),o(M,Bc),o(M,eo),o(eo,qa),o(qa,Oc),o(eo,Hc),o(eo,sl),o(sl,Rc),o(M,zc),o(M,to),o(to,Qa),o(Qa,Zc),o(to,Yc),o(to,Je),o(Je,Dc),o(Je,Ue),o(Ue,Fc),o(Ue,Ka),o(Ka,Vc),o(M,Xc),o(M,rl),o(rl,es),o(es,qc),o(rl,Qc),h(e,ai,n),h(e,il,n),o(il,Kc),h(e,si,n),m(oo,e,n),h(e,ri,n),m(lo,e,n),h(e,ii,n),h(e,nl,n),o(nl,ep),h(e,ni,n),h(e,ao,n),o(ao,tp),o(ao,so),o(so,op),h(e,hi,n),h(e,ro,n),o(ro,lp),o(ro,ts),o(ts,ap),h(e,ui,n),h(e,Q,n),o(Q,Pe),o(Pe,os),m(io,os,null),o(Q,sp),o(Q,ls),o(ls,rp),h(e,fi,n),h(e,Ne,n),o(Ne,ip),o(Ne,as),o(as,np),o(Ne,hp),h(e,ci,n),h(e,hl,n),o(hl,up),h(e,pi,n),h(e,ul,n),o(ul,ss),o(ss,fp),h(e,di,n),h(e,no,n),h(e,mi,n),h(e,fl,n),o(fl,cp),h(e,vi,n),h(e,ho,n),h(e,wi,n),h(e,uo,n),o(uo,fo),o(fo,pp),o(fo,rs),o(rs,dp),o(fo,mp),h(e,yi,n),h(e,co,n),h(e,gi,n),h(e,cl,n),o(cl,vp),h(e,bi,n),h(e,K,n),o(K,Ce),o(Ce,is),m(po,is,null),o(K,wp),o(K,ns),o(ns,yp),h(e,Ei,n),h(e,ee,n),o(ee,gp),o(ee,hs),o(hs,bp),o(ee,Ep),o(ee,mo),o(mo,_p),h(e,_i,n),h(e,pl,n),o(pl,Tp),h(e,Ti,n),h(e,dl,n),o(dl,ml),o(ml,Mp),o(ml,vo),o(vo,Ap),h(e,Mi,n),h(e,xe,n),o(xe,Ip),o(xe,us),o(us,$p),o(xe,Sp),h(e,Ai,n),h(e,vl,n),o(vl,kp),Ii=!0},p:Pw,i(e){Ii||(v(He.$$.fragment,e),v(De.$$.fragment,e),v(ot.$$.fragment,e),v(rt.$$.fragment,e),v(ht.$$.fragment,e),v(ft.$$.fragment,e),v(ct.$$.fragment,e),v(pt.$$.fragment,e),v(dt.$$.fragment,e),v(mt.$$.fragment,e),v(Mt.$$.fragment,e),v($t.$$.fragment,e),v(kt.$$.fragment,e),v(jt.$$.fragment,e),v(Ut.$$.fragment,e),v(xt.$$.fragment,e),v(Ot.$$.fragment,e),v(zt.$$.fragment,e),v(Yt.$$.fragment,e),v(Dt.$$.fragment,e),v(Kt.$$.fragment,e),v(oo.$$.fragment,e),v(lo.$$.fragment,e),v(io.$$.fragment,e),v(po.$$.fragment,e),Ii=!0)},o(e){w(He.$$.fragment,e),w(De.$$.fragment,e),w(ot.$$.fragment,e),w(rt.$$.fragment,e),w(ht.$$.fragment,e),w(ft.$$.fragment,e),w(ct.$$.fragment,e),w(pt.$$.fragment,e),w(dt.$$.fragment,e),w(mt.$$.fragment,e),w(Mt.$$.fragment,e),w($t.$$.fragment,e),w(kt.$$.fragment,e),w(jt.$$.fragment,e),w(Ut.$$.fragment,e),w(xt.$$.fragment,e),w(Ot.$$.fragment,e),w(zt.$$.fragment,e),w(Yt.$$.fragment,e),w(Dt.$$.fragment,e),w(Kt.$$.fragment,e),w(oo.$$.fragment,e),w(lo.$$.fragment,e),w(io.$$.fragment,e),w(po.$$.fragment,e),Ii=!1},d(e){t(N),e&&t(ys),e&&t(C),y(He),e&&t(gs),e&&t(oe),e&&t(bs),e&&t(Re),e&&t(Es),e&&t(ze),e&&t(_s),e&&t(Ze),e&&t(Ts),e&&t(le),e&&t(Ms),e&&t(wo),e&&t(As),e&&t(x),y(De),e&&t(Is),e&&t(yo),e&&t($s),e&&t(Fe),e&&t(Ss),e&&t(go),e&&t(ks),e&&t(I),e&&t(js),e&&t(Qe),e&&t(Js),e&&t(ie),e&&t(Us),e&&t(E),e&&t(Ps),e&&t(L),y(ot),e&&t(Ns),e&&t(_),e&&t(Cs),e&&t(To),e&&t(xs),e&&t($),e&&t(Ls),e&&t(he),e&&t(Ws),e&&t(S),e&&t(Gs),e&&t(W),y(rt),e&&t(Bs),e&&t(fe),e&&t(Os),e&&t(G),y(ht),e&&t(Hs),e&&t(pe),e&&t(Rs),e&&t(de),e&&t(zs),y(ft,e),e&&t(Zs),e&&t(Io),e&&t(Ys),y(ct,e),e&&t(Ds),e&&t($o),e&&t(Fs),y(pt,e),e&&t(Vs),e&&t(So),e&&t(Xs),y(dt,e),e&&t(qs),e&&t(ko),e&&t(Qs),y(mt,e),e&&t(Ks),e&&t(vt),e&&t(er),e&&t(B),e&&t(tr),e&&t(yt),e&&t(or),e&&t(gt),e&&t(lr),e&&t(Et),e&&t(ar),e&&t(k),e&&t(sr),e&&t(O),y(Mt),e&&t(rr),e&&t(H),e&&t(ir),e&&t(It),e&&t(nr),e&&t(ve),e&&t(hr),e&&t(R),y($t),e&&t(ur),e&&t(Po),e&&t(fr),e&&t(St),e&&t(cr),e&&t(z),y(kt),e&&t(pr),e&&t(No),e&&t(dr),e&&t(j),e&&t(mr),e&&t(Z),y(jt),e&&t(vr),e&&t(xo),e&&t(wr),e&&t(Jt),e&&t(yr),e&&t(Y),y(Ut),e&&t(gr),e&&t(Ee),e&&t(br),e&&t(Pt),e&&t(Er),e&&t(J),e&&t(_r),e&&t(_e),e&&t(Tr),e&&t(Te),e&&t(Mr),e&&t(U),e&&t(Ar),e&&t(Me),e&&t(Ir),e&&t(Bo),e&&t($r),e&&t(Ae),e&&t(Sr),e&&t(D),y(xt),e&&t(kr),e&&t(F),e&&t(jr),e&&t(Gt),e&&t(Jr),e&&t($e),e&&t(Ur),y(Ot,e),e&&t(Pr),e&&t(Oo),e&&t(Nr),e&&t(Ht),e&&t(Cr),e&&t(Ro),e&&t(xr),e&&t(V),y(zt),e&&t(Lr),e&&t(Zt),e&&t(Wr),e&&t(zo),e&&t(Gr),e&&t(T),e&&t(Br),e&&t(Vo),e&&t(Or),e&&t(Xo),e&&t(Hr),y(Yt,e),e&&t(Rr),e&&t(qo),e&&t(zr),e&&t(Qo),e&&t(Zr),e&&t(Ko),e&&t(Yr),e&&t(X),y(Dt),e&&t(Dr),e&&t(Ft),e&&t(Fr),e&&t(el),e&&t(Vr),e&&t(Vt),e&&t(Xr),e&&t(tl),e&&t(qr),e&&t(q),e&&t(Qr),e&&t(Qt),e&&t(Kr),e&&t(ol),e&&t(ei),y(Kt,e),e&&t(ti),e&&t(je),e&&t(oi),e&&t(ll),e&&t(li),e&&t(M),e&&t(ai),e&&t(il),e&&t(si),y(oo,e),e&&t(ri),y(lo,e),e&&t(ii),e&&t(nl),e&&t(ni),e&&t(ao),e&&t(hi),e&&t(ro),e&&t(ui),e&&t(Q),y(io),e&&t(fi),e&&t(Ne),e&&t(ci),e&&t(hl),e&&t(pi),e&&t(ul),e&&t(di),e&&t(no),e&&t(mi),e&&t(fl),e&&t(vi),e&&t(ho),e&&t(wi),e&&t(uo),e&&t(yi),e&&t(co),e&&t(gi),e&&t(cl),e&&t(bi),e&&t(K),y(po),e&&t(Ei),e&&t(ee),e&&t(_i),e&&t(pl),e&&t(Ti),e&&t(dl),e&&t(Mi),e&&t(xe),e&&t(Ai),e&&t(vl)}}}const xw={local:"handson",sections:[{local:"what-is-ai-vs-ai",sections:[{local:"competition-rules",title:"Competition Rules"},{local:"chat-with-your-classmates-share-advice-and-ask-questions-on-discord",title:"Chat with your classmates, share advice and ask questions on Discord"}],title:"What is AI vs. AI?"},{local:"step-0-install-mlagents-and-download-the-correct-executable",title:"Step 0: Install MLAgents and download the correct executable"},{local:"step-1-understand-the-environment",sections:[{local:"the-reward-function",title:"The reward function"},{local:"the-observation-space",title:"The observation space"},{local:"the-action-space",title:"The action space"}],title:"Step 1: Understand the environment"},{local:"step-2-understand-mapoca",title:"Step 2: Understand MA-POCA"},{local:"step-3-define-the-config-file",title:"Step 3: Define the config file"},{local:"step-4-start-the-training",title:"Step 4: Start the training"},{local:"step-5-push-the-agent-to-the-hugging-face-hub",title:"Step 5: **Push the agent to the Hugging Face Hub**"},{local:"step-6-verify-that-your-model-is-ready-for-ai-vs-ai-challenge",title:"Step 6: Verify that your model is ready for AI vs AI Challenge"},{local:"step-7-visualize-some-match-in-our-demo",title:"Step 7: Visualize some match in our demo"}],title:"Hands-on"};function Lw(cd){return Nw(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ow extends kw{constructor(N){super();jw(this,N,Lw,Cw,Jw,{})}}export{Ow as default,xw as metadata};
