import{s as _e,n as He,o as Ce}from"../chunks/scheduler.37c15a92.js";import{S as ze,i as Pe,g as l,s as a,r as I,A as ke,h as r,f as n,c as s,j as Me,u as A,x as o,k as Le,y as Ie,a as i,v as E,d as R,t as S,w as q}from"../chunks/index.7cb9c9b8.js";import{Y as Ae}from"../chunks/Youtube.8666c400.js";import{H as re,E as Ee}from"../chunks/EditOnGithub.858acfec.js";function Re(oe){let m,D,P,G,u,U,g,me='For this section, you’re going to watch this excellent introduction to multi-agents made by <a href="https://www.youtube.com/channel/UCq0imsn84ShAe9PBOFnoIrg">Brian Douglas </a>.',F,p,j,c,ue="In this video, Brian talked about how to design multi-agent systems. He specifically took a multi-agents system of vacuum cleaners and asked: <strong>how can can cooperate with each other</strong>?",B,h,ge="We have two solutions to design this multi-agent reinforcement learning system (MARL).",W,f,O,v,pe='<img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/decentralized.png" alt="Decentralized"/> <figcaption>Source: <a href="https://www.youtube.com/watch?v=qgb0gyrpiGk">Introduction to Multi-Agent Reinforcement Learning</a></figcaption>',Y,d,ce="In decentralized learning, <strong>each agent is trained independently from the others</strong>. In the example given, each vacuum learns to clean as many places as it can <strong>without caring about what other vacuums (agents) are doing</strong>.",N,w,he="The benefit is that <strong>since no information is shared between agents, these vacuums can be designed and trained like we train single agents</strong>.",J,y,fe="The idea here is that <strong>our training agent will consider other agents as part of the environment dynamics</strong>. Not as agents.",K,$,ve=`However, the big drawback of this technique is that it will <strong>make the environment non-stationary</strong> since the underlying Markov decision process changes over time as other agents are also interacting in the environment.
And this is problematic for many Reinforcement Learning algorithms <strong>that can’t reach a global optimum with a non-stationary environment</strong>.`,Q,x,V,T,de='<img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/centralized.png" alt="Centralized"/> <figcaption>Source: <a href="https://www.youtube.com/watch?v=qgb0gyrpiGk">Introduction to Multi-Agent Reinforcement Learning</a></figcaption>',X,b,we="In this architecture, <strong>we have a high-level process that collects agents’ experiences</strong>: the experience buffer. And we’ll use these experiences <strong>to learn a common policy</strong>.",Z,M,ye="For instance, in the vacuum cleaner example, the observation will be:",ee,L,$e="<li>The coverage map of the vacuums.</li> <li>The position of all the vacuums.</li>",te,_,xe=`We use that collective experience <strong>to train a policy that will move all three robots in the most beneficial way as a whole</strong>. So each robot is learning from their common experience.
We now have a stationary environment since all the agents are treated as a larger entity, and they know the change of other agents’ policies (since it’s the same as theirs).`,ne,H,Te="If we recap:",ie,C,be="<li><p>In a <em>decentralized approach</em>, we <strong>treat all agents independently without considering the existence of the other agents.</strong></p> <ul><li>In this case, all agents <strong>consider others agents as part of the environment</strong>.</li> <li><strong>It’s a non-stationarity environment condition</strong>, so has no guarantee of convergence.</li></ul></li> <li><p>In a <em>centralized approach</em>:</p> <ul><li>A <strong>single policy is learned from all the agents</strong>.</li> <li>Takes as input the present state of an environment and the policy outputs joint actions.</li> <li>The reward is global.</li></ul></li>",ae,z,se,k,le;return u=new re({props:{title:"Designing Multi-Agents systems",local:"designing-multi-agents-systems",headingTag:"h1"}}),p=new Ae({props:{id:"qgb0gyrpiGk"}}),f=new re({props:{title:"Decentralized system",local:"decentralized-system",headingTag:"h2"}}),x=new re({props:{title:"Centralized approach",local:"centralized-approach",headingTag:"h2"}}),z=new Ee({props:{source:"https://github.com/huggingface/deep-rl-class/blob/main/units/en/unit7/multi-agent-setting.mdx"}}),{c(){m=l("meta"),D=a(),P=l("p"),G=a(),I(u.$$.fragment),U=a(),g=l("p"),g.innerHTML=me,F=a(),I(p.$$.fragment),j=a(),c=l("p"),c.innerHTML=ue,B=a(),h=l("p"),h.textContent=ge,W=a(),I(f.$$.fragment),O=a(),v=l("figure"),v.innerHTML=pe,Y=a(),d=l("p"),d.innerHTML=ce,N=a(),w=l("p"),w.innerHTML=he,J=a(),y=l("p"),y.innerHTML=fe,K=a(),$=l("p"),$.innerHTML=ve,Q=a(),I(x.$$.fragment),V=a(),T=l("figure"),T.innerHTML=de,X=a(),b=l("p"),b.innerHTML=we,Z=a(),M=l("p"),M.textContent=ye,ee=a(),L=l("ul"),L.innerHTML=$e,te=a(),_=l("p"),_.innerHTML=xe,ne=a(),H=l("p"),H.textContent=Te,ie=a(),C=l("ul"),C.innerHTML=be,ae=a(),I(z.$$.fragment),se=a(),k=l("p"),this.h()},l(e){const t=ke("svelte-u9bgzb",document.head);m=r(t,"META",{name:!0,content:!0}),t.forEach(n),D=s(e),P=r(e,"P",{}),Me(P).forEach(n),G=s(e),A(u.$$.fragment,e),U=s(e),g=r(e,"P",{"data-svelte-h":!0}),o(g)!=="svelte-s0u6gv"&&(g.innerHTML=me),F=s(e),A(p.$$.fragment,e),j=s(e),c=r(e,"P",{"data-svelte-h":!0}),o(c)!=="svelte-12o0rkz"&&(c.innerHTML=ue),B=s(e),h=r(e,"P",{"data-svelte-h":!0}),o(h)!=="svelte-h4778x"&&(h.textContent=ge),W=s(e),A(f.$$.fragment,e),O=s(e),v=r(e,"FIGURE",{"data-svelte-h":!0}),o(v)!=="svelte-18z4kzt"&&(v.innerHTML=pe),Y=s(e),d=r(e,"P",{"data-svelte-h":!0}),o(d)!=="svelte-ljzs98"&&(d.innerHTML=ce),N=s(e),w=r(e,"P",{"data-svelte-h":!0}),o(w)!=="svelte-3o4vn0"&&(w.innerHTML=he),J=s(e),y=r(e,"P",{"data-svelte-h":!0}),o(y)!=="svelte-c8dx7v"&&(y.innerHTML=fe),K=s(e),$=r(e,"P",{"data-svelte-h":!0}),o($)!=="svelte-z6wvuu"&&($.innerHTML=ve),Q=s(e),A(x.$$.fragment,e),V=s(e),T=r(e,"FIGURE",{"data-svelte-h":!0}),o(T)!=="svelte-ltt229"&&(T.innerHTML=de),X=s(e),b=r(e,"P",{"data-svelte-h":!0}),o(b)!=="svelte-ypcoh6"&&(b.innerHTML=we),Z=s(e),M=r(e,"P",{"data-svelte-h":!0}),o(M)!=="svelte-1owod73"&&(M.textContent=ye),ee=s(e),L=r(e,"UL",{"data-svelte-h":!0}),o(L)!=="svelte-dln72i"&&(L.innerHTML=$e),te=s(e),_=r(e,"P",{"data-svelte-h":!0}),o(_)!=="svelte-14wt5x8"&&(_.innerHTML=xe),ne=s(e),H=r(e,"P",{"data-svelte-h":!0}),o(H)!=="svelte-1uwte4"&&(H.textContent=Te),ie=s(e),C=r(e,"UL",{"data-svelte-h":!0}),o(C)!=="svelte-1h1njmy"&&(C.innerHTML=be),ae=s(e),A(z.$$.fragment,e),se=s(e),k=r(e,"P",{}),Me(k).forEach(n),this.h()},h(){Le(m,"name","hf:doc:metadata"),Le(m,"content",Se)},m(e,t){Ie(document.head,m),i(e,D,t),i(e,P,t),i(e,G,t),E(u,e,t),i(e,U,t),i(e,g,t),i(e,F,t),E(p,e,t),i(e,j,t),i(e,c,t),i(e,B,t),i(e,h,t),i(e,W,t),E(f,e,t),i(e,O,t),i(e,v,t),i(e,Y,t),i(e,d,t),i(e,N,t),i(e,w,t),i(e,J,t),i(e,y,t),i(e,K,t),i(e,$,t),i(e,Q,t),E(x,e,t),i(e,V,t),i(e,T,t),i(e,X,t),i(e,b,t),i(e,Z,t),i(e,M,t),i(e,ee,t),i(e,L,t),i(e,te,t),i(e,_,t),i(e,ne,t),i(e,H,t),i(e,ie,t),i(e,C,t),i(e,ae,t),E(z,e,t),i(e,se,t),i(e,k,t),le=!0},p:He,i(e){le||(R(u.$$.fragment,e),R(p.$$.fragment,e),R(f.$$.fragment,e),R(x.$$.fragment,e),R(z.$$.fragment,e),le=!0)},o(e){S(u.$$.fragment,e),S(p.$$.fragment,e),S(f.$$.fragment,e),S(x.$$.fragment,e),S(z.$$.fragment,e),le=!1},d(e){e&&(n(D),n(P),n(G),n(U),n(g),n(F),n(j),n(c),n(B),n(h),n(W),n(O),n(v),n(Y),n(d),n(N),n(w),n(J),n(y),n(K),n($),n(Q),n(V),n(T),n(X),n(b),n(Z),n(M),n(ee),n(L),n(te),n(_),n(ne),n(H),n(ie),n(C),n(ae),n(se),n(k)),n(m),q(u,e),q(p,e),q(f,e),q(x,e),q(z,e)}}}const Se='{"title":"Designing Multi-Agents systems","local":"designing-multi-agents-systems","sections":[{"title":"Decentralized system","local":"decentralized-system","sections":[],"depth":2},{"title":"Centralized approach","local":"centralized-approach","sections":[],"depth":2}],"depth":1}';function qe(oe){return Ce(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class je extends ze{constructor(m){super(),Pe(this,m,qe,Re,_e,{})}}export{je as component};
