import{s as st,f as lt,n as at,o as ot}from"../chunks/scheduler.37c15a92.js";import{S as rt,i as ut,g as a,s,r as X,A as ct,h as o,f as i,c as l,j as Y,u as Z,x as u,k as G,y as mt,a as n,v as tt,d as et,t as it,w as nt}from"../chunks/index.7cb9c9b8.js";import{H as pt,E as ht}from"../chunks/EditOnGithub.858acfec.js";function dt(U){let r,L,M,x,c,H,m,W,P,p,j="In unit 4, we learned about our first Policy-Based algorithm called <strong>Reinforce</strong>.",C,h,O="In Policy-Based methods, <strong>we aim to optimize the policy directly without using a value function</strong>. More precisely, Reinforce is part of a subclass of <em>Policy-Based Methods</em> called <em>Policy-Gradient methods</em>. This subclass optimizes the policy directly by <strong>estimating the weights of the optimal policy using Gradient Ascent</strong>.",$,d,V="We saw that Reinforce worked well. However, because we use Monte-Carlo sampling to estimate return (we use an entire episode to calculate the return), <strong>we have significant variance in policy gradient estimation</strong>.",A,f,D="Remember that the policy gradient estimation is <strong>the direction of the steepest increase in return</strong>. In other words, how to update our policy weights so that actions that lead to good returns have a higher probability of being taken. The Monte Carlo variance, which we will further study in this unit, <strong>leads to slower training since we need a lot of samples to mitigate it</strong>.",B,g,F="So today we’ll study <strong>Actor-Critic methods</strong>, a hybrid architecture combining value-based and Policy-Based methods that helps to stabilize the training by reducing the variance using:",k,v,J="<li><em>An Actor</em> that controls <strong>how our agent behaves</strong> (Policy-Based method)</li> <li><em>A Critic</em> that measures <strong>how good the taken action is</strong> (Value-Based method)</li>",z,w,K="We’ll study one of these hybrid methods, Advantage Actor Critic (A2C), <strong>and train our agent using Stable-Baselines3 in robotic environments</strong>. We’ll train:",E,_,N="<li>A robotic arm 🦾 to move to the correct position.</li>",I,b,Q="Sound exciting? Let’s get started!",S,y,R,T,q;return c=new pt({props:{title:"Introduction",local:"introduction",headingTag:"h1"}}),y=new ht({props:{source:"https://github.com/huggingface/deep-rl-class/blob/main/units/en/unit6/introduction.mdx"}}),{c(){r=a("meta"),L=s(),M=a("p"),x=s(),X(c.$$.fragment),H=s(),m=a("img"),P=s(),p=a("p"),p.innerHTML=j,C=s(),h=a("p"),h.innerHTML=O,$=s(),d=a("p"),d.innerHTML=V,A=s(),f=a("p"),f.innerHTML=D,B=s(),g=a("p"),g.innerHTML=F,k=s(),v=a("ul"),v.innerHTML=J,z=s(),w=a("p"),w.innerHTML=K,E=s(),_=a("ul"),_.innerHTML=N,I=s(),b=a("p"),b.textContent=Q,S=s(),X(y.$$.fragment),R=s(),T=a("p"),this.h()},l(t){const e=ct("svelte-u9bgzb",document.head);r=o(e,"META",{name:!0,content:!0}),e.forEach(i),L=l(t),M=o(t,"P",{}),Y(M).forEach(i),x=l(t),Z(c.$$.fragment,t),H=l(t),m=o(t,"IMG",{src:!0,alt:!0}),P=l(t),p=o(t,"P",{"data-svelte-h":!0}),u(p)!=="svelte-dvu1gd"&&(p.innerHTML=j),C=l(t),h=o(t,"P",{"data-svelte-h":!0}),u(h)!=="svelte-z9stkq"&&(h.innerHTML=O),$=l(t),d=o(t,"P",{"data-svelte-h":!0}),u(d)!=="svelte-1p5t49p"&&(d.innerHTML=V),A=l(t),f=o(t,"P",{"data-svelte-h":!0}),u(f)!=="svelte-spae8s"&&(f.innerHTML=D),B=l(t),g=o(t,"P",{"data-svelte-h":!0}),u(g)!=="svelte-a75o96"&&(g.innerHTML=F),k=l(t),v=o(t,"UL",{"data-svelte-h":!0}),u(v)!=="svelte-1g2297h"&&(v.innerHTML=J),z=l(t),w=o(t,"P",{"data-svelte-h":!0}),u(w)!=="svelte-54x6jn"&&(w.innerHTML=K),E=l(t),_=o(t,"UL",{"data-svelte-h":!0}),u(_)!=="svelte-10o7xql"&&(_.innerHTML=N),I=l(t),b=o(t,"P",{"data-svelte-h":!0}),u(b)!=="svelte-1lrk2z4"&&(b.textContent=Q),S=l(t),Z(y.$$.fragment,t),R=l(t),T=o(t,"P",{}),Y(T).forEach(i),this.h()},h(){G(r,"name","hf:doc:metadata"),G(r,"content",ft),lt(m.src,W="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/thumbnail.png")||G(m,"src",W),G(m,"alt","Thumbnail")},m(t,e){mt(document.head,r),n(t,L,e),n(t,M,e),n(t,x,e),tt(c,t,e),n(t,H,e),n(t,m,e),n(t,P,e),n(t,p,e),n(t,C,e),n(t,h,e),n(t,$,e),n(t,d,e),n(t,A,e),n(t,f,e),n(t,B,e),n(t,g,e),n(t,k,e),n(t,v,e),n(t,z,e),n(t,w,e),n(t,E,e),n(t,_,e),n(t,I,e),n(t,b,e),n(t,S,e),tt(y,t,e),n(t,R,e),n(t,T,e),q=!0},p:at,i(t){q||(et(c.$$.fragment,t),et(y.$$.fragment,t),q=!0)},o(t){it(c.$$.fragment,t),it(y.$$.fragment,t),q=!1},d(t){t&&(i(L),i(M),i(x),i(H),i(m),i(P),i(p),i(C),i(h),i($),i(d),i(A),i(f),i(B),i(g),i(k),i(v),i(z),i(w),i(E),i(_),i(I),i(b),i(S),i(R),i(T)),i(r),nt(c,t),nt(y,t)}}}const ft='{"title":"Introduction","local":"introduction","sections":[],"depth":1}';function gt(U){return ot(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class bt extends rt{constructor(r){super(),ut(this,r,gt,dt,st,{})}}export{bt as component};
