import{s as Ce,f as N,n as He,o as Qe}from"../chunks/scheduler.37c15a92.js";import{S as De,i as Ne,g as r,s as n,r as ae,m as je,A as qe,h as o,f as s,c as i,j as be,u as ne,x as l,n as Ie,k as p,y as Ge,a,v as ie,d as re,t as oe,w as le}from"../chunks/index.7cb9c9b8.js";import{H as Pe,E as We}from"../chunks/EditOnGithub.858acfec.js";function ze(pe){let u,j,Q,q,m,I,c,ue="This is the architecture of our Deep Q-Learning network:",G,g,me,W,h,ce="As input, we take a <strong>stack of 4 frames</strong> passed through the network as a state and output a <strong>vector of Q-values for each possible action at that state</strong>. Then, like with Q-Learning, we just need to use our epsilon-greedy policy to select which action to take.",z,f,ge="When the Neural Network is initialized, <strong>the Q-value estimation is terrible</strong>. But during training, our Deep Q-Network agent will associate a situation with the appropriate action and <strong>learn to play the game well</strong>.",E,d,A,w,he="We need to <strong>preprocess the input</strong>. It’s an essential step since we want to <strong>reduce the complexity of our state to reduce the computation time needed for training</strong>.",B,v,fe=`To achieve this, we <strong>reduce the state space to 84x84 and grayscale it</strong>. We can do this since the colors in Atari environments don’t add important information.
This is a big improvement since we <strong>reduce our three color channels (RGB) to 1</strong>.`,S,_,de=`We can also <strong>crop a part of the screen in some games</strong> if it does not contain important information.
Then we stack four frames together.`,R,y,we,U,T,ve=`<strong>Why do we stack four frames together?</strong>
We stack frames together because it helps us <strong>handle the problem of temporal limitation</strong>. Let’s take an example with the game of Pong. When you see this frame:`,F,x,_e,O,k,ye=`Can you tell me where the ball is going?
No, because one frame is not enough to have a sense of motion! But what if I add three more frames? <strong>Here you can see that the ball is going to the right</strong>.`,Y,L,Te,J,$,xe="Then the stacked frames are processed by three convolutional layers. These layers <strong>allow us to capture and exploit spatial relationships in images</strong>. But also, because the frames are stacked together, <strong>we can exploit some temporal properties across those frames</strong>.",K,M,ke='If you don’t know what convolutional layers are, don’t worry. You can check out <a href="https://www.udacity.com/course/deep-learning-pytorch--ud188" rel="nofollow">Lesson 4 of this free Deep Learning Course by Udacity</a>',V,b,Le="Finally, we have a couple of fully connected layers that output a Q-value for each possible action at that state.",X,P,$e,Z,C,Me="So, we see that Deep Q-Learning uses a neural network to approximate, given a state, the different Q-values for each possible action at that state. Now let’s study the Deep Q-Learning algorithm.",ee,H,te,D,se;return m=new Pe({props:{title:"The Deep Q-Network (DQN)",local:"deep-q-network",headingTag:"h1"}}),d=new Pe({props:{title:"Preprocessing the input and temporal limitation",local:"preprocessing",headingTag:"h2"}}),H=new We({props:{source:"https://github.com/huggingface/deep-rl-class/blob/main/units/en/unit3/deep-q-network.mdx"}}),{c(){u=r("meta"),j=n(),Q=r("p"),q=n(),ae(m.$$.fragment),I=n(),c=r("p"),c.textContent=ue,G=n(),g=r("img"),W=n(),h=r("p"),h.innerHTML=ce,z=n(),f=r("p"),f.innerHTML=ge,E=n(),ae(d.$$.fragment),A=n(),w=r("p"),w.innerHTML=he,B=n(),v=r("p"),v.innerHTML=fe,S=n(),_=r("p"),_.innerHTML=de,R=n(),y=r("img"),U=n(),T=r("p"),T.innerHTML=ve,F=n(),x=r("img"),O=n(),k=r("p"),k.innerHTML=ye,Y=n(),L=r("img"),J=je(`
That’s why, to capture temporal information, we stack four frames together.
`),$=r("p"),$.innerHTML=xe,K=n(),M=r("p"),M.innerHTML=ke,V=n(),b=r("p"),b.textContent=Le,X=n(),P=r("img"),Z=n(),C=r("p"),C.textContent=Me,ee=n(),ae(H.$$.fragment),te=n(),D=r("p"),this.h()},l(e){const t=qe("svelte-u9bgzb",document.head);u=o(t,"META",{name:!0,content:!0}),t.forEach(s),j=i(e),Q=o(e,"P",{}),be(Q).forEach(s),q=i(e),ne(m.$$.fragment,e),I=i(e),c=o(e,"P",{"data-svelte-h":!0}),l(c)!=="svelte-1jlvrsz"&&(c.textContent=ue),G=i(e),g=o(e,"IMG",{src:!0,alt:!0}),W=i(e),h=o(e,"P",{"data-svelte-h":!0}),l(h)!=="svelte-1hypp4p"&&(h.innerHTML=ce),z=i(e),f=o(e,"P",{"data-svelte-h":!0}),l(f)!=="svelte-1k6czvr"&&(f.innerHTML=ge),E=i(e),ne(d.$$.fragment,e),A=i(e),w=o(e,"P",{"data-svelte-h":!0}),l(w)!=="svelte-1lyhjdo"&&(w.innerHTML=he),B=i(e),v=o(e,"P",{"data-svelte-h":!0}),l(v)!=="svelte-3unddj"&&(v.innerHTML=fe),S=i(e),_=o(e,"P",{"data-svelte-h":!0}),l(_)!=="svelte-12atbxn"&&(_.innerHTML=de),R=i(e),y=o(e,"IMG",{src:!0,alt:!0}),U=i(e),T=o(e,"P",{"data-svelte-h":!0}),l(T)!=="svelte-1d5ylzm"&&(T.innerHTML=ve),F=i(e),x=o(e,"IMG",{src:!0,alt:!0}),O=i(e),k=o(e,"P",{"data-svelte-h":!0}),l(k)!=="svelte-1gpzepb"&&(k.innerHTML=ye),Y=i(e),L=o(e,"IMG",{src:!0,alt:!0}),J=Ie(e,`
That’s why, to capture temporal information, we stack four frames together.
`),$=o(e,"P",{"data-svelte-h":!0}),l($)!=="svelte-1qrirkm"&&($.innerHTML=xe),K=i(e),M=o(e,"P",{"data-svelte-h":!0}),l(M)!=="svelte-7233xh"&&(M.innerHTML=ke),V=i(e),b=o(e,"P",{"data-svelte-h":!0}),l(b)!=="svelte-2hryy8"&&(b.textContent=Le),X=i(e),P=o(e,"IMG",{src:!0,alt:!0}),Z=i(e),C=o(e,"P",{"data-svelte-h":!0}),l(C)!=="svelte-2wm1wc"&&(C.textContent=Me),ee=i(e),ne(H.$$.fragment,e),te=i(e),D=o(e,"P",{}),be(D).forEach(s),this.h()},h(){p(u,"name","hf:doc:metadata"),p(u,"content",Ee),N(g.src,me="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/deep-q-network.jpg")||p(g,"src",me),p(g,"alt","Deep Q Network"),N(y.src,we="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/preprocessing.jpg")||p(y,"src",we),p(y,"alt","Preprocessing"),N(x.src,_e="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/temporal-limitation.jpg")||p(x,"src",_e),p(x,"alt","Temporal Limitation"),N(L.src,Te="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/temporal-limitation-2.jpg")||p(L,"src",Te),p(L,"alt","Temporal Limitation"),N(P.src,$e="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/deep-q-network.jpg")||p(P,"src",$e),p(P,"alt","Deep Q Network")},m(e,t){Ge(document.head,u),a(e,j,t),a(e,Q,t),a(e,q,t),ie(m,e,t),a(e,I,t),a(e,c,t),a(e,G,t),a(e,g,t),a(e,W,t),a(e,h,t),a(e,z,t),a(e,f,t),a(e,E,t),ie(d,e,t),a(e,A,t),a(e,w,t),a(e,B,t),a(e,v,t),a(e,S,t),a(e,_,t),a(e,R,t),a(e,y,t),a(e,U,t),a(e,T,t),a(e,F,t),a(e,x,t),a(e,O,t),a(e,k,t),a(e,Y,t),a(e,L,t),a(e,J,t),a(e,$,t),a(e,K,t),a(e,M,t),a(e,V,t),a(e,b,t),a(e,X,t),a(e,P,t),a(e,Z,t),a(e,C,t),a(e,ee,t),ie(H,e,t),a(e,te,t),a(e,D,t),se=!0},p:He,i(e){se||(re(m.$$.fragment,e),re(d.$$.fragment,e),re(H.$$.fragment,e),se=!0)},o(e){oe(m.$$.fragment,e),oe(d.$$.fragment,e),oe(H.$$.fragment,e),se=!1},d(e){e&&(s(j),s(Q),s(q),s(I),s(c),s(G),s(g),s(W),s(h),s(z),s(f),s(E),s(A),s(w),s(B),s(v),s(S),s(_),s(R),s(y),s(U),s(T),s(F),s(x),s(O),s(k),s(Y),s(L),s(J),s($),s(K),s(M),s(V),s(b),s(X),s(P),s(Z),s(C),s(ee),s(te),s(D)),s(u),le(m,e),le(d,e),le(H,e)}}}const Ee='{"title":"The Deep Q-Network (DQN)","local":"deep-q-network","sections":[{"title":"Preprocessing the input and temporal limitation","local":"preprocessing","sections":[],"depth":2}],"depth":1}';function Ae(pe){return Qe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ue extends De{constructor(u){super(),Ne(this,u,Ae,ze,Ce,{})}}export{Ue as component};
