import{S as nj,i as Jj,s as Uj,e as a,k as w,w as p,t as J,M as jj,c as e,d as M,m as o,a as t,x as T,h as U,b as j,N as Q,f as yj,G as s,g as n,y as r,L as wj,q as i,o as c,B as I,v as oj}from"../../chunks/vendor-hf-doc-builder.js";import{I as m}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as pj}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function Tj(Fn){let A,Ua,Z,V,DM,ol,at,$M,et,ja,pl,wa,Tl,tt,PM,yt,oa,TM,nt,pa,rM,Jt,Ta,F,B,Ut,rl,jt,wt,il,ot,pt,Tt,iM,rt,cl,it,ra,cM,ct,ia,IM,OM,Il,It,ca,dl,d,dM,Nn,Ia,hM,dt,da,hl,ht,ls,mt,ha,h,Hn,mM,CM,Ct,ma,uM,ut,Ca,bM,bt,ua,AM,ml,ZM,zn,ba,G,N,Ms,Cl,At,ss,Zt,Aa,ul,qn,Za,H,Bt,as,Gt,ft,Ba,BM,Wt,Ga,GM,es,bl,vt,fa,u,Xt,ts,gt,_t,Al,kt,Et,Wa,f,z,ys,Zl,Rt,ns,Yt,va,fM,St,Xa,q,Bl,Qt,Js,Vt,Ft,Nt,Gl,Ht,Us,zt,qt,ga,W,x,js,fl,xt,ws,Lt,_a,WM,Kt,ka,L,Dt,Wl,$t,Pt,Ea,b,Ot,vl,ly,My,os,sy,ay,Ra,Xl,ey,ps,ty,Ya,gl,yy,_l,ny,Sa,v,K,Ts,kl,Jy,rs,Uy,Qa,vM,D,jy,is,wy,oy,cs,py,Va,El,xn,Fa,XM,Is,ds,Ty,Na,Rl,Ln,Ha,X,$,hs,Yl,ry,ms,iy,za,P,cy,Cs,Iy,dy,qa,gM,hy,xa,Sl,La,Ql,Ka,g,O,us,Vl,my,bs,Cy,Da,Fl,$a,_,ll,As,Nl,uy,Zs,by,Pa,Hl,Ay,zl,Zy,Oa,ql,le,k,Ml,Bs,xl,By,Gs,Gy,Me,_M,fs,fy,se,Ll,ae,kM,Kl,Wy,Ws,vy,Xy,ee,Dl,te,sl,vs,Xs,gy,_y,$l,gs,ky,Ey,E,EM,_s,Ry,Yy,Sy,RM,ks,Qy,Vy,Fy,YM,Es,Ny,Hy,ye,Pl,ne,SM,Rs,zy,Je,Ol,Ue,QM,Ys,qy,je,lM,we,VM,xy,oe,MM,Ly,sM,Ky,pe,FM,Dy,Te,NM,al,$y,aM,Py,Oy,Ss,ln,re,eM,Kn,ie,el,Qs,Mn,sn,Vs,an,ce,tM,Ie,yM,en,Fs,tn,de,R,tl,Ns,nM,yn,Hs,nn,he,JM,Jn,zs,Un,me,yl,qs,xs,jn,wn,Ls,HM,on,Ks,pn,Ce,UM,Dn,ue,jM,$n,be,nl,Ds,Jl,Tn,$s,rn,cn,Ps,In,dn,Os,la,hn,Ae,wM,Ze,Y,Ul,Ma,oM,mn,sa,Cn,Be,jl,un,aa,bn,An,Ge,zM,Zn,fe,S,wl,ea,pM,Bn,ta,Gn,We;return ol=new m({}),pl=new pj({props:{classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/deep-rl-class/blob/main/notebooks/unit8/unit8_part1.ipynb"}],askForHelpUrl:"http://hf.co/join/discord"}}),Cl=new m({}),Zl=new m({}),fl=new m({}),kl=new m({}),Yl=new m({}),Sl=new C({props:{code:"YXB0JTIwaW5zdGFsbCUyMHB5dGhvbi1vcGVuZ2wlMEFhcHQlMjBpbnN0YWxsJTIwZmZtcGVnJTBBYXB0JTIwaW5zdGFsbCUyMHh2ZmIlMEFwaXAlMjBpbnN0YWxsJTIwcHlnbGV0JTNEJTNEMS41JTBBcGlwJTIwaW5zdGFsbCUyMHB5dmlydHVhbGRpc3BsYXk=",highlighted:`apt install python-opengl
apt install ffmpeg
apt install xvfb
pip install pyglet==<span class="hljs-number">1.5</span>
pip install pyvirtualdisplay`}}),Ql=new C({props:{code:"JTIzJTIwVmlydHVhbCUyMGRpc3BsYXklMEFmcm9tJTIwcHl2aXJ0dWFsZGlzcGxheSUyMGltcG9ydCUyMERpc3BsYXklMEElMEF2aXJ0dWFsX2Rpc3BsYXklMjAlM0QlMjBEaXNwbGF5KHZpc2libGUlM0QwJTJDJTIwc2l6ZSUzRCgxNDAwJTJDJTIwOTAwKSklMEF2aXJ0dWFsX2Rpc3BsYXkuc3RhcnQoKQ==",highlighted:`<span class="hljs-comment"># Virtual display</span>
<span class="hljs-keyword">from</span> pyvirtualdisplay <span class="hljs-keyword">import</span> Display

virtual_display = Display(visible=<span class="hljs-number">0</span>, size=(<span class="hljs-number">1400</span>, <span class="hljs-number">900</span>))
virtual_display.start()`}}),Vl=new m({}),Fl=new C({props:{code:"cGlwJTIwaW5zdGFsbCUyMGd5bSUzRCUzRDAuMjIlMEFwaXAlMjBpbnN0YWxsJTIwaW1hZ2Vpby1mZm1wZWclMEFwaXAlMjBpbnN0YWxsJTIwaHVnZ2luZ2ZhY2VfaHViJTBBcGlwJTIwaW5zdGFsbCUyMGd5bSU1QmJveDJkJTVEJTNEJTNEMC4yMg==",highlighted:`pip install gym==<span class="hljs-number">0.22</span>
pip install imageio-ffmpeg
pip install huggingface_hub
pip install gym[box2d]==<span class="hljs-number">0.22</span>`}}),Nl=new m({}),ql=new C({props:{code:"ZnJvbSUyMElQeXRob24uZGlzcGxheSUyMGltcG9ydCUyMEhUTUwlMEElMEFIVE1MKCUwQSUyMCUyMCUyMCUyMCclM0NpZnJhbWUlMjB3aWR0aCUzRCUyMjU2MCUyMiUyMGhlaWdodCUzRCUyMjMxNSUyMiUyMHNyYyUzRCUyMmh0dHBzJTNBJTJGJTJGd3d3LnlvdXR1YmUuY29tJTJGZW1iZWQlMkZNRXQ2cnJ4SDhXNCUyMiUyMHRpdGxlJTNEJTIyWW91VHViZSUyMHZpZGVvJTIwcGxheWVyJTIyJTIwZnJhbWVib3JkZXIlM0QlMjIwJTIyJTIwYWxsb3clM0QlMjJhY2NlbGVyb21ldGVyJTNCJTIwYXV0b3BsYXklM0IlMjBjbGlwYm9hcmQtd3JpdGUlM0IlMjBlbmNyeXB0ZWQtbWVkaWElM0IlMjBneXJvc2NvcGUlM0IlMjBwaWN0dXJlLWluLXBpY3R1cmUlMjIlMjBhbGxvd2Z1bGxzY3JlZW4lM0UlM0MlMkZpZnJhbWUlM0UnJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> IPython.display <span class="hljs-keyword">import</span> HTML

HTML(
    <span class="hljs-string">&#x27;&lt;iframe width=&quot;560&quot; height=&quot;315&quot; src=&quot;https://www.youtube.com/embed/MEt6rrxH8W4&quot; title=&quot;YouTube video player&quot; frameborder=&quot;0&quot; allow=&quot;accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture&quot; allowfullscreen&gt;&lt;/iframe&gt;&#x27;</span>
)`}}),xl=new m({}),Ll=new C({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmQXBpJTJDJTIwdXBsb2FkX2ZvbGRlciUwQWZyb20lMjBodWdnaW5nZmFjZV9odWIucmVwb2NhcmQlMjBpbXBvcnQlMjBtZXRhZGF0YV9ldmFsX3Jlc3VsdCUyQyUyMG1ldGFkYXRhX3NhdmUlMEElMEFmcm9tJTIwcGF0aGxpYiUyMGltcG9ydCUyMFBhdGglMEFpbXBvcnQlMjBkYXRldGltZSUwQWltcG9ydCUyMHRlbXBmaWxlJTBBaW1wb3J0JTIwanNvbiUwQWltcG9ydCUyMHNodXRpbCUwQWltcG9ydCUyMGltYWdlaW8lMEElMEFmcm9tJTIwd2FzYWJpJTIwaW1wb3J0JTIwUHJpbnRlciUwQSUwQW1zZyUyMCUzRCUyMFByaW50ZXIoKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi, upload_folder
<span class="hljs-keyword">from</span> huggingface_hub.repocard <span class="hljs-keyword">import</span> metadata_eval_result, metadata_save

<span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
<span class="hljs-keyword">import</span> datetime
<span class="hljs-keyword">import</span> tempfile
<span class="hljs-keyword">import</span> json
<span class="hljs-keyword">import</span> shutil
<span class="hljs-keyword">import</span> imageio

<span class="hljs-keyword">from</span> wasabi <span class="hljs-keyword">import</span> Printer

msg = Printer()`}}),Dl=new C({props:{code:"JTIzJTIwQWRkaW5nJTIwSHVnZ2luZ0ZhY2UlMjBhcmd1bWVudCUwQXBhcnNlci5hZGRfYXJndW1lbnQoJTBBJTIwJTIwJTIwJTIwJTIyLS1yZXBvLWlkJTIyJTJDJTBBJTIwJTIwJTIwJTIwdHlwZSUzRHN0ciUyQyUwQSUyMCUyMCUyMCUyMGRlZmF1bHQlM0QlMjJUaG9tYXNTaW1vbmluaSUyRnBwby1DYXJ0UG9sZS12MSUyMiUyQyUwQSUyMCUyMCUyMCUyMGhlbHAlM0QlMjJpZCUyMG9mJTIwdGhlJTIwbW9kZWwlMjByZXBvc2l0b3J5JTIwZnJvbSUyMHRoZSUyMEh1Z2dpbmclMjBGYWNlJTIwSHViJTIwJTdCdXNlcm5hbWUlMkZyZXBvX25hbWUlN0QlMjIlMkMlMEEp",highlighted:`<span class="hljs-comment"># Adding HuggingFace argument</span>
parser.add_argument(
    <span class="hljs-string">&quot;--repo-id&quot;</span>,
    <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>,
    default=<span class="hljs-string">&quot;ThomasSimonini/ppo-CartPole-v1&quot;</span>,
    <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;id of the model repository from the Hugging Face Hub {username/repo_name}&quot;</span>,
)`}}),Pl=new C({props:{code:" ",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">package_to_hub</span>(<span class="hljs-params">
    repo_id,
    model,
    hyperparameters,
    eval_env,
    video_fps=<span class="hljs-number">30</span>,
    commit_message=<span class="hljs-string">&quot;Push agent to the Hub&quot;</span>,
    token=<span class="hljs-literal">None</span>,
    logs=<span class="hljs-literal">None</span>,
</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Evaluate, Generate a video and Upload a model to Hugging Face Hub.
    This method does the complete pipeline:
    - It evaluates the model
    - It generates the model card
    - It generates a replay video of the agent
    - It pushes everything to the hub
    :param repo_id: id of the model repository from the Hugging Face Hub
    :param model: trained model
    :param eval_env: environment used to evaluate the agent
    :param fps: number of fps for rendering the video
    :param commit_message: commit message
    :param logs: directory on local machine of tensorboard logs you&#x27;d like to upload
    &quot;&quot;&quot;</span>
    msg.info(
        <span class="hljs-string">&quot;This function will save, evaluate, generate a video of your agent, &quot;</span>
        <span class="hljs-string">&quot;create a model card and push everything to the hub. &quot;</span>
        <span class="hljs-string">&quot;It might take up to 1min. \\n &quot;</span>
        <span class="hljs-string">&quot;This is a work in progress: if you encounter a bug, please open an issue.&quot;</span>
    )
    <span class="hljs-comment"># Step 1: Clone or create the repo</span>
    repo_url = HfApi().create_repo(
        repo_id=repo_id,
        token=token,
        private=<span class="hljs-literal">False</span>,
        exist_ok=<span class="hljs-literal">True</span>,
    )

    <span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmpdirname:
        tmpdirname = Path(tmpdirname)

        <span class="hljs-comment"># Step 2: Save the model</span>
        torch.save(model.state_dict(), tmpdirname / <span class="hljs-string">&quot;model.pt&quot;</span>)

        <span class="hljs-comment"># Step 3: Evaluate the model and build JSON</span>
        mean_reward, std_reward = _evaluate_agent(eval_env, <span class="hljs-number">10</span>, model)

        <span class="hljs-comment"># First get datetime</span>
        eval_datetime = datetime.datetime.now()
        eval_form_datetime = eval_datetime.isoformat()

        evaluate_data = {
            <span class="hljs-string">&quot;env_id&quot;</span>: hyperparameters.env_id,
            <span class="hljs-string">&quot;mean_reward&quot;</span>: mean_reward,
            <span class="hljs-string">&quot;std_reward&quot;</span>: std_reward,
            <span class="hljs-string">&quot;n_evaluation_episodes&quot;</span>: <span class="hljs-number">10</span>,
            <span class="hljs-string">&quot;eval_datetime&quot;</span>: eval_form_datetime,
        }

        <span class="hljs-comment"># Write a JSON file</span>
        <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(tmpdirname / <span class="hljs-string">&quot;results.json&quot;</span>, <span class="hljs-string">&quot;w&quot;</span>) <span class="hljs-keyword">as</span> outfile:
            json.dump(evaluate_data, outfile)

        <span class="hljs-comment"># Step 4: Generate a video</span>
        video_path = tmpdirname / <span class="hljs-string">&quot;replay.mp4&quot;</span>
        record_video(eval_env, model, video_path, video_fps)

        <span class="hljs-comment"># Step 5: Generate the model card</span>
        generated_model_card, metadata = _generate_model_card(
            <span class="hljs-string">&quot;PPO&quot;</span>, hyperparameters.env_id, mean_reward, std_reward, hyperparameters
        )
        _save_model_card(tmpdirname, generated_model_card, metadata)

        <span class="hljs-comment"># Step 6: Add logs if needed</span>
        <span class="hljs-keyword">if</span> logs:
            _add_logdir(tmpdirname, Path(logs))

        msg.info(<span class="hljs-string">f&quot;Pushing repo <span class="hljs-subst">{repo_id}</span> to the Hugging Face Hub&quot;</span>)

        repo_url = upload_folder(
            repo_id=repo_id,
            folder_path=tmpdirname,
            path_in_repo=<span class="hljs-string">&quot;&quot;</span>,
            commit_message=commit_message,
            token=token,
        )

        msg.info(<span class="hljs-string">f&quot;Your model is pushed to the Hub. You can view your model here: <span class="hljs-subst">{repo_url}</span>&quot;</span>)
    <span class="hljs-keyword">return</span> repo_url


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_evaluate_agent</span>(<span class="hljs-params">env, n_eval_episodes, policy</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Evaluate the agent for \`\`n_eval_episodes\`\` episodes and returns average reward and std of reward.
    :param env: The evaluation environment
    :param n_eval_episodes: Number of episode to evaluate the agent
    :param policy: The agent
    &quot;&quot;&quot;</span>
    episode_rewards = []
    <span class="hljs-keyword">for</span> episode <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_eval_episodes):
        state = env.reset()
        step = <span class="hljs-number">0</span>
        done = <span class="hljs-literal">False</span>
        total_rewards_ep = <span class="hljs-number">0</span>

        <span class="hljs-keyword">while</span> done <span class="hljs-keyword">is</span> <span class="hljs-literal">False</span>:
            state = torch.Tensor(state).to(device)
            action, _, _, _ = policy.get_action_and_value(state)
            new_state, reward, done, info = env.step(action.cpu().numpy())
            total_rewards_ep += reward
            <span class="hljs-keyword">if</span> done:
                <span class="hljs-keyword">break</span>
            state = new_state
        episode_rewards.append(total_rewards_ep)
    mean_reward = np.mean(episode_rewards)
    std_reward = np.std(episode_rewards)

    <span class="hljs-keyword">return</span> mean_reward, std_reward


<span class="hljs-keyword">def</span> <span class="hljs-title function_">record_video</span>(<span class="hljs-params">env, policy, out_directory, fps=<span class="hljs-number">30</span></span>):
    images = []
    done = <span class="hljs-literal">False</span>
    state = env.reset()
    img = env.render(mode=<span class="hljs-string">&quot;rgb_array&quot;</span>)
    images.append(img)
    <span class="hljs-keyword">while</span> <span class="hljs-keyword">not</span> done:
        state = torch.Tensor(state).to(device)
        <span class="hljs-comment"># Take the action (index) that have the maximum expected future reward given that state</span>
        action, _, _, _ = policy.get_action_and_value(state)
        state, reward, done, info = env.step(
            action.cpu().numpy()
        )  <span class="hljs-comment"># We directly put next_state = state for recording logic</span>
        img = env.render(mode=<span class="hljs-string">&quot;rgb_array&quot;</span>)
        images.append(img)
    imageio.mimsave(out_directory, [np.array(img) <span class="hljs-keyword">for</span> i, img <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(images)], fps=fps)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_generate_model_card</span>(<span class="hljs-params">model_name, env_id, mean_reward, std_reward, hyperparameters</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Generate the model card for the Hub
    :param model_name: name of the model
    :env_id: name of the environment
    :mean_reward: mean reward of the agent
    :std_reward: standard deviation of the mean reward of the agent
    :hyperparameters: training arguments
    &quot;&quot;&quot;</span>
    <span class="hljs-comment"># Step 1: Select the tags</span>
    metadata = generate_metadata(model_name, env_id, mean_reward, std_reward)

    <span class="hljs-comment"># Transform the hyperparams namespace to string</span>
    converted_dict = <span class="hljs-built_in">vars</span>(hyperparameters)
    converted_str = <span class="hljs-built_in">str</span>(converted_dict)
    converted_str = converted_str.split(<span class="hljs-string">&quot;, &quot;</span>)
    converted_str = <span class="hljs-string">&quot;\\n&quot;</span>.join(converted_str)

    <span class="hljs-comment"># Step 2: Generate the model card</span>
    model_card = <span class="hljs-string">f&quot;&quot;&quot;
  # PPO Agent Playing <span class="hljs-subst">{env_id}</span>

  This is a trained model of a PPO agent playing <span class="hljs-subst">{env_id}</span>.

  # Hyperparameters
  &quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> model_card, metadata


<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate_metadata</span>(<span class="hljs-params">model_name, env_id, mean_reward, std_reward</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Define the tags for the model card
    :param model_name: name of the model
    :param env_id: name of the environment
    :mean_reward: mean reward of the agent
    :std_reward: standard deviation of the mean reward of the agent
    &quot;&quot;&quot;</span>
    metadata = {}
    metadata[<span class="hljs-string">&quot;tags&quot;</span>] = [
        env_id,
        <span class="hljs-string">&quot;ppo&quot;</span>,
        <span class="hljs-string">&quot;deep-reinforcement-learning&quot;</span>,
        <span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        <span class="hljs-string">&quot;custom-implementation&quot;</span>,
        <span class="hljs-string">&quot;deep-rl-course&quot;</span>,
    ]

    <span class="hljs-comment"># Add metrics</span>
    <span class="hljs-built_in">eval</span> = metadata_eval_result(
        model_pretty_name=model_name,
        task_pretty_name=<span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        task_id=<span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        metrics_pretty_name=<span class="hljs-string">&quot;mean_reward&quot;</span>,
        metrics_id=<span class="hljs-string">&quot;mean_reward&quot;</span>,
        metrics_value=<span class="hljs-string">f&quot;<span class="hljs-subst">{mean_reward:<span class="hljs-number">.2</span>f}</span> +/- <span class="hljs-subst">{std_reward:<span class="hljs-number">.2</span>f}</span>&quot;</span>,
        dataset_pretty_name=env_id,
        dataset_id=env_id,
    )

    <span class="hljs-comment"># Merges both dictionaries</span>
    metadata = {**metadata, **<span class="hljs-built_in">eval</span>}

    <span class="hljs-keyword">return</span> metadata


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_save_model_card</span>(<span class="hljs-params">local_path, generated_model_card, metadata</span>):
    <span class="hljs-string">&quot;&quot;&quot;Saves a model card for the repository.
    :param local_path: repository directory
    :param generated_model_card: model card generated by _generate_model_card()
    :param metadata: metadata
    &quot;&quot;&quot;</span>
    readme_path = local_path / <span class="hljs-string">&quot;README.md&quot;</span>
    readme = <span class="hljs-string">&quot;&quot;</span>
    <span class="hljs-keyword">if</span> readme_path.exists():
        <span class="hljs-keyword">with</span> readme_path.<span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;r&quot;</span>, encoding=<span class="hljs-string">&quot;utf8&quot;</span>) <span class="hljs-keyword">as</span> f:
            readme = f.read()
    <span class="hljs-keyword">else</span>:
        readme = generated_model_card

    <span class="hljs-keyword">with</span> readme_path.<span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;w&quot;</span>, encoding=<span class="hljs-string">&quot;utf-8&quot;</span>) <span class="hljs-keyword">as</span> f:
        f.write(readme)

    <span class="hljs-comment"># Save our metrics to Readme metadata</span>
    metadata_save(readme_path, metadata)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_add_logdir</span>(<span class="hljs-params">local_path: Path, logdir: Path</span>):
    <span class="hljs-string">&quot;&quot;&quot;Adds a logdir to the repository.
    :param local_path: repository directory
    :param logdir: logdir directory
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">if</span> logdir.exists() <span class="hljs-keyword">and</span> logdir.is_dir():
        <span class="hljs-comment"># Add the logdir to the repository under new dir called logs</span>
        repo_logdir = local_path / <span class="hljs-string">&quot;logs&quot;</span>

        <span class="hljs-comment"># Delete current logs if they exist</span>
        <span class="hljs-keyword">if</span> repo_logdir.exists():
            shutil.rmtree(repo_logdir)

        <span class="hljs-comment"># Copy logdir into repo logdir</span>
        shutil.copytree(logdir, repo_logdir)`}}),Ol=new C({props:{code:"JTIzJTIwQ3JlYXRlJTIwdGhlJTIwZXZhbHVhdGlvbiUyMGVudmlyb25tZW50JTBBZXZhbF9lbnYlMjAlM0QlMjBneW0ubWFrZShhcmdzLmVudl9pZCklMEElMEFwYWNrYWdlX3RvX2h1YiglMEElMjAlMjAlMjAlMjByZXBvX2lkJTNEYXJncy5yZXBvX2lkJTJDJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0RhZ2VudCUyQyUyMCUyMCUyMyUyMFRoZSUyMG1vZGVsJTIwd2UlMjB3YW50JTIwdG8lMjBzYXZlJTBBJTIwJTIwJTIwJTIwaHlwZXJwYXJhbWV0ZXJzJTNEYXJncyUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfZW52JTNEZ3ltLm1ha2UoYXJncy5lbnZfaWQpJTJDJTBBJTIwJTIwJTIwJTIwbG9ncyUzRGYlMjJydW5zJTJGJTdCcnVuX25hbWUlN0QlMjIlMkMlMEEp",highlighted:`<span class="hljs-comment"># Create the evaluation environment</span>
eval_env = gym.make(args.env_id)

package_to_hub(
    repo_id=args.repo_id,
    model=agent,  <span class="hljs-comment"># The model we want to save</span>
    hyperparameters=args,
    eval_env=gym.make(args.env_id),
    logs=<span class="hljs-string">f&quot;runs/<span class="hljs-subst">{run_name}</span>&quot;</span>,
)`}}),lM=new C({props:{code:" ",highlighted:`<span class="hljs-comment"># docs and experiment results can be found at https://docs.cleanrl.dev/rl-algorithms/ppo/#ppopy</span>

<span class="hljs-keyword">import</span> argparse
<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> random
<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">from</span> distutils.util <span class="hljs-keyword">import</span> strtobool

<span class="hljs-keyword">import</span> gym
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.nn <span class="hljs-keyword">as</span> nn
<span class="hljs-keyword">import</span> torch.optim <span class="hljs-keyword">as</span> optim
<span class="hljs-keyword">from</span> torch.distributions.categorical <span class="hljs-keyword">import</span> Categorical
<span class="hljs-keyword">from</span> torch.utils.tensorboard <span class="hljs-keyword">import</span> SummaryWriter

<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi, upload_folder
<span class="hljs-keyword">from</span> huggingface_hub.repocard <span class="hljs-keyword">import</span> metadata_eval_result, metadata_save

<span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
<span class="hljs-keyword">import</span> datetime
<span class="hljs-keyword">import</span> tempfile
<span class="hljs-keyword">import</span> json
<span class="hljs-keyword">import</span> shutil
<span class="hljs-keyword">import</span> imageio

<span class="hljs-keyword">from</span> wasabi <span class="hljs-keyword">import</span> Printer

msg = Printer()


<span class="hljs-keyword">def</span> <span class="hljs-title function_">parse_args</span>():
    <span class="hljs-comment"># fmt: off</span>
    parser = argparse.ArgumentParser()
    parser.add_argument(<span class="hljs-string">&quot;--exp-name&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>, default=os.path.basename(__file__).rstrip(<span class="hljs-string">&quot;.py&quot;</span>),
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the name of this experiment&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--seed&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">1</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;seed of the experiment&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--torch-deterministic&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;if toggled, \`torch.backends.cudnn.deterministic=False\`&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--cuda&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;if toggled, cuda will be enabled by default&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--track&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">False</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;if toggled, this experiment will be tracked with Weights and Biases&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--wandb-project-name&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>, default=<span class="hljs-string">&quot;cleanRL&quot;</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the wandb&#x27;s project name&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--wandb-entity&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>, default=<span class="hljs-literal">None</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the entity (team) of wandb&#x27;s project&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--capture-video&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">False</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;weather to capture videos of the agent performances (check out \`videos\` folder)&quot;</span>)

    <span class="hljs-comment"># Algorithm specific arguments</span>
    parser.add_argument(<span class="hljs-string">&quot;--env-id&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>, default=<span class="hljs-string">&quot;CartPole-v1&quot;</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the id of the environment&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--total-timesteps&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">50000</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;total timesteps of the experiments&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--learning-rate&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">2.5e-4</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the learning rate of the optimizer&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--num-envs&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">4</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the number of parallel game environments&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--num-steps&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">128</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the number of steps to run in each environment per policy rollout&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--anneal-lr&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;Toggle learning rate annealing for policy and value networks&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--gae&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;Use GAE for advantage computation&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--gamma&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.99</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the discount factor gamma&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--gae-lambda&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.95</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the lambda for the general advantage estimation&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--num-minibatches&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">4</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the number of mini-batches&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--update-epochs&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">4</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the K epochs to update the policy&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--norm-adv&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;Toggles advantages normalization&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--clip-coef&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.2</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the surrogate clipping coefficient&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--clip-vloss&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;Toggles whether or not to use a clipped loss for the value function, as per the paper.&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--ent-coef&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.01</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;coefficient of the entropy&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--vf-coef&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.5</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;coefficient of the value function&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--max-grad-norm&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.5</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the maximum norm for the gradient clipping&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--target-kl&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-literal">None</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the target KL divergence threshold&quot;</span>)

    <span class="hljs-comment"># Adding HuggingFace argument</span>
    parser.add_argument(<span class="hljs-string">&quot;--repo-id&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>, default=<span class="hljs-string">&quot;ThomasSimonini/ppo-CartPole-v1&quot;</span>, <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;id of the model repository from the Hugging Face Hub {username/repo_name}&quot;</span>)

    args = parser.parse_args()
    args.batch_size = <span class="hljs-built_in">int</span>(args.num_envs * args.num_steps)
    args.minibatch_size = <span class="hljs-built_in">int</span>(args.batch_size // args.num_minibatches)
    <span class="hljs-comment"># fmt: on</span>
    <span class="hljs-keyword">return</span> args


<span class="hljs-keyword">def</span> <span class="hljs-title function_">package_to_hub</span>(<span class="hljs-params">
    repo_id,
    model,
    hyperparameters,
    eval_env,
    video_fps=<span class="hljs-number">30</span>,
    commit_message=<span class="hljs-string">&quot;Push agent to the Hub&quot;</span>,
    token=<span class="hljs-literal">None</span>,
    logs=<span class="hljs-literal">None</span>,
</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Evaluate, Generate a video and Upload a model to Hugging Face Hub.
    This method does the complete pipeline:
    - It evaluates the model
    - It generates the model card
    - It generates a replay video of the agent
    - It pushes everything to the hub
    :param repo_id: id of the model repository from the Hugging Face Hub
    :param model: trained model
    :param eval_env: environment used to evaluate the agent
    :param fps: number of fps for rendering the video
    :param commit_message: commit message
    :param logs: directory on local machine of tensorboard logs you&#x27;d like to upload
    &quot;&quot;&quot;</span>
    msg.info(
        <span class="hljs-string">&quot;This function will save, evaluate, generate a video of your agent, &quot;</span>
        <span class="hljs-string">&quot;create a model card and push everything to the hub. &quot;</span>
        <span class="hljs-string">&quot;It might take up to 1min. \\n &quot;</span>
        <span class="hljs-string">&quot;This is a work in progress: if you encounter a bug, please open an issue.&quot;</span>
    )
    <span class="hljs-comment"># Step 1: Clone or create the repo</span>
    repo_url = HfApi().create_repo(
        repo_id=repo_id,
        token=token,
        private=<span class="hljs-literal">False</span>,
        exist_ok=<span class="hljs-literal">True</span>,
    )

    <span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmpdirname:
        tmpdirname = Path(tmpdirname)

        <span class="hljs-comment"># Step 2: Save the model</span>
        torch.save(model.state_dict(), tmpdirname / <span class="hljs-string">&quot;model.pt&quot;</span>)

        <span class="hljs-comment"># Step 3: Evaluate the model and build JSON</span>
        mean_reward, std_reward = _evaluate_agent(eval_env, <span class="hljs-number">10</span>, model)

        <span class="hljs-comment"># First get datetime</span>
        eval_datetime = datetime.datetime.now()
        eval_form_datetime = eval_datetime.isoformat()

        evaluate_data = {
            <span class="hljs-string">&quot;env_id&quot;</span>: hyperparameters.env_id,
            <span class="hljs-string">&quot;mean_reward&quot;</span>: mean_reward,
            <span class="hljs-string">&quot;std_reward&quot;</span>: std_reward,
            <span class="hljs-string">&quot;n_evaluation_episodes&quot;</span>: <span class="hljs-number">10</span>,
            <span class="hljs-string">&quot;eval_datetime&quot;</span>: eval_form_datetime,
        }

        <span class="hljs-comment"># Write a JSON file</span>
        <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(tmpdirname / <span class="hljs-string">&quot;results.json&quot;</span>, <span class="hljs-string">&quot;w&quot;</span>) <span class="hljs-keyword">as</span> outfile:
            json.dump(evaluate_data, outfile)

        <span class="hljs-comment"># Step 4: Generate a video</span>
        video_path = tmpdirname / <span class="hljs-string">&quot;replay.mp4&quot;</span>
        record_video(eval_env, model, video_path, video_fps)

        <span class="hljs-comment"># Step 5: Generate the model card</span>
        generated_model_card, metadata = _generate_model_card(
            <span class="hljs-string">&quot;PPO&quot;</span>, hyperparameters.env_id, mean_reward, std_reward, hyperparameters
        )
        _save_model_card(tmpdirname, generated_model_card, metadata)

        <span class="hljs-comment"># Step 6: Add logs if needed</span>
        <span class="hljs-keyword">if</span> logs:
            _add_logdir(tmpdirname, Path(logs))

        msg.info(<span class="hljs-string">f&quot;Pushing repo <span class="hljs-subst">{repo_id}</span> to the Hugging Face Hub&quot;</span>)

        repo_url = upload_folder(
            repo_id=repo_id,
            folder_path=tmpdirname,
            path_in_repo=<span class="hljs-string">&quot;&quot;</span>,
            commit_message=commit_message,
            token=token,
        )

        msg.info(<span class="hljs-string">f&quot;Your model is pushed to the Hub. You can view your model here: <span class="hljs-subst">{repo_url}</span>&quot;</span>)
    <span class="hljs-keyword">return</span> repo_url


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_evaluate_agent</span>(<span class="hljs-params">env, n_eval_episodes, policy</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Evaluate the agent for \`\`n_eval_episodes\`\` episodes and returns average reward and std of reward.
    :param env: The evaluation environment
    :param n_eval_episodes: Number of episode to evaluate the agent
    :param policy: The agent
    &quot;&quot;&quot;</span>
    episode_rewards = []
    <span class="hljs-keyword">for</span> episode <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_eval_episodes):
        state = env.reset()
        step = <span class="hljs-number">0</span>
        done = <span class="hljs-literal">False</span>
        total_rewards_ep = <span class="hljs-number">0</span>

        <span class="hljs-keyword">while</span> done <span class="hljs-keyword">is</span> <span class="hljs-literal">False</span>:
            state = torch.Tensor(state).to(device)
            action, _, _, _ = policy.get_action_and_value(state)
            new_state, reward, done, info = env.step(action.cpu().numpy())
            total_rewards_ep += reward
            <span class="hljs-keyword">if</span> done:
                <span class="hljs-keyword">break</span>
            state = new_state
        episode_rewards.append(total_rewards_ep)
    mean_reward = np.mean(episode_rewards)
    std_reward = np.std(episode_rewards)

    <span class="hljs-keyword">return</span> mean_reward, std_reward


<span class="hljs-keyword">def</span> <span class="hljs-title function_">record_video</span>(<span class="hljs-params">env, policy, out_directory, fps=<span class="hljs-number">30</span></span>):
    images = []
    done = <span class="hljs-literal">False</span>
    state = env.reset()
    img = env.render(mode=<span class="hljs-string">&quot;rgb_array&quot;</span>)
    images.append(img)
    <span class="hljs-keyword">while</span> <span class="hljs-keyword">not</span> done:
        state = torch.Tensor(state).to(device)
        <span class="hljs-comment"># Take the action (index) that have the maximum expected future reward given that state</span>
        action, _, _, _ = policy.get_action_and_value(state)
        state, reward, done, info = env.step(
            action.cpu().numpy()
        )  <span class="hljs-comment"># We directly put next_state = state for recording logic</span>
        img = env.render(mode=<span class="hljs-string">&quot;rgb_array&quot;</span>)
        images.append(img)
    imageio.mimsave(out_directory, [np.array(img) <span class="hljs-keyword">for</span> i, img <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(images)], fps=fps)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_generate_model_card</span>(<span class="hljs-params">model_name, env_id, mean_reward, std_reward, hyperparameters</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Generate the model card for the Hub
    :param model_name: name of the model
    :env_id: name of the environment
    :mean_reward: mean reward of the agent
    :std_reward: standard deviation of the mean reward of the agent
    :hyperparameters: training arguments
    &quot;&quot;&quot;</span>
    <span class="hljs-comment"># Step 1: Select the tags</span>
    metadata = generate_metadata(model_name, env_id, mean_reward, std_reward)

    <span class="hljs-comment"># Transform the hyperparams namespace to string</span>
    converted_dict = <span class="hljs-built_in">vars</span>(hyperparameters)
    converted_str = <span class="hljs-built_in">str</span>(converted_dict)
    converted_str = converted_str.split(<span class="hljs-string">&quot;, &quot;</span>)
    converted_str = <span class="hljs-string">&quot;\\n&quot;</span>.join(converted_str)

    <span class="hljs-comment"># Step 2: Generate the model card</span>
    model_card = <span class="hljs-string">f&quot;&quot;&quot;
  # PPO Agent Playing <span class="hljs-subst">{env_id}</span>

  This is a trained model of a PPO agent playing <span class="hljs-subst">{env_id}</span>.

  # Hyperparameters
  &quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> model_card, metadata


<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate_metadata</span>(<span class="hljs-params">model_name, env_id, mean_reward, std_reward</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Define the tags for the model card
    :param model_name: name of the model
    :param env_id: name of the environment
    :mean_reward: mean reward of the agent
    :std_reward: standard deviation of the mean reward of the agent
    &quot;&quot;&quot;</span>
    metadata = {}
    metadata[<span class="hljs-string">&quot;tags&quot;</span>] = [
        env_id,
        <span class="hljs-string">&quot;ppo&quot;</span>,
        <span class="hljs-string">&quot;deep-reinforcement-learning&quot;</span>,
        <span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        <span class="hljs-string">&quot;custom-implementation&quot;</span>,
        <span class="hljs-string">&quot;deep-rl-course&quot;</span>,
    ]

    <span class="hljs-comment"># Add metrics</span>
    <span class="hljs-built_in">eval</span> = metadata_eval_result(
        model_pretty_name=model_name,
        task_pretty_name=<span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        task_id=<span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        metrics_pretty_name=<span class="hljs-string">&quot;mean_reward&quot;</span>,
        metrics_id=<span class="hljs-string">&quot;mean_reward&quot;</span>,
        metrics_value=<span class="hljs-string">f&quot;<span class="hljs-subst">{mean_reward:<span class="hljs-number">.2</span>f}</span> +/- <span class="hljs-subst">{std_reward:<span class="hljs-number">.2</span>f}</span>&quot;</span>,
        dataset_pretty_name=env_id,
        dataset_id=env_id,
    )

    <span class="hljs-comment"># Merges both dictionaries</span>
    metadata = {**metadata, **<span class="hljs-built_in">eval</span>}

    <span class="hljs-keyword">return</span> metadata


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_save_model_card</span>(<span class="hljs-params">local_path, generated_model_card, metadata</span>):
    <span class="hljs-string">&quot;&quot;&quot;Saves a model card for the repository.
    :param local_path: repository directory
    :param generated_model_card: model card generated by _generate_model_card()
    :param metadata: metadata
    &quot;&quot;&quot;</span>
    readme_path = local_path / <span class="hljs-string">&quot;README.md&quot;</span>
    readme = <span class="hljs-string">&quot;&quot;</span>
    <span class="hljs-keyword">if</span> readme_path.exists():
        <span class="hljs-keyword">with</span> readme_path.<span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;r&quot;</span>, encoding=<span class="hljs-string">&quot;utf8&quot;</span>) <span class="hljs-keyword">as</span> f:
            readme = f.read()
    <span class="hljs-keyword">else</span>:
        readme = generated_model_card

    <span class="hljs-keyword">with</span> readme_path.<span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;w&quot;</span>, encoding=<span class="hljs-string">&quot;utf-8&quot;</span>) <span class="hljs-keyword">as</span> f:
        f.write(readme)

    <span class="hljs-comment"># Save our metrics to Readme metadata</span>
    metadata_save(readme_path, metadata)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_add_logdir</span>(<span class="hljs-params">local_path: Path, logdir: Path</span>):
    <span class="hljs-string">&quot;&quot;&quot;Adds a logdir to the repository.
    :param local_path: repository directory
    :param logdir: logdir directory
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">if</span> logdir.exists() <span class="hljs-keyword">and</span> logdir.is_dir():
        <span class="hljs-comment"># Add the logdir to the repository under new dir called logs</span>
        repo_logdir = local_path / <span class="hljs-string">&quot;logs&quot;</span>

        <span class="hljs-comment"># Delete current logs if they exist</span>
        <span class="hljs-keyword">if</span> repo_logdir.exists():
            shutil.rmtree(repo_logdir)

        <span class="hljs-comment"># Copy logdir into repo logdir</span>
        shutil.copytree(logdir, repo_logdir)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">make_env</span>(<span class="hljs-params">env_id, seed, idx, capture_video, run_name</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">thunk</span>():
        env = gym.make(env_id)
        env = gym.wrappers.RecordEpisodeStatistics(env)
        <span class="hljs-keyword">if</span> capture_video:
            <span class="hljs-keyword">if</span> idx == <span class="hljs-number">0</span>:
                env = gym.wrappers.RecordVideo(env, <span class="hljs-string">f&quot;videos/<span class="hljs-subst">{run_name}</span>&quot;</span>)
        env.seed(seed)
        env.action_space.seed(seed)
        env.observation_space.seed(seed)
        <span class="hljs-keyword">return</span> env

    <span class="hljs-keyword">return</span> thunk


<span class="hljs-keyword">def</span> <span class="hljs-title function_">layer_init</span>(<span class="hljs-params">layer, std=np.sqrt(<span class="hljs-params"><span class="hljs-number">2</span></span>), bias_const=<span class="hljs-number">0.0</span></span>):
    torch.nn.init.orthogonal_(layer.weight, std)
    torch.nn.init.constant_(layer.bias, bias_const)
    <span class="hljs-keyword">return</span> layer


<span class="hljs-keyword">class</span> <span class="hljs-title class_">Agent</span>(nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, envs</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.critic = nn.Sequential(
            layer_init(nn.Linear(np.array(envs.single_observation_space.shape).prod(), <span class="hljs-number">64</span>)),
            nn.Tanh(),
            layer_init(nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>)),
            nn.Tanh(),
            layer_init(nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">1</span>), std=<span class="hljs-number">1.0</span>),
        )
        self.actor = nn.Sequential(
            layer_init(nn.Linear(np.array(envs.single_observation_space.shape).prod(), <span class="hljs-number">64</span>)),
            nn.Tanh(),
            layer_init(nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>)),
            nn.Tanh(),
            layer_init(nn.Linear(<span class="hljs-number">64</span>, envs.single_action_space.n), std=<span class="hljs-number">0.01</span>),
        )

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_value</span>(<span class="hljs-params">self, x</span>):
        <span class="hljs-keyword">return</span> self.critic(x)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_action_and_value</span>(<span class="hljs-params">self, x, action=<span class="hljs-literal">None</span></span>):
        logits = self.actor(x)
        probs = Categorical(logits=logits)
        <span class="hljs-keyword">if</span> action <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
            action = probs.sample()
        <span class="hljs-keyword">return</span> action, probs.log_prob(action), probs.entropy(), self.critic(x)


<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
    args = parse_args()
    run_name = <span class="hljs-string">f&quot;<span class="hljs-subst">{args.env_id}</span>__<span class="hljs-subst">{args.exp_name}</span>__<span class="hljs-subst">{args.seed}</span>__<span class="hljs-subst">{<span class="hljs-built_in">int</span>(time.time())}</span>&quot;</span>
    <span class="hljs-keyword">if</span> args.track:
        <span class="hljs-keyword">import</span> wandb

        wandb.init(
            project=args.wandb_project_name,
            entity=args.wandb_entity,
            sync_tensorboard=<span class="hljs-literal">True</span>,
            config=<span class="hljs-built_in">vars</span>(args),
            name=run_name,
            monitor_gym=<span class="hljs-literal">True</span>,
            save_code=<span class="hljs-literal">True</span>,
        )
    writer = SummaryWriter(<span class="hljs-string">f&quot;runs/<span class="hljs-subst">{run_name}</span>&quot;</span>)
    writer.add_text(
        <span class="hljs-string">&quot;hyperparameters&quot;</span>,
        <span class="hljs-string">&quot;|param|value|\\n|-|-|\\n%s&quot;</span> % (<span class="hljs-string">&quot;\\n&quot;</span>.join([<span class="hljs-string">f&quot;|<span class="hljs-subst">{key}</span>|<span class="hljs-subst">{value}</span>|&quot;</span> <span class="hljs-keyword">for</span> key, value <span class="hljs-keyword">in</span> <span class="hljs-built_in">vars</span>(args).items()])),
    )

    <span class="hljs-comment"># TRY NOT TO MODIFY: seeding</span>
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.backends.cudnn.deterministic = args.torch_deterministic

    device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">and</span> args.cuda <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>)

    <span class="hljs-comment"># env setup</span>
    envs = gym.vector.SyncVectorEnv(
        [make_env(args.env_id, args.seed + i, i, args.capture_video, run_name) <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(args.num_envs)]
    )
    <span class="hljs-keyword">assert</span> <span class="hljs-built_in">isinstance</span>(envs.single_action_space, gym.spaces.Discrete), <span class="hljs-string">&quot;only discrete action space is supported&quot;</span>

    agent = Agent(envs).to(device)
    optimizer = optim.Adam(agent.parameters(), lr=args.learning_rate, eps=<span class="hljs-number">1e-5</span>)

    <span class="hljs-comment"># ALGO Logic: Storage setup</span>
    obs = torch.zeros((args.num_steps, args.num_envs) + envs.single_observation_space.shape).to(device)
    actions = torch.zeros((args.num_steps, args.num_envs) + envs.single_action_space.shape).to(device)
    logprobs = torch.zeros((args.num_steps, args.num_envs)).to(device)
    rewards = torch.zeros((args.num_steps, args.num_envs)).to(device)
    dones = torch.zeros((args.num_steps, args.num_envs)).to(device)
    values = torch.zeros((args.num_steps, args.num_envs)).to(device)

    <span class="hljs-comment"># TRY NOT TO MODIFY: start the game</span>
    global_step = <span class="hljs-number">0</span>
    start_time = time.time()
    next_obs = torch.Tensor(envs.reset()).to(device)
    next_done = torch.zeros(args.num_envs).to(device)
    num_updates = args.total_timesteps // args.batch_size

    <span class="hljs-keyword">for</span> update <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, num_updates + <span class="hljs-number">1</span>):
        <span class="hljs-comment"># Annealing the rate if instructed to do so.</span>
        <span class="hljs-keyword">if</span> args.anneal_lr:
            frac = <span class="hljs-number">1.0</span> - (update - <span class="hljs-number">1.0</span>) / num_updates
            lrnow = frac * args.learning_rate
            optimizer.param_groups[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;lr&quot;</span>] = lrnow

        <span class="hljs-keyword">for</span> step <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, args.num_steps):
            global_step += <span class="hljs-number">1</span> * args.num_envs
            obs[step] = next_obs
            dones[step] = next_done

            <span class="hljs-comment"># ALGO LOGIC: action logic</span>
            <span class="hljs-keyword">with</span> torch.no_grad():
                action, logprob, _, value = agent.get_action_and_value(next_obs)
                values[step] = value.flatten()
            actions[step] = action
            logprobs[step] = logprob

            <span class="hljs-comment"># TRY NOT TO MODIFY: execute the game and log data.</span>
            next_obs, reward, done, info = envs.step(action.cpu().numpy())
            rewards[step] = torch.tensor(reward).to(device).view(-<span class="hljs-number">1</span>)
            next_obs, next_done = torch.Tensor(next_obs).to(device), torch.Tensor(done).to(device)

            <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> info:
                <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;episode&quot;</span> <span class="hljs-keyword">in</span> item.keys():
                    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;global_step=<span class="hljs-subst">{global_step}</span>, episodic_return=<span class="hljs-subst">{item[<span class="hljs-string">&#x27;episode&#x27;</span>][<span class="hljs-string">&#x27;r&#x27;</span>]}</span>&quot;</span>)
                    writer.add_scalar(<span class="hljs-string">&quot;charts/episodic_return&quot;</span>, item[<span class="hljs-string">&quot;episode&quot;</span>][<span class="hljs-string">&quot;r&quot;</span>], global_step)
                    writer.add_scalar(<span class="hljs-string">&quot;charts/episodic_length&quot;</span>, item[<span class="hljs-string">&quot;episode&quot;</span>][<span class="hljs-string">&quot;l&quot;</span>], global_step)
                    <span class="hljs-keyword">break</span>

        <span class="hljs-comment"># bootstrap value if not done</span>
        <span class="hljs-keyword">with</span> torch.no_grad():
            next_value = agent.get_value(next_obs).reshape(<span class="hljs-number">1</span>, -<span class="hljs-number">1</span>)
            <span class="hljs-keyword">if</span> args.gae:
                advantages = torch.zeros_like(rewards).to(device)
                lastgaelam = <span class="hljs-number">0</span>
                <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> <span class="hljs-built_in">reversed</span>(<span class="hljs-built_in">range</span>(args.num_steps)):
                    <span class="hljs-keyword">if</span> t == args.num_steps - <span class="hljs-number">1</span>:
                        nextnonterminal = <span class="hljs-number">1.0</span> - next_done
                        nextvalues = next_value
                    <span class="hljs-keyword">else</span>:
                        nextnonterminal = <span class="hljs-number">1.0</span> - dones[t + <span class="hljs-number">1</span>]
                        nextvalues = values[t + <span class="hljs-number">1</span>]
                    delta = rewards[t] + args.gamma * nextvalues * nextnonterminal - values[t]
                    advantages[t] = lastgaelam = delta + args.gamma * args.gae_lambda * nextnonterminal * lastgaelam
                returns = advantages + values
            <span class="hljs-keyword">else</span>:
                returns = torch.zeros_like(rewards).to(device)
                <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> <span class="hljs-built_in">reversed</span>(<span class="hljs-built_in">range</span>(args.num_steps)):
                    <span class="hljs-keyword">if</span> t == args.num_steps - <span class="hljs-number">1</span>:
                        nextnonterminal = <span class="hljs-number">1.0</span> - next_done
                        next_return = next_value
                    <span class="hljs-keyword">else</span>:
                        nextnonterminal = <span class="hljs-number">1.0</span> - dones[t + <span class="hljs-number">1</span>]
                        next_return = returns[t + <span class="hljs-number">1</span>]
                    returns[t] = rewards[t] + args.gamma * nextnonterminal * next_return
                advantages = returns - values

        <span class="hljs-comment"># flatten the batch</span>
        b_obs = obs.reshape((-<span class="hljs-number">1</span>,) + envs.single_observation_space.shape)
        b_logprobs = logprobs.reshape(-<span class="hljs-number">1</span>)
        b_actions = actions.reshape((-<span class="hljs-number">1</span>,) + envs.single_action_space.shape)
        b_advantages = advantages.reshape(-<span class="hljs-number">1</span>)
        b_returns = returns.reshape(-<span class="hljs-number">1</span>)
        b_values = values.reshape(-<span class="hljs-number">1</span>)

        <span class="hljs-comment"># Optimizing the policy and value network</span>
        b_inds = np.arange(args.batch_size)
        clipfracs = []
        <span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(args.update_epochs):
            np.random.shuffle(b_inds)
            <span class="hljs-keyword">for</span> start <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, args.batch_size, args.minibatch_size):
                end = start + args.minibatch_size
                mb_inds = b_inds[start:end]

                _, newlogprob, entropy, newvalue = agent.get_action_and_value(
                    b_obs[mb_inds], b_actions.long()[mb_inds]
                )
                logratio = newlogprob - b_logprobs[mb_inds]
                ratio = logratio.exp()

                <span class="hljs-keyword">with</span> torch.no_grad():
                    <span class="hljs-comment"># calculate approx_kl http://joschu.net/blog/kl-approx.html</span>
                    old_approx_kl = (-logratio).mean()
                    approx_kl = ((ratio - <span class="hljs-number">1</span>) - logratio).mean()
                    clipfracs += [((ratio - <span class="hljs-number">1.0</span>).<span class="hljs-built_in">abs</span>() &gt; args.clip_coef).<span class="hljs-built_in">float</span>().mean().item()]

                mb_advantages = b_advantages[mb_inds]
                <span class="hljs-keyword">if</span> args.norm_adv:
                    mb_advantages = (mb_advantages - mb_advantages.mean()) / (mb_advantages.std() + <span class="hljs-number">1e-8</span>)

                <span class="hljs-comment"># Policy loss</span>
                pg_loss1 = -mb_advantages * ratio
                pg_loss2 = -mb_advantages * torch.clamp(ratio, <span class="hljs-number">1</span> - args.clip_coef, <span class="hljs-number">1</span> + args.clip_coef)
                pg_loss = torch.<span class="hljs-built_in">max</span>(pg_loss1, pg_loss2).mean()

                <span class="hljs-comment"># Value loss</span>
                newvalue = newvalue.view(-<span class="hljs-number">1</span>)
                <span class="hljs-keyword">if</span> args.clip_vloss:
                    v_loss_unclipped = (newvalue - b_returns[mb_inds]) ** <span class="hljs-number">2</span>
                    v_clipped = b_values[mb_inds] + torch.clamp(
                        newvalue - b_values[mb_inds],
                        -args.clip_coef,
                        args.clip_coef,
                    )
                    v_loss_clipped = (v_clipped - b_returns[mb_inds]) ** <span class="hljs-number">2</span>
                    v_loss_max = torch.<span class="hljs-built_in">max</span>(v_loss_unclipped, v_loss_clipped)
                    v_loss = <span class="hljs-number">0.5</span> * v_loss_max.mean()
                <span class="hljs-keyword">else</span>:
                    v_loss = <span class="hljs-number">0.5</span> * ((newvalue - b_returns[mb_inds]) ** <span class="hljs-number">2</span>).mean()

                entropy_loss = entropy.mean()
                loss = pg_loss - args.ent_coef * entropy_loss + v_loss * args.vf_coef

                optimizer.zero_grad()
                loss.backward()
                nn.utils.clip_grad_norm_(agent.parameters(), args.max_grad_norm)
                optimizer.step()

            <span class="hljs-keyword">if</span> args.target_kl <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                <span class="hljs-keyword">if</span> approx_kl &gt; args.target_kl:
                    <span class="hljs-keyword">break</span>

        y_pred, y_true = b_values.cpu().numpy(), b_returns.cpu().numpy()
        var_y = np.var(y_true)
        explained_var = np.nan <span class="hljs-keyword">if</span> var_y == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-number">1</span> - np.var(y_true - y_pred) / var_y

        <span class="hljs-comment"># TRY NOT TO MODIFY: record rewards for plotting purposes</span>
        writer.add_scalar(<span class="hljs-string">&quot;charts/learning_rate&quot;</span>, optimizer.param_groups[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;lr&quot;</span>], global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/value_loss&quot;</span>, v_loss.item(), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/policy_loss&quot;</span>, pg_loss.item(), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/entropy&quot;</span>, entropy_loss.item(), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/old_approx_kl&quot;</span>, old_approx_kl.item(), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/approx_kl&quot;</span>, approx_kl.item(), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/clipfrac&quot;</span>, np.mean(clipfracs), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/explained_variance&quot;</span>, explained_var, global_step)
        <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;SPS:&quot;</span>, <span class="hljs-built_in">int</span>(global_step / (time.time() - start_time)))
        writer.add_scalar(<span class="hljs-string">&quot;charts/SPS&quot;</span>, <span class="hljs-built_in">int</span>(global_step / (time.time() - start_time)), global_step)

    envs.close()
    writer.close()

    <span class="hljs-comment"># Create the evaluation environment</span>
    eval_env = gym.make(args.env_id)

    package_to_hub(
        repo_id=args.repo_id,
        model=agent,  <span class="hljs-comment"># The model we want to save</span>
        hyperparameters=args,
        eval_env=gym.make(args.env_id),
        logs=<span class="hljs-string">f&quot;runs/<span class="hljs-subst">{run_name}</span>&quot;</span>,
    )`}}),tM=new C({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBbm90ZWJvb2tfbG9naW4oKSUwQSFnaXQlMjBjb25maWclMjAtLWdsb2JhbCUyMGNyZWRlbnRpYWwuaGVscGVyJTIwc3RvcmU=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login
notebook_login()
!git config --<span class="hljs-keyword">global</span> credential.helper store`}}),nM=new m({}),wM=new C({props:{code:"IXB5dGhvbiUyMHBwby5weSUyMC0tZW52LWlkJTNEJTIyTHVuYXJMYW5kZXItdjIlMjIlMjAtLXJlcG8taWQlM0QlMjJZT1VSX1JFUE9fSUQlMjIlMjAtLXRvdGFsLXRpbWVzdGVwcyUzRDUwMDAw",highlighted:'!python ppo.py --env-<span class="hljs-built_in">id</span>=<span class="hljs-string">&quot;LunarLander-v2&quot;</span> --repo-<span class="hljs-built_in">id</span>=<span class="hljs-string">&quot;YOUR_REPO_ID&quot;</span> --total-timesteps=<span class="hljs-number">50000</span>'}}),oM=new m({}),pM=new m({}),{c(){A=a("meta"),Ua=w(),Z=a("h1"),V=a("a"),DM=a("span"),p(ol.$$.fragment),at=w(),$M=a("span"),et=J("Hands-on"),ja=w(),p(pl.$$.fragment),wa=w(),Tl=a("p"),tt=J("Now that we studied the theory behind PPO, the best way to understand how it works\xA0"),PM=a("strong"),yt=J("is to implement it from scratch."),oa=w(),TM=a("p"),nt=J("Implementing an architecture from scratch is the best way to understand it, and it\u2019s a good habit. We have already done it for a value-based method with Q-Learning and a Policy-based method with Reinforce."),pa=w(),rM=a("p"),Jt=J("So, to be able to code it, we\u2019re going to use two resources:"),Ta=w(),F=a("ul"),B=a("li"),Ut=J("A tutorial made by "),rl=a("a"),jt=J("Costa Huang"),wt=J(". Costa is behind "),il=a("a"),ot=J("CleanRL"),pt=J(", a Deep Reinforcement Learning library that provides high-quality single-file implementation with research-friendly features."),Tt=w(),iM=a("li"),rt=J("In addition to the tutorial, to go deeper, you can read the 13 core implementation details:\xA0"),cl=a("a"),it=J("https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/"),ra=w(),cM=a("p"),ct=J("Then, to test its robustness, we\u2019re going to train it in:"),ia=w(),IM=a("ul"),OM=a("li"),Il=a("a"),It=J("LunarLander-v2"),ca=w(),dl=a("figure"),d=a("video"),dM=a("source"),Ia=w(),hM=a("p"),dt=J("And finally, we will push the trained model to the Hub to evaluate and visualize your agent playing."),da=w(),hl=a("p"),ht=J("LunarLander-v2 is the first environment you used when you started this course. At that time, you didn\u2019t know how it worked, and now you can code it from scratch and train it.\xA0"),ls=a("strong"),mt=J("How incredible is that \u{1F929}."),ha=w(),h=a("iframe"),mM=a("p"),CM=a("a"),Ct=J("via GIPHY"),ma=w(),uM=a("p"),ut=J("Let\u2019s get started! \u{1F680}"),Ca=w(),bM=a("p"),bt=J("The colab notebook:"),ua=w(),AM=a("p"),ml=a("a"),ZM=a("img"),ba=w(),G=a("h1"),N=a("a"),Ms=a("span"),p(Cl.$$.fragment),At=w(),ss=a("span"),Zt=J("Unit 8: Proximal Policy Gradient (PPO) with PyTorch \u{1F916}"),Aa=w(),ul=a("img"),Za=w(),H=a("p"),Bt=J("In this notebook, you\u2019ll learn to "),as=a("strong"),Gt=J("code your PPO agent from scratch with PyTorch using CleanRL implementation as model"),ft=J("."),Ba=w(),BM=a("p"),Wt=J("To test its robustness, we\u2019re going to train it in:"),Ga=w(),GM=a("ul"),es=a("li"),bl=a("a"),vt=J("LunarLander-v2 \u{1F680}"),fa=w(),u=a("p"),Xt=J("We\u2019re constantly trying to improve our tutorials, so "),ts=a("strong"),gt=J("if you find some issues in this notebook"),_t=J(", please "),Al=a("a"),kt=J("open an issue on the GitHub Repo"),Et=J("."),Wa=w(),f=a("h2"),z=a("a"),ys=a("span"),p(Zl.$$.fragment),Rt=w(),ns=a("span"),Yt=J("Objectives of this notebook \u{1F3C6}"),va=w(),fM=a("p"),St=J("At the end of the notebook, you will:"),Xa=w(),q=a("ul"),Bl=a("li"),Qt=J("Be able to "),Js=a("strong"),Vt=J("code your PPO agent from scratch using PyTorch"),Ft=J("."),Nt=w(),Gl=a("li"),Ht=J("Be able to "),Us=a("strong"),zt=J("push your trained agent and the code to the Hub"),qt=J(" with a nice video replay and an evaluation score \u{1F525}."),ga=w(),W=a("h2"),x=a("a"),js=a("span"),p(fl.$$.fragment),xt=w(),ws=a("span"),Lt=J("Prerequisites \u{1F3D7}\uFE0F"),_a=w(),WM=a("p"),Kt=J("Before diving into the notebook, you need to:"),ka=w(),L=a("p"),Dt=J("\u{1F532} \u{1F4DA} Study "),Wl=a("a"),$t=J("PPO by reading Unit 8"),Pt=J(" \u{1F917}"),Ea=w(),b=a("p"),Ot=J("To validate this hands-on for the "),vl=a("a"),ly=J("certification process"),My=J(", you need to push one model, we don\u2019t ask for a minimal result but we "),os=a("strong"),sy=J("advise you to try different hyperparameters settings to get better results"),ay=J("."),Ra=w(),Xl=a("p"),ey=J("If you don\u2019t find your model, "),ps=a("strong"),ty=J("go to the bottom of the page and click on the refresh button"),Ya=w(),gl=a("p"),yy=J("For more information about the certification process, check this section \u{1F449} "),_l=a("a"),ny=J("https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),Sa=w(),v=a("h2"),K=a("a"),Ts=a("span"),p(kl.$$.fragment),Jy=w(),rs=a("span"),Uy=J("Set the GPU \u{1F4AA}"),Qa=w(),vM=a("ul"),D=a("li"),jy=J("To "),is=a("strong"),wy=J("accelerate the agent\u2019s training, we\u2019ll use a GPU"),oy=J(". To do that, go to "),cs=a("code"),py=J("Runtime > Change Runtime type"),Va=w(),El=a("img"),Fa=w(),XM=a("ul"),Is=a("li"),ds=a("code"),Ty=J("Hardware Accelerator > GPU"),Na=w(),Rl=a("img"),Ha=w(),X=a("h2"),$=a("a"),hs=a("span"),p(Yl.$$.fragment),ry=w(),ms=a("span"),iy=J("Create a virtual display \u{1F53D}"),za=w(),P=a("p"),cy=J("During the notebook, we\u2019ll need to generate a replay video. To do so, with colab, "),Cs=a("strong"),Iy=J("we need to have a virtual screen to be able to render the environment"),dy=J(" (and thus record the frames)."),qa=w(),gM=a("p"),hy=J("Hence the following cell will install the librairies and create and run a virtual screen \u{1F5A5}"),xa=w(),p(Sl.$$.fragment),La=w(),p(Ql.$$.fragment),Ka=w(),g=a("h2"),O=a("a"),us=a("span"),p(Vl.$$.fragment),my=w(),bs=a("span"),Cy=J("Install dependencies \u{1F53D}"),Da=J(`

For this exercise, we use \`gym==0.21\` because the video was recorded with Gym.

	`),p(Fl.$$.fragment),$a=w(),_=a("h2"),ll=a("a"),As=a("span"),p(Nl.$$.fragment),uy=w(),Zs=a("span"),by=J("Let's code PPO from scratch with Costa Huang's tutorial"),Pa=J(`

- For the core implementation of PPO we're going to use the excellent [Costa Huang](https://costa.sh/) tutorial.
- In addition to the tutorial, to go deeper you can read the 37 core implementation details: https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/
`),Hl=a("p"),Ay=J("\u{1F449} The video tutorial: "),zl=a("a"),Zy=J("https://youtu.be/MEt6rrxH8W4"),Oa=w(),p(ql.$$.fragment),le=w(),k=a("h2"),Ml=a("a"),Bs=a("span"),p(xl.$$.fragment),By=w(),Gs=a("span"),Gy=J("Add the Hugging Face Integration \u{1F917}"),Me=J(`

- In order to push our model to the Hub, we need to define a function \`package_to_hub\`
`),_M=a("ul"),fs=a("li"),fy=J("Add dependencies we need to push our model to the Hub"),se=w(),p(Ll.$$.fragment),ae=w(),kM=a("ul"),Kl=a("li"),Wy=J("Add new argument in "),Ws=a("code"),vy=J("parse_args()"),Xy=J(" function to define the repo-id where we want to push the model."),ee=w(),p(Dl.$$.fragment),te=w(),sl=a("ul"),vs=a("li"),Xs=a("p"),gy=J("Next, we add the methods needed to push the model to the Hub"),_y=w(),$l=a("li"),gs=a("p"),ky=J("These methods will:"),Ey=w(),E=a("ul"),EM=a("li"),_s=a("code"),Ry=J("_evalutate_agent()"),Yy=J(": evaluate the agent."),Sy=w(),RM=a("li"),ks=a("code"),Qy=J("_generate_model_card()"),Vy=J(": generate the model card of your agent."),Fy=w(),YM=a("li"),Es=a("code"),Ny=J("_record_video()"),Hy=J(": record a video of your agent."),ye=w(),p(Pl.$$.fragment),ne=w(),SM=a("ul"),Rs=a("li"),zy=J("Finally, we call this function at the end of the PPO training"),Je=w(),p(Ol.$$.fragment),Ue=w(),QM=a("ul"),Ys=a("li"),qy=J("Here\u2019s what the final ppo.py file looks like:"),je=w(),p(lM.$$.fragment),we=w(),VM=a("p"),xy=J("To be able to share your model with the community there are three more steps to follow:"),oe=w(),MM=a("p"),Ly=J("1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),sM=a("a"),Ky=J("https://huggingface.co/join"),pe=w(),FM=a("p"),Dy=J("2\uFE0F\u20E3 Sign in and get your authentication token from the Hugging Face website."),Te=w(),NM=a("ul"),al=a("li"),$y=J("Create a new token ("),aM=a("a"),Py=J("https://huggingface.co/settings/tokens"),Oy=J(") "),Ss=a("strong"),ln=J("with write role"),re=w(),eM=a("img"),ie=w(),el=a("ul"),Qs=a("li"),Mn=J("Copy the token"),sn=w(),Vs=a("li"),an=J("Run the cell below and paste the token"),ce=w(),p(tM.$$.fragment),Ie=w(),yM=a("p"),en=J("If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),Fs=a("code"),tn=J("huggingface-cli login"),de=w(),R=a("h2"),tl=a("a"),Ns=a("span"),p(nM.$$.fragment),yn=w(),Hs=a("span"),nn=J("Let's start the training \u{1F525}"),he=w(),JM=a("p"),Jn=J("\u26A0\uFE0F \u26A0\uFE0F \u26A0\uFE0F  Don\u2019t use "),zs=a("strong"),Un=J("the same repo id with the one you used for the Unit 1"),me=w(),yl=a("ul"),qs=a("li"),xs=a("p"),jn=J("Now that you\u2019ve coded PPO from scratch and added the Hugging Face Integration, we\u2019re ready to start the training \u{1F525}"),wn=w(),Ls=a("li"),HM=a("p"),on=J("First, you need to copy all your code to a file you create called "),Ks=a("code"),pn=J("ppo.py"),Ce=w(),UM=a("img"),ue=w(),jM=a("img"),be=w(),nl=a("ul"),Ds=a("li"),Jl=a("p"),Tn=J("Now we just need to run this python script using "),$s=a("code"),rn=J("python <name-of-python-script>.py"),cn=J(" with the additional parameters we defined using "),Ps=a("code"),In=J("argparse"),dn=w(),Os=a("li"),la=a("p"),hn=J("You should modify more hyperparameters otherwise the training will not be super stable."),Ae=w(),p(wM.$$.fragment),Ze=w(),Y=a("h2"),Ul=a("a"),Ma=a("span"),p(oM.$$.fragment),mn=w(),sa=a("span"),Cn=J("Some additional challenges \u{1F3C6}"),Be=w(),jl=a("p"),un=J("The best way to learn "),aa=a("strong"),bn=J("is to try things on your own"),An=J("! Why not try another environment? Or why not trying to modify the implementation to work with Gymnasium?"),Ge=w(),zM=a("p"),Zn=J("See you in Unit 8, part 2 where we\u2019re going to train agents to play Doom \u{1F525}"),fe=w(),S=a("h2"),wl=a("a"),ea=a("span"),p(pM.$$.fragment),Bn=w(),ta=a("span"),Gn=J("Keep learning, stay awesome \u{1F917}"),this.h()},l(l){const y=jj('[data-svelte="svelte-1phssyn"]',document.head);A=e(y,"META",{name:!0,content:!0}),y.forEach(M),Ua=o(l),Z=e(l,"H1",{class:!0});var ve=t(Z);V=e(ve,"A",{id:!0,class:!0,href:!0});var Pn=t(V);DM=e(Pn,"SPAN",{});var On=t(DM);T(ol.$$.fragment,On),On.forEach(M),Pn.forEach(M),at=o(ve),$M=e(ve,"SPAN",{});var lJ=t($M);et=U(lJ,"Hands-on"),lJ.forEach(M),ve.forEach(M),ja=o(l),T(pl.$$.fragment,l),wa=o(l),Tl=e(l,"P",{});var fn=t(Tl);tt=U(fn,"Now that we studied the theory behind PPO, the best way to understand how it works\xA0"),PM=e(fn,"STRONG",{});var MJ=t(PM);yt=U(MJ,"is to implement it from scratch."),MJ.forEach(M),fn.forEach(M),oa=o(l),TM=e(l,"P",{});var sJ=t(TM);nt=U(sJ,"Implementing an architecture from scratch is the best way to understand it, and it\u2019s a good habit. We have already done it for a value-based method with Q-Learning and a Policy-based method with Reinforce."),sJ.forEach(M),pa=o(l),rM=e(l,"P",{});var aJ=t(rM);Jt=U(aJ,"So, to be able to code it, we\u2019re going to use two resources:"),aJ.forEach(M),Ta=o(l),F=e(l,"UL",{});var Xe=t(F);B=e(Xe,"LI",{});var qM=t(B);Ut=U(qM,"A tutorial made by "),rl=e(qM,"A",{href:!0,rel:!0});var eJ=t(rl);jt=U(eJ,"Costa Huang"),eJ.forEach(M),wt=U(qM,". Costa is behind "),il=e(qM,"A",{href:!0,rel:!0});var tJ=t(il);ot=U(tJ,"CleanRL"),tJ.forEach(M),pt=U(qM,", a Deep Reinforcement Learning library that provides high-quality single-file implementation with research-friendly features."),qM.forEach(M),Tt=o(Xe),iM=e(Xe,"LI",{});var Wn=t(iM);rt=U(Wn,"In addition to the tutorial, to go deeper, you can read the 13 core implementation details:\xA0"),cl=e(Wn,"A",{href:!0,rel:!0});var yJ=t(cl);it=U(yJ,"https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/"),yJ.forEach(M),Wn.forEach(M),Xe.forEach(M),ra=o(l),cM=e(l,"P",{});var nJ=t(cM);ct=U(nJ,"Then, to test its robustness, we\u2019re going to train it in:"),nJ.forEach(M),ia=o(l),IM=e(l,"UL",{});var JJ=t(IM);OM=e(JJ,"LI",{});var UJ=t(OM);Il=e(UJ,"A",{href:!0,rel:!0});var jJ=t(Il);It=U(jJ,"LunarLander-v2"),jJ.forEach(M),UJ.forEach(M),JJ.forEach(M),ca=o(l),dl=e(l,"FIGURE",{class:!0});var wJ=t(dl);d=e(wJ,"VIDEO",{alt:!0,style:!0,autobuffer:!0});var oJ=t(d);dM=e(oJ,"SOURCE",{src:!0,type:!0}),oJ.forEach(M),wJ.forEach(M),Ia=o(l),hM=e(l,"P",{});var pJ=t(hM);dt=U(pJ,"And finally, we will push the trained model to the Hub to evaluate and visualize your agent playing."),pJ.forEach(M),da=o(l),hl=e(l,"P",{});var vn=t(hl);ht=U(vn,"LunarLander-v2 is the first environment you used when you started this course. At that time, you didn\u2019t know how it worked, and now you can code it from scratch and train it.\xA0"),ls=e(vn,"STRONG",{});var TJ=t(ls);mt=U(TJ,"How incredible is that \u{1F929}."),TJ.forEach(M),vn.forEach(M),ha=o(l),h=e(l,"IFRAME",{src:!0,width:!0,height:!0,frameborder:!0,class:!0}),t(h).forEach(M),mM=e(l,"P",{});var rJ=t(mM);CM=e(rJ,"A",{href:!0});var iJ=t(CM);Ct=U(iJ,"via GIPHY"),iJ.forEach(M),rJ.forEach(M),ma=o(l),uM=e(l,"P",{});var cJ=t(uM);ut=U(cJ,"Let\u2019s get started! \u{1F680}"),cJ.forEach(M),Ca=o(l),bM=e(l,"P",{});var IJ=t(bM);bt=U(IJ,"The colab notebook:"),IJ.forEach(M),ua=o(l),AM=e(l,"P",{});var dJ=t(AM);ml=e(dJ,"A",{href:!0,rel:!0});var hJ=t(ml);ZM=e(hJ,"IMG",{src:!0,alt:!0}),hJ.forEach(M),dJ.forEach(M),ba=o(l),G=e(l,"H1",{class:!0});var ge=t(G);N=e(ge,"A",{id:!0,class:!0,href:!0});var mJ=t(N);Ms=e(mJ,"SPAN",{});var CJ=t(Ms);T(Cl.$$.fragment,CJ),CJ.forEach(M),mJ.forEach(M),At=o(ge),ss=e(ge,"SPAN",{});var uJ=t(ss);Zt=U(uJ,"Unit 8: Proximal Policy Gradient (PPO) with PyTorch \u{1F916}"),uJ.forEach(M),ge.forEach(M),Aa=o(l),ul=e(l,"IMG",{src:!0,alt:!0}),Za=o(l),H=e(l,"P",{});var _e=t(H);Bt=U(_e,"In this notebook, you\u2019ll learn to "),as=e(_e,"STRONG",{});var bJ=t(as);Gt=U(bJ,"code your PPO agent from scratch with PyTorch using CleanRL implementation as model"),bJ.forEach(M),ft=U(_e,"."),_e.forEach(M),Ba=o(l),BM=e(l,"P",{});var AJ=t(BM);Wt=U(AJ,"To test its robustness, we\u2019re going to train it in:"),AJ.forEach(M),Ga=o(l),GM=e(l,"UL",{});var ZJ=t(GM);es=e(ZJ,"LI",{});var BJ=t(es);bl=e(BJ,"A",{href:!0,rel:!0});var GJ=t(bl);vt=U(GJ,"LunarLander-v2 \u{1F680}"),GJ.forEach(M),BJ.forEach(M),ZJ.forEach(M),fa=o(l),u=e(l,"P",{});var xM=t(u);Xt=U(xM,"We\u2019re constantly trying to improve our tutorials, so "),ts=e(xM,"STRONG",{});var fJ=t(ts);gt=U(fJ,"if you find some issues in this notebook"),fJ.forEach(M),_t=U(xM,", please "),Al=e(xM,"A",{href:!0,rel:!0});var WJ=t(Al);kt=U(WJ,"open an issue on the GitHub Repo"),WJ.forEach(M),Et=U(xM,"."),xM.forEach(M),Wa=o(l),f=e(l,"H2",{class:!0});var ke=t(f);z=e(ke,"A",{id:!0,class:!0,href:!0});var vJ=t(z);ys=e(vJ,"SPAN",{});var XJ=t(ys);T(Zl.$$.fragment,XJ),XJ.forEach(M),vJ.forEach(M),Rt=o(ke),ns=e(ke,"SPAN",{});var gJ=t(ns);Yt=U(gJ,"Objectives of this notebook \u{1F3C6}"),gJ.forEach(M),ke.forEach(M),va=o(l),fM=e(l,"P",{});var _J=t(fM);St=U(_J,"At the end of the notebook, you will:"),_J.forEach(M),Xa=o(l),q=e(l,"UL",{});var Ee=t(q);Bl=e(Ee,"LI",{});var Re=t(Bl);Qt=U(Re,"Be able to "),Js=e(Re,"STRONG",{});var kJ=t(Js);Vt=U(kJ,"code your PPO agent from scratch using PyTorch"),kJ.forEach(M),Ft=U(Re,"."),Re.forEach(M),Nt=o(Ee),Gl=e(Ee,"LI",{});var Ye=t(Gl);Ht=U(Ye,"Be able to "),Us=e(Ye,"STRONG",{});var EJ=t(Us);zt=U(EJ,"push your trained agent and the code to the Hub"),EJ.forEach(M),qt=U(Ye," with a nice video replay and an evaluation score \u{1F525}."),Ye.forEach(M),Ee.forEach(M),ga=o(l),W=e(l,"H2",{class:!0});var Se=t(W);x=e(Se,"A",{id:!0,class:!0,href:!0});var RJ=t(x);js=e(RJ,"SPAN",{});var YJ=t(js);T(fl.$$.fragment,YJ),YJ.forEach(M),RJ.forEach(M),xt=o(Se),ws=e(Se,"SPAN",{});var SJ=t(ws);Lt=U(SJ,"Prerequisites \u{1F3D7}\uFE0F"),SJ.forEach(M),Se.forEach(M),_a=o(l),WM=e(l,"P",{});var QJ=t(WM);Kt=U(QJ,"Before diving into the notebook, you need to:"),QJ.forEach(M),ka=o(l),L=e(l,"P",{});var Qe=t(L);Dt=U(Qe,"\u{1F532} \u{1F4DA} Study "),Wl=e(Qe,"A",{href:!0,rel:!0});var VJ=t(Wl);$t=U(VJ,"PPO by reading Unit 8"),VJ.forEach(M),Pt=U(Qe," \u{1F917}"),Qe.forEach(M),Ea=o(l),b=e(l,"P",{});var LM=t(b);Ot=U(LM,"To validate this hands-on for the "),vl=e(LM,"A",{href:!0,rel:!0});var FJ=t(vl);ly=U(FJ,"certification process"),FJ.forEach(M),My=U(LM,", you need to push one model, we don\u2019t ask for a minimal result but we "),os=e(LM,"STRONG",{});var NJ=t(os);sy=U(NJ,"advise you to try different hyperparameters settings to get better results"),NJ.forEach(M),ay=U(LM,"."),LM.forEach(M),Ra=o(l),Xl=e(l,"P",{});var Xn=t(Xl);ey=U(Xn,"If you don\u2019t find your model, "),ps=e(Xn,"STRONG",{});var HJ=t(ps);ty=U(HJ,"go to the bottom of the page and click on the refresh button"),HJ.forEach(M),Xn.forEach(M),Ya=o(l),gl=e(l,"P",{});var gn=t(gl);yy=U(gn,"For more information about the certification process, check this section \u{1F449} "),_l=e(gn,"A",{href:!0,rel:!0});var zJ=t(_l);ny=U(zJ,"https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),zJ.forEach(M),gn.forEach(M),Sa=o(l),v=e(l,"H2",{class:!0});var Ve=t(v);K=e(Ve,"A",{id:!0,class:!0,href:!0});var qJ=t(K);Ts=e(qJ,"SPAN",{});var xJ=t(Ts);T(kl.$$.fragment,xJ),xJ.forEach(M),qJ.forEach(M),Jy=o(Ve),rs=e(Ve,"SPAN",{});var LJ=t(rs);Uy=U(LJ,"Set the GPU \u{1F4AA}"),LJ.forEach(M),Ve.forEach(M),Qa=o(l),vM=e(l,"UL",{});var KJ=t(vM);D=e(KJ,"LI",{});var ya=t(D);jy=U(ya,"To "),is=e(ya,"STRONG",{});var DJ=t(is);wy=U(DJ,"accelerate the agent\u2019s training, we\u2019ll use a GPU"),DJ.forEach(M),oy=U(ya,". To do that, go to "),cs=e(ya,"CODE",{});var $J=t(cs);py=U($J,"Runtime > Change Runtime type"),$J.forEach(M),ya.forEach(M),KJ.forEach(M),Va=o(l),El=e(l,"IMG",{src:!0,alt:!0}),Fa=o(l),XM=e(l,"UL",{});var PJ=t(XM);Is=e(PJ,"LI",{});var OJ=t(Is);ds=e(OJ,"CODE",{});var lU=t(ds);Ty=U(lU,"Hardware Accelerator > GPU"),lU.forEach(M),OJ.forEach(M),PJ.forEach(M),Na=o(l),Rl=e(l,"IMG",{src:!0,alt:!0}),Ha=o(l),X=e(l,"H2",{class:!0});var Fe=t(X);$=e(Fe,"A",{id:!0,class:!0,href:!0});var MU=t($);hs=e(MU,"SPAN",{});var sU=t(hs);T(Yl.$$.fragment,sU),sU.forEach(M),MU.forEach(M),ry=o(Fe),ms=e(Fe,"SPAN",{});var aU=t(ms);iy=U(aU,"Create a virtual display \u{1F53D}"),aU.forEach(M),Fe.forEach(M),za=o(l),P=e(l,"P",{});var Ne=t(P);cy=U(Ne,"During the notebook, we\u2019ll need to generate a replay video. To do so, with colab, "),Cs=e(Ne,"STRONG",{});var eU=t(Cs);Iy=U(eU,"we need to have a virtual screen to be able to render the environment"),eU.forEach(M),dy=U(Ne," (and thus record the frames)."),Ne.forEach(M),qa=o(l),gM=e(l,"P",{});var tU=t(gM);hy=U(tU,"Hence the following cell will install the librairies and create and run a virtual screen \u{1F5A5}"),tU.forEach(M),xa=o(l),T(Sl.$$.fragment,l),La=o(l),T(Ql.$$.fragment,l),Ka=o(l),g=e(l,"H2",{class:!0});var He=t(g);O=e(He,"A",{id:!0,class:!0,href:!0});var yU=t(O);us=e(yU,"SPAN",{});var nU=t(us);T(Vl.$$.fragment,nU),nU.forEach(M),yU.forEach(M),my=o(He),bs=e(He,"SPAN",{});var JU=t(bs);Cy=U(JU,"Install dependencies \u{1F53D}"),JU.forEach(M),He.forEach(M),Da=U(l,`

For this exercise, we use \`gym==0.21\` because the video was recorded with Gym.

	`),T(Fl.$$.fragment,l),$a=o(l),_=e(l,"H2",{class:!0});var ze=t(_);ll=e(ze,"A",{id:!0,class:!0,href:!0});var UU=t(ll);As=e(UU,"SPAN",{});var jU=t(As);T(Nl.$$.fragment,jU),jU.forEach(M),UU.forEach(M),uy=o(ze),Zs=e(ze,"SPAN",{});var wU=t(Zs);by=U(wU,"Let's code PPO from scratch with Costa Huang's tutorial"),wU.forEach(M),ze.forEach(M),Pa=U(l,`

- For the core implementation of PPO we're going to use the excellent [Costa Huang](https://costa.sh/) tutorial.
- In addition to the tutorial, to go deeper you can read the 37 core implementation details: https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/
`),Hl=e(l,"P",{});var _n=t(Hl);Ay=U(_n,"\u{1F449} The video tutorial: "),zl=e(_n,"A",{href:!0,rel:!0});var oU=t(zl);Zy=U(oU,"https://youtu.be/MEt6rrxH8W4"),oU.forEach(M),_n.forEach(M),Oa=o(l),T(ql.$$.fragment,l),le=o(l),k=e(l,"H2",{class:!0});var qe=t(k);Ml=e(qe,"A",{id:!0,class:!0,href:!0});var pU=t(Ml);Bs=e(pU,"SPAN",{});var TU=t(Bs);T(xl.$$.fragment,TU),TU.forEach(M),pU.forEach(M),By=o(qe),Gs=e(qe,"SPAN",{});var rU=t(Gs);Gy=U(rU,"Add the Hugging Face Integration \u{1F917}"),rU.forEach(M),qe.forEach(M),Me=U(l,`

- In order to push our model to the Hub, we need to define a function \`package_to_hub\`
`),_M=e(l,"UL",{});var iU=t(_M);fs=e(iU,"LI",{});var cU=t(fs);fy=U(cU,"Add dependencies we need to push our model to the Hub"),cU.forEach(M),iU.forEach(M),se=o(l),T(Ll.$$.fragment,l),ae=o(l),kM=e(l,"UL",{});var IU=t(kM);Kl=e(IU,"LI",{});var xe=t(Kl);Wy=U(xe,"Add new argument in "),Ws=e(xe,"CODE",{});var dU=t(Ws);vy=U(dU,"parse_args()"),dU.forEach(M),Xy=U(xe," function to define the repo-id where we want to push the model."),xe.forEach(M),IU.forEach(M),ee=o(l),T(Dl.$$.fragment,l),te=o(l),sl=e(l,"UL",{});var Le=t(sl);vs=e(Le,"LI",{});var hU=t(vs);Xs=e(hU,"P",{});var mU=t(Xs);gy=U(mU,"Next, we add the methods needed to push the model to the Hub"),mU.forEach(M),hU.forEach(M),_y=o(Le),$l=e(Le,"LI",{});var Ke=t($l);gs=e(Ke,"P",{});var CU=t(gs);ky=U(CU,"These methods will:"),CU.forEach(M),Ey=o(Ke),E=e(Ke,"UL",{});var KM=t(E);EM=e(KM,"LI",{});var kn=t(EM);_s=e(kn,"CODE",{});var uU=t(_s);Ry=U(uU,"_evalutate_agent()"),uU.forEach(M),Yy=U(kn,": evaluate the agent."),kn.forEach(M),Sy=o(KM),RM=e(KM,"LI",{});var En=t(RM);ks=e(En,"CODE",{});var bU=t(ks);Qy=U(bU,"_generate_model_card()"),bU.forEach(M),Vy=U(En,": generate the model card of your agent."),En.forEach(M),Fy=o(KM),YM=e(KM,"LI",{});var Rn=t(YM);Es=e(Rn,"CODE",{});var AU=t(Es);Ny=U(AU,"_record_video()"),AU.forEach(M),Hy=U(Rn,": record a video of your agent."),Rn.forEach(M),KM.forEach(M),Ke.forEach(M),Le.forEach(M),ye=o(l),T(Pl.$$.fragment,l),ne=o(l),SM=e(l,"UL",{});var ZU=t(SM);Rs=e(ZU,"LI",{});var BU=t(Rs);zy=U(BU,"Finally, we call this function at the end of the PPO training"),BU.forEach(M),ZU.forEach(M),Je=o(l),T(Ol.$$.fragment,l),Ue=o(l),QM=e(l,"UL",{});var GU=t(QM);Ys=e(GU,"LI",{});var fU=t(Ys);qy=U(fU,"Here\u2019s what the final ppo.py file looks like:"),fU.forEach(M),GU.forEach(M),je=o(l),T(lM.$$.fragment,l),we=o(l),VM=e(l,"P",{});var WU=t(VM);xy=U(WU,"To be able to share your model with the community there are three more steps to follow:"),WU.forEach(M),oe=o(l),MM=e(l,"P",{});var Yn=t(MM);Ly=U(Yn,"1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),sM=e(Yn,"A",{href:!0,rel:!0});var vU=t(sM);Ky=U(vU,"https://huggingface.co/join"),vU.forEach(M),Yn.forEach(M),pe=o(l),FM=e(l,"P",{});var XU=t(FM);Dy=U(XU,"2\uFE0F\u20E3 Sign in and get your authentication token from the Hugging Face website."),XU.forEach(M),Te=o(l),NM=e(l,"UL",{});var gU=t(NM);al=e(gU,"LI",{});var na=t(al);$y=U(na,"Create a new token ("),aM=e(na,"A",{href:!0,rel:!0});var _U=t(aM);Py=U(_U,"https://huggingface.co/settings/tokens"),_U.forEach(M),Oy=U(na,") "),Ss=e(na,"STRONG",{});var kU=t(Ss);ln=U(kU,"with write role"),kU.forEach(M),na.forEach(M),gU.forEach(M),re=o(l),eM=e(l,"IMG",{src:!0,alt:!0}),ie=o(l),el=e(l,"UL",{});var De=t(el);Qs=e(De,"LI",{});var EU=t(Qs);Mn=U(EU,"Copy the token"),EU.forEach(M),sn=o(De),Vs=e(De,"LI",{});var RU=t(Vs);an=U(RU,"Run the cell below and paste the token"),RU.forEach(M),De.forEach(M),ce=o(l),T(tM.$$.fragment,l),Ie=o(l),yM=e(l,"P",{});var Sn=t(yM);en=U(Sn,"If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),Fs=e(Sn,"CODE",{});var YU=t(Fs);tn=U(YU,"huggingface-cli login"),YU.forEach(M),Sn.forEach(M),de=o(l),R=e(l,"H2",{class:!0});var $e=t(R);tl=e($e,"A",{id:!0,class:!0,href:!0});var SU=t(tl);Ns=e(SU,"SPAN",{});var QU=t(Ns);T(nM.$$.fragment,QU),QU.forEach(M),SU.forEach(M),yn=o($e),Hs=e($e,"SPAN",{});var VU=t(Hs);nn=U(VU,"Let's start the training \u{1F525}"),VU.forEach(M),$e.forEach(M),he=o(l),JM=e(l,"P",{});var Qn=t(JM);Jn=U(Qn,"\u26A0\uFE0F \u26A0\uFE0F \u26A0\uFE0F  Don\u2019t use "),zs=e(Qn,"STRONG",{});var FU=t(zs);Un=U(FU,"the same repo id with the one you used for the Unit 1"),FU.forEach(M),Qn.forEach(M),me=o(l),yl=e(l,"UL",{});var Pe=t(yl);qs=e(Pe,"LI",{});var NU=t(qs);xs=e(NU,"P",{});var HU=t(xs);jn=U(HU,"Now that you\u2019ve coded PPO from scratch and added the Hugging Face Integration, we\u2019re ready to start the training \u{1F525}"),HU.forEach(M),NU.forEach(M),wn=o(Pe),Ls=e(Pe,"LI",{});var zU=t(Ls);HM=e(zU,"P",{});var Vn=t(HM);on=U(Vn,"First, you need to copy all your code to a file you create called "),Ks=e(Vn,"CODE",{});var qU=t(Ks);pn=U(qU,"ppo.py"),qU.forEach(M),Vn.forEach(M),zU.forEach(M),Pe.forEach(M),Ce=o(l),UM=e(l,"IMG",{src:!0,alt:!0}),ue=o(l),jM=e(l,"IMG",{src:!0,alt:!0}),be=o(l),nl=e(l,"UL",{});var Oe=t(nl);Ds=e(Oe,"LI",{});var xU=t(Ds);Jl=e(xU,"P",{});var Ja=t(Jl);Tn=U(Ja,"Now we just need to run this python script using "),$s=e(Ja,"CODE",{});var LU=t($s);rn=U(LU,"python <name-of-python-script>.py"),LU.forEach(M),cn=U(Ja," with the additional parameters we defined using "),Ps=e(Ja,"CODE",{});var KU=t(Ps);In=U(KU,"argparse"),KU.forEach(M),Ja.forEach(M),xU.forEach(M),dn=o(Oe),Os=e(Oe,"LI",{});var DU=t(Os);la=e(DU,"P",{});var $U=t(la);hn=U($U,"You should modify more hyperparameters otherwise the training will not be super stable."),$U.forEach(M),DU.forEach(M),Oe.forEach(M),Ae=o(l),T(wM.$$.fragment,l),Ze=o(l),Y=e(l,"H2",{class:!0});var lt=t(Y);Ul=e(lt,"A",{id:!0,class:!0,href:!0});var PU=t(Ul);Ma=e(PU,"SPAN",{});var OU=t(Ma);T(oM.$$.fragment,OU),OU.forEach(M),PU.forEach(M),mn=o(lt),sa=e(lt,"SPAN",{});var lj=t(sa);Cn=U(lj,"Some additional challenges \u{1F3C6}"),lj.forEach(M),lt.forEach(M),Be=o(l),jl=e(l,"P",{});var Mt=t(jl);un=U(Mt,"The best way to learn "),aa=e(Mt,"STRONG",{});var Mj=t(aa);bn=U(Mj,"is to try things on your own"),Mj.forEach(M),An=U(Mt,"! Why not try another environment? Or why not trying to modify the implementation to work with Gymnasium?"),Mt.forEach(M),Ge=o(l),zM=e(l,"P",{});var sj=t(zM);Zn=U(sj,"See you in Unit 8, part 2 where we\u2019re going to train agents to play Doom \u{1F525}"),sj.forEach(M),fe=o(l),S=e(l,"H2",{class:!0});var st=t(S);wl=e(st,"A",{id:!0,class:!0,href:!0});var aj=t(wl);ea=e(aj,"SPAN",{});var ej=t(ea);T(pM.$$.fragment,ej),ej.forEach(M),aj.forEach(M),Bn=o(st),ta=e(st,"SPAN",{});var tj=t(ta);Gn=U(tj,"Keep learning, stay awesome \u{1F917}"),tj.forEach(M),st.forEach(M),this.h()},h(){j(A,"name","hf:doc:metadata"),j(A,"content",JSON.stringify(rj)),j(V,"id","handson"),j(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(V,"href","#handson"),j(Z,"class","relative group"),j(rl,"href","https://github.com/vwxyzjn"),j(rl,"rel","nofollow"),j(il,"href","https://github.com/vwxyzjn/cleanrl"),j(il,"rel","nofollow"),j(cl,"href","https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/"),j(cl,"rel","nofollow"),j(Il,"href","https://www.gymlibrary.ml/environments/box2d/lunar_lander/"),j(Il,"rel","nofollow"),Q(dM.src,Nn="assets/63_deep_rl_intro/lunarlander.mp4")||j(dM,"src",Nn),j(dM,"type","video/mp4"),j(d,"alt","LunarLander"),yj(d,"max-width","70%"),yj(d,"margin","auto"),d.autoplay=!0,d.loop=!0,j(d,"autobuffer",""),d.muted=!0,d.playsInline=!0,j(dl,"class","image table text-center m-0 w-full"),Q(h.src,Hn="https://giphy.com/embed/pynZagVcYxVUk")||j(h,"src",Hn),j(h,"width","480"),j(h,"height","480"),j(h,"frameborder","0"),j(h,"class","giphy-embed"),h.allowFullscreen=!0,j(CM,"href","https://giphy.com/gifs/the-office-michael-heartbreak-pynZagVcYxVUk"),Q(ZM.src,zn="https://colab.research.google.com/assets/colab-badge.svg")||j(ZM,"src",zn),j(ZM,"alt","Open In Colab"),j(ml,"href","https://colab.research.google.com/github/huggingface/deep-rl-class/blob/master/notebooks/unit8/unit8_part1.ipynb"),j(ml,"rel","nofollow"),j(N,"id","unit-8-proximal-policy-gradient-ppo-with-pytorch"),j(N,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(N,"href","#unit-8-proximal-policy-gradient-ppo-with-pytorch"),j(G,"class","relative group"),Q(ul.src,qn="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit9/thumbnail.png")||j(ul,"src",qn),j(ul,"alt","Unit 8"),j(bl,"href","https://www.gymlibrary.dev/environments/box2d/lunar_lander/"),j(bl,"rel","nofollow"),j(Al,"href","https://github.com/huggingface/deep-rl-class/issues"),j(Al,"rel","nofollow"),j(z,"id","objectives-of-this-notebook"),j(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(z,"href","#objectives-of-this-notebook"),j(f,"class","relative group"),j(x,"id","prerequisites"),j(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(x,"href","#prerequisites"),j(W,"class","relative group"),j(Wl,"href","https://huggingface.co/deep-rl-course/unit8/introduction"),j(Wl,"rel","nofollow"),j(vl,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),j(vl,"rel","nofollow"),j(_l,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),j(_l,"rel","nofollow"),j(K,"id","set-the-gpu"),j(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(K,"href","#set-the-gpu"),j(v,"class","relative group"),Q(El.src,xn="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step1.jpg")||j(El,"src",xn),j(El,"alt","GPU Step 1"),Q(Rl.src,Ln="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step2.jpg")||j(Rl,"src",Ln),j(Rl,"alt","GPU Step 2"),j($,"id","create-a-virtual-display"),j($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j($,"href","#create-a-virtual-display"),j(X,"class","relative group"),j(O,"id","install-dependencies"),j(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(O,"href","#install-dependencies"),j(g,"class","relative group"),j(ll,"id","lets-code-ppo-from-scratch-with-costa-huangs-tutorial"),j(ll,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(ll,"href","#lets-code-ppo-from-scratch-with-costa-huangs-tutorial"),j(_,"class","relative group"),j(zl,"href","https://youtu.be/MEt6rrxH8W4"),j(zl,"rel","nofollow"),j(Ml,"id","add-the-hugging-face-integration"),j(Ml,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(Ml,"href","#add-the-hugging-face-integration"),j(k,"class","relative group"),j(sM,"href","https://huggingface.co/join"),j(sM,"rel","nofollow"),j(aM,"href","https://huggingface.co/settings/tokens"),j(aM,"rel","nofollow"),Q(eM.src,Kn="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/create-token.jpg")||j(eM,"src",Kn),j(eM,"alt","Create HF Token"),j(tl,"id","lets-start-the-training"),j(tl,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(tl,"href","#lets-start-the-training"),j(R,"class","relative group"),Q(UM.src,Dn="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit9/step1.png")||j(UM,"src",Dn),j(UM,"alt","PPO"),Q(jM.src,$n="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit9/step2.png")||j(jM,"src",$n),j(jM,"alt","PPO"),j(Ul,"id","some-additional-challenges"),j(Ul,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(Ul,"href","#some-additional-challenges"),j(Y,"class","relative group"),j(wl,"id","keep-learning-stay-awesome"),j(wl,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(wl,"href","#keep-learning-stay-awesome"),j(S,"class","relative group")},m(l,y){s(document.head,A),n(l,Ua,y),n(l,Z,y),s(Z,V),s(V,DM),r(ol,DM,null),s(Z,at),s(Z,$M),s($M,et),n(l,ja,y),r(pl,l,y),n(l,wa,y),n(l,Tl,y),s(Tl,tt),s(Tl,PM),s(PM,yt),n(l,oa,y),n(l,TM,y),s(TM,nt),n(l,pa,y),n(l,rM,y),s(rM,Jt),n(l,Ta,y),n(l,F,y),s(F,B),s(B,Ut),s(B,rl),s(rl,jt),s(B,wt),s(B,il),s(il,ot),s(B,pt),s(F,Tt),s(F,iM),s(iM,rt),s(iM,cl),s(cl,it),n(l,ra,y),n(l,cM,y),s(cM,ct),n(l,ia,y),n(l,IM,y),s(IM,OM),s(OM,Il),s(Il,It),n(l,ca,y),n(l,dl,y),s(dl,d),s(d,dM),n(l,Ia,y),n(l,hM,y),s(hM,dt),n(l,da,y),n(l,hl,y),s(hl,ht),s(hl,ls),s(ls,mt),n(l,ha,y),n(l,h,y),n(l,mM,y),s(mM,CM),s(CM,Ct),n(l,ma,y),n(l,uM,y),s(uM,ut),n(l,Ca,y),n(l,bM,y),s(bM,bt),n(l,ua,y),n(l,AM,y),s(AM,ml),s(ml,ZM),n(l,ba,y),n(l,G,y),s(G,N),s(N,Ms),r(Cl,Ms,null),s(G,At),s(G,ss),s(ss,Zt),n(l,Aa,y),n(l,ul,y),n(l,Za,y),n(l,H,y),s(H,Bt),s(H,as),s(as,Gt),s(H,ft),n(l,Ba,y),n(l,BM,y),s(BM,Wt),n(l,Ga,y),n(l,GM,y),s(GM,es),s(es,bl),s(bl,vt),n(l,fa,y),n(l,u,y),s(u,Xt),s(u,ts),s(ts,gt),s(u,_t),s(u,Al),s(Al,kt),s(u,Et),n(l,Wa,y),n(l,f,y),s(f,z),s(z,ys),r(Zl,ys,null),s(f,Rt),s(f,ns),s(ns,Yt),n(l,va,y),n(l,fM,y),s(fM,St),n(l,Xa,y),n(l,q,y),s(q,Bl),s(Bl,Qt),s(Bl,Js),s(Js,Vt),s(Bl,Ft),s(q,Nt),s(q,Gl),s(Gl,Ht),s(Gl,Us),s(Us,zt),s(Gl,qt),n(l,ga,y),n(l,W,y),s(W,x),s(x,js),r(fl,js,null),s(W,xt),s(W,ws),s(ws,Lt),n(l,_a,y),n(l,WM,y),s(WM,Kt),n(l,ka,y),n(l,L,y),s(L,Dt),s(L,Wl),s(Wl,$t),s(L,Pt),n(l,Ea,y),n(l,b,y),s(b,Ot),s(b,vl),s(vl,ly),s(b,My),s(b,os),s(os,sy),s(b,ay),n(l,Ra,y),n(l,Xl,y),s(Xl,ey),s(Xl,ps),s(ps,ty),n(l,Ya,y),n(l,gl,y),s(gl,yy),s(gl,_l),s(_l,ny),n(l,Sa,y),n(l,v,y),s(v,K),s(K,Ts),r(kl,Ts,null),s(v,Jy),s(v,rs),s(rs,Uy),n(l,Qa,y),n(l,vM,y),s(vM,D),s(D,jy),s(D,is),s(is,wy),s(D,oy),s(D,cs),s(cs,py),n(l,Va,y),n(l,El,y),n(l,Fa,y),n(l,XM,y),s(XM,Is),s(Is,ds),s(ds,Ty),n(l,Na,y),n(l,Rl,y),n(l,Ha,y),n(l,X,y),s(X,$),s($,hs),r(Yl,hs,null),s(X,ry),s(X,ms),s(ms,iy),n(l,za,y),n(l,P,y),s(P,cy),s(P,Cs),s(Cs,Iy),s(P,dy),n(l,qa,y),n(l,gM,y),s(gM,hy),n(l,xa,y),r(Sl,l,y),n(l,La,y),r(Ql,l,y),n(l,Ka,y),n(l,g,y),s(g,O),s(O,us),r(Vl,us,null),s(g,my),s(g,bs),s(bs,Cy),n(l,Da,y),r(Fl,l,y),n(l,$a,y),n(l,_,y),s(_,ll),s(ll,As),r(Nl,As,null),s(_,uy),s(_,Zs),s(Zs,by),n(l,Pa,y),n(l,Hl,y),s(Hl,Ay),s(Hl,zl),s(zl,Zy),n(l,Oa,y),r(ql,l,y),n(l,le,y),n(l,k,y),s(k,Ml),s(Ml,Bs),r(xl,Bs,null),s(k,By),s(k,Gs),s(Gs,Gy),n(l,Me,y),n(l,_M,y),s(_M,fs),s(fs,fy),n(l,se,y),r(Ll,l,y),n(l,ae,y),n(l,kM,y),s(kM,Kl),s(Kl,Wy),s(Kl,Ws),s(Ws,vy),s(Kl,Xy),n(l,ee,y),r(Dl,l,y),n(l,te,y),n(l,sl,y),s(sl,vs),s(vs,Xs),s(Xs,gy),s(sl,_y),s(sl,$l),s($l,gs),s(gs,ky),s($l,Ey),s($l,E),s(E,EM),s(EM,_s),s(_s,Ry),s(EM,Yy),s(E,Sy),s(E,RM),s(RM,ks),s(ks,Qy),s(RM,Vy),s(E,Fy),s(E,YM),s(YM,Es),s(Es,Ny),s(YM,Hy),n(l,ye,y),r(Pl,l,y),n(l,ne,y),n(l,SM,y),s(SM,Rs),s(Rs,zy),n(l,Je,y),r(Ol,l,y),n(l,Ue,y),n(l,QM,y),s(QM,Ys),s(Ys,qy),n(l,je,y),r(lM,l,y),n(l,we,y),n(l,VM,y),s(VM,xy),n(l,oe,y),n(l,MM,y),s(MM,Ly),s(MM,sM),s(sM,Ky),n(l,pe,y),n(l,FM,y),s(FM,Dy),n(l,Te,y),n(l,NM,y),s(NM,al),s(al,$y),s(al,aM),s(aM,Py),s(al,Oy),s(al,Ss),s(Ss,ln),n(l,re,y),n(l,eM,y),n(l,ie,y),n(l,el,y),s(el,Qs),s(Qs,Mn),s(el,sn),s(el,Vs),s(Vs,an),n(l,ce,y),r(tM,l,y),n(l,Ie,y),n(l,yM,y),s(yM,en),s(yM,Fs),s(Fs,tn),n(l,de,y),n(l,R,y),s(R,tl),s(tl,Ns),r(nM,Ns,null),s(R,yn),s(R,Hs),s(Hs,nn),n(l,he,y),n(l,JM,y),s(JM,Jn),s(JM,zs),s(zs,Un),n(l,me,y),n(l,yl,y),s(yl,qs),s(qs,xs),s(xs,jn),s(yl,wn),s(yl,Ls),s(Ls,HM),s(HM,on),s(HM,Ks),s(Ks,pn),n(l,Ce,y),n(l,UM,y),n(l,ue,y),n(l,jM,y),n(l,be,y),n(l,nl,y),s(nl,Ds),s(Ds,Jl),s(Jl,Tn),s(Jl,$s),s($s,rn),s(Jl,cn),s(Jl,Ps),s(Ps,In),s(nl,dn),s(nl,Os),s(Os,la),s(la,hn),n(l,Ae,y),r(wM,l,y),n(l,Ze,y),n(l,Y,y),s(Y,Ul),s(Ul,Ma),r(oM,Ma,null),s(Y,mn),s(Y,sa),s(sa,Cn),n(l,Be,y),n(l,jl,y),s(jl,un),s(jl,aa),s(aa,bn),s(jl,An),n(l,Ge,y),n(l,zM,y),s(zM,Zn),n(l,fe,y),n(l,S,y),s(S,wl),s(wl,ea),r(pM,ea,null),s(S,Bn),s(S,ta),s(ta,Gn),We=!0},p:wj,i(l){We||(i(ol.$$.fragment,l),i(pl.$$.fragment,l),i(Cl.$$.fragment,l),i(Zl.$$.fragment,l),i(fl.$$.fragment,l),i(kl.$$.fragment,l),i(Yl.$$.fragment,l),i(Sl.$$.fragment,l),i(Ql.$$.fragment,l),i(Vl.$$.fragment,l),i(Fl.$$.fragment,l),i(Nl.$$.fragment,l),i(ql.$$.fragment,l),i(xl.$$.fragment,l),i(Ll.$$.fragment,l),i(Dl.$$.fragment,l),i(Pl.$$.fragment,l),i(Ol.$$.fragment,l),i(lM.$$.fragment,l),i(tM.$$.fragment,l),i(nM.$$.fragment,l),i(wM.$$.fragment,l),i(oM.$$.fragment,l),i(pM.$$.fragment,l),We=!0)},o(l){c(ol.$$.fragment,l),c(pl.$$.fragment,l),c(Cl.$$.fragment,l),c(Zl.$$.fragment,l),c(fl.$$.fragment,l),c(kl.$$.fragment,l),c(Yl.$$.fragment,l),c(Sl.$$.fragment,l),c(Ql.$$.fragment,l),c(Vl.$$.fragment,l),c(Fl.$$.fragment,l),c(Nl.$$.fragment,l),c(ql.$$.fragment,l),c(xl.$$.fragment,l),c(Ll.$$.fragment,l),c(Dl.$$.fragment,l),c(Pl.$$.fragment,l),c(Ol.$$.fragment,l),c(lM.$$.fragment,l),c(tM.$$.fragment,l),c(nM.$$.fragment,l),c(wM.$$.fragment,l),c(oM.$$.fragment,l),c(pM.$$.fragment,l),We=!1},d(l){M(A),l&&M(Ua),l&&M(Z),I(ol),l&&M(ja),I(pl,l),l&&M(wa),l&&M(Tl),l&&M(oa),l&&M(TM),l&&M(pa),l&&M(rM),l&&M(Ta),l&&M(F),l&&M(ra),l&&M(cM),l&&M(ia),l&&M(IM),l&&M(ca),l&&M(dl),l&&M(Ia),l&&M(hM),l&&M(da),l&&M(hl),l&&M(ha),l&&M(h),l&&M(mM),l&&M(ma),l&&M(uM),l&&M(Ca),l&&M(bM),l&&M(ua),l&&M(AM),l&&M(ba),l&&M(G),I(Cl),l&&M(Aa),l&&M(ul),l&&M(Za),l&&M(H),l&&M(Ba),l&&M(BM),l&&M(Ga),l&&M(GM),l&&M(fa),l&&M(u),l&&M(Wa),l&&M(f),I(Zl),l&&M(va),l&&M(fM),l&&M(Xa),l&&M(q),l&&M(ga),l&&M(W),I(fl),l&&M(_a),l&&M(WM),l&&M(ka),l&&M(L),l&&M(Ea),l&&M(b),l&&M(Ra),l&&M(Xl),l&&M(Ya),l&&M(gl),l&&M(Sa),l&&M(v),I(kl),l&&M(Qa),l&&M(vM),l&&M(Va),l&&M(El),l&&M(Fa),l&&M(XM),l&&M(Na),l&&M(Rl),l&&M(Ha),l&&M(X),I(Yl),l&&M(za),l&&M(P),l&&M(qa),l&&M(gM),l&&M(xa),I(Sl,l),l&&M(La),I(Ql,l),l&&M(Ka),l&&M(g),I(Vl),l&&M(Da),I(Fl,l),l&&M($a),l&&M(_),I(Nl),l&&M(Pa),l&&M(Hl),l&&M(Oa),I(ql,l),l&&M(le),l&&M(k),I(xl),l&&M(Me),l&&M(_M),l&&M(se),I(Ll,l),l&&M(ae),l&&M(kM),l&&M(ee),I(Dl,l),l&&M(te),l&&M(sl),l&&M(ye),I(Pl,l),l&&M(ne),l&&M(SM),l&&M(Je),I(Ol,l),l&&M(Ue),l&&M(QM),l&&M(je),I(lM,l),l&&M(we),l&&M(VM),l&&M(oe),l&&M(MM),l&&M(pe),l&&M(FM),l&&M(Te),l&&M(NM),l&&M(re),l&&M(eM),l&&M(ie),l&&M(el),l&&M(ce),I(tM,l),l&&M(Ie),l&&M(yM),l&&M(de),l&&M(R),I(nM),l&&M(he),l&&M(JM),l&&M(me),l&&M(yl),l&&M(Ce),l&&M(UM),l&&M(ue),l&&M(jM),l&&M(be),l&&M(nl),l&&M(Ae),I(wM,l),l&&M(Ze),l&&M(Y),I(oM),l&&M(Be),l&&M(jl),l&&M(Ge),l&&M(zM),l&&M(fe),l&&M(S),I(pM)}}}const rj={local:"unit-8-proximal-policy-gradient-ppo-with-pytorch",sections:[{local:"objectives-of-this-notebook",title:"Objectives of this notebook \u{1F3C6}"},{local:"prerequisites",title:"Prerequisites \u{1F3D7}\uFE0F"},{local:"set-the-gpu",title:"Set the GPU \u{1F4AA}"},{local:"create-a-virtual-display",title:"Create a virtual display \u{1F53D}"},{local:"install-dependencies",title:"Install dependencies \u{1F53D}"},{local:"lets-code-ppo-from-scratch-with-costa-huangs-tutorial",title:"Let's code PPO from scratch with Costa Huang's tutorial"},{local:"add-the-hugging-face-integration",title:"Add the Hugging Face Integration \u{1F917}"},{local:"lets-start-the-training",title:"Let's start the training \u{1F525}"},{local:"some-additional-challenges",title:"Some additional challenges \u{1F3C6}"},{local:"keep-learning-stay-awesome",title:"Keep learning, stay awesome \u{1F917}"}],title:"Unit 8: Proximal Policy Gradient (PPO) with PyTorch \u{1F916}"};function ij(Fn){return oj(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class mj extends nj{constructor(A){super();Jj(this,A,ij,Tj,Uj,{})}}export{mj as default,rj as metadata};
