import{s as Ae,n as Ee,o as je}from"../chunks/scheduler.37c15a92.js";import{S as Se,i as Be,g as r,s as a,r as z,A as Ie,h as l,f as n,c as o,j as qe,u as A,x as s,k as ze,y as Ge,a as i,v as E,d as j,t as S,w as B}from"../chunks/index.7cb9c9b8.js";import{H as D,E as De}from"../chunks/getInferenceSnippets.15b814d1.js";function Fe(he){let m,F,I,N,p,O,d,ce=`In this advanced topic, we address the question: <strong>how should we monitor and keep track of powerful reinforcement learning agents that we are training in the real world and
interfacing with humans?</strong>`,U,u,ge="As machine learning systems have increasingly impacted modern life, the <strong>call for the documentation of these systems has grown</strong>.",W,f,we=`Such documentation can cover aspects such as the training data used — where it is stored, when it was collected, who was involved, etc.
— or the model optimization framework — the architecture, evaluation metrics, relevant papers, etc. — and more.`,J,h,ve=`Today, model cards and datasheets are becoming increasingly available. For example, on the Hub
(see documentation <a href="https://huggingface.co/docs/hub/model-cards" rel="nofollow">here</a>).`,K,c,be='If you click on a <a href="https://huggingface.co/models" rel="nofollow">popular model on the Hub</a>, you can learn about its creation process.',Q,g,$e=`These model and data specific logs are designed to be completed when the model or dataset are created, leaving them to go un-updated when these models are built into evolving systems in the future.
​`,V,w,X,v,xe=`Reinforcement learning systems are fundamentally designed to optimize based on measurements of reward and time.
While the notion of a reward function can be mapped nicely to many well-understood fields of supervised learning (via a loss function),
understanding of how machine learning systems evolve over time is limited.`,Y,b,ye=`To that end, the authors introduce <a href="https://www.notion.so/Brief-introduction-to-RL-documentation-b8cbda5a6f5242338e0756e6bef72af4" rel="nofollow"><em>Reward Reports for Reinforcement Learning</em></a> (the pithy naming is designed to mirror the popular papers <em>Model Cards for Model Reporting</em> and <em>Datasheets for Datasets</em>).
The goal is to propose a type of documentation focused on the <strong>human factors of reward</strong> and <strong>time-varying feedback systems</strong>.`,Z,$,Ce='Building on the documentation frameworks for <a href="https://arxiv.org/abs/1810.03993" rel="nofollow">model cards</a> and <a href="https://arxiv.org/abs/1803.09010" rel="nofollow">datasheets</a> proposed by Mitchell et al. and Gebru et al., we argue the need for Reward Reports for AI systems.',ee,x,Te="<strong>Reward Reports</strong> are living documents for proposed RL deployments that demarcate design choices.",te,y,Le=`However, many questions remain about the applicability of this framework to different RL applications, roadblocks to system interpretability,
and the resonances between deployed supervised machine learning systems and the sequential decision-making utilized in RL.`,ne,C,Re=`At a minimum, Reward Reports are an opportunity for RL practitioners to deliberate on these questions and begin the work of deciding how to resolve them in practice.
​`,ie,T,ae,L,_e=`The core piece specific to documentation designed for RL and feedback-driven ML systems is a <em>change-log</em>. The change-log updates information
from the designer (changed training parameters, data, etc.) along with noticed changes from the user (harmful behavior, unexpected responses, etc.).`,oe,R,Me="The change log is accompanied by update triggers that encourage monitoring these effects.",re,_,le,M,He=`Some of the most impactful RL-driven systems are multi-stakeholder in nature and behind the closed doors of private corporations.
These corporations are largely without regulation, so the burden of documentation falls on the public.`,se,H,Pe=`If you are interested in contributing, we are building Reward Reports for popular machine learning systems on a public
record on <a href="https://github.com/RewardReports/reward-reports" rel="nofollow">GitHub</a>.
​
For further reading, you can visit the Reward Reports <a href="https://arxiv.org/abs/2204.10817" rel="nofollow">paper</a>
or look <a href="https://github.com/RewardReports/reward-reports/tree/main/examples" rel="nofollow">an example report</a>.`,me,P,pe,k,ke='This section was written by <a href="https://twitter.com/natolambert">Nathan Lambert</a>',de,q,ue,G,fe;return p=new D({props:{title:"Brief introduction to RL documentation",local:"brief-introduction-to-rl-documentation",headingTag:"h1"}}),w=new D({props:{title:"Motivating Reward Reports",local:"motivating-reward-reports",headingTag:"h2"}}),T=new D({props:{title:"Capturing temporal behavior with documentation",local:"capturing-temporal-behavior-with-documentation",headingTag:"h2"}}),_=new D({props:{title:"Contributing",local:"contributing",headingTag:"h2"}}),P=new D({props:{title:"Author",local:"author",headingTag:"h2"}}),q=new De({props:{source:"https://github.com/huggingface/deep-rl-class/blob/main/units/en/unitbonus3/rl-documentation.mdx"}}),{c(){m=r("meta"),F=a(),I=r("p"),N=a(),z(p.$$.fragment),O=a(),d=r("p"),d.innerHTML=ce,U=a(),u=r("p"),u.innerHTML=ge,W=a(),f=r("p"),f.textContent=we,J=a(),h=r("p"),h.innerHTML=ve,K=a(),c=r("p"),c.innerHTML=be,Q=a(),g=r("p"),g.textContent=$e,V=a(),z(w.$$.fragment),X=a(),v=r("p"),v.textContent=xe,Y=a(),b=r("p"),b.innerHTML=ye,Z=a(),$=r("p"),$.innerHTML=Ce,ee=a(),x=r("p"),x.innerHTML=Te,te=a(),y=r("p"),y.textContent=Le,ne=a(),C=r("p"),C.textContent=Re,ie=a(),z(T.$$.fragment),ae=a(),L=r("p"),L.innerHTML=_e,oe=a(),R=r("p"),R.textContent=Me,re=a(),z(_.$$.fragment),le=a(),M=r("p"),M.textContent=He,se=a(),H=r("p"),H.innerHTML=Pe,me=a(),z(P.$$.fragment),pe=a(),k=r("p"),k.innerHTML=ke,de=a(),z(q.$$.fragment),ue=a(),G=r("p"),this.h()},l(e){const t=Ie("svelte-u9bgzb",document.head);m=l(t,"META",{name:!0,content:!0}),t.forEach(n),F=o(e),I=l(e,"P",{}),qe(I).forEach(n),N=o(e),A(p.$$.fragment,e),O=o(e),d=l(e,"P",{"data-svelte-h":!0}),s(d)!=="svelte-vtjhob"&&(d.innerHTML=ce),U=o(e),u=l(e,"P",{"data-svelte-h":!0}),s(u)!=="svelte-1fronss"&&(u.innerHTML=ge),W=o(e),f=l(e,"P",{"data-svelte-h":!0}),s(f)!=="svelte-18x38zx"&&(f.textContent=we),J=o(e),h=l(e,"P",{"data-svelte-h":!0}),s(h)!=="svelte-1dlzh15"&&(h.innerHTML=ve),K=o(e),c=l(e,"P",{"data-svelte-h":!0}),s(c)!=="svelte-11x8tuw"&&(c.innerHTML=be),Q=o(e),g=l(e,"P",{"data-svelte-h":!0}),s(g)!=="svelte-1m80jls"&&(g.textContent=$e),V=o(e),A(w.$$.fragment,e),X=o(e),v=l(e,"P",{"data-svelte-h":!0}),s(v)!=="svelte-uuu9we"&&(v.textContent=xe),Y=o(e),b=l(e,"P",{"data-svelte-h":!0}),s(b)!=="svelte-163ow0b"&&(b.innerHTML=ye),Z=o(e),$=l(e,"P",{"data-svelte-h":!0}),s($)!=="svelte-iy1n1"&&($.innerHTML=Ce),ee=o(e),x=l(e,"P",{"data-svelte-h":!0}),s(x)!=="svelte-1q2xcek"&&(x.innerHTML=Te),te=o(e),y=l(e,"P",{"data-svelte-h":!0}),s(y)!=="svelte-j1zl9k"&&(y.textContent=Le),ne=o(e),C=l(e,"P",{"data-svelte-h":!0}),s(C)!=="svelte-1w6v3rz"&&(C.textContent=Re),ie=o(e),A(T.$$.fragment,e),ae=o(e),L=l(e,"P",{"data-svelte-h":!0}),s(L)!=="svelte-5kkp2w"&&(L.innerHTML=_e),oe=o(e),R=l(e,"P",{"data-svelte-h":!0}),s(R)!=="svelte-1j0m0fx"&&(R.textContent=Me),re=o(e),A(_.$$.fragment,e),le=o(e),M=l(e,"P",{"data-svelte-h":!0}),s(M)!=="svelte-i3qq7q"&&(M.textContent=He),se=o(e),H=l(e,"P",{"data-svelte-h":!0}),s(H)!=="svelte-ihbc3x"&&(H.innerHTML=Pe),me=o(e),A(P.$$.fragment,e),pe=o(e),k=l(e,"P",{"data-svelte-h":!0}),s(k)!=="svelte-17c4ke5"&&(k.innerHTML=ke),de=o(e),A(q.$$.fragment,e),ue=o(e),G=l(e,"P",{}),qe(G).forEach(n),this.h()},h(){ze(m,"name","hf:doc:metadata"),ze(m,"content",Ne)},m(e,t){Ge(document.head,m),i(e,F,t),i(e,I,t),i(e,N,t),E(p,e,t),i(e,O,t),i(e,d,t),i(e,U,t),i(e,u,t),i(e,W,t),i(e,f,t),i(e,J,t),i(e,h,t),i(e,K,t),i(e,c,t),i(e,Q,t),i(e,g,t),i(e,V,t),E(w,e,t),i(e,X,t),i(e,v,t),i(e,Y,t),i(e,b,t),i(e,Z,t),i(e,$,t),i(e,ee,t),i(e,x,t),i(e,te,t),i(e,y,t),i(e,ne,t),i(e,C,t),i(e,ie,t),E(T,e,t),i(e,ae,t),i(e,L,t),i(e,oe,t),i(e,R,t),i(e,re,t),E(_,e,t),i(e,le,t),i(e,M,t),i(e,se,t),i(e,H,t),i(e,me,t),E(P,e,t),i(e,pe,t),i(e,k,t),i(e,de,t),E(q,e,t),i(e,ue,t),i(e,G,t),fe=!0},p:Ee,i(e){fe||(j(p.$$.fragment,e),j(w.$$.fragment,e),j(T.$$.fragment,e),j(_.$$.fragment,e),j(P.$$.fragment,e),j(q.$$.fragment,e),fe=!0)},o(e){S(p.$$.fragment,e),S(w.$$.fragment,e),S(T.$$.fragment,e),S(_.$$.fragment,e),S(P.$$.fragment,e),S(q.$$.fragment,e),fe=!1},d(e){e&&(n(F),n(I),n(N),n(O),n(d),n(U),n(u),n(W),n(f),n(J),n(h),n(K),n(c),n(Q),n(g),n(V),n(X),n(v),n(Y),n(b),n(Z),n($),n(ee),n(x),n(te),n(y),n(ne),n(C),n(ie),n(ae),n(L),n(oe),n(R),n(re),n(le),n(M),n(se),n(H),n(me),n(pe),n(k),n(de),n(ue),n(G)),n(m),B(p,e),B(w,e),B(T,e),B(_,e),B(P,e),B(q,e)}}}const Ne='{"title":"Brief introduction to RL documentation","local":"brief-introduction-to-rl-documentation","sections":[{"title":"Motivating Reward Reports","local":"motivating-reward-reports","sections":[],"depth":2},{"title":"Capturing temporal behavior with documentation","local":"capturing-temporal-behavior-with-documentation","sections":[],"depth":2},{"title":"Contributing","local":"contributing","sections":[],"depth":2},{"title":"Author","local":"author","sections":[],"depth":2}],"depth":1}';function Oe(he){return je(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ke extends Se{constructor(m){super(),Be(this,m,Oe,Fe,Ae,{})}}export{Ke as component};
