Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / deep-rl-course /pr_721 /en /unit1 /rl-framework.html
HuggingFaceDocBuilder's picture
HuggingFaceDocBuilder
download
raw
28.4 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;The Reinforcement Learning Framework&quot;,&quot;local&quot;:&quot;the-reinforcement-learning-framework&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;The RL Process&quot;,&quot;local&quot;:&quot;the-rl-process&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;The reward hypothesis: the central idea of Reinforcement Learning&quot;,&quot;local&quot;:&quot;reward-hypothesis&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Markov Property&quot;,&quot;local&quot;:&quot;markov-property&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Observations/States Space&quot;,&quot;local&quot;:&quot;obs-space&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Action Space&quot;,&quot;local&quot;:&quot;action-space&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Rewards and the discounting&quot;,&quot;local&quot;:&quot;rewards&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/entry/start.sDXluo3R.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/chunks/B22JROst.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/chunks/BDzc2A-J.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/entry/app.DFlLpriw.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/chunks/dL6ONiCo.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/chunks/CDpT6ySu.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/chunks/BiuJz3Hd.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/nodes/0.CdQ7t_nh.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/chunks/DV9AJ6VV.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/nodes/16.za1SBjHs.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/chunks/B408aFXw.js" rel="modulepreload">
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/chunks/CXKJogc7.js" rel="modulepreload">
<!--urs00m--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;The Reinforcement Learning Framework&quot;,&quot;local&quot;:&quot;the-reinforcement-learning-framework&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;The RL Process&quot;,&quot;local&quot;:&quot;the-rl-process&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;The reward hypothesis: the central idea of Reinforcement Learning&quot;,&quot;local&quot;:&quot;reward-hypothesis&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Markov Property&quot;,&quot;local&quot;:&quot;markov-property&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Observations/States Space&quot;,&quot;local&quot;:&quot;obs-space&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Action Space&quot;,&quot;local&quot;:&quot;action-space&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Rewards and the discounting&quot;,&quot;local&quot;:&quot;rewards&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
<link href="/docs/deep-rl-course/pr_721/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="the-reinforcement-learning-framework" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#the-reinforcement-learning-framework"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->The Reinforcement Learning Framework<!----></span></h1><!--]--><!----> <!--[1--><h2 class="relative group"><a id="the-rl-process" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#the-rl-process"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->The RL Process<!----></span></h2><!--]--><!----> <figure><img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/RL_process.jpg" alt="The RL process" width="100%"/> <figcaption>The RL Process: a loop of state, action, reward and next state</figcaption> <figcaption>Source: <a href="http://incompleteideas.net/book/RLbook2020.pdf">Reinforcement Learning: An Introduction, Richard Sutton and Andrew G. Barto</a></figcaption></figure> <p>To understand the RL process, let’s imagine an agent learning to play a platform game:</p> <img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/RL_process_game.jpg" alt="The RL process" width="100%"/> <ul><li>Our Agent receives <strong>state <!----><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>S</mi><mn>0</mn></msub></mrow><annotation encoding="application/x-tex">S_0</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0576em;">S</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0576em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">0</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span><!----></strong> from the <strong>Environment</strong> — we receive the first frame of our game (Environment).</li> <li>Based on that <strong>state<!----><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>S</mi><mn>0</mn></msub></mrow><annotation encoding="application/x-tex">S_0</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0576em;">S</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0576em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">0</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span><!---->,</strong> the Agent takes <strong>action<!----><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>A</mi><mn>0</mn></msub></mrow><annotation encoding="application/x-tex">A_0</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">A</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">0</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span><!----></strong> — our Agent will move to the right.</li> <li>The environment goes to a <strong>new</strong> <strong>state<!----><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>S</mi><mn>1</mn></msub></mrow><annotation encoding="application/x-tex">S_1</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0576em;">S</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0576em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span><!----></strong> — new frame.</li> <li>The environment gives some <strong>reward<!----><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>R</mi><mn>1</mn></msub></mrow><annotation encoding="application/x-tex">R_1</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8333em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.0077em;">R</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0077em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span><!----></strong> to the Agent — we’re not dead <em>(Positive Reward +1)</em>.</li></ul> <p>This RL loop outputs a sequence of <strong>state, action, reward and next state.</strong></p> <img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/sars.jpg" alt="State, Action, Reward, Next State" width="100%"/> <p>The agent’s goal is to <em>maximize</em> its cumulative reward, <strong>called the expected return.</strong></p> <!--[1--><h2 class="relative group"><a id="reward-hypothesis" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#reward-hypothesis"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->The reward hypothesis: the central idea of Reinforcement Learning<!----></span></h2><!--]--><!----> <p>⇒ Why is the goal of the agent to maximize the expected return?</p> <p>Because RL is based on the <strong>reward hypothesis</strong>, which is that all goals can be described as the <strong>maximization of the expected return</strong> (expected cumulative reward).</p> <p>That’s why in Reinforcement Learning, <strong>to have the best behavior,</strong> we aim to learn to take actions that <strong>maximize the expected cumulative reward.</strong></p> <!--[1--><h2 class="relative group"><a id="markov-property" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#markov-property"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Markov Property<!----></span></h2><!--]--><!----> <p>In papers, you’ll see that the RL process is called a <strong>Markov Decision Process</strong> (MDP).</p> <p>We’ll talk again about the Markov Property in the following units. But if you need to remember something today about it, it’s this: the Markov Property implies that our agent needs <strong>only the current state to decide</strong> what action to take and <strong>not the history of all the states and actions</strong> they took before.</p> <!--[1--><h2 class="relative group"><a id="obs-space" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#obs-space"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Observations/States Space<!----></span></h2><!--]--><!----> <p>Observations/States are the <strong>information our agent gets from the environment.</strong> In the case of a video game, it can be a frame (a screenshot). In the case of the trading agent, it can be the value of a certain stock, etc.</p> <p>There is a differentiation to make between <em>observation</em> and <em>state</em>, however:</p> <ul><li><em>State s</em>: is <strong>a complete description of the state of the world</strong> (there is no hidden information). In a fully observed environment.</li></ul> <figure><img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/chess.jpg" alt="Chess"/> <figcaption>In chess game, we receive a state from the environment since we have access to the whole check board information.</figcaption></figure> <p>In a chess game, we have access to the whole board information, so we receive a state from the environment. In other words, the environment is fully observed.</p> <ul><li><em>Observation o</em>: is a <strong>partial description of the state.</strong> In a partially observed environment.</li></ul> <figure><img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/mario.jpg" alt="Mario"/> <figcaption>In Super Mario Bros, we only see the part of the level close to the player, so we receive an observation.</figcaption></figure> <p>In Super Mario Bros, we only see the part of the level close to the player, so we receive an observation.</p> <p>In Super Mario Bros, we are in a partially observed environment. We receive an observation <strong>since we only see a part of the level.</strong></p> <blockquote class="tip"><!---->In this course, we use the term "state" to denote both state and observation, but we will make the distinction in implementations.<!----></blockquote><!----> <p>To recap:</p> <img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/obs_space_recap.jpg" alt="Obs space recap" width="100%"/> <!--[1--><h2 class="relative group"><a id="action-space" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#action-space"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Action Space<!----></span></h2><!--]--><!----> <p>The Action space is the set of <strong>all possible actions in an environment.</strong></p> <p>The actions can come from a <em>discrete</em> or <em>continuous space</em>:</p> <ul><li><em>Discrete space</em>: the number of possible actions is <strong>finite</strong>.</li></ul> <figure><img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/mario.jpg" alt="Mario"/> <figcaption>In Super Mario Bros, we have only 4 possible actions: left, right, up (jumping) and down (crouching).</figcaption></figure> <p>Again, in Super Mario Bros, we have a finite set of actions since we have only 4 directions.</p> <ul><li><em>Continuous space</em>: the number of possible actions is <strong>infinite</strong>.</li></ul> <figure><img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/self_driving_car.jpg" alt="Self Driving Car"/> <figcaption>A Self Driving Car agent has an infinite number of possible actions since it can turn left 20°, 21,1°, 21,2°, honk, turn right 20°…</figcaption></figure> <p>To recap:</p> <img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/action_space.jpg" alt="Action space recap" width="100%"/> <p>Taking this information into consideration is crucial because it will <strong>have importance when choosing the RL algorithm in the future.</strong></p> <!--[1--><h2 class="relative group"><a id="rewards" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#rewards"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Rewards and the discounting<!----></span></h2><!--]--><!----> <p>The reward is fundamental in RL because it’s <strong>the only feedback</strong> for the agent. Thanks to it, our agent knows <strong>if the action taken was good or not.</strong></p> <p>The cumulative reward at each time step <strong>t</strong> can be written as:</p> <figure><img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/rewards_1.jpg" alt="Rewards"/> <figcaption>The cumulative reward equals the sum of all rewards in the sequence.</figcaption></figure> <p>Which is equivalent to:</p> <figure><img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/rewards_2.jpg" alt="Rewards"/> <figcaption>The cumulative reward = rt+1 (rt+k+1 = rt+0+1 = rt+1)+ rt+2 (rt+k+1 = rt+1+1 = rt+2) + ...</figcaption></figure> <p>However, in reality, <strong>we can’t just add them like that.</strong> The rewards that come sooner (at the beginning of the game) <strong>are more likely to happen</strong> since they are more predictable than the long-term future reward.</p> <p>Let’s say your agent is this tiny mouse that can move one tile each time step, and your opponent is the cat (that can move too). The mouse’s goal is <strong>to eat the maximum amount of cheese before being eaten by the cat.</strong></p> <img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/rewards_3.jpg" alt="Rewards" width="100%"/> <p>As we can see in the diagram, <strong>it’s more probable to eat the cheese near us than the cheese close to the cat</strong> (the closer we are to the cat, the more dangerous it is).</p> <p>Consequently, <strong>the reward near the cat, even if it is bigger (more cheese), will be more discounted</strong> since we’re not really sure we’ll be able to eat it.</p> <p>To discount the rewards, we proceed like this:</p> <ol><li>We define a discount rate called gamma. <strong>It must be between 0 and 1.</strong> Most of the time between <strong>0.95 and 0.99</strong>.</li></ol> <ul><li>The larger the gamma, the smaller the discount. This means our agent <strong>cares more about the long-term reward.</strong></li> <li>On the other hand, the smaller the gamma, the bigger the discount. This means our <strong>agent cares more about the short term reward (the nearest cheese).</strong></li></ul> <p>2. Then, each reward will be discounted by gamma to the exponent of the time step. As the time step increases, the cat gets closer to us, <strong>so the future reward is less and less likely to happen.</strong></p> <p>Our discounted expected cumulative reward is:</p> <img src="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit1/rewards_4.jpg" alt="Rewards" width="100%"/> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/deep-rl-class/blob/main/units/en/unit1/rl-framework.mdx" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
<script>
{
__sveltekit_zezkeo = {
base: "/docs/deep-rl-course/pr_721/en",
assets: "/docs/deep-rl-course/pr_721/en"
};
const element = document.currentScript.parentElement;
Promise.all([
import("/docs/deep-rl-course/pr_721/en/_app/immutable/entry/start.sDXluo3R.js"),
import("/docs/deep-rl-course/pr_721/en/_app/immutable/entry/app.DFlLpriw.js")
]).then(([kit, app]) => {
kit.start(app, element, {
node_ids: [0, 16],
data: [null,null],
form: null,
error: null
});
});
}
</script>

Xet Storage Details

Size:
28.4 kB
·
Xet hash:
fd261a363a33e184a67b706c6272d90fe459ac7687cd7225d2e4069b19d20b16

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.