{"data": [{"hoverinfo": "skip", "mode": "markers", "x": [null], "y": [null], "type": "scatter"}], "layout": {"template": {"data": {"histogram2dcontour": [{"type": "histogram2dcontour", "colorbar": {"outlinewidth": 0, "ticks": ""}, "colorscale": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]]}], "choropleth": [{"type": "choropleth", "colorbar": {"outlinewidth": 0, "ticks": ""}}], "histogram2d": [{"type": "histogram2d", "colorbar": {"outlinewidth": 0, "ticks": ""}, "colorscale": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]]}], "heatmap": [{"type": "heatmap", "colorbar": {"outlinewidth": 0, "ticks": ""}, "colorscale": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]]}], "contourcarpet": [{"type": "contourcarpet", "colorbar": {"outlinewidth": 0, "ticks": ""}}], "contour": [{"type": "contour", "colorbar": {"outlinewidth": 0, "ticks": ""}, "colorscale": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]]}], "surface": [{"type": "surface", "colorbar": {"outlinewidth": 0, "ticks": ""}, "colorscale": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]]}], "mesh3d": [{"type": "mesh3d", "colorbar": {"outlinewidth": 0, "ticks": ""}}], "scatter": [{"fillpattern": {"fillmode": "overlay", "size": 10, "solidity": 0.2}, "type": "scatter"}], "parcoords": [{"type": "parcoords", "line": {"colorbar": {"outlinewidth": 0, "ticks": ""}}}], "scatterpolargl": [{"type": "scatterpolargl", "marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}}], "bar": [{"error_x": {"color": "#2a3f5f"}, "error_y": {"color": "#2a3f5f"}, "marker": {"line": {"color": "#E5ECF6", "width": 0.5}, "pattern": {"fillmode": "overlay", "size": 10, "solidity": 0.2}}, "type": "bar"}], "scattergeo": [{"type": "scattergeo", "marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}}], "scatterpolar": [{"type": "scatterpolar", "marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}}], "histogram": [{"marker": {"pattern": {"fillmode": "overlay", "size": 10, "solidity": 0.2}}, "type": "histogram"}], "scattergl": [{"type": "scattergl", "marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}}], "scatter3d": [{"type": "scatter3d", "line": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}}], "scattermap": [{"type": "scattermap", "marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}}], "scattermapbox": [{"type": "scattermapbox", "marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}}], "scatterternary": [{"type": "scatterternary", "marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}}], "scattercarpet": [{"type": "scattercarpet", "marker": {"colorbar": {"outlinewidth": 0, "ticks": ""}}}], "carpet": [{"aaxis": {"endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f"}, "baxis": {"endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f"}, "type": "carpet"}], "table": [{"cells": {"fill": {"color": "#EBF0F8"}, "line": {"color": "white"}}, "header": {"fill": {"color": "#C8D4E3"}, "line": {"color": "white"}}, "type": "table"}], "barpolar": [{"marker": {"line": {"color": "#E5ECF6", "width": 0.5}, "pattern": {"fillmode": "overlay", "size": 10, "solidity": 0.2}}, "type": "barpolar"}], "pie": [{"automargin": true, "type": "pie"}]}, "layout": {"autotypenumbers": "strict", "colorway": ["#636efa", "#EF553B", "#00cc96", "#ab63fa", "#FFA15A", "#19d3f3", "#FF6692", "#B6E880", "#FF97FF", "#FECB52"], "font": {"color": "#2a3f5f"}, "hovermode": "closest", "hoverlabel": {"align": "left"}, "paper_bgcolor": "white", "plot_bgcolor": "#E5ECF6", "polar": {"bgcolor": "#E5ECF6", "angularaxis": {"gridcolor": "white", "linecolor": "white", "ticks": ""}, "radialaxis": {"gridcolor": "white", "linecolor": "white", "ticks": ""}}, "ternary": {"bgcolor": "#E5ECF6", "aaxis": {"gridcolor": "white", "linecolor": "white", "ticks": ""}, "baxis": {"gridcolor": "white", "linecolor": "white", "ticks": ""}, "caxis": {"gridcolor": "white", "linecolor": "white", "ticks": ""}}, "coloraxis": {"colorbar": {"outlinewidth": 0, "ticks": ""}}, "colorscale": {"sequential": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]], "sequentialminus": [[0.0, "#0d0887"], [0.1111111111111111, "#46039f"], [0.2222222222222222, "#7201a8"], [0.3333333333333333, "#9c179e"], [0.4444444444444444, "#bd3786"], [0.5555555555555556, "#d8576b"], [0.6666666666666666, "#ed7953"], [0.7777777777777778, "#fb9f3a"], [0.8888888888888888, "#fdca26"], [1.0, "#f0f921"]], "diverging": [[0, "#8e0152"], [0.1, "#c51b7d"], [0.2, "#de77ae"], [0.3, "#f1b6da"], [0.4, "#fde0ef"], [0.5, "#f7f7f7"], [0.6, "#e6f5d0"], [0.7, "#b8e186"], [0.8, "#7fbc41"], [0.9, "#4d9221"], [1, "#276419"]]}, "xaxis": {"gridcolor": "white", "linecolor": "white", "ticks": "", "title": {"standoff": 15}, "zerolinecolor": "white", "automargin": true, "zerolinewidth": 2}, "yaxis": {"gridcolor": "white", "linecolor": "white", "ticks": "", "title": {"standoff": 15}, "zerolinecolor": "white", "automargin": true, "zerolinewidth": 2}, "scene": {"xaxis": {"backgroundcolor": "#E5ECF6", "gridcolor": "white", "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white", "gridwidth": 2}, "yaxis": {"backgroundcolor": "#E5ECF6", "gridcolor": "white", "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white", "gridwidth": 2}, "zaxis": {"backgroundcolor": "#E5ECF6", "gridcolor": "white", "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white", "gridwidth": 2}}, "shapedefaults": {"line": {"color": "#2a3f5f"}}, "annotationdefaults": {"arrowcolor": "#2a3f5f", "arrowhead": 0, "arrowwidth": 1}, "geo": {"bgcolor": "white", "landcolor": "#E5ECF6", "subunitcolor": "white", "showland": true, "showlakes": true, "lakecolor": "white"}, "title": {"x": 0.05}, "mapbox": {"style": "light"}}}, "title": {"font": {"size": 16, "color": "#212121"}, "text": "QMIX / HAD3QN: Value Decomposition Methods", "x": 0.5}, "xaxis": {"range": [-0.5, 10.5], "visible": false, "fixedrange": true}, "yaxis": {"range": [-0.5, 8.5], "visible": false, "fixedrange": true, "scaleanchor": "x"}, "margin": {"l": 20, "r": 20, "t": 50, "b": 20}, "width": 900, "height": 580, "plot_bgcolor": "#FFFFFF", "paper_bgcolor": "#FFFFFF", "showlegend": false, "annotations": [{"font": {"color": "#212121", "size": 14}, "showarrow": false, "text": "<b>QMIX</b>", "x": 2.2, "y": 8}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>o\u2081</b>", "x": 0.75, "y": 6.5}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>o\u2082</b>", "x": 2.55, "y": 6.5}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>o\u2099</b>", "x": 4.35, "y": 6.5}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>Q\u2081 Net</b><br><span style='font-size:10px;color:#555'>Q\u1d62(o\u1d62, a\u1d62)</span>", "x": 0.75, "y": 4.8}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>Q\u2082 Net</b><br><span style='font-size:10px;color:#555'>per-agent Q</span>", "x": 2.55, "y": 4.8}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>Q\u2099 Net</b><br><span style='font-size:10px;color:#555'>per-agent Q</span>", "x": 4.35, "y": 4.8}, {"arrowcolor": "#546E7A", "arrowhead": 3, "arrowsize": 1.2, "arrowwidth": 1.8, "ax": 0.75, "axref": "x", "ay": 6, "ayref": "y", "showarrow": true, "standoff": 4, "startstandoff": 4, "x": 0.75, "xref": "x", "y": 5.4, "yref": "y"}, {"arrowcolor": "#546E7A", "arrowhead": 3, "arrowsize": 1.2, "arrowwidth": 1.8, "ax": 2.55, "axref": "x", "ay": 6, "ayref": "y", "showarrow": true, "standoff": 4, "startstandoff": 4, "x": 2.55, "xref": "x", "y": 5.4, "yref": "y"}, {"arrowcolor": "#546E7A", "arrowhead": 3, "arrowsize": 1.2, "arrowwidth": 1.8, "ax": 4.35, "axref": "x", "ay": 6, "ayref": "y", "showarrow": true, "standoff": 4, "startstandoff": 4, "x": 4.35, "xref": "x", "y": 5.4, "yref": "y"}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>QMixer</b><br><span style='font-size:10px;color:#555'>Monotonic mixing network<br>Q_tot = f(Q\u2081,...,Q\u2099; s)<br>\u2202Q_tot/\u2202Q\u1d62 \u2265 0 (monotonicity)</span>", "x": 2.5, "y": 2.9000000000000004}, {"arrowcolor": "#546E7A", "arrowhead": 3, "arrowsize": 1.2, "arrowwidth": 1.8, "ax": 0.75, "axref": "x", "ay": 4.2, "ayref": "y", "showarrow": true, "standoff": 4, "startstandoff": 4, "x": 1.5, "xref": "x", "y": 3.6, "yref": "y"}, {"bgcolor": "rgba(255,255,255,0.85)", "borderpad": 2, "font": {"color": "#666", "size": 9}, "showarrow": false, "text": "<i>Q\u2081</i>", "x": 1.125, "y": 3.9000000000000004}, {"arrowcolor": "#546E7A", "arrowhead": 3, "arrowsize": 1.2, "arrowwidth": 1.8, "ax": 2.55, "axref": "x", "ay": 4.2, "ayref": "y", "showarrow": true, "standoff": 4, "startstandoff": 4, "x": 2.5, "xref": "x", "y": 3.6, "yref": "y"}, {"bgcolor": "rgba(255,255,255,0.85)", "borderpad": 2, "font": {"color": "#666", "size": 9}, "showarrow": false, "text": "<i>Q\u2082</i>", "x": 2.525, "y": 3.9000000000000004}, {"arrowcolor": "#546E7A", "arrowhead": 3, "arrowsize": 1.2, "arrowwidth": 1.8, "ax": 4.35, "axref": "x", "ay": 4.2, "ayref": "y", "showarrow": true, "standoff": 4, "startstandoff": 4, "x": 3.5, "xref": "x", "y": 3.6, "yref": "y"}, {"bgcolor": "rgba(255,255,255,0.85)", "borderpad": 2, "font": {"color": "#666", "size": 9}, "showarrow": false, "text": "<i>Q\u2099</i>", "x": 3.925, "y": 3.9000000000000004}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>TD Loss</b><br><span style='font-size:10px;color:#555'>L = (Q_tot \u2212 y)\u00b2,  y = r + \u03b3\u00b7Q_tot'</span>", "x": 2.5, "y": 1.0}, {"arrowcolor": "#546E7A", "arrowhead": 3, "arrowsize": 1.2, "arrowwidth": 1.8, "ax": 2.5, "axref": "x", "ay": 2.2, "ayref": "y", "showarrow": true, "standoff": 4, "startstandoff": 4, "x": 2.5, "xref": "x", "y": 1.5, "yref": "y"}, {"bgcolor": "rgba(255,255,255,0.85)", "borderpad": 2, "font": {"color": "#666", "size": 9}, "showarrow": false, "text": "<i>Q_tot</i>", "x": 2.5, "y": 1.85}, {"font": {"color": "#212121", "size": 14}, "showarrow": false, "text": "<b>HAD3QN</b>", "x": 8, "y": 8}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>Agent i Obs</b><br><span style='font-size:10px;color:#555'>o\u1d62</span>", "x": 8.0, "y": 6.5}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>Dueling DQN</b><br><span style='font-size:10px;color:#555'>V(o\u1d62) \u2014 state value stream<br>A(o\u1d62,a) \u2014 advantage stream<br>Q = V + A \u2212 mean(A)</span>", "x": 8.0, "y": 4.75}, {"arrowcolor": "#546E7A", "arrowhead": 3, "arrowsize": 1.2, "arrowwidth": 1.8, "ax": 8, "axref": "x", "ay": 6, "ayref": "y", "showarrow": true, "standoff": 4, "startstandoff": 4, "x": 8, "xref": "x", "y": 5.5, "yref": "y"}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>Double Q-Learning</b><br><span style='font-size:10px;color:#555'>Select: a* = argmax Q_online<br>Evaluate: Q_target(s', a*)</span>", "x": 8.0, "y": 2.8000000000000003}, {"arrowcolor": "#546E7A", "arrowhead": 3, "arrowsize": 1.2, "arrowwidth": 1.8, "ax": 8, "axref": "x", "ay": 4, "ayref": "y", "showarrow": true, "standoff": 4, "startstandoff": 4, "x": 8, "xref": "x", "y": 3.4, "yref": "y"}, {"align": "center", "font": {"color": "#212121", "size": 12}, "showarrow": false, "text": "<b>TD Loss</b><br><span style='font-size:10px;color:#555'>Per-agent TD error<br>\u03b5-greedy exploration</span>", "x": 8.0, "y": 1.0}, {"arrowcolor": "#546E7A", "arrowhead": 3, "arrowsize": 1.2, "arrowwidth": 1.8, "ax": 8, "axref": "x", "ay": 2.2, "ayref": "y", "showarrow": true, "standoff": 4, "startstandoff": 4, "x": 8, "xref": "x", "y": 1.5, "yref": "y"}], "shapes": [{"fillcolor": "#E3F2FD", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 0, "x1": 1.5, "y0": 6, "y1": 7}, {"fillcolor": "#E3F2FD", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 1.8, "x1": 3.3, "y0": 6, "y1": 7}, {"fillcolor": "#E3F2FD", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 3.6, "x1": 5.1, "y0": 6, "y1": 7}, {"fillcolor": "#42A5F5", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 0, "x1": 1.5, "y0": 4.2, "y1": 5.4}, {"fillcolor": "#42A5F5", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 1.8, "x1": 3.3, "y0": 4.2, "y1": 5.4}, {"fillcolor": "#42A5F5", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 3.6, "x1": 5.1, "y0": 4.2, "y1": 5.4}, {"fillcolor": "#AB47BC", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 0.5, "x1": 4.5, "y0": 2.2, "y1": 3.6}, {"fillcolor": "#FFA726", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 0.5, "x1": 4.5, "y0": 0.5, "y1": 1.5}, {"fillcolor": "#E3F2FD", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 6.5, "x1": 9.5, "y0": 6, "y1": 7}, {"fillcolor": "#42A5F5", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 6.5, "x1": 9.5, "y0": 4, "y1": 5.5}, {"fillcolor": "#AB47BC", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 6.5, "x1": 9.5, "y0": 2.2, "y1": 3.4000000000000004}, {"fillcolor": "#FFA726", "layer": "below", "line": {"color": "#78909C", "width": 1.5}, "opacity": 1.0, "type": "rect", "x0": 6.5, "x1": 9.5, "y0": 0.5, "y1": 1.5}, {"line": {"color": "#ddd", "dash": "dash", "width": 1}, "type": "line", "x0": 5.5, "x1": 5.5, "y0": 0, "y1": 8.2}]}}