transformers/pr_33962/zh/_app/immutable/nodes/16.44ea45ea.js

import{s as oe,o as fe,n as kl}from"../chunks/scheduler.25b97de1.js";import{S as Ce,i as Ze,g as p,s as a,r as w,A as Ae,h as M,f as e,c as t,j as Ie,u as r,x as j,k as Ue,y as Ge,a as n,v as T,d as J,t as m,w as b}from"../chunks/index.d9030fc9.js";import{T as _l}from"../chunks/Tip.baa67368.js";import{C as d}from"../chunks/CodeBlock.e6cd0d95.js";import{H as Ll,E as Ve}from"../chunks/EditOnGithub.91d95064.js";function Be(y){let u,i="目前，此功能仅适用于PyTorch。";return{c(){u=p("p"),u.textContent=i},l(c){u=M(c,"P",{"data-svelte-h":!0}),j(u)!=="svelte-1x37u39"&&(u.textContent=i)},m(c,h){n(c,u,h)},p:kl,d(c){c&&e(u)}}}function $e(y){let u,i="对于多GPU训练，它需要使用DDP（<code>torch.distributed.launch</code>）。";return{c(){u=p("p"),u.innerHTML=i},l(c){u=M(c,"P",{"data-svelte-h":!0}),j(u)!=="svelte-tpgxv2"&&(u.innerHTML=i)},m(c,h){n(c,u,h)},p:kl,d(c){c&&e(u)}}}function Re(y){let u,i="此功能可以与任何基于<code>nn.Module</code>的模型一起使用。";return{c(){u=p("p"),u.innerHTML=i},l(c){u=M(c,"P",{"data-svelte-h":!0}),j(u)!=="svelte-1rh6rqd"&&(u.innerHTML=i)},m(c,h){n(c,u,h)},p:kl,d(c){c&&e(u)}}}function Le(y){let u,i,c,h,f,As,C,Gs,Z,Nl="当使用<code>DistributedDataParallel</code>和多个GPU进行训练或推理时，如果遇到进程和（或）节点之间的互联问题，您可以使用以下脚本来诊断网络问题。",Vs,A,Bs,G,Wl="例如，要测试两个GPU之间的互联，请执行以下操作：",$s,V,Rs,B,vl="如果两个进程能够相互通信并分配GPU内存，它们各自将打印出 “OK” 状态。",Ls,$,El="对于更多的GPU或节点，可以根据脚本中的参数进行调整。",_s,R,xl="在诊断脚本内部，您将找到更多详细信息，甚至有关如何在SLURM环境中运行它的说明。",ks,L,Xl="另一种级别的调试是添加 <code>NCCL_DEBUG=INFO</code> 环境变量，如下所示：",Ns,_,Ws,k,Ql="这将产生大量与NCCL相关的调试信息，如果发现有问题报告，您可以在线搜索以获取相关信息。或者，如果您不确定如何解释输出，可以在<code>issue</code>中分享日志文件。",vs,N,Es,I,xs,U,Xs,o,Qs,W,Hl="如果您开始发现<code>loss=NaN</code>或模型因激活值或权重中的<code>inf</code>或<code>nan</code>而出现一些异常行为，就需要发现第一个下溢或上溢发生的地方以及导致它的原因。幸运的是，您可以通过激活一个特殊模块来自动进行检测。",Hs,v,Dl="如果您正在使用<code>Trainer</code>，只需把以下内容：",Ds,E,Ys,x,Yl="添加到常规命令行参数中，或在创建<code>TrainingArguments</code>对象时传递 <code>debug=&quot;underflow_overflow&quot;</code>。",gs,X,gl="如果您正在使用自己的训练循环或其他Trainer，您可以通过以下方式实现相同的功能：",Ss,Q,zs,H,Sl='<a href="/docs/transformers/pr_33962/zh/internal/trainer_utils#transformers.debug_utils.DebugUnderflowOverflow">debug_utils.DebugUnderflowOverflow</a> 将<code>hooks</code>插入模型，紧跟在每次前向调用之后，进而测试输入和输出变量，以及相应模块的权重。一旦在激活值或权重的至少一个元素中检测到<code>inf</code>或<code>nan</code>，程序将执行<code>assert</code>并打印报告，就像这样（这是在<code>google/mt5-small</code>下使用fp16混合精度捕获的）：',Fs,D,Ps,Y,zl="由于篇幅原因，示例输出中间的部分已经被缩减。",Ks,g,Fl="第二列显示了绝对最大元素的值，因此，如果您仔细查看最后<code>frame</code>，输入和输出都在<code>1e4</code>的范围内。因此，在使用fp16混合精度进行训练时，最后一步发生了溢出（因为在<code>fp16</code>下，在<code>inf</code>之前的最大数字是<code>64e3</code>）。为了避免在<code>fp16</code>下发生溢出，激活值必须保持低于<code>1e4</code>，因为<code>1e4 * 1e4 = 1e8</code>，因此任何具有大激活值的矩阵乘法都会导致数值溢出。",qs,S,Pl="在跟踪的开始处，您可以发现问题发生在哪个批次（这里的<code>Detected inf/nan during batch_number=0</code>表示问题发生在第一个批次）。",Os,z,Kl="每个报告的<code>frame</code>都以声明相应模块的层信息为开头，说明这一<code>frame</code>是为哪个模块报告的。如果只看这个<code>frame</code>：",sl,F,ll,P,ql="在这里，<code>encoder.block.2.layer.1.layer_norm</code> 表示它是编码器的第二个块中第一层的<code>layer norm</code>。而 <code>forward</code> 的具体调用是 <code>T5LayerNorm</code>。",el,K,Ol="让我们看看该报告的最后几个<code>frame</code>：",nl,q,al,O,se="最后一个<code>frame</code>报告了<code>Dropout.forward</code>函数，第一个条目是唯一的输入，第二个条目是唯一的输出。您可以看到，它是从<code>DenseReluDense</code>类内的属性<code>dropout</code>中调用的。我们可以看到它发生在第2个块的第1层，也就是在第一个批次期间。最后，绝对最大的输入元素值为<code>6.27e+04</code>，输出也是<code>inf</code>。",tl,ss,le="您可以在这里看到，<code>T5DenseGatedGeluDense.forward</code>产生了输出激活值，其绝对最大值约为62.7K，非常接近fp16的上限64K。在下一个<code>frame</code>中，我们有<code>Dropout</code>对权重进行重新归一化，之后将某些元素归零，将绝对最大值推到了64K以上，导致溢出（<code>inf</code>）。",pl,ls,ee="正如你所看到的，我们需要查看前面的<code>frame</code>, 从那里fp16数字开始变得非常大。",Ml,es,ne="让我们将报告与<code>models/t5/modeling_t5.py</code>中的代码匹配：",jl,ns,ul,as,ae="现在很容易看到<code>dropout</code>调用，以及所有之前的调用。",cl,ts,te="由于检测是在前向<code>hook</code>中进行的，这些报告将立即在每个<code>forward</code>返回后打印出来。",wl,ps,pe="回到完整的报告，要采取措施并解决问题，我们需要往回看几个<code>frame</code>，在那里数字开始上升，并且最有可能切换到fp32模式以便在乘法或求和时数字不会溢出。当然，可能还有其他解决方案。例如，如果启用了<code>amp</code>，我们可以在将原始<code>forward</code>移到<code>helper wrapper</code>中后，暂时关闭它，如下所示：",rl,Ms,Tl,js,Me="由于自动检测器仅报告完整<code>frame</code>的输入和输出，一旦知道在哪里查找，您可能还希望分析特定<code>forward</code>函数的中间阶段。在这种情况下，您可以使用<code>detect_overflow</code>辅助函数将检测器放到希望的位置，例如：",Jl,us,ml,cs,je="可以看到，我们添加了2个检测器，现在我们可以跟踪是否在<code>forwarded_states</code>中间的某个地方检测到了<code>inf</code>或<code>nan</code>。",bl,ws,ue="实际上，检测器已经报告了这些，因为上面示例中的每个调用都是一个<code>nn.Module</code>，但假设如果您有一些本地的直接计算，这就是您将如何执行的方式。",dl,rs,ce="此外，如果您在自己的代码中实例化调试器，您可以调整从其默认打印的<code>frame</code>数，例如：",il,Ts,hl,Js,yl,ms,we="当关闭下溢/上溢检测功能, 同样的调试类可以用于批处理跟踪。",Il,bs,re="假设您想要监视给定批次的每个<code>forward</code>调用的所有成分的绝对最小值和最大值，并且仅对批次1和3执行此操作，您可以这样实例化这个类：",Ul,ds,ol,is,Te="现在，完整的批次1和3将以与下溢/上溢检测器相同的格式进行跟踪。",fl,hs,Je="批次从0开始计数。",Cl,ys,me="如果您知道程序在某个批次编号之后开始出现问题，那么您可以直接快进到该区域。以下是一个截取的配置示例输出：",Zl,Is,Al,Us,be="在这里，您将获得大量的<code>frame</code>被<code>dump</code> - 与您的模型中的前向调用一样多，它有可能符合也可能不符合您的要求，但有时对于调试目的来说，它可能比正常的调试器更容易使用。例如，如果问题开始发生在批次号150上，您可以<code>dump</code>批次149和150的跟踪，并比较数字开始发散的地方。",Gl,os,de="你还可以使用以下命令指定停止训练的批次号：",Vl,fs,Bl,Cs,$l,Zs,Rl;return f=new Ll({props:{title:"调试",local:"调试",headingTag:"h1"}}),C=new Ll({props:{title:"多GPU网络问题调试",local:"多gpu网络问题调试",headingTag:"h2"}}),A=new d({props:{code:"d2dldCUyMGh0dHBzJTNBJTJGJTJGcmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSUyRmh1Z2dpbmdmYWNlJTJGdHJhbnNmb3JtZXJzJTJGbWFpbiUyRnNjcmlwdHMlMkZkaXN0cmlidXRlZCUyRnRvcmNoLWRpc3RyaWJ1dGVkLWdwdS10ZXN0LnB5",highlighted:"wget https://raw.githubusercontent.com/huggingface/transformers/main/scripts/distributed/torch-distributed-gpu-test.py",wrap:!1}}),V=new d({props:{code:"cHl0aG9uJTIwLW0lMjB0b3JjaC5kaXN0cmlidXRlZC5ydW4lMjAtLW5wcm9jX3Blcl9ub2RlJTIwMiUyMC0tbm5vZGVzJTIwMSUyMHRvcmNoLWRpc3RyaWJ1dGVkLWdwdS10ZXN0LnB5",highlighted:"python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py",wrap:!1}}),_=new d({props:{code:"TkNDTF9ERUJVRyUzRElORk8lMjBweXRob24lMjAtbSUyMHRvcmNoLmRpc3RyaWJ1dGVkLnJ1biUyMC0tbnByb2NfcGVyX25vZGUlMjAyJTIwLS1ubm9kZXMlMjAxJTIwdG9yY2gtZGlzdHJpYnV0ZWQtZ3B1LXRlc3QucHk=",highlighted:"NCCL_DEBUG=INFO python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py",wrap:!1}}),N=new Ll({props:{title:"下溢和上溢检测",local:"下溢和上溢检测",headingTag:"h2"}}),I=new _l({props:{$$slots:{default:[Be]},$$scope:{ctx:y}}}),U=new _l({props:{$$slots:{default:[$e]},$$scope:{ctx:y}}}),o=new _l({props:{$$slots:{default:[Re]},$$scope:{ctx:y}}}),E=new d({props:{code:"LS1kZWJ1ZyUyMHVuZGVyZmxvd19vdmVyZmxvdw==",highlighted:"--debug underflow_overflow",wrap:!1}}),Q=new d({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5kZWJ1Z191dGlscyUyMGltcG9ydCUyMERlYnVnVW5kZXJmbG93T3ZlcmZsb3clMEElMEFkZWJ1Z19vdmVyZmxvdyUyMCUzRCUyMERlYnVnVW5kZXJmbG93T3ZlcmZsb3cobW9kZWwp",highlighted:`<span class="hljs-keyword">from</span> transformers.debug_utils <span class="hljs-keyword">import</span> DebugUnderflowOverflow

debug_overflow = DebugUnderflowOverflow(model)`,wrap:!1}}),D=new d({props:{code:"RGV0ZWN0ZWQlMjBpbmYlMkZuYW4lMjBkdXJpbmclMjBiYXRjaF9udW1iZXIlM0QwJTBBTGFzdCUyMDIxJTIwZm9yd2FyZCUyMGZyYW1lcyUzQSUwQWFicyUyMG1pbiUyMCUyMGFicyUyMG1heCUyMCUyMG1ldGFkYXRhJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4xLmxheWVyLjEuRGVuc2VSZWx1RGVuc2UuZHJvcG91dCUyMERyb3BvdXQlMEEwLjAwZSUyQjAwJTIwMi41N2UlMkIwMiUyMGlucHV0JTVCMCU1RCUwQTAuMDBlJTJCMDAlMjAyLjg1ZSUyQjAyJTIwb3V0cHV0JTBBJTVCLi4uJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjAlMjBUNUxheWVyU2VsZkF0dGVudGlvbiUwQTYuNzhlLTA0JTIwMy4xNWUlMkIwMyUyMGlucHV0JTVCMCU1RCUwQTIuNjVlLTA0JTIwMy40MmUlMkIwMyUyMG91dHB1dCU1QjAlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBOb25lJTIwb3V0cHV0JTVCMSU1RCUwQTIuMjVlLTAxJTIwMS4wMGUlMkIwNCUyMG91dHB1dCU1QjIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5sYXllcl9ub3JtJTIwVDVMYXllck5vcm0lMEE4LjY5ZS0wMiUyMDQuMThlLTAxJTIwd2VpZ2h0JTBBMi42NWUtMDQlMjAzLjQyZSUyQjAzJTIwaW5wdXQlNUIwJTVEJTBBMS43OWUtMDYlMjA0LjY1ZSUyQjAwJTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjEuRGVuc2VSZWx1RGVuc2Uud2lfMCUyMExpbmVhciUwQTIuMTdlLTA3JTIwNC41MGUlMkIwMCUyMHdlaWdodCUwQTEuNzllLTA2JTIwNC42NWUlMkIwMCUyMGlucHV0JTVCMCU1RCUwQTIuNjhlLTA2JTIwMy43MGUlMkIwMSUyMG91dHB1dCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVuY29kZXIuYmxvY2suMi5sYXllci4xLkRlbnNlUmVsdURlbnNlLndpXzElMjBMaW5lYXIlMEE4LjA4ZS0wNyUyMDIuNjZlJTJCMDElMjB3ZWlnaHQlMEExLjc5ZS0wNiUyMDQuNjVlJTJCMDAlMjBpbnB1dCU1QjAlNUQlMEExLjI3ZS0wNCUyMDIuMzdlJTJCMDIlMjBvdXRwdXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5EZW5zZVJlbHVEZW5zZS5kcm9wb3V0JTIwRHJvcG91dCUwQTAuMDBlJTJCMDAlMjA4Ljc2ZSUyQjAzJTIwaW5wdXQlNUIwJTVEJTBBMC4wMGUlMkIwMCUyMDkuNzRlJTJCMDMlMjBvdXRwdXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5EZW5zZVJlbHVEZW5zZS53byUyMExpbmVhciUwQTEuMDFlLTA2JTIwNi40NGUlMkIwMCUyMHdlaWdodCUwQTAuMDBlJTJCMDAlMjA5Ljc0ZSUyQjAzJTIwaW5wdXQlNUIwJTVEJTBBMy4xOGUtMDQlMjA2LjI3ZSUyQjA0JTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjEuRGVuc2VSZWx1RGVuc2UlMjBUNURlbnNlR2F0ZWRHZWx1RGVuc2UlMEExLjc5ZS0wNiUyMDQuNjVlJTJCMDAlMjBpbnB1dCU1QjAlNUQlMEEzLjE4ZS0wNCUyMDYuMjdlJTJCMDQlMjBvdXRwdXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5kcm9wb3V0JTIwRHJvcG91dCUwQTMuMThlLTA0JTIwNi4yN2UlMkIwNCUyMGlucHV0JTVCMCU1RCUwQTAuMDBlJTJCMDAlMjAlMjAlMjAlMjAlMjAlMjBpbmYlMjBvdXRwdXQ=",highlighted:`<span class="hljs-attribute">Detected</span> inf/nan during batch_number=<span class="hljs-number">0</span>
<span class="hljs-attribute">Last</span> <span class="hljs-number">21</span> forward frames:
<span class="hljs-attribute">abs</span> min  abs max  metadata
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">1</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.dropout Dropout
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">2</span>.<span class="hljs-number">57</span>e+<span class="hljs-number">02</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">2</span>.<span class="hljs-number">85</span>e+<span class="hljs-number">02</span> output<span class="hljs-meta">
[...]</span>
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">0</span> T5LayerSelfAttention
<span class="hljs-attribute">6</span>.<span class="hljs-number">78</span>e-<span class="hljs-number">04</span> <span class="hljs-number">3</span>.<span class="hljs-number">15</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">2</span>.<span class="hljs-number">65</span>e-<span class="hljs-number">04</span> <span class="hljs-number">3</span>.<span class="hljs-number">42</span>e+<span class="hljs-number">03</span> output[<span class="hljs-number">0</span>]
             <span class="hljs-attribute">None</span> output[<span class="hljs-number">1</span>]
<span class="hljs-attribute">2</span>.<span class="hljs-number">25</span>e-<span class="hljs-number">01</span> <span class="hljs-number">1</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">04</span> output[<span class="hljs-number">2</span>]
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.layer_norm T5LayerNorm
<span class="hljs-attribute">8</span>.<span class="hljs-number">69</span>e-<span class="hljs-number">02</span> <span class="hljs-number">4</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">01</span> weight
<span class="hljs-attribute">2</span>.<span class="hljs-number">65</span>e-<span class="hljs-number">04</span> <span class="hljs-number">3</span>.<span class="hljs-number">42</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wi_0 Linear
<span class="hljs-attribute">2</span>.<span class="hljs-number">17</span>e-<span class="hljs-number">07</span> <span class="hljs-number">4</span>.<span class="hljs-number">50</span>e+<span class="hljs-number">00</span> weight
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">2</span>.<span class="hljs-number">68</span>e-<span class="hljs-number">06</span> <span class="hljs-number">3</span>.<span class="hljs-number">70</span>e+<span class="hljs-number">01</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wi_1 Linear
<span class="hljs-attribute">8</span>.<span class="hljs-number">08</span>e-<span class="hljs-number">07</span> <span class="hljs-number">2</span>.<span class="hljs-number">66</span>e+<span class="hljs-number">01</span> weight
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">1</span>.<span class="hljs-number">27</span>e-<span class="hljs-number">04</span> <span class="hljs-number">2</span>.<span class="hljs-number">37</span>e+<span class="hljs-number">02</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.dropout Dropout
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">8</span>.<span class="hljs-number">76</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">9</span>.<span class="hljs-number">74</span>e+<span class="hljs-number">03</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wo Linear
<span class="hljs-attribute">1</span>.<span class="hljs-number">01</span>e-<span class="hljs-number">06</span> <span class="hljs-number">6</span>.<span class="hljs-number">44</span>e+<span class="hljs-number">00</span> weight
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">9</span>.<span class="hljs-number">74</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense T5DenseGatedGeluDense
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.dropout Dropout
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span>      inf output`,wrap:!1}}),F=new d({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjEubGF5ZXJfbm9ybSUyMFQ1TGF5ZXJOb3JtJTBBOC42OWUtMDIlMjA0LjE4ZS0wMSUyMHdlaWdodCUwQTIuNjVlLTA0JTIwMy40MmUlMkIwMyUyMGlucHV0JTVCMCU1RCUwQTEuNzllLTA2JTIwNC42NWUlMkIwMCUyMG91dHB1dA==",highlighted:`                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.layer_norm T5LayerNorm
<span class="hljs-attribute">8</span>.<span class="hljs-number">69</span>e-<span class="hljs-number">02</span> <span class="hljs-number">4</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">01</span> weight
<span class="hljs-attribute">2</span>.<span class="hljs-number">65</span>e-<span class="hljs-number">04</span> <span class="hljs-number">3</span>.<span class="hljs-number">42</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> output`,wrap:!1}}),q=new d({props:{code:"RGV0ZWN0ZWQlMjBpbmYlMkZuYW4lMjBkdXJpbmclMjBiYXRjaF9udW1iZXIlM0QwJTBBTGFzdCUyMDIxJTIwZm9yd2FyZCUyMGZyYW1lcyUzQSUwQWFicyUyMG1pbiUyMCUyMGFicyUyMG1heCUyMCUyMG1ldGFkYXRhJTBBJTVCLi4uJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjEuRGVuc2VSZWx1RGVuc2Uud2lfMCUyMExpbmVhciUwQTIuMTdlLTA3JTIwNC41MGUlMkIwMCUyMHdlaWdodCUwQTEuNzllLTA2JTIwNC42NWUlMkIwMCUyMGlucHV0JTVCMCU1RCUwQTIuNjhlLTA2JTIwMy43MGUlMkIwMSUyMG91dHB1dCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVuY29kZXIuYmxvY2suMi5sYXllci4xLkRlbnNlUmVsdURlbnNlLndpXzElMjBMaW5lYXIlMEE4LjA4ZS0wNyUyMDIuNjZlJTJCMDElMjB3ZWlnaHQlMEExLjc5ZS0wNiUyMDQuNjVlJTJCMDAlMjBpbnB1dCU1QjAlNUQlMEExLjI3ZS0wNCUyMDIuMzdlJTJCMDIlMjBvdXRwdXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5EZW5zZVJlbHVEZW5zZS53byUyMExpbmVhciUwQTEuMDFlLTA2JTIwNi40NGUlMkIwMCUyMHdlaWdodCUwQTAuMDBlJTJCMDAlMjA5Ljc0ZSUyQjAzJTIwaW5wdXQlNUIwJTVEJTBBMy4xOGUtMDQlMjA2LjI3ZSUyQjA0JTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjEuRGVuc2VSZWx1RGVuc2UlMjBUNURlbnNlR2F0ZWRHZWx1RGVuc2UlMEExLjc5ZS0wNiUyMDQuNjVlJTJCMDAlMjBpbnB1dCU1QjAlNUQlMEEzLjE4ZS0wNCUyMDYuMjdlJTJCMDQlMjBvdXRwdXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5kcm9wb3V0JTIwRHJvcG91dCUwQTMuMThlLTA0JTIwNi4yN2UlMkIwNCUyMGlucHV0JTVCMCU1RCUwQTAuMDBlJTJCMDAlMjAlMjAlMjAlMjAlMjAlMjBpbmYlMjBvdXRwdXQ=",highlighted:`<span class="hljs-attribute">Detected</span> inf/nan during batch_number=<span class="hljs-number">0</span>
<span class="hljs-attribute">Last</span> <span class="hljs-number">21</span> forward frames:
<span class="hljs-attribute">abs</span> min  abs max  metadata<span class="hljs-meta">
[...]</span>
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wi_0 Linear
<span class="hljs-attribute">2</span>.<span class="hljs-number">17</span>e-<span class="hljs-number">07</span> <span class="hljs-number">4</span>.<span class="hljs-number">50</span>e+<span class="hljs-number">00</span> weight
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">2</span>.<span class="hljs-number">68</span>e-<span class="hljs-number">06</span> <span class="hljs-number">3</span>.<span class="hljs-number">70</span>e+<span class="hljs-number">01</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wi_1 Linear
<span class="hljs-attribute">8</span>.<span class="hljs-number">08</span>e-<span class="hljs-number">07</span> <span class="hljs-number">2</span>.<span class="hljs-number">66</span>e+<span class="hljs-number">01</span> weight
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">1</span>.<span class="hljs-number">27</span>e-<span class="hljs-number">04</span> <span class="hljs-number">2</span>.<span class="hljs-number">37</span>e+<span class="hljs-number">02</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wo Linear
<span class="hljs-attribute">1</span>.<span class="hljs-number">01</span>e-<span class="hljs-number">06</span> <span class="hljs-number">6</span>.<span class="hljs-number">44</span>e+<span class="hljs-number">00</span> weight
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">9</span>.<span class="hljs-number">74</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense T5DenseGatedGeluDense
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.dropout Dropout
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span>      inf output`,wrap:!1}}),ns=new d({props:{code:"Y2xhc3MlMjBUNURlbnNlR2F0ZWRHZWx1RGVuc2Uobm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYlMkMlMjBjb25maWcpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VwZXIoKS5fX2luaXRfXygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi53aV8wJTIwJTNEJTIwbm4uTGluZWFyKGNvbmZpZy5kX21vZGVsJTJDJTIwY29uZmlnLmRfZmYlMkMlMjBiaWFzJTNERmFsc2UpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi53aV8xJTIwJTNEJTIwbm4uTGluZWFyKGNvbmZpZy5kX21vZGVsJTJDJTIwY29uZmlnLmRfZmYlMkMlMjBiaWFzJTNERmFsc2UpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi53byUyMCUzRCUyMG5uLkxpbmVhcihjb25maWcuZF9mZiUyQyUyMGNvbmZpZy5kX21vZGVsJTJDJTIwYmlhcyUzREZhbHNlKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYuZHJvcG91dCUyMCUzRCUyMG5uLkRyb3BvdXQoY29uZmlnLmRyb3BvdXRfcmF0ZSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLmdlbHVfYWN0JTIwJTNEJTIwQUNUMkZOJTVCJTIyZ2VsdV9uZXclMjIlNUQlMEElMEElMjAlMjAlMjAlMjBkZWYlMjBmb3J3YXJkKHNlbGYlMkMlMjBoaWRkZW5fc3RhdGVzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGhpZGRlbl9nZWx1JTIwJTNEJTIwc2VsZi5nZWx1X2FjdChzZWxmLndpXzAoaGlkZGVuX3N0YXRlcykpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaGlkZGVuX2xpbmVhciUyMCUzRCUyMHNlbGYud2lfMShoaWRkZW5fc3RhdGVzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGhpZGRlbl9zdGF0ZXMlMjAlM0QlMjBoaWRkZW5fZ2VsdSUyMColMjBoaWRkZW5fbGluZWFyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaGlkZGVuX3N0YXRlcyUyMCUzRCUyMHNlbGYuZHJvcG91dChoaWRkZW5fc3RhdGVzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGhpZGRlbl9zdGF0ZXMlMjAlM0QlMjBzZWxmLndvKGhpZGRlbl9zdGF0ZXMpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwaGlkZGVuX3N0YXRlcw==",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">T5DenseGatedGeluDense</span>(nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.wi_0 = nn.Linear(config.d_model, config.d_ff, bias=<span class="hljs-literal">False</span>)
        self.wi_1 = nn.Linear(config.d_model, config.d_ff, bias=<span class="hljs-literal">False</span>)
        self.wo = nn.Linear(config.d_ff, config.d_model, bias=<span class="hljs-literal">False</span>)
        self.dropout = nn.Dropout(config.dropout_rate)
        self.gelu_act = ACT2FN[<span class="hljs-string">&quot;gelu_new&quot;</span>]

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, hidden_states</span>):
        hidden_gelu = self.gelu_act(self.wi_0(hidden_states))
        hidden_linear = self.wi_1(hidden_states)
        hidden_states = hidden_gelu * hidden_linear
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.wo(hidden_states)
        <span class="hljs-keyword">return</span> hidden_states`,wrap:!1}}),Ms=new d({props:{code:"ZGVmJTIwX2ZvcndhcmQoc2VsZiUyQyUyMGhpZGRlbl9zdGF0ZXMpJTNBJTBBJTIwJTIwJTIwJTIwaGlkZGVuX2dlbHUlMjAlM0QlMjBzZWxmLmdlbHVfYWN0KHNlbGYud2lfMChoaWRkZW5fc3RhdGVzKSklMEElMjAlMjAlMjAlMjBoaWRkZW5fbGluZWFyJTIwJTNEJTIwc2VsZi53aV8xKGhpZGRlbl9zdGF0ZXMpJTBBJTIwJTIwJTIwJTIwaGlkZGVuX3N0YXRlcyUyMCUzRCUyMGhpZGRlbl9nZWx1JTIwKiUyMGhpZGRlbl9saW5lYXIlMEElMjAlMjAlMjAlMjBoaWRkZW5fc3RhdGVzJTIwJTNEJTIwc2VsZi5kcm9wb3V0KGhpZGRlbl9zdGF0ZXMpJTBBJTIwJTIwJTIwJTIwaGlkZGVuX3N0YXRlcyUyMCUzRCUyMHNlbGYud28oaGlkZGVuX3N0YXRlcyklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBoaWRkZW5fc3RhdGVzJTBBJTBBJTBBaW1wb3J0JTIwdG9yY2glMEElMEElMEFkZWYlMjBmb3J3YXJkKHNlbGYlMkMlMjBoaWRkZW5fc3RhdGVzKSUzQSUwQSUyMCUyMCUyMCUyMGlmJTIwdG9yY2guaXNfYXV0b2Nhc3RfZW5hYmxlZCgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2l0aCUyMHRvcmNoLmN1ZGEuYW1wLmF1dG9jYXN0KGVuYWJsZWQlM0RGYWxzZSklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBzZWxmLl9mb3J3YXJkKGhpZGRlbl9zdGF0ZXMpJTBBJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMHNlbGYuX2ZvcndhcmQoaGlkZGVuX3N0YXRlcyk=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">_forward</span>(<span class="hljs-params">self, hidden_states</span>):
    hidden_gelu = self.gelu_act(self.wi_0(hidden_states))
    hidden_linear = self.wi_1(hidden_states)
    hidden_states = hidden_gelu * hidden_linear
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.wo(hidden_states)
    <span class="hljs-keyword">return</span> hidden_states


<span class="hljs-keyword">import</span> torch


<span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, hidden_states</span>):
    <span class="hljs-keyword">if</span> torch.is_autocast_enabled():
        <span class="hljs-keyword">with</span> torch.cuda.amp.autocast(enabled=<span class="hljs-literal">False</span>):
            <span class="hljs-keyword">return</span> self._forward(hidden_states)
    <span class="hljs-keyword">else</span>:
        <span class="hljs-keyword">return</span> self._forward(hidden_states)`,wrap:!1}}),us=new d({props:{code:"ZnJvbSUyMGRlYnVnX3V0aWxzJTIwaW1wb3J0JTIwZGV0ZWN0X292ZXJmbG93JTBBJTBBJTBBY2xhc3MlMjBUNUxheWVyRkYobm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMCU1Qi4uLiU1RCUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMGhpZGRlbl9zdGF0ZXMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yd2FyZGVkX3N0YXRlcyUyMCUzRCUyMHNlbGYubGF5ZXJfbm9ybShoaWRkZW5fc3RhdGVzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGRldGVjdF9vdmVyZmxvdyhmb3J3YXJkZWRfc3RhdGVzJTJDJTIwJTIyYWZ0ZXIlMjBsYXllcl9ub3JtJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvcndhcmRlZF9zdGF0ZXMlMjAlM0QlMjBzZWxmLkRlbnNlUmVsdURlbnNlKGZvcndhcmRlZF9zdGF0ZXMpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZGV0ZWN0X292ZXJmbG93KGZvcndhcmRlZF9zdGF0ZXMlMkMlMjAlMjJhZnRlciUyMERlbnNlUmVsdURlbnNlJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMGhpZGRlbl9zdGF0ZXMlMjAlMkIlMjBzZWxmLmRyb3BvdXQoZm9yd2FyZGVkX3N0YXRlcyk=",highlighted:`<span class="hljs-keyword">from</span> debug_utils <span class="hljs-keyword">import</span> detect_overflow


<span class="hljs-keyword">class</span> <span class="hljs-title class_">T5LayerFF</span>(nn.Module):
    [...]

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, hidden_states</span>):
        forwarded_states = self.layer_norm(hidden_states)
        detect_overflow(forwarded_states, <span class="hljs-string">&quot;after layer_norm&quot;</span>)
        forwarded_states = self.DenseReluDense(forwarded_states)
        detect_overflow(forwarded_states, <span class="hljs-string">&quot;after DenseReluDense&quot;</span>)
        <span class="hljs-keyword">return</span> hidden_states + self.dropout(forwarded_states)`,wrap:!1}}),Ts=new d({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5kZWJ1Z191dGlscyUyMGltcG9ydCUyMERlYnVnVW5kZXJmbG93T3ZlcmZsb3clMEElMEFkZWJ1Z19vdmVyZmxvdyUyMCUzRCUyMERlYnVnVW5kZXJmbG93T3ZlcmZsb3cobW9kZWwlMkMlMjBtYXhfZnJhbWVzX3RvX3NhdmUlM0QxMDAp",highlighted:`<span class="hljs-keyword">from</span> transformers.debug_utils <span class="hljs-keyword">import</span> DebugUnderflowOverflow

debug_overflow = DebugUnderflowOverflow(model, max_frames_to_save=<span class="hljs-number">100</span>)`,wrap:!1}}),Js=new Ll({props:{title:"特定批次的绝对最小值和最大值跟踪",local:"特定批次的绝对最小值和最大值跟踪",headingTag:"h3"}}),ds=new d({props:{code:"ZGVidWdfb3ZlcmZsb3clMjAlM0QlMjBEZWJ1Z1VuZGVyZmxvd092ZXJmbG93KG1vZGVsJTJDJTIwdHJhY2VfYmF0Y2hfbnVtcyUzRCU1QjElMkMlMjAzJTVEKQ==",highlighted:'debug_overflow = DebugUnderflowOverflow(model, trace_batch_nums=[<span class="hljs-number">1</span>, <span class="hljs-number">3</span>])',wrap:!1}}),Is=new d({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKioqJTIwU3RhcnRpbmclMjBiYXRjaCUyMG51bWJlciUzRDElMjAqKiolMEFhYnMlMjBtaW4lMjAlMjBhYnMlMjBtYXglMjAlMjBtZXRhZGF0YSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNoYXJlZCUyMEVtYmVkZGluZyUwQTEuMDFlLTA2JTIwNy45MmUlMkIwMiUyMHdlaWdodCUwQTAuMDBlJTJCMDAlMjAyLjQ3ZSUyQjA0JTIwaW5wdXQlNUIwJTVEJTBBNS4zNmUtMDUlMjA3LjkyZSUyQjAyJTIwb3V0cHV0JTBBJTVCLi4uJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZGVjb2Rlci5kcm9wb3V0JTIwRHJvcG91dCUwQTEuNjBlLTA3JTIwMi4yN2UlMkIwMSUyMGlucHV0JTVCMCU1RCUwQTAuMDBlJTJCMDAlMjAyLjUyZSUyQjAxJTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZGVjb2RlciUyMFQ1U3RhY2slMEElMjAlMjAlMjAlMjAlMjBub3QlMjBhJTIwdGVuc29yJTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbG1faGVhZCUyMExpbmVhciUwQTEuMDFlLTA2JTIwNy45MmUlMkIwMiUyMHdlaWdodCUwQTAuMDBlJTJCMDAlMjAxLjExZSUyQjAwJTIwaW5wdXQlNUIwJTVEJTBBNi4wNmUtMDIlMjA4LjM5ZSUyQjAxJTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwVDVGb3JDb25kaXRpb25hbEdlbmVyYXRpb24lMEElMjAlMjAlMjAlMjAlMjBub3QlMjBhJTIwdGVuc29yJTIwb3V0cHV0JTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKioqJTIwU3RhcnRpbmclMjBiYXRjaCUyMG51bWJlciUzRDMlMjAqKiolMEFhYnMlMjBtaW4lMjAlMjBhYnMlMjBtYXglMjAlMjBtZXRhZGF0YSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNoYXJlZCUyMEVtYmVkZGluZyUwQTEuMDFlLTA2JTIwNy45MmUlMkIwMiUyMHdlaWdodCUwQTAuMDBlJTJCMDAlMjAyLjc4ZSUyQjA0JTIwaW5wdXQlNUIwJTVEJTBBNS4zNmUtMDUlMjA3LjkyZSUyQjAyJTIwb3V0cHV0JTBBJTVCLi4uJTVE",highlighted:`                  *** Starting batch number=1 ***
abs min  abs max  metadata
                  shared Embedding
1.01e<span class="hljs-string">-06</span> 7.92e<span class="hljs-string">+02</span> weight
0.00e<span class="hljs-string">+00</span> 2.47e<span class="hljs-string">+04</span> input[0]
5.36e<span class="hljs-string">-05</span> 7.92e<span class="hljs-string">+02</span> output
[...]
                  decoder.dropout Dropout
1.60e<span class="hljs-string">-07</span> 2.27e<span class="hljs-string">+01</span> input[0]
0.00e<span class="hljs-string">+00</span> 2.52e<span class="hljs-string">+01</span> output
                  decoder T5Stack
     not a tensor output
                  lm_head Linear
1.01e<span class="hljs-string">-06</span> 7.92e<span class="hljs-string">+02</span> weight
0.00e<span class="hljs-string">+00</span> 1.11e<span class="hljs-string">+00</span> input[0]
6.06e<span class="hljs-string">-02</span> 8.39e<span class="hljs-string">+01</span> output
                   T5ForConditionalGeneration
     not a tensor output

                  *** Starting batch number=3 ***
abs min  abs max  metadata
                  shared Embedding
1.01e<span class="hljs-string">-06</span> 7.92e<span class="hljs-string">+02</span> weight
0.00e<span class="hljs-string">+00</span> 2.78e<span class="hljs-string">+04</span> input[0]
5.36e<span class="hljs-string">-05</span> 7.92e<span class="hljs-string">+02</span> output
[...]`,wrap:!1}}),fs=new d({props:{code:"ZGVidWdfb3ZlcmZsb3clMjAlM0QlMjBEZWJ1Z1VuZGVyZmxvd092ZXJmbG93KG1vZGVsJTJDJTIwdHJhY2VfYmF0Y2hfbnVtcyUzRCU1QjElMkMlMjAzJTVEJTJDJTIwYWJvcnRfYWZ0ZXJfYmF0Y2hfbnVtJTNEMyk=",highlighted:'debug_overflow = DebugUnderflowOverflow(model, trace_batch_nums=[<span class="hljs-number">1</span>, <span class="hljs-number">3</span>], abort_after_batch_num=<span class="hljs-number">3</span>)',wrap:!1}}),Cs=new Ve({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/zh/debugging.md"}}),{c(){u=p("meta"),i=a(),c=p("p"),h=a(),w(f.$$.fragment),As=a(),w(C.$$.fragment),Gs=a(),Z=p("p"),Z.innerHTML=Nl,Vs=a(),w(A.$$.fragment),Bs=a(),G=p("p"),G.textContent=Wl,$s=a(),w(V.$$.fragment),Rs=a(),B=p("p"),B.textContent=vl,Ls=a(),$=p("p"),$.textContent=El,_s=a(),R=p("p"),R.textContent=xl,ks=a(),L=p("p"),L.innerHTML=Xl,Ns=a(),w(_.$$.fragment),Ws=a(),k=p("p"),k.innerHTML=Ql,vs=a(),w(N.$$.fragment),Es=a(),w(I.$$.fragment),xs=a(),w(U.$$.fragment),Xs=a(),w(o.$$.fragment),Qs=a(),W=p("p"),W.innerHTML=Hl,Hs=a(),v=p("p"),v.innerHTML=Dl,Ds=a(),w(E.$$.fragment),Ys=a(),x=p("p"),x.innerHTML=Yl,gs=a(),X=p("p"),X.textContent=gl,Ss=a(),w(Q.$$.fragment),zs=a(),H=p("p"),H.innerHTML=Sl,Fs=a(),w(D.$$.fragment),Ps=a(),Y=p("p"),Y.textContent=zl,Ks=a(),g=p("p"),g.innerHTML=Fl,qs=a(),S=p("p"),S.innerHTML=Pl,Os=a(),z=p("p"),z.innerHTML=Kl,sl=a(),w(F.$$.fragment),ll=a(),P=p("p"),P.innerHTML=ql,el=a(),K=p("p"),K.innerHTML=Ol,nl=a(),w(q.$$.fragment),al=a(),O=p("p"),O.innerHTML=se,tl=a(),ss=p("p"),ss.innerHTML=le,pl=a(),ls=p("p"),ls.innerHTML=ee,Ml=a(),es=p("p"),es.innerHTML=ne,jl=a(),w(ns.$$.fragment),ul=a(),as=p("p"),as.innerHTML=ae,cl=a(),ts=p("p"),ts.innerHTML=te,wl=a(),ps=p("p"),ps.innerHTML=pe,rl=a(),w(Ms.$$.fragment),Tl=a(),js=p("p"),js.innerHTML=Me,Jl=a(),w(us.$$.fragment),ml=a(),cs=p("p"),cs.innerHTML=je,bl=a(),ws=p("p"),ws.innerHTML=ue,dl=a(),rs=p("p"),rs.innerHTML=ce,il=a(),w(Ts.$$.fragment),hl=a(),w(Js.$$.fragment),yl=a(),ms=p("p"),ms.textContent=we,Il=a(),bs=p("p"),bs.innerHTML=re,Ul=a(),w(ds.$$.fragment),ol=a(),is=p("p"),is.textContent=Te,fl=a(),hs=p("p"),hs.textContent=Je,Cl=a(),ys=p("p"),ys.textContent=me,Zl=a(),w(Is.$$.fragment),Al=a(),Us=p("p"),Us.innerHTML=be,Gl=a(),os=p("p"),os.textContent=de,Vl=a(),w(fs.$$.fragment),Bl=a(),w(Cs.$$.fragment),$l=a(),Zs=p("p"),this.h()},l(s){const l=Ae("svelte-u9bgzb",document.head);u=M(l,"META",{name:!0,content:!0}),l.forEach(e),i=t(s),c=M(s,"P",{}),Ie(c).forEach(e),h=t(s),r(f.$$.fragment,s),As=t(s),r(C.$$.fragment,s),Gs=t(s),Z=M(s,"P",{"data-svelte-h":!0}),j(Z)!=="svelte-67xt30"&&(Z.innerHTML=Nl),Vs=t(s),r(A.$$.fragment,s),Bs=t(s),G=M(s,"P",{"data-svelte-h":!0}),j(G)!=="svelte-1s4g8yw"&&(G.textContent=Wl),$s=t(s),r(V.$$.fragment,s),Rs=t(s),B=M(s,"P",{"data-svelte-h":!0}),j(B)!=="svelte-qy3f9"&&(B.textContent=vl),Ls=t(s),$=M(s,"P",{"data-svelte-h":!0}),j($)!=="svelte-14wiu9e"&&($.textContent=El),_s=t(s),R=M(s,"P",{"data-svelte-h":!0}),j(R)!=="svelte-z3falj"&&(R.textContent=xl),ks=t(s),L=M(s,"P",{"data-svelte-h":!0}),j(L)!=="svelte-i3lae5"&&(L.innerHTML=Xl),Ns=t(s),r(_.$$.fragment,s),Ws=t(s),k=M(s,"P",{"data-svelte-h":!0}),j(k)!=="svelte-1tjfq9j"&&(k.innerHTML=Ql),vs=t(s),r(N.$$.fragment,s),Es=t(s),r(I.$$.fragment,s),xs=t(s),r(U.$$.fragment,s),Xs=t(s),r(o.$$.fragment,s),Qs=t(s),W=M(s,"P",{"data-svelte-h":!0}),j(W)!=="svelte-11jqvlw"&&(W.innerHTML=Hl),Hs=t(s),v=M(s,"P",{"data-svelte-h":!0}),j(v)!=="svelte-g31qm2"&&(v.innerHTML=Dl),Ds=t(s),r(E.$$.fragment,s),Ys=t(s),x=M(s,"P",{"data-svelte-h":!0}),j(x)!=="svelte-cy1a69"&&(x.innerHTML=Yl),gs=t(s),X=M(s,"P",{"data-svelte-h":!0}),j(X)!=="svelte-dlmaea"&&(X.textContent=gl),Ss=t(s),r(Q.$$.fragment,s),zs=t(s),H=M(s,"P",{"data-svelte-h":!0}),j(H)!=="svelte-1ee18dl"&&(H.innerHTML=Sl),Fs=t(s),r(D.$$.fragment,s),Ps=t(s),Y=M(s,"P",{"data-svelte-h":!0}),j(Y)!=="svelte-m2e293"&&(Y.textContent=zl),Ks=t(s),g=M(s,"P",{"data-svelte-h":!0}),j(g)!=="svelte-iz2hps"&&(g.innerHTML=Fl),qs=t(s),S=M(s,"P",{"data-svelte-h":!0}),j(S)!=="svelte-ilrczr"&&(S.innerHTML=Pl),Os=t(s),z=M(s,"P",{"data-svelte-h":!0}),j(z)!=="svelte-1oprv25"&&(z.innerHTML=Kl),sl=t(s),r(F.$$.fragment,s),ll=t(s),P=M(s,"P",{"data-svelte-h":!0}),j(P)!=="svelte-shsskp"&&(P.innerHTML=ql),el=t(s),K=M(s,"P",{"data-svelte-h":!0}),j(K)!=="svelte-2wn20y"&&(K.innerHTML=Ol),nl=t(s),r(q.$$.fragment,s),al=t(s),O=M(s,"P",{"data-svelte-h":!0}),j(O)!=="svelte-w4aer1"&&(O.innerHTML=se),tl=t(s),ss=M(s,"P",{"data-svelte-h":!0}),j(ss)!=="svelte-1i1156u"&&(ss.innerHTML=le),pl=t(s),ls=M(s,"P",{"data-svelte-h":!0}),j(ls)!=="svelte-xkw1jz"&&(ls.innerHTML=ee),Ml=t(s),es=M(s,"P",{"data-svelte-h":!0}),j(es)!=="svelte-1i25iqy"&&(es.innerHTML=ne),jl=t(s),r(ns.$$.fragment,s),ul=t(s),as=M(s,"P",{"data-svelte-h":!0}),j(as)!=="svelte-1ruyyp3"&&(as.innerHTML=ae),cl=t(s),ts=M(s,"P",{"data-svelte-h":!0}),j(ts)!=="svelte-1l7ieub"&&(ts.innerHTML=te),wl=t(s),ps=M(s,"P",{"data-svelte-h":!0}),j(ps)!=="svelte-7cjiyk"&&(ps.innerHTML=pe),rl=t(s),r(Ms.$$.fragment,s),Tl=t(s),js=M(s,"P",{"data-svelte-h":!0}),j(js)!=="svelte-1o54bz9"&&(js.innerHTML=Me),Jl=t(s),r(us.$$.fragment,s),ml=t(s),cs=M(s,"P",{"data-svelte-h":!0}),j(cs)!=="svelte-7w1v62"&&(cs.innerHTML=je),bl=t(s),ws=M(s,"P",{"data-svelte-h":!0}),j(ws)!=="svelte-1qn1k7g"&&(ws.innerHTML=ue),dl=t(s),rs=M(s,"P",{"data-svelte-h":!0}),j(rs)!=="svelte-iwaw7y"&&(rs.innerHTML=ce),il=t(s),r(Ts.$$.fragment,s),hl=t(s),r(Js.$$.fragment,s),yl=t(s),ms=M(s,"P",{"data-svelte-h":!0}),j(ms)!=="svelte-ilqc08"&&(ms.textContent=we),Il=t(s),bs=M(s,"P",{"data-svelte-h":!0}),j(bs)!=="svelte-tojn2x"&&(bs.innerHTML=re),Ul=t(s),r(ds.$$.fragment,s),ol=t(s),is=M(s,"P",{"data-svelte-h":!0}),j(is)!=="svelte-frrq89"&&(is.textContent=Te),fl=t(s),hs=M(s,"P",{"data-svelte-h":!0}),j(hs)!=="svelte-1478tpq"&&(hs.textContent=Je),Cl=t(s),ys=M(s,"P",{"data-svelte-h":!0}),j(ys)!=="svelte-i9lsvp"&&(ys.textContent=me),Zl=t(s),r(Is.$$.fragment,s),Al=t(s),Us=M(s,"P",{"data-svelte-h":!0}),j(Us)!=="svelte-5ghfmp"&&(Us.innerHTML=be),Gl=t(s),os=M(s,"P",{"data-svelte-h":!0}),j(os)!=="svelte-qx2qdk"&&(os.textContent=de),Vl=t(s),r(fs.$$.fragment,s),Bl=t(s),r(Cs.$$.fragment,s),$l=t(s),Zs=M(s,"P",{}),Ie(Zs).forEach(e),this.h()},h(){Ue(u,"name","hf:doc:metadata"),Ue(u,"content",_e)},m(s,l){Ge(document.head,u),n(s,i,l),n(s,c,l),n(s,h,l),T(f,s,l),n(s,As,l),T(C,s,l),n(s,Gs,l),n(s,Z,l),n(s,Vs,l),T(A,s,l),n(s,Bs,l),n(s,G,l),n(s,$s,l),T(V,s,l),n(s,Rs,l),n(s,B,l),n(s,Ls,l),n(s,$,l),n(s,_s,l),n(s,R,l),n(s,ks,l),n(s,L,l),n(s,Ns,l),T(_,s,l),n(s,Ws,l),n(s,k,l),n(s,vs,l),T(N,s,l),n(s,Es,l),T(I,s,l),n(s,xs,l),T(U,s,l),n(s,Xs,l),T(o,s,l),n(s,Qs,l),n(s,W,l),n(s,Hs,l),n(s,v,l),n(s,Ds,l),T(E,s,l),n(s,Ys,l),n(s,x,l),n(s,gs,l),n(s,X,l),n(s,Ss,l),T(Q,s,l),n(s,zs,l),n(s,H,l),n(s,Fs,l),T(D,s,l),n(s,Ps,l),n(s,Y,l),n(s,Ks,l),n(s,g,l),n(s,qs,l),n(s,S,l),n(s,Os,l),n(s,z,l),n(s,sl,l),T(F,s,l),n(s,ll,l),n(s,P,l),n(s,el,l),n(s,K,l),n(s,nl,l),T(q,s,l),n(s,al,l),n(s,O,l),n(s,tl,l),n(s,ss,l),n(s,pl,l),n(s,ls,l),n(s,Ml,l),n(s,es,l),n(s,jl,l),T(ns,s,l),n(s,ul,l),n(s,as,l),n(s,cl,l),n(s,ts,l),n(s,wl,l),n(s,ps,l),n(s,rl,l),T(Ms,s,l),n(s,Tl,l),n(s,js,l),n(s,Jl,l),T(us,s,l),n(s,ml,l),n(s,cs,l),n(s,bl,l),n(s,ws,l),n(s,dl,l),n(s,rs,l),n(s,il,l),T(Ts,s,l),n(s,hl,l),T(Js,s,l),n(s,yl,l),n(s,ms,l),n(s,Il,l),n(s,bs,l),n(s,Ul,l),T(ds,s,l),n(s,ol,l),n(s,is,l),n(s,fl,l),n(s,hs,l),n(s,Cl,l),n(s,ys,l),n(s,Zl,l),T(Is,s,l),n(s,Al,l),n(s,Us,l),n(s,Gl,l),n(s,os,l),n(s,Vl,l),T(fs,s,l),n(s,Bl,l),T(Cs,s,l),n(s,$l,l),n(s,Zs,l),Rl=!0},p(s,[l]){const ie={};l&2&&(ie.$$scope={dirty:l,ctx:s}),I.$set(ie);const he={};l&2&&(he.$$scope={dirty:l,ctx:s}),U.$set(he);const ye={};l&2&&(ye.$$scope={dirty:l,ctx:s}),o.$set(ye)},i(s){Rl||(J(f.$$.fragment,s),J(C.$$.fragment,s),J(A.$$.fragment,s),J(V.$$.fragment,s),J(_.$$.fragment,s),J(N.$$.fragment,s),J(I.$$.fragment,s),J(U.$$.fragment,s),J(o.$$.fragment,s),J(E.$$.fragment,s),J(Q.$$.fragment,s),J(D.$$.fragment,s),J(F.$$.fragment,s),J(q.$$.fragment,s),J(ns.$$.fragment,s),J(Ms.$$.fragment,s),J(us.$$.fragment,s),J(Ts.$$.fragment,s),J(Js.$$.fragment,s),J(ds.$$.fragment,s),J(Is.$$.fragment,s),J(fs.$$.fragment,s),J(Cs.$$.fragment,s),Rl=!0)},o(s){m(f.$$.fragment,s),m(C.$$.fragment,s),m(A.$$.fragment,s),m(V.$$.fragment,s),m(_.$$.fragment,s),m(N.$$.fragment,s),m(I.$$.fragment,s),m(U.$$.fragment,s),m(o.$$.fragment,s),m(E.$$.fragment,s),m(Q.$$.fragment,s),m(D.$$.fragment,s),m(F.$$.fragment,s),m(q.$$.fragment,s),m(ns.$$.fragment,s),m(Ms.$$.fragment,s),m(us.$$.fragment,s),m(Ts.$$.fragment,s),m(Js.$$.fragment,s),m(ds.$$.fragment,s),m(Is.$$.fragment,s),m(fs.$$.fragment,s),m(Cs.$$.fragment,s),Rl=!1},d(s){s&&(e(i),e(c),e(h),e(As),e(Gs),e(Z),e(Vs),e(Bs),e(G),e($s),e(Rs),e(B),e(Ls),e($),e(_s),e(R),e(ks),e(L),e(Ns),e(Ws),e(k),e(vs),e(Es),e(xs),e(Xs),e(Qs),e(W),e(Hs),e(v),e(Ds),e(Ys),e(x),e(gs),e(X),e(Ss),e(zs),e(H),e(Fs),e(Ps),e(Y),e(Ks),e(g),e(qs),e(S),e(Os),e(z),e(sl),e(ll),e(P),e(el),e(K),e(nl),e(al),e(O),e(tl),e(ss),e(pl),e(ls),e(Ml),e(es),e(jl),e(ul),e(as),e(cl),e(ts),e(wl),e(ps),e(rl),e(Tl),e(js),e(Jl),e(ml),e(cs),e(bl),e(ws),e(dl),e(rs),e(il),e(hl),e(yl),e(ms),e(Il),e(bs),e(Ul),e(ol),e(is),e(fl),e(hs),e(Cl),e(ys),e(Zl),e(Al),e(Us),e(Gl),e(os),e(Vl),e(Bl),e($l),e(Zs)),e(u),b(f,s),b(C,s),b(A,s),b(V,s),b(_,s),b(N,s),b(I,s),b(U,s),b(o,s),b(E,s),b(Q,s),b(D,s),b(F,s),b(q,s),b(ns,s),b(Ms,s),b(us,s),b(Ts,s),b(Js,s),b(ds,s),b(Is,s),b(fs,s),b(Cs,s)}}}const _e='{"title":"调试","local":"调试","sections":[{"title":"多GPU网络问题调试","local":"多gpu网络问题调试","sections":[],"depth":2},{"title":"下溢和上溢检测","local":"下溢和上溢检测","sections":[{"title":"特定批次的绝对最小值和最大值跟踪","local":"特定批次的绝对最小值和最大值跟踪","sections":[],"depth":3}],"depth":2}],"depth":1}';function ke(y){return fe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Xe extends Ce{constructor(u){super(),Ze(this,u,ke,Le,oe,{})}}export{Xe as component};