transformers/pr_36049/zh/_app/immutable/nodes/8.e63ac3cc.js

import{s as K,n as N,o as q}from"../chunks/scheduler.25b97de1.js";import{S as F,i as W,g as s,s as n,r as k,A as J,h as i,f as a,c as o,j as S,u as U,x as T,k as j,y as Q,a as l,v as D,d as O,t as z,w as I}from"../chunks/index.d9030fc9.js";import{H as V,E as X}from"../chunks/EditOnGithub.91d95064.js";function Y(R){let r,x,b,d,p,y,h,B="当前，一个新兴的研究领域正致力于探索大规模 transformer 模型（如BERT）的内部工作机制，一些人称之为“BERTology”。以下是这个领域的一些典型示例：",_,m,C=`<li>BERT Rediscovers the Classical NLP Pipeline by Ian Tenney, Dipanjan Das, Ellie Pavlick:
<a href="https://arxiv.org/abs/1905.05950" rel="nofollow">https://arxiv.org/abs/1905.05950</a></li> <li>Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: <a href="https://arxiv.org/abs/1905.10650" rel="nofollow">https://arxiv.org/abs/1905.10650</a></li> <li>What Does BERT Look At? An Analysis of BERT’s Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D.
Manning: <a href="https://arxiv.org/abs/1906.04341" rel="nofollow">https://arxiv.org/abs/1906.04341</a></li> <li>CAT-probing: A Metric-based Approach to Interpret How Pre-trained Models for Programming Language Attend Code Structure: <a href="https://arxiv.org/abs/2210.04633" rel="nofollow">https://arxiv.org/abs/2210.04633</a></li>`,E,f,A='为了助力这一新兴领域的发展，我们在BERT/GPT/GPT-2模型中增加了一些附加功能，方便人们访问其内部表示，这些功能主要借鉴了Paul Michel的杰出工作(<a href="https://arxiv.org/abs/1905.10650)%EF%BC%9A" rel="nofollow">https://arxiv.org/abs/1905.10650)：</a>',P,u,H='<li>访问BERT/GPT/GPT-2的所有隐藏状态，</li> <li>访问BERT/GPT/GPT-2每个注意力头的所有注意力权重，</li> <li>检索注意力头的输出值和梯度，以便计算头的重要性得分并对头进行剪枝，详情可见论文：<a href="https://arxiv.org/abs/1905.10650%E3%80%82" rel="nofollow">https://arxiv.org/abs/1905.10650。</a></li>',w,c,G='为了帮助您理解和使用这些功能，我们添加了一个具体的示例脚本：<a href="https://github.com/huggingface/transformers/tree/main/examples/research_projects/bertology/run_bertology.py" rel="nofollow">bertology.py</a>，该脚本可以对一个在 GLUE 数据集上预训练的模型进行信息提取与剪枝。',$,g,L,v,M;return p=new V({props:{title:"基于BERT进行的相关研究（BERTology）",local:"基于bert进行的相关研究bertology",headingTag:"h1"}}),g=new X({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/zh/bertology.md"}}),{c(){r=s("meta"),x=n(),b=s("p"),d=n(),k(p.$$.fragment),y=n(),h=s("p"),h.textContent=B,_=n(),m=s("ul"),m.innerHTML=C,E=n(),f=s("p"),f.innerHTML=A,P=n(),u=s("ul"),u.innerHTML=H,w=n(),c=s("p"),c.innerHTML=G,$=n(),k(g.$$.fragment),L=n(),v=s("p"),this.h()},l(t){const e=J("svelte-u9bgzb",document.head);r=i(e,"META",{name:!0,content:!0}),e.forEach(a),x=o(t),b=i(t,"P",{}),S(b).forEach(a),d=o(t),U(p.$$.fragment,t),y=o(t),h=i(t,"P",{"data-svelte-h":!0}),T(h)!=="svelte-122li14"&&(h.textContent=B),_=o(t),m=i(t,"UL",{"data-svelte-h":!0}),T(m)!=="svelte-1e34lw"&&(m.innerHTML=C),E=o(t),f=i(t,"P",{"data-svelte-h":!0}),T(f)!=="svelte-kha6wh"&&(f.innerHTML=A),P=o(t),u=i(t,"UL",{"data-svelte-h":!0}),T(u)!=="svelte-1dvlo65"&&(u.innerHTML=H),w=o(t),c=i(t,"P",{"data-svelte-h":!0}),T(c)!=="svelte-j2vorw"&&(c.innerHTML=G),$=o(t),U(g.$$.fragment,t),L=o(t),v=i(t,"P",{}),S(v).forEach(a),this.h()},h(){j(r,"name","hf:doc:metadata"),j(r,"content",Z)},m(t,e){Q(document.head,r),l(t,x,e),l(t,b,e),l(t,d,e),D(p,t,e),l(t,y,e),l(t,h,e),l(t,_,e),l(t,m,e),l(t,E,e),l(t,f,e),l(t,P,e),l(t,u,e),l(t,w,e),l(t,c,e),l(t,$,e),D(g,t,e),l(t,L,e),l(t,v,e),M=!0},p:N,i(t){M||(O(p.$$.fragment,t),O(g.$$.fragment,t),M=!0)},o(t){z(p.$$.fragment,t),z(g.$$.fragment,t),M=!1},d(t){t&&(a(x),a(b),a(d),a(y),a(h),a(_),a(m),a(E),a(f),a(P),a(u),a(w),a(c),a($),a(L),a(v)),a(r),I(p,t),I(g,t)}}}const Z='{"title":"基于BERT进行的相关研究（BERTology）","local":"基于bert进行的相关研究bertology","sections":[],"depth":1}';function tt(R){return q(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class rt extends F{constructor(r){super(),W(this,r,tt,Y,K,{})}}export{rt as component};