import{s as z,n as J,o as W}from"../chunks/scheduler.9bc65507.js";import{S as F,i as Q,g as o,s,u as V,A as X,h as r,f as t,c as h,j as _,v as Y,y as x,k as y,o as P,a as l,w as Z,d as ee,t as te,x as ae}from"../chunks/index.78c82d43.js";import{I as le}from"../chunks/IconCopyLink.bedaa44d.js";function oe(G){let n,E,w,L,i,p,b,f,S,g,I="BERTology",C,c,N=`There is a growing field of study concerned with investigating the inner working of large-scale transformers like BERT
(that some call “BERTology”). Some good examples of this field are:`,M,d,U=`<li>BERT Rediscovers the Classical NLP Pipeline by Ian Tenney, Dipanjan Das, Ellie Pavlick:
<a href="https://arxiv.org/abs/1905.05950" rel="nofollow">https://arxiv.org/abs/1905.05950</a></li> <li>Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: <a href="https://arxiv.org/abs/1905.10650" rel="nofollow">https://arxiv.org/abs/1905.10650</a></li> <li>What Does BERT Look At? An Analysis of BERT’s Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D.
Manning: <a href="https://arxiv.org/abs/1906.04341" rel="nofollow">https://arxiv.org/abs/1906.04341</a></li> <li>CAT-probing: A Metric-based Approach to Interpret How Pre-trained Models for Programming Language Attend Code Structure: <a href="https://arxiv.org/abs/2210.04633" rel="nofollow">https://arxiv.org/abs/2210.04633</a></li>`,A,u,D=`In order to help this new field develop, we have included a few additional features in the BERT/GPT/GPT-2 models to
help people access the inner representations, mainly adapted from the great work of Paul Michel
(<a href="https://arxiv.org/abs/1905.10650" rel="nofollow">https://arxiv.org/abs/1905.10650</a>):`,R,m,O=`<li>accessing all the hidden-states of BERT/GPT/GPT-2,</li> <li>accessing all the attention weights for each head of BERT/GPT/GPT-2,</li> <li>retrieving heads output values and gradients to be able to compute head importance score and prune head as explained
in <a href="https://arxiv.org/abs/1905.10650" rel="nofollow">https://arxiv.org/abs/1905.10650</a>.</li>`,k,v,j=`To help you understand and use these features, we have added a specific example script: <a href="https://github.com/huggingface/transformers/tree/main/examples/research_projects/bertology/run_bertology.py" rel="nofollow">bertology.py</a> while extract information and prune a model pre-trained on
GLUE.`,B,T,H;return f=new le({}),{c(){n=o("meta"),E=s(),w=o("p"),L=s(),i=o("h1"),p=o("a"),b=o("span"),V(f.$$.fragment),S=s(),g=o("span"),g.textContent=I,C=s(),c=o("p"),c.textContent=N,M=s(),d=o("ul"),d.innerHTML=U,A=s(),u=o("p"),u.innerHTML=D,R=s(),m=o("ul"),m.innerHTML=O,k=s(),v=o("p"),v.innerHTML=j,B=s(),T=o("p"),this.h()},l(e){const a=X("svelte-1phssyn",document.head);n=r(a,"META",{name:!0,content:!0}),a.forEach(t),E=h(e),w=r(e,"P",{}),_(w).forEach(t),L=h(e),i=r(e,"H1",{class:!0});var $=_(i);p=r($,"A",{id:!0,class:!0,href:!0});var K=_(p);b=r(K,"SPAN",{});var q=_(b);Y(f.$$.fragment,q),q.forEach(t),K.forEach(t),S=h($),g=r($,"SPAN",{"data-svelte-h":!0}),x(g)!=="svelte-1pr1yoi"&&(g.textContent=I),$.forEach(t),C=h(e),c=r(e,"P",{"data-svelte-h":!0}),x(c)!=="svelte-e5vxoh"&&(c.textContent=N),M=h(e),d=r(e,"UL",{"data-svelte-h":!0}),x(d)!=="svelte-1e34lw"&&(d.innerHTML=U),A=h(e),u=r(e,"P",{"data-svelte-h":!0}),x(u)!=="svelte-1bavv9a"&&(u.innerHTML=D),R=h(e),m=r(e,"UL",{"data-svelte-h":!0}),x(m)!=="svelte-14erpmf"&&(m.innerHTML=O),k=h(e),v=r(e,"P",{"data-svelte-h":!0}),x(v)!=="svelte-7ztyw0"&&(v.innerHTML=j),B=h(e),T=r(e,"P",{}),_(T).forEach(t),this.h()},h(){y(n,"name","hf:doc:metadata"),y(n,"content",JSON.stringify(re)),y(p,"id","bertology"),y(p,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y(p,"href","#bertology"),y(i,"class","relative group")},m(e,a){P(document.head,n),l(e,E,a),l(e,w,a),l(e,L,a),l(e,i,a),P(i,p),P(p,b),Z(f,b,null),P(i,S),P(i,g),l(e,C,a),l(e,c,a),l(e,M,a),l(e,d,a),l(e,A,a),l(e,u,a),l(e,R,a),l(e,m,a),l(e,k,a),l(e,v,a),l(e,B,a),l(e,T,a),H=!0},p:J,i(e){H||(ee(f.$$.fragment,e),H=!0)},o(e){te(f.$$.fragment,e),H=!1},d(e){e&&(t(E),t(w),t(L),t(i),t(C),t(c),t(M),t(d),t(A),t(u),t(R),t(m),t(k),t(v),t(B),t(T)),t(n),ae(f)}}}const re={local:"bertology",title:"BERTology"};function ne(G){return W(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class pe extends F{constructor(n){super(),Q(this,n,ne,oe,z,{})}}export{pe as component};
