import{s as ge,n as we,o as ve}from"../chunks/scheduler.7bc62968.js";import{S as be,i as ye,g as s,s as l,r as G,A as Te,h as a,f as i,c as o,j as ce,u as J,x as r,k as F,y as ke,a as n,v as Q,d as X,t as Y,w as Z}from"../chunks/index.2f8492b0.js";import{H as ee}from"../chunks/Heading.14fb2476.js";function xe(te){let h,M,H,P,u,C,d,ie='We are going to learn about Knowledge Distillation, the method behind <a href="https://huggingface.co/distilgpt2" rel="nofollow">distilGPT</a> and <a href="https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english" rel="nofollow">distilbert</a>, two of <em>the most downloaded models on the Hugging Face Hub!</em>',D,f,ne=`Presumably, we’ve all had teachers who “teach” by simply providing us the correct answers and then testing us on questions we haven’t seen before, analogous to supervised learning
of machine learning models where we provide a labeled dataset to train on. Instead of having a model train on labels, however,
we can pursue <a href="https://arxiv.org/abs/1503.02531" rel="nofollow">Knowledge Distillation</a> as an alternative to arriving at a much smaller model that can perform comparably to the larger model and much faster to boot.`,K,p,I,c,le="Imagine you were given this multiple-choice question:",q,g,oe='<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/multiple-choice-question.png" alt="Multiple Choice Question"/>',N,w,se="If you had someone just tell you, “The answer is Draco Malfoy,” that doesn’t teach you a whole lot about each of the characters’ relative relationships with Harry Potter.",E,v,ae=`On the other hand, if someone tells you, “I am very confident it is not Ron Weasley, I am somewhat confident it is not Neville Longbottom, and
I am very confident that it <em>is</em> Draco Malfoy”, this gives you some information about these characters’ relationships to Harry Potter!
This is precisely the kind of information that gets passed down to our student model under the Knowledge Distillation paradigm.`,j,b,z,y,re=`In the paper <a href="https://arxiv.org/abs/1503.02531" rel="nofollow"><em>Distilling the Knowledge in a Neural Network</em></a>, Hinton et al. introduced the training methodology known as knowledge distillation,
taking inspiration from <em>insects</em>, of all things. Just as insects transition from larval to adult forms that are optimized for different tasks, large-scale machine learning models can
initially be cumbersome, like larvae, for extracting structure from data but can distill their knowledge into smaller, more efficient models for deployment.`,V,T,he=`The essence of Knowledge Distillation is using the predicted logits from a teacher network to pass information to a smaller, more efficient student model. We do this
by re-writing the loss function to contain a <em>distillation loss</em>, which encourages the student model’s distribution over the output space to approximate the teacher’s.`,A,k,me="The distillation loss is formulated as:",S,x,ue='<img src="https://huggingface.co/datasets/hf-vision/course-assets/resolve/main/KL-Loss.png" alt="Distillation Loss"/>',U,_,de=`The KL loss refers to the <a href="https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence" rel="nofollow">Kullback-Leibler Divergence</a> between the teacher and the student’s output distributions.
The overall loss for the student model is then formulated as the sum of this distillation loss with the standard cross-entropy loss over the ground-truth labels.`,W,L,fe='To see this loss function implemented in Python and a fully worked out example in Python, let’s check out the <a href="https://github.com/johko/computer-vision-course/blob/main/notebooks/Unit%203%20-%20Vision%20Transformers/KnowledgeDistillation.ipynb" rel="nofollow">notebook for this section</a>.',B,m,pe='<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>',O,$,R;return u=new ee({props:{title:"Knowledge Distillation with Vision Transformers",local:"knowledge-distillation-with-vision-transformers",headingTag:"h1"}}),p=new ee({props:{title:"Intuition Behind Knowledge Distillation",local:"intuition-behind-knowledge-distillation",headingTag:"h2"}}),b=new ee({props:{title:"Distilling the Knowledge in a Neural Network",local:"distilling-the-knowledge-in-a-neural-network",headingTag:"h2"}}),{c(){h=s("meta"),M=l(),H=s("p"),P=l(),G(u.$$.fragment),C=l(),d=s("p"),d.innerHTML=ie,D=l(),f=s("p"),f.innerHTML=ne,K=l(),G(p.$$.fragment),I=l(),c=s("p"),c.textContent=le,q=l(),g=s("p"),g.innerHTML=oe,N=l(),w=s("p"),w.textContent=se,E=l(),v=s("p"),v.innerHTML=ae,j=l(),G(b.$$.fragment),z=l(),y=s("p"),y.innerHTML=re,V=l(),T=s("p"),T.innerHTML=he,A=l(),k=s("p"),k.textContent=me,S=l(),x=s("p"),x.innerHTML=ue,U=l(),_=s("p"),_.innerHTML=de,W=l(),L=s("p"),L.innerHTML=fe,B=l(),m=s("a"),m.innerHTML=pe,O=l(),$=s("p"),this.h()},l(e){const t=Te("svelte-u9bgzb",document.head);h=a(t,"META",{name:!0,content:!0}),t.forEach(i),M=o(e),H=a(e,"P",{}),ce(H).forEach(i),P=o(e),J(u.$$.fragment,e),C=o(e),d=a(e,"P",{"data-svelte-h":!0}),r(d)!=="svelte-1ofcq6q"&&(d.innerHTML=ie),D=o(e),f=a(e,"P",{"data-svelte-h":!0}),r(f)!=="svelte-18scrnu"&&(f.innerHTML=ne),K=o(e),J(p.$$.fragment,e),I=o(e),c=a(e,"P",{"data-svelte-h":!0}),r(c)!=="svelte-v5zyyx"&&(c.textContent=le),q=o(e),g=a(e,"P",{"data-svelte-h":!0}),r(g)!=="svelte-1vpf9y2"&&(g.innerHTML=oe),N=o(e),w=a(e,"P",{"data-svelte-h":!0}),r(w)!=="svelte-193lyi0"&&(w.textContent=se),E=o(e),v=a(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-11ohx5b"&&(v.innerHTML=ae),j=o(e),J(b.$$.fragment,e),z=o(e),y=a(e,"P",{"data-svelte-h":!0}),r(y)!=="svelte-1m1x8ev"&&(y.innerHTML=re),V=o(e),T=a(e,"P",{"data-svelte-h":!0}),r(T)!=="svelte-1q33f44"&&(T.innerHTML=he),A=o(e),k=a(e,"P",{"data-svelte-h":!0}),r(k)!=="svelte-1l2wqyt"&&(k.textContent=me),S=o(e),x=a(e,"P",{"data-svelte-h":!0}),r(x)!=="svelte-y84sde"&&(x.innerHTML=ue),U=o(e),_=a(e,"P",{"data-svelte-h":!0}),r(_)!=="svelte-1mhwb2d"&&(_.innerHTML=de),W=o(e),L=a(e,"P",{"data-svelte-h":!0}),r(L)!=="svelte-1onfg6s"&&(L.innerHTML=fe),B=o(e),m=a(e,"A",{target:!0,href:!0,"data-svelte-h":!0}),r(m)!=="svelte-rzabs7"&&(m.innerHTML=pe),O=o(e),$=a(e,"P",{}),ce($).forEach(i),this.h()},h(){F(h,"name","hf:doc:metadata"),F(h,"content",_e),F(m,"target","_blank"),F(m,"href","https://colab.research.google.com/github/johko/computer-vision-course/blob/main/notebooks/Unit%203%20-%20Vision%20Transformers/KnowledgeDistillation.ipynb")},m(e,t){ke(document.head,h),n(e,M,t),n(e,H,t),n(e,P,t),Q(u,e,t),n(e,C,t),n(e,d,t),n(e,D,t),n(e,f,t),n(e,K,t),Q(p,e,t),n(e,I,t),n(e,c,t),n(e,q,t),n(e,g,t),n(e,N,t),n(e,w,t),n(e,E,t),n(e,v,t),n(e,j,t),Q(b,e,t),n(e,z,t),n(e,y,t),n(e,V,t),n(e,T,t),n(e,A,t),n(e,k,t),n(e,S,t),n(e,x,t),n(e,U,t),n(e,_,t),n(e,W,t),n(e,L,t),n(e,B,t),n(e,m,t),n(e,O,t),n(e,$,t),R=!0},p:we,i(e){R||(X(u.$$.fragment,e),X(p.$$.fragment,e),X(b.$$.fragment,e),R=!0)},o(e){Y(u.$$.fragment,e),Y(p.$$.fragment,e),Y(b.$$.fragment,e),R=!1},d(e){e&&(i(M),i(H),i(P),i(C),i(d),i(D),i(f),i(K),i(I),i(c),i(q),i(g),i(N),i(w),i(E),i(v),i(j),i(z),i(y),i(V),i(T),i(A),i(k),i(S),i(x),i(U),i(_),i(W),i(L),i(B),i(m),i(O),i($)),i(h),Z(u,e),Z(p,e),Z(b,e)}}}const _e='{"title":"Knowledge Distillation with Vision Transformers","local":"knowledge-distillation-with-vision-transformers","sections":[{"title":"Intuition Behind Knowledge Distillation","local":"intuition-behind-knowledge-distillation","sections":[],"depth":2},{"title":"Distilling the Knowledge in a Neural Network","local":"distilling-the-knowledge-in-a-neural-network","sections":[],"depth":2}],"depth":1}';function Le(te){return ve(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pe extends be{constructor(h){super(),ye(this,h,Le,xe,ge,{})}}export{Pe as component};
