import{s as Xt,o as Nt,n as Et}from"../chunks/scheduler.56730f09.js";import{S as Gt,i as Bt,g as p,s as t,u as r,A as Ft,h as i,f as l,c as e,j as m,v as o,y as c,k as g,o as d,a as n,w as h,d as j,t as M,x}from"../chunks/index.ba464511.js";import{T as Wt}from"../chunks/Tip.70442610.js";import{I as B}from"../chunks/IconCopyLink.89bde88e.js";import{C as U}from"../chunks/CodeBlock.2302fd0d.js";import{D as zt}from"../chunks/DocNotebookDropdown.68fd8653.js";function St(Ra){let J,F="이 튜토리얼에서 설명하는 태스크는 다음과 같은 모델 아키텍처에서 지원됩니다:",T,u,w='<a href="../model_doc/layoutlm">LayoutLM</a>, <a href="../model_doc/layoutlmv2">LayoutLMv2</a>, <a href="../model_doc/layoutlmv3">LayoutLMv3</a>';return{c(){J=p("p"),J.textContent=F,T=t(),u=p("p"),u.innerHTML=w},l(y){J=i(y,"P",{"data-svelte-h":!0}),c(J)!=="svelte-6xt7gu"&&(J.textContent=F),T=e(y),u=i(y,"P",{"data-svelte-h":!0}),c(u)!=="svelte-4nltzx"&&(u.innerHTML=w)},m(y,C){n(y,J,C),n(y,T,C),n(y,u,C)},p:Et,d(y){y&&(l(J),l(T),l(u))}}}function qt(Ra){let J,F,T,u,w,y,C,W,dn,da,Cn="문서 질의 응답(Document Question Answering)",Za,z,Aa,S,bn=`문서 시각적 질의 응답(Document Visual Question Answering)이라고도 하는
문서 질의 응답(Document Question Answering)은 문서 이미지에 대한 질문에 답변을 주는 태스크입니다.
이 태스크를 지원하는 모델의 입력은 일반적으로 이미지와 질문의 조합이고, 출력은 자연어로 된 답변입니다. 이러한 모델은 텍스트, 단어의 위치(바운딩 박스), 이미지 등 다양한 모달리티를 활용합니다.`,Qa,q,fn="이 가이드는 다음 내용을 설명합니다:",$a,Y,In='<li><a href="https://huggingface.co/datasets/nielsr/docvqa_1200_examples_donut" rel="nofollow">DocVQA dataset</a>을 사용해 <a href="../model_doc/layoutlmv2">LayoutLMv2</a> 미세 조정하기</li> <li>추론을 위해 미세 조정된 모델을 사용하기</li>',Xa,R,Na,H,_n=`LayoutLMv2는 토큰의 마지막 은닉층 위에 질의 응답 헤드를 추가해 답변의 시작 토큰과 끝 토큰의 위치를 예측함으로써 문서 질의 응답 태스크를 해결합니다. 즉, 문맥이 주어졌을 때 질문에 답하는 정보를 추출하는 추출형 질의 응답(Extractive question answering)으로 문제를 처리합니다.
문맥은 OCR 엔진의 출력에서 가져오며, 여기서는 Google의 Tesseract를 사용합니다.`,Ea,D,vn="시작하기 전에 필요한 라이브러리가 모두 설치되어 있는지 확인하세요. LayoutLMv2는 detectron2, torchvision 및 테서랙트를 필요로 합니다.",Ga,L,Ba,O,Fa,P,Wa,K,Vn="필요한 라이브러리들을 모두 설치한 후 런타임을 다시 시작합니다.",za,ss,kn=`커뮤니티에 당신의 모델을 공유하는 것을 권장합니다. Hugging Face 계정에 로그인해서 모델을 🤗 Hub에 업로드하세요.
프롬프트가 실행되면, 로그인을 위해 토큰을 입력하세요:`,Sa,as,qa,ls,Rn="몇 가지 전역 변수를 정의해 보겠습니다.",Ya,ns,Ha,b,Z,Ca,ts,Un,Ua,Zn="데이터 불러오기",Da,es,An=`이 가이드에서는 🤗 Hub에서 찾을 수 있는 전처리된 DocVQA의 작은 샘플을 사용합니다.
DocVQA의 전체 데이터 세트를 사용하고 싶다면, <a href="https://rrc.cvc.uab.es/?ch=17" rel="nofollow">DocVQA homepage</a>에 가입 후 다운로드 할 수 있습니다. 전체 데이터 세트를 다운로드 했다면, 이 가이드를 계속 진행하기 위해 <a href="https://huggingface.co/docs/datasets/loading#local-and-remote-files" rel="nofollow">🤗 dataset에 파일을 가져오는 방법</a>을 확인하세요.`,La,ps,Oa,is,Qn="보시다시피, 데이터 세트는 이미 훈련 세트와 테스트 세트로 나누어져 있습니다. 무작위로 예제를 살펴보면서 특성을 확인해보세요.",Pa,cs,Ka,rs,$n="각 필드가 나타내는 내용은 다음과 같습니다:",sl,os,Xn="<li><code>id</code>: 예제의 id</li> <li><code>image</code>: 문서 이미지를 포함하는 PIL.Image.Image 객체</li> <li><code>query</code>: 질문 문자열 - 여러 언어의 자연어로 된 질문</li> <li><code>answers</code>: 사람이 주석을 단 정답 리스트</li> <li><code>words</code> and <code>bounding_boxes</code>: OCR의 결과값들이며 이 가이드에서는 사용하지 않을 예정</li> <li><code>answer</code>: 다른 모델과 일치하는 답변이며 이 가이드에서는 사용하지 않을 예정</li>",al,hs,Nn=`영어로 된 질문만 남기고 다른 모델에 대한 예측을 포함하는 <code>answer</code> 특성을 삭제하겠습니다.
그리고 주석 작성자가 제공한 데이터 세트에서 첫 번째 답변을 가져옵니다. 또는 무작위로 샘플을 추출할 수도 있습니다.`,ll,js,nl,Ms,En=`이 가이드에서 사용하는 LayoutLMv2 체크포인트는 <code>max_position_embeddings = 512</code>로 훈련되었습니다(이 정보는 <a href="https://huggingface.co/microsoft/layoutlmv2-base-uncased/blob/main/config.json#L18" rel="nofollow">체크포인트의 <code>config.json</code> 파일</a>에서 확인할 수 있습니다).
바로 예제를 잘라낼 수도 있지만, 긴 문서의 끝에 답변이 있어 잘리는 상황을 피하기 위해 여기서는 임베딩이 512보다 길어질 가능성이 있는 몇 가지 예제를 제거하겠습니다.
데이터 세트에 있는 대부분의 문서가 긴 경우 슬라이딩 윈도우 방법을 사용할 수 있습니다 - 자세한 내용을 확인하고 싶으면 이 <a href="https://github.com/huggingface/notebooks/blob/main/examples/question_answering.ipynb" rel="nofollow">노트북</a>을 확인하세요.`,tl,xs,el,gs,Gn=`이 시점에서 이 데이터 세트의 OCR 특성도 제거해 보겠습니다. OCR 특성은 다른 모델을 미세 조정하기 위한 것으로, 이 가이드에서 사용하는 모델의 입력 요구 사항과 일치하지 않기 때문에 이 특성을 사용하기 위해서는 일부 처리가 필요합니다.
대신, 원본 데이터에 <code>LayoutLMv2Processor</code>를 사용하여 OCR 및 토큰화를 모두 수행할 수 있습니다.
이렇게 하면 모델이 요구하는 입력을 얻을 수 있습니다.
이미지를 수동으로 처리하려면, <a href="../model_doc/layoutlmv2"><code>LayoutLMv2</code> model documentation</a>에서 모델이 요구하는 입력 포맷을 확인해보세요.`,pl,ds,il,Us,Bn="마지막으로, 데이터 탐색을 완료하기 위해 이미지 예시를 살펴봅시다.",cl,ms,rl,A,Fn='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/docvqa_example.jpg" alt="DocVQA Image Example"/>',ol,f,Q,ba,ys,mn,ma,Wn="데이터 전처리",hl,Js,zn=`문서 질의 응답 태스크는 멀티모달 태스크이며, 각 모달리티의 입력이 모델의 요구에 맞게 전처리 되었는지 확인해야 합니다.
이미지 데이터를 처리할 수 있는 이미지 프로세서와 텍스트 데이터를 인코딩할 수 있는 토크나이저를 결합한 <code>LayoutLMv2Processor</code>를 가져오는 것부터 시작해 보겠습니다.`,jl,us,Ml,I,$,fa,ws,yn,ya,Sn="문서 이미지 전처리",xl,Ts,qn=`먼저, 프로세서의 <code>image_processor</code>를 사용해 모델에 대한 문서 이미지를 준비해 보겠습니다.
기본값으로, 이미지 프로세서는 이미지 크기를 224x224로 조정하고 색상 채널의 순서가 올바른지 확인한 후 단어와 정규화된 바운딩 박스를 얻기 위해 테서랙트를 사용해 OCR를 적용합니다.
이 튜토리얼에서 우리가 필요한 것과 기본값은 완전히 동일합니다. 이미지 배치에 기본 이미지 처리를 적용하고 OCR의 결과를 변환하는 함수를 작성합니다.`,gl,Cs,dl,bs,Yn="이 전처리를 데이터 세트 전체에 빠르게 적용하려면 <code>map</code>를 사용하세요.",Ul,fs,ml,_,X,Ia,Is,Jn,Ja,Hn="텍스트 데이터 전처리",yl,_s,Dn=`이미지에 OCR을 적용했으면 데이터 세트의 텍스트 부분을 모델에 맞게 인코딩해야 합니다.
이 인코딩에는 이전 단계에서 가져온 단어와 박스를 토큰 수준의 <code>input_ids</code>, <code>attention_mask</code>, <code>token_type_ids</code> 및 <code>bbox</code>로 변환하는 작업이 포함됩니다.
텍스트를 전처리하려면 프로세서의 <code>tokenizer</code>가 필요합니다.`,Jl,vs,ul,Vs,Ln="위에서 언급한 전처리 외에도 모델을 위해 레이블을 추가해야 합니다. 🤗 Transformers의 <code>xxxForQuestionAnswering</code> 모델의 경우, 레이블은 <code>start_positions</code>와 <code>end_positions</code>로 구성되며 어떤 토큰이 답변의 시작과 끝에 있는지를 나타냅니다.",wl,ks,On="레이블 추가를 위해서, 먼저 더 큰 리스트(단어 리스트)에서 하위 리스트(단어로 분할된 답변)을 찾을 수 있는 헬퍼 함수를 정의합니다.",Tl,Rs,Pn=`이 함수는 <code>words_list</code>와 <code>answer_list</code>, 이렇게 두 리스트를 입력으로 받습니다.
그런 다음 <code>words_list</code>를 반복하여 <code>words_list</code>의 현재 단어(words_list[i])가 <code>answer_list</code>의 첫 번째 단어(answer_list[0])와 같은지,
현재 단어에서 시작해 <code>answer_list</code>와 같은 길이만큼의 <code>words_list</code>의 하위 리스트가 <code>answer_list</code>와 일치하는지 확인합니다.
이 조건이 참이라면 일치하는 항목을 발견했음을 의미하며, 함수는 일치 항목, 시작 인덱스(idx) 및 종료 인덱스(idx + len(answer_list) - 1)를 기록합니다. 일치하는 항목이 두 개 이상 발견되면 함수는 첫 번째 항목만 반환합니다. 일치하는 항목이 없다면 함수는 (<code>None</code>, 0, 0)을 반환합니다.`,Cl,Zs,bl,As,Kn="이 함수가 어떻게 정답의 위치를 찾는지 설명하기 위해 다음 예제에서 함수를 사용해 보겠습니다:",fl,Qs,Il,$s,st="한편, 위 예제가 인코딩되면 다음과 같이 표시됩니다:",_l,Xs,vl,Ns,at="이제 인코딩된 입력에서 정답의 위치를 찾아야 합니다.",Vl,Es,lt="<li><code>token_type_ids</code>는 어떤 토큰이 질문에 속하는지, 그리고 어떤 토큰이 문서의 단어에 포함되는지를 알려줍니다.</li> <li><code>tokenizer.cls_token_id</code> 입력의 시작 부분에 있는 특수 토큰을 찾는 데 도움을 줍니다.</li> <li><code>word_ids</code>는 원본 <code>words</code>에서 찾은 답변을 전체 인코딩된 입력의 동일한 답과 일치시키고 인코딩된 입력에서 답변의 시작/끝 위치를 결정합니다.</li>",kl,Gs,nt="위 내용들을 염두에 두고 데이터 세트 예제의 배치를 인코딩하는 함수를 만들어 보겠습니다:",Rl,Bs,Zl,Fs,tt="이제 이 전처리 함수가 있으니 전체 데이터 세트를 인코딩할 수 있습니다:",Al,Ws,Ql,zs,et="인코딩된 데이터 세트의 특성이 어떻게 생겼는지 확인해 보겠습니다:",$l,Ss,Xl,v,N,_a,qs,un,ua,pt="평가",Nl,Ys,it=`문서 질의 응답을 평가하려면 상당한 양의 후처리가 필요합니다. 시간이 너무 많이 걸리지 않도록 이 가이드에서는 평가 단계를 생략합니다.
<code>Trainer</code>가 훈련 과정에서 평가 손실(evaluation loss)을 계속 계산하기 때문에 모델의 성능을 대략적으로 알 수 있습니다.
추출적(Extractive) 질의 응답은 보통 F1/exact match 방법을 사용해 평가됩니다.
직접 구현해보고 싶으시다면, Hugging Face course의 <a href="https://huggingface.co/course/chapter7/7?fw=pt#postprocessing" rel="nofollow">Question Answering chapter</a>을 참고하세요.`,El,V,E,va,Hs,wn,wa,ct="훈련",Gl,Ds,rt=`축하합니다! 이 가이드의 가장 어려운 부분을 성공적으로 처리했으니 이제 나만의 모델을 훈련할 준비가 되었습니다.
훈련은 다음과 같은 단계로 이루어져 있습니다:`,Bl,Ls,ot="<li>전처리에서의 동일한 체크포인트를 사용하기 위해 <code>AutoModelForDocumentQuestionAnswering</code>으로 모델을 가져옵니다.</li> <li><code>TrainingArguments</code>로 훈련 하이퍼파라미터를 정합니다.</li> <li>예제를 배치 처리하는 함수를 정의합니다. 여기서는 <code>DefaultDataCollator</code>가 적당합니다.</li> <li>모델, 데이터 세트, 데이터 콜레이터(Data collator)와 함께 <code>Trainer</code>에 훈련 인수들을 전달합니다.</li> <li><code>train()</code>을 호출해서 모델을 미세 조정합니다.</li>",Fl,Os,Wl,Ps,ht=`<code>TrainingArguments</code>에서 <code>output_dir</code>을 사용하여 모델을 저장할 위치를 지정하고, 적절한 하이퍼파라미터를 설정합니다.
모델을 커뮤니티와 공유하려면 <code>push_to_hub</code>를 <code>True</code>로 설정하세요 (모델을 업로드하려면 Hugging Face에 로그인해야 합니다).
이 경우 <code>output_dir</code>은 모델의 체크포인트를 푸시할 레포지토리의 이름이 됩니다.`,zl,Ks,Sl,sa,jt="간단한 데이터 콜레이터를 정의하여 예제를 함께 배치합니다.",ql,aa,Yl,la,Mt="마지막으로, 모든 것을 한 곳에 모아 <code>train()</code>을 호출합니다:",Hl,na,Dl,ta,xt="최종 모델을 🤗 Hub에 추가하려면, 모델 카드를 생성하고 <code>push_to_hub</code>를 호출합니다:",Ll,ea,Ol,k,G,Va,pa,Tn,Ta,gt="추론",Pl,ia,dt=`이제 LayoutLMv2 모델을 미세 조정하고 🤗 Hub에 업로드했으니 추론에도 사용할 수 있습니다.
추론을 위해 미세 조정된 모델을 사용해 보는 가장 간단한 방법은 <code>Pipeline</code>을 사용하는 것 입니다.`,Kl,ca,Ut="예를 들어 보겠습니다:",sn,ra,an,oa,mt="그 다음, 모델로 문서 질의 응답을 하기 위해 파이프라인을 인스턴스화하고 이미지 + 질문 조합을 전달합니다.",ln,ha,nn,ja,yt="원한다면 파이프라인의 결과를 수동으로 복제할 수도 있습니다:",tn,Ma,Jt="<li>이미지와 질문을 가져와 모델의 프로세서를 사용해 모델에 맞게 준비합니다.</li> <li>모델을 통해 결과 또는 전처리를 전달합니다.</li> <li>모델은 어떤 토큰이 답변의 시작에 있는지, 어떤 토큰이 답변이 끝에 있는지를 나타내는 <code>start_logits</code>와 <code>end_logits</code>를 반환합니다. 둘 다 (batch_size, sequence_length) 형태를 갖습니다.</li> <li><code>start_logits</code>와 <code>end_logits</code>의 마지막 차원을 최대로 만드는 값을 찾아 예상 <code>start_idx</code>와 <code>end_idx</code>를 얻습니다.</li> <li>토크나이저로 답변을 디코딩합니다.</li>",en,xa,pn,ka,cn;return W=new B({}),z=new zt({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/document_question_answering.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/pytorch/document_question_answering.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/tensorflow/document_question_answering.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/document_question_answering.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/pytorch/document_question_answering.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/tensorflow/document_question_answering.ipynb"}]}}),R=new Wt({props:{$$slots:{default:[St]},$$scope:{ctx:Ra}}}),L=new U({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1xJTIwdHJhbnNmb3JtZXJzJTIwZGF0YXNldHM=",highlighted:"pip install -q transformers datasets",wrap:!1}}),O=new U({props:{code:"cGlwJTIwaW5zdGFsbCUyMCdnaXQlMkJodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZmYWNlYm9va3Jlc2VhcmNoJTJGZGV0ZWN0cm9uMi5naXQnJTBBcGlwJTIwaW5zdGFsbCUyMHRvcmNodmlzaW9u",highlighted:`pip install <span class="hljs-string">&#x27;git+https://github.com/facebookresearch/detectron2.git&#x27;</span>
pip install torchvision`,wrap:!1}}),P=new U({props:{code:"c3VkbyUyMGFwdCUyMGluc3RhbGwlMjB0ZXNzZXJhY3Qtb2NyJTBBcGlwJTIwaW5zdGFsbCUyMC1xJTIwcHl0ZXNzZXJhY3Q=",highlighted:`sudo apt install tesseract-ocr
pip install -q pytesseract`,wrap:!1}}),as=new U({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

<span class="hljs-meta">&gt;&gt;&gt; </span>notebook_login()`,wrap:!1}}),ns=new U({props:{code:"bW9kZWxfY2hlY2twb2ludCUyMCUzRCUyMCUyMm1pY3Jvc29mdCUyRmxheW91dGxtdjItYmFzZS11bmNhc2VkJTIyJTBBYmF0Y2hfc2l6ZSUyMCUzRCUyMDQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>model_checkpoint = <span class="hljs-string">&quot;microsoft/layoutlmv2-base-uncased&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">4</span>`,wrap:!1}}),ts=new B({}),ps=new U({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJuaWVsc3IlMkZkb2N2cWFfMTIwMF9leGFtcGxlcyUyMiklMEFkYXRhc2V0",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;nielsr/docvqa_1200_examples&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset
DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;image&#x27;</span>, <span class="hljs-string">&#x27;query&#x27;</span>, <span class="hljs-string">&#x27;answers&#x27;</span>, <span class="hljs-string">&#x27;words&#x27;</span>, <span class="hljs-string">&#x27;bounding_boxes&#x27;</span>, <span class="hljs-string">&#x27;answer&#x27;</span>],
        num_rows: <span class="hljs-number">1000</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;image&#x27;</span>, <span class="hljs-string">&#x27;query&#x27;</span>, <span class="hljs-string">&#x27;answers&#x27;</span>, <span class="hljs-string">&#x27;words&#x27;</span>, <span class="hljs-string">&#x27;bounding_boxes&#x27;</span>, <span class="hljs-string">&#x27;answer&#x27;</span>],
        num_rows: <span class="hljs-number">200</span>
    })
})`,wrap:!1}}),cs=new U({props:{code:"ZGF0YXNldCU1QiUyMnRyYWluJTIyJTVELmZlYXR1cmVz",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-string">&quot;train&quot;</span>].features',wrap:!1}}),js=new U({props:{code:"dXBkYXRlZF9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldC5tYXAobGFtYmRhJTIwZXhhbXBsZSUzQSUyMCU3QiUyMnF1ZXN0aW9uJTIyJTNBJTIwZXhhbXBsZSU1QiUyMnF1ZXJ5JTIyJTVEJTVCJTIyZW4lMjIlNUQlN0QlMkMlMjByZW1vdmVfY29sdW1ucyUzRCU1QiUyMnF1ZXJ5JTIyJTVEKSUwQXVwZGF0ZWRfZGF0YXNldCUyMCUzRCUyMHVwZGF0ZWRfZGF0YXNldC5tYXAoJTBBJTIwJTIwJTIwJTIwbGFtYmRhJTIwZXhhbXBsZSUzQSUyMCU3QiUyMmFuc3dlciUyMiUzQSUyMGV4YW1wbGUlNUIlMjJhbnN3ZXJzJTIyJTVEJTVCMCU1RCU3RCUyQyUyMHJlbW92ZV9jb2x1bW5zJTNEJTVCJTIyYW5zd2VyJTIyJTJDJTIwJTIyYW5zd2VycyUyMiU1RCUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>updated_dataset = dataset.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> example: {<span class="hljs-string">&quot;question&quot;</span>: example[<span class="hljs-string">&quot;query&quot;</span>][<span class="hljs-string">&quot;en&quot;</span>]}, remove_columns=[<span class="hljs-string">&quot;query&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>updated_dataset = updated_dataset.<span class="hljs-built_in">map</span>(
<span class="hljs-meta">... </span>    <span class="hljs-keyword">lambda</span> example: {<span class="hljs-string">&quot;answer&quot;</span>: example[<span class="hljs-string">&quot;answers&quot;</span>][<span class="hljs-number">0</span>]}, remove_columns=[<span class="hljs-string">&quot;answer&quot;</span>, <span class="hljs-string">&quot;answers&quot;</span>]
<span class="hljs-meta">... </span>)`,wrap:!1}}),xs=new U({props:{code:"dXBkYXRlZF9kYXRhc2V0JTIwJTNEJTIwdXBkYXRlZF9kYXRhc2V0LmZpbHRlcihsYW1iZGElMjB4JTNBJTIwbGVuKHglNUIlMjJ3b3JkcyUyMiU1RCklMjAlMkIlMjBsZW4oeCU1QiUyMnF1ZXN0aW9uJTIyJTVELnNwbGl0KCkpJTIwJTNDJTIwNTEyKQ==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>updated_dataset = updated_dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;words&quot;</span>]) + <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;question&quot;</span>].split()) &lt; <span class="hljs-number">512</span>)',wrap:!1}}),ds=new U({props:{code:"dXBkYXRlZF9kYXRhc2V0JTIwJTNEJTIwdXBkYXRlZF9kYXRhc2V0LnJlbW92ZV9jb2x1bW5zKCUyMndvcmRzJTIyKSUwQXVwZGF0ZWRfZGF0YXNldCUyMCUzRCUyMHVwZGF0ZWRfZGF0YXNldC5yZW1vdmVfY29sdW1ucyglMjJib3VuZGluZ19ib3hlcyUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>updated_dataset = updated_dataset.remove_columns(<span class="hljs-string">&quot;words&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>updated_dataset = updated_dataset.remove_columns(<span class="hljs-string">&quot;bounding_boxes&quot;</span>)`,wrap:!1}}),ms=new U({props:{code:"dXBkYXRlZF9kYXRhc2V0JTVCJTIydHJhaW4lMjIlNUQlNUIxMSU1RCU1QiUyMmltYWdlJTIyJTVE",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>updated_dataset[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">11</span>][<span class="hljs-string">&quot;image&quot;</span>]',wrap:!1}}),ys=new B({}),us=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Qcm9jZXNzb3IlMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZChtb2RlbF9jaGVja3BvaW50KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(model_checkpoint)`,wrap:!1}}),ws=new B({}),Cs=new U({props:{code:"aW1hZ2VfcHJvY2Vzc29yJTIwJTNEJTIwcHJvY2Vzc29yLmltYWdlX3Byb2Nlc3NvciUwQSUwQSUwQWRlZiUyMGdldF9vY3Jfd29yZHNfYW5kX2JveGVzKGV4YW1wbGVzKSUzQSUwQSUyMCUyMCUyMCUyMGltYWdlcyUyMCUzRCUyMCU1QmltYWdlLmNvbnZlcnQoJTIyUkdCJTIyKSUyMGZvciUyMGltYWdlJTIwaW4lMjBleGFtcGxlcyU1QiUyMmltYWdlJTIyJTVEJTVEJTBBJTIwJTIwJTIwJTIwZW5jb2RlZF9pbnB1dHMlMjAlM0QlMjBpbWFnZV9wcm9jZXNzb3IoaW1hZ2VzKSUwQSUwQSUyMCUyMCUyMCUyMGV4YW1wbGVzJTVCJTIyaW1hZ2UlMjIlNUQlMjAlM0QlMjBlbmNvZGVkX2lucHV0cy5waXhlbF92YWx1ZXMlMEElMjAlMjAlMjAlMjBleGFtcGxlcyU1QiUyMndvcmRzJTIyJTVEJTIwJTNEJTIwZW5jb2RlZF9pbnB1dHMud29yZHMlMEElMjAlMjAlMjAlMjBleGFtcGxlcyU1QiUyMmJveGVzJTIyJTVEJTIwJTNEJTIwZW5jb2RlZF9pbnB1dHMuYm94ZXMlMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBleGFtcGxlcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>image_processor = processor.image_processor


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">get_ocr_words_and_boxes</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    images = [image.convert(<span class="hljs-string">&quot;RGB&quot;</span>) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    encoded_inputs = image_processor(images)

<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;image&quot;</span>] = encoded_inputs.pixel_values
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;words&quot;</span>] = encoded_inputs.words
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;boxes&quot;</span>] = encoded_inputs.boxes

<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`,wrap:!1}}),fs=new U({props:{code:"ZGF0YXNldF93aXRoX29jciUyMCUzRCUyMHVwZGF0ZWRfZGF0YXNldC5tYXAoZ2V0X29jcl93b3Jkc19hbmRfYm94ZXMlMkMlMjBiYXRjaGVkJTNEVHJ1ZSUyQyUyMGJhdGNoX3NpemUlM0QyKQ==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset_with_ocr = updated_dataset.<span class="hljs-built_in">map</span>(get_ocr_words_and_boxes, batched=<span class="hljs-literal">True</span>, batch_size=<span class="hljs-number">2</span>)',wrap:!1}}),Is=new B({}),vs=new U({props:{code:"dG9rZW5pemVyJTIwJTNEJTIwcHJvY2Vzc29yLnRva2VuaXplcg==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = processor.tokenizer',wrap:!1}}),Zs=new U({props:{code:"ZGVmJTIwc3ViZmluZGVyKHdvcmRzX2xpc3QlMkMlMjBhbnN3ZXJfbGlzdCklM0ElMEElMjAlMjAlMjAlMjBtYXRjaGVzJTIwJTNEJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwc3RhcnRfaW5kaWNlcyUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGVuZF9pbmRpY2VzJTIwJTNEJTIwJTVCJTVEJTBBJTIwJTIwJTIwJTIwZm9yJTIwaWR4JTJDJTIwaSUyMGluJTIwZW51bWVyYXRlKHJhbmdlKGxlbih3b3Jkc19saXN0KSkpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjB3b3Jkc19saXN0JTVCaSU1RCUyMCUzRCUzRCUyMGFuc3dlcl9saXN0JTVCMCU1RCUyMGFuZCUyMHdvcmRzX2xpc3QlNUJpJTIwJTNBJTIwaSUyMCUyQiUyMGxlbihhbnN3ZXJfbGlzdCklNUQlMjAlM0QlM0QlMjBhbnN3ZXJfbGlzdCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1hdGNoZXMuYXBwZW5kKGFuc3dlcl9saXN0KSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0YXJ0X2luZGljZXMuYXBwZW5kKGlkeCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmRfaW5kaWNlcy5hcHBlbmQoaWR4JTIwJTJCJTIwbGVuKGFuc3dlcl9saXN0KSUyMC0lMjAxKSUwQSUyMCUyMCUyMCUyMGlmJTIwbWF0Y2hlcyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMG1hdGNoZXMlNUIwJTVEJTJDJTIwc3RhcnRfaW5kaWNlcyU1QjAlNUQlMkMlMjBlbmRfaW5kaWNlcyU1QjAlNUQlMEElMjAlMjAlMjAlMjBlbHNlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwTm9uZSUyQyUyMDAlMkMlMjAw",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">subfinder</span>(<span class="hljs-params">words_list, answer_list</span>):
<span class="hljs-meta">... </span>    matches = []
<span class="hljs-meta">... </span>    start_indices = []
<span class="hljs-meta">... </span>    end_indices = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> idx, i <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(<span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(words_list))):
<span class="hljs-meta">... </span>        <span class="hljs-keyword">if</span> words_list[i] == answer_list[<span class="hljs-number">0</span>] <span class="hljs-keyword">and</span> words_list[i : i + <span class="hljs-built_in">len</span>(answer_list)] == answer_list:
<span class="hljs-meta">... </span>            matches.append(answer_list)
<span class="hljs-meta">... </span>            start_indices.append(idx)
<span class="hljs-meta">... </span>            end_indices.append(idx + <span class="hljs-built_in">len</span>(answer_list) - <span class="hljs-number">1</span>)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">if</span> matches:
<span class="hljs-meta">... </span>        <span class="hljs-keyword">return</span> matches[<span class="hljs-number">0</span>], start_indices[<span class="hljs-number">0</span>], end_indices[<span class="hljs-number">0</span>]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>        <span class="hljs-keyword">return</span> <span class="hljs-literal">None</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>`,wrap:!1}}),Qs=new U({props:{code:"ZXhhbXBsZSUyMCUzRCUyMGRhdGFzZXRfd2l0aF9vY3IlNUIlMjJ0cmFpbiUyMiU1RCU1QjElNUQlMEF3b3JkcyUyMCUzRCUyMCU1QndvcmQubG93ZXIoKSUyMGZvciUyMHdvcmQlMjBpbiUyMGV4YW1wbGUlNUIlMjJ3b3JkcyUyMiU1RCU1RCUwQW1hdGNoJTJDJTIwd29yZF9pZHhfc3RhcnQlMkMlMjB3b3JkX2lkeF9lbmQlMjAlM0QlMjBzdWJmaW5kZXIod29yZHMlMkMlMjBleGFtcGxlJTVCJTIyYW5zd2VyJTIyJTVELmxvd2VyKCkuc3BsaXQoKSklMEFwcmludCglMjJRdWVzdGlvbiUzQSUyMCUyMiUyQyUyMGV4YW1wbGUlNUIlMjJxdWVzdGlvbiUyMiU1RCklMEFwcmludCglMjJXb3JkcyUzQSUyMiUyQyUyMHdvcmRzKSUwQXByaW50KCUyMkFuc3dlciUzQSUyMCUyMiUyQyUyMGV4YW1wbGUlNUIlMjJhbnN3ZXIlMjIlNUQpJTBBcHJpbnQoJTIyc3RhcnRfaW5kZXglMjIlMkMlMjB3b3JkX2lkeF9zdGFydCklMEFwcmludCglMjJlbmRfaW5kZXglMjIlMkMlMjB3b3JkX2lkeF9lbmQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>example = dataset_with_ocr[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>words = [word.lower() <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> example[<span class="hljs-string">&quot;words&quot;</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">match</span>, word_idx_start, word_idx_end = subfinder(words, example[<span class="hljs-string">&quot;answer&quot;</span>].lower().split())
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Question: &quot;</span>, example[<span class="hljs-string">&quot;question&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Words:&quot;</span>, words)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Answer: &quot;</span>, example[<span class="hljs-string">&quot;answer&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;start_index&quot;</span>, word_idx_start)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;end_index&quot;</span>, word_idx_end)
Question:  Who <span class="hljs-keyword">is</span> <span class="hljs-keyword">in</span>  cc <span class="hljs-keyword">in</span> this letter?
Words: [<span class="hljs-string">&#x27;wie&#x27;</span>, <span class="hljs-string">&#x27;baw&#x27;</span>, <span class="hljs-string">&#x27;brown&#x27;</span>, <span class="hljs-string">&#x27;&amp;&#x27;</span>, <span class="hljs-string">&#x27;williamson&#x27;</span>, <span class="hljs-string">&#x27;tobacco&#x27;</span>, <span class="hljs-string">&#x27;corporation&#x27;</span>, <span class="hljs-string">&#x27;research&#x27;</span>, <span class="hljs-string">&#x27;&amp;&#x27;</span>, <span class="hljs-string">&#x27;development&#x27;</span>, <span class="hljs-string">&#x27;internal&#x27;</span>, <span class="hljs-string">&#x27;correspondence&#x27;</span>, <span class="hljs-string">&#x27;to:&#x27;</span>, <span class="hljs-string">&#x27;r.&#x27;</span>, <span class="hljs-string">&#x27;h.&#x27;</span>, <span class="hljs-string">&#x27;honeycutt&#x27;</span>, <span class="hljs-string">&#x27;ce:&#x27;</span>, <span class="hljs-string">&#x27;t.f.&#x27;</span>, <span class="hljs-string">&#x27;riehl&#x27;</span>, <span class="hljs-string">&#x27;from:&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;c.j.&#x27;</span>, <span class="hljs-string">&#x27;cook&#x27;</span>, <span class="hljs-string">&#x27;date:&#x27;</span>, <span class="hljs-string">&#x27;may&#x27;</span>, <span class="hljs-string">&#x27;8,&#x27;</span>, <span class="hljs-string">&#x27;1995&#x27;</span>, <span class="hljs-string">&#x27;subject:&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;existing&#x27;</span>, <span class="hljs-string">&#x27;brainstorming&#x27;</span>, <span class="hljs-string">&#x27;ideas/483&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;major&#x27;</span>, <span class="hljs-string">&#x27;function&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;product&#x27;</span>, <span class="hljs-string">&#x27;innovation&#x27;</span>, <span class="hljs-string">&#x27;graup&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-string">&#x27;develop&#x27;</span>, <span class="hljs-string">&#x27;marketable&#x27;</span>, <span class="hljs-string">&#x27;nove!&#x27;</span>, <span class="hljs-string">&#x27;products&#x27;</span>, <span class="hljs-string">&#x27;that&#x27;</span>, <span class="hljs-string">&#x27;would&#x27;</span>, <span class="hljs-string">&#x27;be&#x27;</span>, <span class="hljs-string">&#x27;profitable&#x27;</span>, <span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-string">&#x27;manufacture&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;sell.&#x27;</span>, <span class="hljs-string">&#x27;novel&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;defined&#x27;</span>, <span class="hljs-string">&#x27;as:&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;new&#x27;</span>, <span class="hljs-string">&#x27;kind,&#x27;</span>, <span class="hljs-string">&#x27;or&#x27;</span>, <span class="hljs-string">&#x27;different&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;anything&#x27;</span>, <span class="hljs-string">&#x27;seen&#x27;</span>, <span class="hljs-string">&#x27;or&#x27;</span>, <span class="hljs-string">&#x27;known&#x27;</span>, <span class="hljs-string">&#x27;before.&#x27;</span>, <span class="hljs-string">&#x27;innovation&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;defined&#x27;</span>, <span class="hljs-string">&#x27;as:&#x27;</span>, <span class="hljs-string">&#x27;something&#x27;</span>, <span class="hljs-string">&#x27;new&#x27;</span>, <span class="hljs-string">&#x27;or&#x27;</span>, <span class="hljs-string">&#x27;different&#x27;</span>, <span class="hljs-string">&#x27;introduced;&#x27;</span>, <span class="hljs-string">&#x27;act&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;innovating;&#x27;</span>, <span class="hljs-string">&#x27;introduction&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;new&#x27;</span>, <span class="hljs-string">&#x27;things&#x27;</span>, <span class="hljs-string">&#x27;or&#x27;</span>, <span class="hljs-string">&#x27;methods.&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;products&#x27;</span>, <span class="hljs-string">&#x27;may&#x27;</span>, <span class="hljs-string">&#x27;incorporate&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;latest&#x27;</span>, <span class="hljs-string">&#x27;technologies,&#x27;</span>, <span class="hljs-string">&#x27;materials&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;know-how&#x27;</span>, <span class="hljs-string">&#x27;available&#x27;</span>, <span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-string">&#x27;give&#x27;</span>, <span class="hljs-string">&#x27;then&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;unique&#x27;</span>, <span class="hljs-string">&#x27;taste&#x27;</span>, <span class="hljs-string">&#x27;or&#x27;</span>, <span class="hljs-string">&#x27;look.&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;first&#x27;</span>, <span class="hljs-string">&#x27;task&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;product&#x27;</span>, <span class="hljs-string">&#x27;innovation&#x27;</span>, <span class="hljs-string">&#x27;group&#x27;</span>, <span class="hljs-string">&#x27;was&#x27;</span>, <span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-string">&#x27;assemble,&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;categorize&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;list&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;existing&#x27;</span>, <span class="hljs-string">&#x27;brainstorming&#x27;</span>, <span class="hljs-string">&#x27;ideas.&#x27;</span>, <span class="hljs-string">&#x27;ideas&#x27;</span>, <span class="hljs-string">&#x27;were&#x27;</span>, <span class="hljs-string">&#x27;grouped&#x27;</span>, <span class="hljs-string">&#x27;into&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;major&#x27;</span>, <span class="hljs-string">&#x27;categories&#x27;</span>, <span class="hljs-string">&#x27;labeled&#x27;</span>, <span class="hljs-string">&#x27;appearance&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;taste/aroma.&#x27;</span>, <span class="hljs-string">&#x27;these&#x27;</span>, <span class="hljs-string">&#x27;categories&#x27;</span>, <span class="hljs-string">&#x27;are&#x27;</span>, <span class="hljs-string">&#x27;used&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;novel&#x27;</span>, <span class="hljs-string">&#x27;products&#x27;</span>, <span class="hljs-string">&#x27;that&#x27;</span>, <span class="hljs-string">&#x27;may&#x27;</span>, <span class="hljs-string">&#x27;differ&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;visual&#x27;</span>, <span class="hljs-string">&#x27;and/or&#x27;</span>, <span class="hljs-string">&#x27;taste/aroma&#x27;</span>, <span class="hljs-string">&#x27;point&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;compared&#x27;</span>, <span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-string">&#x27;canventional&#x27;</span>, <span class="hljs-string">&#x27;cigarettes.&#x27;</span>, <span class="hljs-string">&#x27;other&#x27;</span>, <span class="hljs-string">&#x27;categories&#x27;</span>, <span class="hljs-string">&#x27;include&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;combination&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;above,&#x27;</span>, <span class="hljs-string">&#x27;filters,&#x27;</span>, <span class="hljs-string">&#x27;packaging&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;brand&#x27;</span>, <span class="hljs-string">&#x27;extensions.&#x27;</span>, <span class="hljs-string">&#x27;appearance&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;category&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;used&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;novel&#x27;</span>, <span class="hljs-string">&#x27;cigarette&#x27;</span>, <span class="hljs-string">&#x27;constructions&#x27;</span>, <span class="hljs-string">&#x27;that&#x27;</span>, <span class="hljs-string">&#x27;yield&#x27;</span>, <span class="hljs-string">&#x27;visually&#x27;</span>, <span class="hljs-string">&#x27;different&#x27;</span>, <span class="hljs-string">&#x27;products&#x27;</span>, <span class="hljs-string">&#x27;with&#x27;</span>, <span class="hljs-string">&#x27;minimal&#x27;</span>, <span class="hljs-string">&#x27;changes&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>, <span class="hljs-string">&#x27;smoke&#x27;</span>, <span class="hljs-string">&#x27;chemistry&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;cigarettes&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>, <span class="hljs-string">&#x27;cne.&#x27;</span>, <span class="hljs-string">&#x27;emulti-plug&#x27;</span>, <span class="hljs-string">&#x27;te&#x27;</span>, <span class="hljs-string">&#x27;build&#x27;</span>, <span class="hljs-string">&#x27;yaur&#x27;</span>, <span class="hljs-string">&#x27;awn&#x27;</span>, <span class="hljs-string">&#x27;cigarette.&#x27;</span>, <span class="hljs-string">&#x27;eswitchable&#x27;</span>, <span class="hljs-string">&#x27;menthol&#x27;</span>, <span class="hljs-string">&#x27;or&#x27;</span>, <span class="hljs-string">&#x27;non&#x27;</span>, <span class="hljs-string">&#x27;menthol&#x27;</span>, <span class="hljs-string">&#x27;cigarette.&#x27;</span>, <span class="hljs-string">&#x27;*cigarettes&#x27;</span>, <span class="hljs-string">&#x27;with&#x27;</span>, <span class="hljs-string">&#x27;interspaced&#x27;</span>, <span class="hljs-string">&#x27;perforations&#x27;</span>, <span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-string">&#x27;enable&#x27;</span>, <span class="hljs-string">&#x27;smoker&#x27;</span>, <span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-string">&#x27;separate&#x27;</span>, <span class="hljs-string">&#x27;unburned&#x27;</span>, <span class="hljs-string">&#x27;section&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;future&#x27;</span>, <span class="hljs-string">&#x27;smoking.&#x27;</span>, <span class="hljs-string">&#x27;«short&#x27;</span>, <span class="hljs-string">&#x27;cigarette,&#x27;</span>, <span class="hljs-string">&#x27;tobacco&#x27;</span>, <span class="hljs-string">&#x27;section&#x27;</span>, <span class="hljs-string">&#x27;30&#x27;</span>, <span class="hljs-string">&#x27;mm.&#x27;</span>, <span class="hljs-string">&#x27;«extremely&#x27;</span>, <span class="hljs-string">&#x27;fast&#x27;</span>, <span class="hljs-string">&#x27;buming&#x27;</span>, <span class="hljs-string">&#x27;cigarette.&#x27;</span>, <span class="hljs-string">&#x27;«novel&#x27;</span>, <span class="hljs-string">&#x27;cigarette&#x27;</span>, <span class="hljs-string">&#x27;constructions&#x27;</span>, <span class="hljs-string">&#x27;that&#x27;</span>, <span class="hljs-string">&#x27;permit&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;significant&#x27;</span>, <span class="hljs-string">&#x27;reduction&#x27;</span>, <span class="hljs-string">&#x27;iretobacco&#x27;</span>, <span class="hljs-string">&#x27;weight&#x27;</span>, <span class="hljs-string">&#x27;while&#x27;</span>, <span class="hljs-string">&#x27;maintaining&#x27;</span>, <span class="hljs-string">&#x27;smoking&#x27;</span>, <span class="hljs-string">&#x27;mechanics&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;visual&#x27;</span>, <span class="hljs-string">&#x27;characteristics.&#x27;</span>, <span class="hljs-string">&#x27;higher&#x27;</span>, <span class="hljs-string">&#x27;basis&#x27;</span>, <span class="hljs-string">&#x27;weight&#x27;</span>, <span class="hljs-string">&#x27;paper:&#x27;</span>, <span class="hljs-string">&#x27;potential&#x27;</span>, <span class="hljs-string">&#x27;reduction&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>, <span class="hljs-string">&#x27;tobacco&#x27;</span>, <span class="hljs-string">&#x27;weight.&#x27;</span>, <span class="hljs-string">&#x27;«more&#x27;</span>, <span class="hljs-string">&#x27;rigid&#x27;</span>, <span class="hljs-string">&#x27;tobacco&#x27;</span>, <span class="hljs-string">&#x27;column;&#x27;</span>, <span class="hljs-string">&#x27;stiffing&#x27;</span>, <span class="hljs-string">&#x27;agent&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;tobacco;&#x27;</span>, <span class="hljs-string">&#x27;e.g.&#x27;</span>, <span class="hljs-string">&#x27;starch&#x27;</span>, <span class="hljs-string">&#x27;*colored&#x27;</span>, <span class="hljs-string">&#x27;tow&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;cigarette&#x27;</span>, <span class="hljs-string">&#x27;papers;&#x27;</span>, <span class="hljs-string">&#x27;seasonal&#x27;</span>, <span class="hljs-string">&#x27;promotions,&#x27;</span>, <span class="hljs-string">&#x27;e.g.&#x27;</span>, <span class="hljs-string">&#x27;pastel&#x27;</span>, <span class="hljs-string">&#x27;colored&#x27;</span>, <span class="hljs-string">&#x27;cigarettes&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;easter&#x27;</span>, <span class="hljs-string">&#x27;or&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>, <span class="hljs-string">&#x27;an&#x27;</span>, <span class="hljs-string">&#x27;ebony&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;ivory&#x27;</span>, <span class="hljs-string">&#x27;brand&#x27;</span>, <span class="hljs-string">&#x27;containing&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;mixture&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;all&#x27;</span>, <span class="hljs-string">&#x27;black&#x27;</span>, <span class="hljs-string">&#x27;(black&#x27;</span>, <span class="hljs-string">&#x27;paper&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;tow)&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;ail&#x27;</span>, <span class="hljs-string">&#x27;white&#x27;</span>, <span class="hljs-string">&#x27;cigarettes.&#x27;</span>, <span class="hljs-string">&#x27;499150498&#x27;</span>]
Answer:  T.F. Riehl
start_index <span class="hljs-number">17</span>
end_index <span class="hljs-number">18</span>`,wrap:!1}}),Xs=new U({props:{code:"ZW5jb2RpbmclMjAlM0QlMjB0b2tlbml6ZXIoZXhhbXBsZSU1QiUyMnF1ZXN0aW9uJTIyJTVEJTJDJTIwZXhhbXBsZSU1QiUyMndvcmRzJTIyJTVEJTJDJTIwZXhhbXBsZSU1QiUyMmJveGVzJTIyJTVEKSUwQXRva2VuaXplci5kZWNvZGUoZW5jb2RpbmclNUIlMjJpbnB1dF9pZHMlMjIlNUQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoding = tokenizer(example[<span class="hljs-string">&quot;question&quot;</span>], example[<span class="hljs-string">&quot;words&quot;</span>], example[<span class="hljs-string">&quot;boxes&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(encoding[<span class="hljs-string">&quot;input_ids&quot;</span>])
[CLS] who <span class="hljs-keyword">is</span> <span class="hljs-keyword">in</span> cc <span class="hljs-keyword">in</span> this letter? [SEP] wie baw brown &amp; williamson tobacco corporation research &amp; development ...`,wrap:!1}}),Bs=new U({props:{code:" ",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_dataset</span>(<span class="hljs-params">examples, max_length=<span class="hljs-number">512</span></span>):
<span class="hljs-meta">... </span>    questions = examples[<span class="hljs-string">&quot;question&quot;</span>]
<span class="hljs-meta">... </span>    words = examples[<span class="hljs-string">&quot;words&quot;</span>]
<span class="hljs-meta">... </span>    boxes = examples[<span class="hljs-string">&quot;boxes&quot;</span>]
<span class="hljs-meta">... </span>    answers = examples[<span class="hljs-string">&quot;answer&quot;</span>]

<span class="hljs-meta">... </span>    <span class="hljs-comment"># 예제 배치를 인코딩하고 start_positions와 end_positions를 초기화합니다</span>
<span class="hljs-meta">... </span>    encoding = tokenizer(questions, words, boxes, max_length=max_length, padding=<span class="hljs-string">&quot;max_length&quot;</span>, truncation=<span class="hljs-literal">True</span>)
<span class="hljs-meta">... </span>    start_positions = []
<span class="hljs-meta">... </span>    end_positions = []

<span class="hljs-meta">... </span>    <span class="hljs-comment"># 배치의 예제를 반복합니다</span>
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(questions)):
<span class="hljs-meta">... </span>        cls_index = encoding[<span class="hljs-string">&quot;input_ids&quot;</span>][i].index(tokenizer.cls_token_id)

<span class="hljs-meta">... </span>        <span class="hljs-comment"># 예제의 words에서 답변의 위치를 찾습니다</span>
<span class="hljs-meta">... </span>        words_example = [word.lower() <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> words[i]]
<span class="hljs-meta">... </span>        answer = answers[i]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">match</span>, word_idx_start, word_idx_end = subfinder(words_example, answer.lower().split())

<span class="hljs-meta">... </span>        <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span>:
<span class="hljs-meta">... </span>            <span class="hljs-comment"># 일치하는 항목을 발견하면, \`token_type_ids\`를 사용해 인코딩에서 단어가 시작하는 위치를 찾습니다</span>
<span class="hljs-meta">... </span>            token_type_ids = encoding[<span class="hljs-string">&quot;token_type_ids&quot;</span>][i]
<span class="hljs-meta">... </span>            token_start_index = <span class="hljs-number">0</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">while</span> token_type_ids[token_start_index] != <span class="hljs-number">1</span>:
<span class="hljs-meta">... </span>                token_start_index += <span class="hljs-number">1</span>

<span class="hljs-meta">... </span>            token_end_index = <span class="hljs-built_in">len</span>(encoding[<span class="hljs-string">&quot;input_ids&quot;</span>][i]) - <span class="hljs-number">1</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">while</span> token_type_ids[token_end_index] != <span class="hljs-number">1</span>:
<span class="hljs-meta">... </span>                token_end_index -= <span class="hljs-number">1</span>

<span class="hljs-meta">... </span>            word_ids = encoding.word_ids(i)[token_start_index : token_end_index + <span class="hljs-number">1</span>]
<span class="hljs-meta">... </span>            start_position = cls_index
<span class="hljs-meta">... </span>            end_position = cls_index

<span class="hljs-meta">... </span>            <span class="hljs-comment"># words의 답변 위치와 일치할 때까지 word_ids를 반복하고 \`token_start_index\`를 늘립니다</span>
<span class="hljs-meta">... </span>            <span class="hljs-comment"># 일치하면 \`token_start_index\`를 인코딩에서 답변의 \`start_position\`으로 저장합니다</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">for</span> <span class="hljs-built_in">id</span> <span class="hljs-keyword">in</span> word_ids:
<span class="hljs-meta">... </span>                <span class="hljs-keyword">if</span> <span class="hljs-built_in">id</span> == word_idx_start:
<span class="hljs-meta">... </span>                    start_position = token_start_index
<span class="hljs-meta">... </span>                <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                    token_start_index += <span class="hljs-number">1</span>

<span class="hljs-meta">... </span>            <span class="hljs-comment"># 비슷하게, 끝에서 시작해 \`word_ids\`를 반복하며 답변의 \`end_position\`을 찾습니다</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">for</span> <span class="hljs-built_in">id</span> <span class="hljs-keyword">in</span> word_ids[::-<span class="hljs-number">1</span>]:
<span class="hljs-meta">... </span>                <span class="hljs-keyword">if</span> <span class="hljs-built_in">id</span> == word_idx_end:
<span class="hljs-meta">... </span>                    end_position = token_end_index
<span class="hljs-meta">... </span>                <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                    token_end_index -= <span class="hljs-number">1</span>

<span class="hljs-meta">... </span>            start_positions.append(start_position)
<span class="hljs-meta">... </span>            end_positions.append(end_position)

<span class="hljs-meta">... </span>        <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>            start_positions.append(cls_index)
<span class="hljs-meta">... </span>            end_positions.append(cls_index)

<span class="hljs-meta">... </span>    encoding[<span class="hljs-string">&quot;image&quot;</span>] = examples[<span class="hljs-string">&quot;image&quot;</span>]
<span class="hljs-meta">... </span>    encoding[<span class="hljs-string">&quot;start_positions&quot;</span>] = start_positions
<span class="hljs-meta">... </span>    encoding[<span class="hljs-string">&quot;end_positions&quot;</span>] = end_positions

<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> encoding`,wrap:!1}}),Ws=new U({props:{code:"ZW5jb2RlZF90cmFpbl9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldF93aXRoX29jciU1QiUyMnRyYWluJTIyJTVELm1hcCglMEElMjAlMjAlMjAlMjBlbmNvZGVfZGF0YXNldCUyQyUyMGJhdGNoZWQlM0RUcnVlJTJDJTIwYmF0Y2hfc2l6ZSUzRDIlMkMlMjByZW1vdmVfY29sdW1ucyUzRGRhdGFzZXRfd2l0aF9vY3IlNUIlMjJ0cmFpbiUyMiU1RC5jb2x1bW5fbmFtZXMlMEEpJTBBZW5jb2RlZF90ZXN0X2RhdGFzZXQlMjAlM0QlMjBkYXRhc2V0X3dpdGhfb2NyJTVCJTIydGVzdCUyMiU1RC5tYXAoJTBBJTIwJTIwJTIwJTIwZW5jb2RlX2RhdGFzZXQlMkMlMjBiYXRjaGVkJTNEVHJ1ZSUyQyUyMGJhdGNoX3NpemUlM0QyJTJDJTIwcmVtb3ZlX2NvbHVtbnMlM0RkYXRhc2V0X3dpdGhfb2NyJTVCJTIydGVzdCUyMiU1RC5jb2x1bW5fbmFtZXMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_train_dataset = dataset_with_ocr[<span class="hljs-string">&quot;train&quot;</span>].<span class="hljs-built_in">map</span>(
<span class="hljs-meta">... </span>    encode_dataset, batched=<span class="hljs-literal">True</span>, batch_size=<span class="hljs-number">2</span>, remove_columns=dataset_with_ocr[<span class="hljs-string">&quot;train&quot;</span>].column_names
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_test_dataset = dataset_with_ocr[<span class="hljs-string">&quot;test&quot;</span>].<span class="hljs-built_in">map</span>(
<span class="hljs-meta">... </span>    encode_dataset, batched=<span class="hljs-literal">True</span>, batch_size=<span class="hljs-number">2</span>, remove_columns=dataset_with_ocr[<span class="hljs-string">&quot;test&quot;</span>].column_names
<span class="hljs-meta">... </span>)`,wrap:!1}}),Ss=new U({props:{code:"ZW5jb2RlZF90cmFpbl9kYXRhc2V0LmZlYXR1cmVz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_train_dataset.features
{<span class="hljs-string">&#x27;image&#x27;</span>: <span class="hljs-type">Sequence</span>(feature=<span class="hljs-type">Sequence</span>(feature=<span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&#x27;uint8&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;input_ids&#x27;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&#x27;int32&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&#x27;int8&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&#x27;int8&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;bbox&#x27;</span>: <span class="hljs-type">Sequence</span>(feature=<span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&#x27;int64&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;start_positions&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;int64&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;end_positions&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;int64&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}`,wrap:!1}}),qs=new B({}),Hs=new B({}),Os=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckRvY3VtZW50UXVlc3Rpb25BbnN3ZXJpbmclMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckRvY3VtZW50UXVlc3Rpb25BbnN3ZXJpbmcuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2NoZWNrcG9pbnQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForDocumentQuestionAnswering

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForDocumentQuestionAnswering.from_pretrained(model_checkpoint)`,wrap:!1}}),Ks=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTBBJTBBJTIzJTIwJUVCJUIzJUI4JUVDJTlEJUI4JUVDJTlEJTk4JTIwJUVCJUEwJTg4JUVEJThGJUFDJUVDJUE3JTgwJUVEJTg2JUEwJUVCJUE2JUFDJTIwSUQlRUIlQTElOUMlMjAlRUIlQjAlOTQlRUElQkUlQjglRUMlODQlQjglRUMlOUElOTQlMEFyZXBvX2lkJTIwJTNEJTIwJTIyTWFyaWFLJTJGbGF5b3V0bG12Mi1iYXNlLXVuY2FzZWRfZmluZXR1bmVkX2RvY3ZxYSUyMiUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEcmVwb19pZCUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDQlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMjAlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3N0ZXBzJTNEMjAwJTJDJTBBJTIwJTIwJTIwJTIwbG9nZ2luZ19zdGVwcyUzRDUwJTJDJTBBJTIwJTIwJTIwJTIwZXZhbHVhdGlvbl9zdHJhdGVneSUzRCUyMnN0ZXBzJTIyJTJDJTBBJTIwJTIwJTIwJTIwbGVhcm5pbmdfcmF0ZSUzRDVlLTUlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3RvdGFsX2xpbWl0JTNEMiUyQyUwQSUyMCUyMCUyMCUyMHJlbW92ZV91bnVzZWRfY29sdW1ucyUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># 본인의 레포지토리 ID로 바꾸세요</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>repo_id = <span class="hljs-string">&quot;MariaK/layoutlmv2-base-uncased_finetuned_docvqa&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=repo_id,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">20</span>,
<span class="hljs-meta">... </span>    save_steps=<span class="hljs-number">200</span>,
<span class="hljs-meta">... </span>    logging_steps=<span class="hljs-number">50</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;steps&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">5e-5</span>,
<span class="hljs-meta">... </span>    save_total_limit=<span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    remove_unused_columns=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    push_to_hub=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)`,wrap:!1}}),aa=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERlZmF1bHREYXRhQ29sbGF0b3IlMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGVmYXVsdERhdGFDb2xsYXRvcigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DefaultDataCollator

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DefaultDataCollator()`,wrap:!1}}),na=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2NvbGxhdG9yJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0RlbmNvZGVkX3RyYWluX2RhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBldmFsX2RhdGFzZXQlM0RlbmNvZGVkX3Rlc3RfZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHByb2Nlc3NvciUyQyUwQSklMEElMEF0cmFpbmVyLnRyYWluKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>    train_dataset=encoded_train_dataset,
<span class="hljs-meta">... </span>    eval_dataset=encoded_test_dataset,
<span class="hljs-meta">... </span>    tokenizer=processor,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`,wrap:!1}}),ea=new U({props:{code:"dHJhaW5lci5jcmVhdGVfbW9kZWxfY2FyZCgpJTBBdHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.create_model_card()
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.push_to_hub()`,wrap:!1}}),pa=new B({}),ra=new U({props:{code:"ZXhhbXBsZSUyMCUzRCUyMGRhdGFzZXQlNUIlMjJ0ZXN0JTIyJTVEJTVCMiU1RCUwQXF1ZXN0aW9uJTIwJTNEJTIwZXhhbXBsZSU1QiUyMnF1ZXJ5JTIyJTVEJTVCJTIyZW4lMjIlNUQlMEFpbWFnZSUyMCUzRCUyMGV4YW1wbGUlNUIlMjJpbWFnZSUyMiU1RCUwQXByaW50KHF1ZXN0aW9uKSUwQXByaW50KGV4YW1wbGUlNUIlMjJhbnN3ZXJzJTIyJTVEKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>example = dataset[<span class="hljs-string">&quot;test&quot;</span>][<span class="hljs-number">2</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>question = example[<span class="hljs-string">&quot;query&quot;</span>][<span class="hljs-string">&quot;en&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>image = example[<span class="hljs-string">&quot;image&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(question)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(example[<span class="hljs-string">&quot;answers&quot;</span>])
<span class="hljs-string">&#x27;Who is ‘presiding’ TRRF GENERAL SESSION (PART 1)?&#x27;</span>
[<span class="hljs-string">&#x27;TRRF Vice President&#x27;</span>, <span class="hljs-string">&#x27;lee a. waller&#x27;</span>]`,wrap:!1}}),ha=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBcWFfcGlwZWxpbmUlMjAlM0QlMjBwaXBlbGluZSglMjJkb2N1bWVudC1xdWVzdGlvbi1hbnN3ZXJpbmclMjIlMkMlMjBtb2RlbCUzRCUyMk1hcmlhSyUyRmxheW91dGxtdjItYmFzZS11bmNhc2VkX2ZpbmV0dW5lZF9kb2N2cWElMjIpJTBBcWFfcGlwZWxpbmUoaW1hZ2UlMkMlMjBxdWVzdGlvbik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>qa_pipeline = pipeline(<span class="hljs-string">&quot;document-question-answering&quot;</span>, model=<span class="hljs-string">&quot;MariaK/layoutlmv2-base-uncased_finetuned_docvqa&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>qa_pipeline(image, question)
[{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9949808120727539</span>,
  <span class="hljs-string">&#x27;answer&#x27;</span>: <span class="hljs-string">&#x27;Lee A. Waller&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">55</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`,wrap:!1}}),xa=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Byb2Nlc3NvciUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JEb2N1bWVudFF1ZXN0aW9uQW5zd2VyaW5nJTBBJTBBcHJvY2Vzc29yJTIwJTNEJTIwQXV0b1Byb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIyTWFyaWFLJTJGbGF5b3V0bG12Mi1iYXNlLXVuY2FzZWRfZmluZXR1bmVkX2RvY3ZxYSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckRvY3VtZW50UXVlc3Rpb25BbnN3ZXJpbmcuZnJvbV9wcmV0cmFpbmVkKCUyMk1hcmlhSyUyRmxheW91dGxtdjItYmFzZS11bmNhc2VkX2ZpbmV0dW5lZF9kb2N2cWElMjIpJTBBJTBBd2l0aCUyMHRvcmNoLm5vX2dyYWQoKSUzQSUwQSUyMCUyMCUyMCUyMGVuY29kaW5nJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlLmNvbnZlcnQoJTIyUkdCJTIyKSUyQyUyMHF1ZXN0aW9uJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKiplbmNvZGluZyklMEElMjAlMjAlMjAlMjBzdGFydF9sb2dpdHMlMjAlM0QlMjBvdXRwdXRzLnN0YXJ0X2xvZ2l0cyUwQSUyMCUyMCUyMCUyMGVuZF9sb2dpdHMlMjAlM0QlMjBvdXRwdXRzLmVuZF9sb2dpdHMlMEElMjAlMjAlMjAlMjBwcmVkaWN0ZWRfc3RhcnRfaWR4JTIwJTNEJTIwc3RhcnRfbG9naXRzLmFyZ21heCgtMSkuaXRlbSgpJTBBJTIwJTIwJTIwJTIwcHJlZGljdGVkX2VuZF9pZHglMjAlM0QlMjBlbmRfbG9naXRzLmFyZ21heCgtMSkuaXRlbSgpJTBBJTBBcHJvY2Vzc29yLnRva2VuaXplci5kZWNvZGUoZW5jb2RpbmcuaW5wdXRfaWRzLnNxdWVlemUoKSU1QnByZWRpY3RlZF9zdGFydF9pZHglMjAlM0ElMjBwcmVkaWN0ZWRfZW5kX2lkeCUyMCUyQiUyMDElNUQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForDocumentQuestionAnswering

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;MariaK/layoutlmv2-base-uncased_finetuned_docvqa&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForDocumentQuestionAnswering.from_pretrained(<span class="hljs-string">&quot;MariaK/layoutlmv2-base-uncased_finetuned_docvqa&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    encoding = processor(image.convert(<span class="hljs-string">&quot;RGB&quot;</span>), question, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">... </span>    outputs = model(**encoding)
<span class="hljs-meta">... </span>    start_logits = outputs.start_logits
<span class="hljs-meta">... </span>    end_logits = outputs.end_logits
<span class="hljs-meta">... </span>    predicted_start_idx = start_logits.argmax(-<span class="hljs-number">1</span>).item()
<span class="hljs-meta">... </span>    predicted_end_idx = end_logits.argmax(-<span class="hljs-number">1</span>).item()

<span class="hljs-meta">&gt;&gt;&gt; </span>processor.tokenizer.decode(encoding.input_ids.squeeze()[predicted_start_idx : predicted_end_idx + <span class="hljs-number">1</span>])
<span class="hljs-string">&#x27;lee a. waller&#x27;</span>`,wrap:!1}}),{c(){J=p("meta"),F=t(),T=p("p"),u=t(),w=p("h1"),y=p("a"),C=p("span"),r(W.$$.fragment),dn=t(),da=p("span"),da.textContent=Cn,Za=t(),r(z.$$.fragment),Aa=t(),S=p("p"),S.textContent=bn,Qa=t(),q=p("p"),q.textContent=fn,$a=t(),Y=p("ul"),Y.innerHTML=In,Xa=t(),r(R.$$.fragment),Na=t(),H=p("p"),H.textContent=_n,Ea=t(),D=p("p"),D.textContent=vn,Ga=t(),r(L.$$.fragment),Ba=t(),r(O.$$.fragment),Fa=t(),r(P.$$.fragment),Wa=t(),K=p("p"),K.textContent=Vn,za=t(),ss=p("p"),ss.textContent=kn,Sa=t(),r(as.$$.fragment),qa=t(),ls=p("p"),ls.textContent=Rn,Ya=t(),r(ns.$$.fragment),Ha=t(),b=p("h2"),Z=p("a"),Ca=p("span"),r(ts.$$.fragment),Un=t(),Ua=p("span"),Ua.textContent=Zn,Da=t(),es=p("p"),es.innerHTML=An,La=t(),r(ps.$$.fragment),Oa=t(),is=p("p"),is.textContent=Qn,Pa=t(),r(cs.$$.fragment),Ka=t(),rs=p("p"),rs.textContent=$n,sl=t(),os=p("ul"),os.innerHTML=Xn,al=t(),hs=p("p"),hs.innerHTML=Nn,ll=t(),r(js.$$.fragment),nl=t(),Ms=p("p"),Ms.innerHTML=En,tl=t(),r(xs.$$.fragment),el=t(),gs=p("p"),gs.innerHTML=Gn,pl=t(),r(ds.$$.fragment),il=t(),Us=p("p"),Us.textContent=Bn,cl=t(),r(ms.$$.fragment),rl=t(),A=p("div"),A.innerHTML=Fn,ol=t(),f=p("h2"),Q=p("a"),ba=p("span"),r(ys.$$.fragment),mn=t(),ma=p("span"),ma.textContent=Wn,hl=t(),Js=p("p"),Js.innerHTML=zn,jl=t(),r(us.$$.fragment),Ml=t(),I=p("h3"),$=p("a"),fa=p("span"),r(ws.$$.fragment),yn=t(),ya=p("span"),ya.textContent=Sn,xl=t(),Ts=p("p"),Ts.innerHTML=qn,gl=t(),r(Cs.$$.fragment),dl=t(),bs=p("p"),bs.innerHTML=Yn,Ul=t(),r(fs.$$.fragment),ml=t(),_=p("h3"),X=p("a"),Ia=p("span"),r(Is.$$.fragment),Jn=t(),Ja=p("span"),Ja.textContent=Hn,yl=t(),_s=p("p"),_s.innerHTML=Dn,Jl=t(),r(vs.$$.fragment),ul=t(),Vs=p("p"),Vs.innerHTML=Ln,wl=t(),ks=p("p"),ks.textContent=On,Tl=t(),Rs=p("p"),Rs.innerHTML=Pn,Cl=t(),r(Zs.$$.fragment),bl=t(),As=p("p"),As.textContent=Kn,fl=t(),r(Qs.$$.fragment),Il=t(),$s=p("p"),$s.textContent=st,_l=t(),r(Xs.$$.fragment),vl=t(),Ns=p("p"),Ns.textContent=at,Vl=t(),Es=p("ul"),Es.innerHTML=lt,kl=t(),Gs=p("p"),Gs.textContent=nt,Rl=t(),r(Bs.$$.fragment),Zl=t(),Fs=p("p"),Fs.textContent=tt,Al=t(),r(Ws.$$.fragment),Ql=t(),zs=p("p"),zs.textContent=et,$l=t(),r(Ss.$$.fragment),Xl=t(),v=p("h2"),N=p("a"),_a=p("span"),r(qs.$$.fragment),un=t(),ua=p("span"),ua.textContent=pt,Nl=t(),Ys=p("p"),Ys.innerHTML=it,El=t(),V=p("h2"),E=p("a"),va=p("span"),r(Hs.$$.fragment),wn=t(),wa=p("span"),wa.textContent=ct,Gl=t(),Ds=p("p"),Ds.textContent=rt,Bl=t(),Ls=p("ul"),Ls.innerHTML=ot,Fl=t(),r(Os.$$.fragment),Wl=t(),Ps=p("p"),Ps.innerHTML=ht,zl=t(),r(Ks.$$.fragment),Sl=t(),sa=p("p"),sa.textContent=jt,ql=t(),r(aa.$$.fragment),Yl=t(),la=p("p"),la.innerHTML=Mt,Hl=t(),r(na.$$.fragment),Dl=t(),ta=p("p"),ta.innerHTML=xt,Ll=t(),r(ea.$$.fragment),Ol=t(),k=p("h2"),G=p("a"),Va=p("span"),r(pa.$$.fragment),Tn=t(),Ta=p("span"),Ta.textContent=gt,Pl=t(),ia=p("p"),ia.innerHTML=dt,Kl=t(),ca=p("p"),ca.textContent=Ut,sn=t(),r(ra.$$.fragment),an=t(),oa=p("p"),oa.textContent=mt,ln=t(),r(ha.$$.fragment),nn=t(),ja=p("p"),ja.textContent=yt,tn=t(),Ma=p("ol"),Ma.innerHTML=Jt,en=t(),r(xa.$$.fragment),pn=t(),ka=p("p"),this.h()},l(s){const a=Ft("svelte-1phssyn",document.head);J=i(a,"META",{name:!0,content:!0}),a.forEach(l),F=e(s),T=i(s,"P",{}),m(T).forEach(l),u=e(s),w=i(s,"H1",{class:!0});var ga=m(w);y=i(ga,"A",{id:!0,class:!0,href:!0});var ut=m(y);C=i(ut,"SPAN",{});var wt=m(C);o(W.$$.fragment,wt),wt.forEach(l),ut.forEach(l),dn=e(ga),da=i(ga,"SPAN",{"data-svelte-h":!0}),c(da)!=="svelte-13l1svr"&&(da.textContent=Cn),ga.forEach(l),Za=e(s),o(z.$$.fragment,s),Aa=e(s),S=i(s,"P",{"data-svelte-h":!0}),c(S)!=="svelte-oqnn9h"&&(S.textContent=bn),Qa=e(s),q=i(s,"P",{"data-svelte-h":!0}),c(q)!=="svelte-k9bbb9"&&(q.textContent=fn),$a=e(s),Y=i(s,"UL",{"data-svelte-h":!0}),c(Y)!=="svelte-1t9y1pd"&&(Y.innerHTML=In),Xa=e(s),o(R.$$.fragment,s),Na=e(s),H=i(s,"P",{"data-svelte-h":!0}),c(H)!=="svelte-1t45j8o"&&(H.textContent=_n),Ea=e(s),D=i(s,"P",{"data-svelte-h":!0}),c(D)!=="svelte-ddzvb0"&&(D.textContent=vn),Ga=e(s),o(L.$$.fragment,s),Ba=e(s),o(O.$$.fragment,s),Fa=e(s),o(P.$$.fragment,s),Wa=e(s),K=i(s,"P",{"data-svelte-h":!0}),c(K)!=="svelte-1uvmu64"&&(K.textContent=Vn),za=e(s),ss=i(s,"P",{"data-svelte-h":!0}),c(ss)!=="svelte-xyvcw8"&&(ss.textContent=kn),Sa=e(s),o(as.$$.fragment,s),qa=e(s),ls=i(s,"P",{"data-svelte-h":!0}),c(ls)!=="svelte-13bifrs"&&(ls.textContent=Rn),Ya=e(s),o(ns.$$.fragment,s),Ha=e(s),b=i(s,"H2",{class:!0});var rn=m(b);Z=i(rn,"A",{id:!0,class:!0,href:!0});var Tt=m(Z);Ca=i(Tt,"SPAN",{});var Ct=m(Ca);o(ts.$$.fragment,Ct),Ct.forEach(l),Tt.forEach(l),Un=e(rn),Ua=i(rn,"SPAN",{"data-svelte-h":!0}),c(Ua)!=="svelte-18g5ms7"&&(Ua.textContent=Zn),rn.forEach(l),Da=e(s),es=i(s,"P",{"data-svelte-h":!0}),c(es)!=="svelte-12y9xdp"&&(es.innerHTML=An),La=e(s),o(ps.$$.fragment,s),Oa=e(s),is=i(s,"P",{"data-svelte-h":!0}),c(is)!=="svelte-9rg4tz"&&(is.textContent=Qn),Pa=e(s),o(cs.$$.fragment,s),Ka=e(s),rs=i(s,"P",{"data-svelte-h":!0}),c(rs)!=="svelte-zq0ej4"&&(rs.textContent=$n),sl=e(s),os=i(s,"UL",{"data-svelte-h":!0}),c(os)!=="svelte-g2ws24"&&(os.innerHTML=Xn),al=e(s),hs=i(s,"P",{"data-svelte-h":!0}),c(hs)!=="svelte-xeapog"&&(hs.innerHTML=Nn),ll=e(s),o(js.$$.fragment,s),nl=e(s),Ms=i(s,"P",{"data-svelte-h":!0}),c(Ms)!=="svelte-m4zkx9"&&(Ms.innerHTML=En),tl=e(s),o(xs.$$.fragment,s),el=e(s),gs=i(s,"P",{"data-svelte-h":!0}),c(gs)!=="svelte-1xnig6m"&&(gs.innerHTML=Gn),pl=e(s),o(ds.$$.fragment,s),il=e(s),Us=i(s,"P",{"data-svelte-h":!0}),c(Us)!=="svelte-lp4iy7"&&(Us.textContent=Bn),cl=e(s),o(ms.$$.fragment,s),rl=e(s),A=i(s,"DIV",{class:!0,"data-svelte-h":!0}),c(A)!=="svelte-q63tj1"&&(A.innerHTML=Fn),ol=e(s),f=i(s,"H2",{class:!0});var on=m(f);Q=i(on,"A",{id:!0,class:!0,href:!0});var bt=m(Q);ba=i(bt,"SPAN",{});var ft=m(ba);o(ys.$$.fragment,ft),ft.forEach(l),bt.forEach(l),mn=e(on),ma=i(on,"SPAN",{"data-svelte-h":!0}),c(ma)!=="svelte-5kizkd"&&(ma.textContent=Wn),on.forEach(l),hl=e(s),Js=i(s,"P",{"data-svelte-h":!0}),c(Js)!=="svelte-1y93l9d"&&(Js.innerHTML=zn),jl=e(s),o(us.$$.fragment,s),Ml=e(s),I=i(s,"H3",{class:!0});var hn=m(I);$=i(hn,"A",{id:!0,class:!0,href:!0});var It=m($);fa=i(It,"SPAN",{});var _t=m(fa);o(ws.$$.fragment,_t),_t.forEach(l),It.forEach(l),yn=e(hn),ya=i(hn,"SPAN",{"data-svelte-h":!0}),c(ya)!=="svelte-j32csz"&&(ya.textContent=Sn),hn.forEach(l),xl=e(s),Ts=i(s,"P",{"data-svelte-h":!0}),c(Ts)!=="svelte-1kcsbqg"&&(Ts.innerHTML=qn),gl=e(s),o(Cs.$$.fragment,s),dl=e(s),bs=i(s,"P",{"data-svelte-h":!0}),c(bs)!=="svelte-1rksgl2"&&(bs.innerHTML=Yn),Ul=e(s),o(fs.$$.fragment,s),ml=e(s),_=i(s,"H3",{class:!0});var jn=m(_);X=i(jn,"A",{id:!0,class:!0,href:!0});var vt=m(X);Ia=i(vt,"SPAN",{});var Vt=m(Ia);o(Is.$$.fragment,Vt),Vt.forEach(l),vt.forEach(l),Jn=e(jn),Ja=i(jn,"SPAN",{"data-svelte-h":!0}),c(Ja)!=="svelte-28ab6q"&&(Ja.textContent=Hn),jn.forEach(l),yl=e(s),_s=i(s,"P",{"data-svelte-h":!0}),c(_s)!=="svelte-1l26czp"&&(_s.innerHTML=Dn),Jl=e(s),o(vs.$$.fragment,s),ul=e(s),Vs=i(s,"P",{"data-svelte-h":!0}),c(Vs)!=="svelte-1yiqpne"&&(Vs.innerHTML=Ln),wl=e(s),ks=i(s,"P",{"data-svelte-h":!0}),c(ks)!=="svelte-rkm9nf"&&(ks.textContent=On),Tl=e(s),Rs=i(s,"P",{"data-svelte-h":!0}),c(Rs)!=="svelte-1pxhw5r"&&(Rs.innerHTML=Pn),Cl=e(s),o(Zs.$$.fragment,s),bl=e(s),As=i(s,"P",{"data-svelte-h":!0}),c(As)!=="svelte-mdy9jm"&&(As.textContent=Kn),fl=e(s),o(Qs.$$.fragment,s),Il=e(s),$s=i(s,"P",{"data-svelte-h":!0}),c($s)!=="svelte-1gff4qz"&&($s.textContent=st),_l=e(s),o(Xs.$$.fragment,s),vl=e(s),Ns=i(s,"P",{"data-svelte-h":!0}),c(Ns)!=="svelte-zzn8kk"&&(Ns.textContent=at),Vl=e(s),Es=i(s,"UL",{"data-svelte-h":!0}),c(Es)!=="svelte-f0s3pn"&&(Es.innerHTML=lt),kl=e(s),Gs=i(s,"P",{"data-svelte-h":!0}),c(Gs)!=="svelte-1h1oq5v"&&(Gs.textContent=nt),Rl=e(s),o(Bs.$$.fragment,s),Zl=e(s),Fs=i(s,"P",{"data-svelte-h":!0}),c(Fs)!=="svelte-10ovv0a"&&(Fs.textContent=tt),Al=e(s),o(Ws.$$.fragment,s),Ql=e(s),zs=i(s,"P",{"data-svelte-h":!0}),c(zs)!=="svelte-1cmvv07"&&(zs.textContent=et),$l=e(s),o(Ss.$$.fragment,s),Xl=e(s),v=i(s,"H2",{class:!0});var Mn=m(v);N=i(Mn,"A",{id:!0,class:!0,href:!0});var kt=m(N);_a=i(kt,"SPAN",{});var Rt=m(_a);o(qs.$$.fragment,Rt),Rt.forEach(l),kt.forEach(l),un=e(Mn),ua=i(Mn,"SPAN",{"data-svelte-h":!0}),c(ua)!=="svelte-6suli0"&&(ua.textContent=pt),Mn.forEach(l),Nl=e(s),Ys=i(s,"P",{"data-svelte-h":!0}),c(Ys)!=="svelte-8iuplh"&&(Ys.innerHTML=it),El=e(s),V=i(s,"H2",{class:!0});var xn=m(V);E=i(xn,"A",{id:!0,class:!0,href:!0});var Zt=m(E);va=i(Zt,"SPAN",{});var At=m(va);o(Hs.$$.fragment,At),At.forEach(l),Zt.forEach(l),wn=e(xn),wa=i(xn,"SPAN",{"data-svelte-h":!0}),c(wa)!=="svelte-19hn5sr"&&(wa.textContent=ct),xn.forEach(l),Gl=e(s),Ds=i(s,"P",{"data-svelte-h":!0}),c(Ds)!=="svelte-1r96dak"&&(Ds.textContent=rt),Bl=e(s),Ls=i(s,"UL",{"data-svelte-h":!0}),c(Ls)!=="svelte-dhr9ud"&&(Ls.innerHTML=ot),Fl=e(s),o(Os.$$.fragment,s),Wl=e(s),Ps=i(s,"P",{"data-svelte-h":!0}),c(Ps)!=="svelte-1wyv6ig"&&(Ps.innerHTML=ht),zl=e(s),o(Ks.$$.fragment,s),Sl=e(s),sa=i(s,"P",{"data-svelte-h":!0}),c(sa)!=="svelte-1hdarm6"&&(sa.textContent=jt),ql=e(s),o(aa.$$.fragment,s),Yl=e(s),la=i(s,"P",{"data-svelte-h":!0}),c(la)!=="svelte-1heolva"&&(la.innerHTML=Mt),Hl=e(s),o(na.$$.fragment,s),Dl=e(s),ta=i(s,"P",{"data-svelte-h":!0}),c(ta)!=="svelte-1gum9w7"&&(ta.innerHTML=xt),Ll=e(s),o(ea.$$.fragment,s),Ol=e(s),k=i(s,"H2",{class:!0});var gn=m(k);G=i(gn,"A",{id:!0,class:!0,href:!0});var Qt=m(G);Va=i(Qt,"SPAN",{});var $t=m(Va);o(pa.$$.fragment,$t),$t.forEach(l),Qt.forEach(l),Tn=e(gn),Ta=i(gn,"SPAN",{"data-svelte-h":!0}),c(Ta)!=="svelte-1dnn72f"&&(Ta.textContent=gt),gn.forEach(l),Pl=e(s),ia=i(s,"P",{"data-svelte-h":!0}),c(ia)!=="svelte-1flysn8"&&(ia.innerHTML=dt),Kl=e(s),ca=i(s,"P",{"data-svelte-h":!0}),c(ca)!=="svelte-1dcjn75"&&(ca.textContent=Ut),sn=e(s),o(ra.$$.fragment,s),an=e(s),oa=i(s,"P",{"data-svelte-h":!0}),c(oa)!=="svelte-gaec9h"&&(oa.textContent=mt),ln=e(s),o(ha.$$.fragment,s),nn=e(s),ja=i(s,"P",{"data-svelte-h":!0}),c(ja)!=="svelte-4epvs7"&&(ja.textContent=yt),tn=e(s),Ma=i(s,"OL",{"data-svelte-h":!0}),c(Ma)!=="svelte-10zqnq5"&&(Ma.innerHTML=Jt),en=e(s),o(xa.$$.fragment,s),pn=e(s),ka=i(s,"P",{}),m(ka).forEach(l),this.h()},h(){g(J,"name","hf:doc:metadata"),g(J,"content",JSON.stringify(Yt)),g(y,"id","document_question_answering"),g(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(y,"href","#document_question_answering"),g(w,"class","relative group"),g(Z,"id","load-the-data"),g(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Z,"href","#load-the-data"),g(b,"class","relative group"),g(A,"class","flex justify-center"),g(Q,"id","preprocess-the-data"),g(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Q,"href","#preprocess-the-data"),g(f,"class","relative group"),g($,"id","preprocessing-document-images"),g($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g($,"href","#preprocessing-document-images"),g(I,"class","relative group"),g(X,"id","preprocessing-text-data"),g(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(X,"href","#preprocessing-text-data"),g(_,"class","relative group"),g(N,"id","evaluation"),g(N,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(N,"href","#evaluation"),g(v,"class","relative group"),g(E,"id","train"),g(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(E,"href","#train"),g(V,"class","relative group"),g(G,"id","inference"),g(G,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(G,"href","#inference"),g(k,"class","relative group")},m(s,a){d(document.head,J),n(s,F,a),n(s,T,a),n(s,u,a),n(s,w,a),d(w,y),d(y,C),h(W,C,null),d(w,dn),d(w,da),n(s,Za,a),h(z,s,a),n(s,Aa,a),n(s,S,a),n(s,Qa,a),n(s,q,a),n(s,$a,a),n(s,Y,a),n(s,Xa,a),h(R,s,a),n(s,Na,a),n(s,H,a),n(s,Ea,a),n(s,D,a),n(s,Ga,a),h(L,s,a),n(s,Ba,a),h(O,s,a),n(s,Fa,a),h(P,s,a),n(s,Wa,a),n(s,K,a),n(s,za,a),n(s,ss,a),n(s,Sa,a),h(as,s,a),n(s,qa,a),n(s,ls,a),n(s,Ya,a),h(ns,s,a),n(s,Ha,a),n(s,b,a),d(b,Z),d(Z,Ca),h(ts,Ca,null),d(b,Un),d(b,Ua),n(s,Da,a),n(s,es,a),n(s,La,a),h(ps,s,a),n(s,Oa,a),n(s,is,a),n(s,Pa,a),h(cs,s,a),n(s,Ka,a),n(s,rs,a),n(s,sl,a),n(s,os,a),n(s,al,a),n(s,hs,a),n(s,ll,a),h(js,s,a),n(s,nl,a),n(s,Ms,a),n(s,tl,a),h(xs,s,a),n(s,el,a),n(s,gs,a),n(s,pl,a),h(ds,s,a),n(s,il,a),n(s,Us,a),n(s,cl,a),h(ms,s,a),n(s,rl,a),n(s,A,a),n(s,ol,a),n(s,f,a),d(f,Q),d(Q,ba),h(ys,ba,null),d(f,mn),d(f,ma),n(s,hl,a),n(s,Js,a),n(s,jl,a),h(us,s,a),n(s,Ml,a),n(s,I,a),d(I,$),d($,fa),h(ws,fa,null),d(I,yn),d(I,ya),n(s,xl,a),n(s,Ts,a),n(s,gl,a),h(Cs,s,a),n(s,dl,a),n(s,bs,a),n(s,Ul,a),h(fs,s,a),n(s,ml,a),n(s,_,a),d(_,X),d(X,Ia),h(Is,Ia,null),d(_,Jn),d(_,Ja),n(s,yl,a),n(s,_s,a),n(s,Jl,a),h(vs,s,a),n(s,ul,a),n(s,Vs,a),n(s,wl,a),n(s,ks,a),n(s,Tl,a),n(s,Rs,a),n(s,Cl,a),h(Zs,s,a),n(s,bl,a),n(s,As,a),n(s,fl,a),h(Qs,s,a),n(s,Il,a),n(s,$s,a),n(s,_l,a),h(Xs,s,a),n(s,vl,a),n(s,Ns,a),n(s,Vl,a),n(s,Es,a),n(s,kl,a),n(s,Gs,a),n(s,Rl,a),h(Bs,s,a),n(s,Zl,a),n(s,Fs,a),n(s,Al,a),h(Ws,s,a),n(s,Ql,a),n(s,zs,a),n(s,$l,a),h(Ss,s,a),n(s,Xl,a),n(s,v,a),d(v,N),d(N,_a),h(qs,_a,null),d(v,un),d(v,ua),n(s,Nl,a),n(s,Ys,a),n(s,El,a),n(s,V,a),d(V,E),d(E,va),h(Hs,va,null),d(V,wn),d(V,wa),n(s,Gl,a),n(s,Ds,a),n(s,Bl,a),n(s,Ls,a),n(s,Fl,a),h(Os,s,a),n(s,Wl,a),n(s,Ps,a),n(s,zl,a),h(Ks,s,a),n(s,Sl,a),n(s,sa,a),n(s,ql,a),h(aa,s,a),n(s,Yl,a),n(s,la,a),n(s,Hl,a),h(na,s,a),n(s,Dl,a),n(s,ta,a),n(s,Ll,a),h(ea,s,a),n(s,Ol,a),n(s,k,a),d(k,G),d(G,Va),h(pa,Va,null),d(k,Tn),d(k,Ta),n(s,Pl,a),n(s,ia,a),n(s,Kl,a),n(s,ca,a),n(s,sn,a),h(ra,s,a),n(s,an,a),n(s,oa,a),n(s,ln,a),h(ha,s,a),n(s,nn,a),n(s,ja,a),n(s,tn,a),n(s,Ma,a),n(s,en,a),h(xa,s,a),n(s,pn,a),n(s,ka,a),cn=!0},p(s,[a]){const ga={};a&2&&(ga.$$scope={dirty:a,ctx:s}),R.$set(ga)},i(s){cn||(j(W.$$.fragment,s),j(z.$$.fragment,s),j(R.$$.fragment,s),j(L.$$.fragment,s),j(O.$$.fragment,s),j(P.$$.fragment,s),j(as.$$.fragment,s),j(ns.$$.fragment,s),j(ts.$$.fragment,s),j(ps.$$.fragment,s),j(cs.$$.fragment,s),j(js.$$.fragment,s),j(xs.$$.fragment,s),j(ds.$$.fragment,s),j(ms.$$.fragment,s),j(ys.$$.fragment,s),j(us.$$.fragment,s),j(ws.$$.fragment,s),j(Cs.$$.fragment,s),j(fs.$$.fragment,s),j(Is.$$.fragment,s),j(vs.$$.fragment,s),j(Zs.$$.fragment,s),j(Qs.$$.fragment,s),j(Xs.$$.fragment,s),j(Bs.$$.fragment,s),j(Ws.$$.fragment,s),j(Ss.$$.fragment,s),j(qs.$$.fragment,s),j(Hs.$$.fragment,s),j(Os.$$.fragment,s),j(Ks.$$.fragment,s),j(aa.$$.fragment,s),j(na.$$.fragment,s),j(ea.$$.fragment,s),j(pa.$$.fragment,s),j(ra.$$.fragment,s),j(ha.$$.fragment,s),j(xa.$$.fragment,s),cn=!0)},o(s){M(W.$$.fragment,s),M(z.$$.fragment,s),M(R.$$.fragment,s),M(L.$$.fragment,s),M(O.$$.fragment,s),M(P.$$.fragment,s),M(as.$$.fragment,s),M(ns.$$.fragment,s),M(ts.$$.fragment,s),M(ps.$$.fragment,s),M(cs.$$.fragment,s),M(js.$$.fragment,s),M(xs.$$.fragment,s),M(ds.$$.fragment,s),M(ms.$$.fragment,s),M(ys.$$.fragment,s),M(us.$$.fragment,s),M(ws.$$.fragment,s),M(Cs.$$.fragment,s),M(fs.$$.fragment,s),M(Is.$$.fragment,s),M(vs.$$.fragment,s),M(Zs.$$.fragment,s),M(Qs.$$.fragment,s),M(Xs.$$.fragment,s),M(Bs.$$.fragment,s),M(Ws.$$.fragment,s),M(Ss.$$.fragment,s),M(qs.$$.fragment,s),M(Hs.$$.fragment,s),M(Os.$$.fragment,s),M(Ks.$$.fragment,s),M(aa.$$.fragment,s),M(na.$$.fragment,s),M(ea.$$.fragment,s),M(pa.$$.fragment,s),M(ra.$$.fragment,s),M(ha.$$.fragment,s),M(xa.$$.fragment,s),cn=!1},d(s){s&&(l(F),l(T),l(u),l(w),l(Za),l(Aa),l(S),l(Qa),l(q),l($a),l(Y),l(Xa),l(Na),l(H),l(Ea),l(D),l(Ga),l(Ba),l(Fa),l(Wa),l(K),l(za),l(ss),l(Sa),l(qa),l(ls),l(Ya),l(Ha),l(b),l(Da),l(es),l(La),l(Oa),l(is),l(Pa),l(Ka),l(rs),l(sl),l(os),l(al),l(hs),l(ll),l(nl),l(Ms),l(tl),l(el),l(gs),l(pl),l(il),l(Us),l(cl),l(rl),l(A),l(ol),l(f),l(hl),l(Js),l(jl),l(Ml),l(I),l(xl),l(Ts),l(gl),l(dl),l(bs),l(Ul),l(ml),l(_),l(yl),l(_s),l(Jl),l(ul),l(Vs),l(wl),l(ks),l(Tl),l(Rs),l(Cl),l(bl),l(As),l(fl),l(Il),l($s),l(_l),l(vl),l(Ns),l(Vl),l(Es),l(kl),l(Gs),l(Rl),l(Zl),l(Fs),l(Al),l(Ql),l(zs),l($l),l(Xl),l(v),l(Nl),l(Ys),l(El),l(V),l(Gl),l(Ds),l(Bl),l(Ls),l(Fl),l(Wl),l(Ps),l(zl),l(Sl),l(sa),l(ql),l(Yl),l(la),l(Hl),l(Dl),l(ta),l(Ll),l(Ol),l(k),l(Pl),l(ia),l(Kl),l(ca),l(sn),l(an),l(oa),l(ln),l(nn),l(ja),l(tn),l(Ma),l(en),l(pn),l(ka)),l(J),x(W),x(z,s),x(R,s),x(L,s),x(O,s),x(P,s),x(as,s),x(ns,s),x(ts),x(ps,s),x(cs,s),x(js,s),x(xs,s),x(ds,s),x(ms,s),x(ys),x(us,s),x(ws),x(Cs,s),x(fs,s),x(Is),x(vs,s),x(Zs,s),x(Qs,s),x(Xs,s),x(Bs,s),x(Ws,s),x(Ss,s),x(qs),x(Hs),x(Os,s),x(Ks,s),x(aa,s),x(na,s),x(ea,s),x(pa),x(ra,s),x(ha,s),x(xa,s)}}}const Yt={local:"document_question_answering",sections:[{local:"load-the-data",title:"데이터 불러오기 "},{local:"preprocess-the-data",sections:[{local:"preprocessing-document-images",title:"문서 이미지 전처리 "},{local:"preprocessing-text-data",title:"텍스트 데이터 전처리 "}],title:"데이터 전처리 "},{local:"evaluation",title:"평가 "},{local:"train",title:"훈련 "},{local:"inference",title:"추론 "}],title:"문서 질의 응답(Document Question Answering) "};function Ht(Ra){return Nt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ae extends Gt{constructor(J){super(),Bt(this,J,Ht,qt,Xt,{})}}export{ae as component};
