import{S as ao,i as ro,s as oo,e as o,k as m,w as c,t as n,M as lo,c as l,d as s,m as f,a as i,x as h,h as p,b as u,N as to,G as a,g as r,y as d,q as _,o as g,B as v,v as io}from"../../chunks/vendor-hf-doc-builder.js";import{T as no}from"../../chunks/Tip-hf-doc-builder.js";import{I as Ae}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as $}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as po}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function mo(ys){let b,S;return{c(){b=o("p"),S=n("Many image captioning datasets contain multiple captions per image. In those cases, a common strategy is to randomly sample a caption amongst the available ones during training.")},l(w){b=l(w,"P",{});var y=i(b);S=p(y,"Many image captioning datasets contain multiple captions per image. In those cases, a common strategy is to randomly sample a caption amongst the available ones during training."),y.forEach(s)},m(w,y){r(w,b,y),a(b,S)},d(w){w&&s(b)}}}function fo(ys){let b,S,w,y,ss,J,Ct,ts,Dt,ks,K,js,Ie,zt,Es,Le,Wt,Ts,M,as,Rt,Ht,rs,Ot,xs,Se,Bt,Ps,Q,qs,Me,Ut,As,X,Is,x,F,os,Z,Gt,ls,Yt,Ls,N,Vt,ee,Jt,Kt,Ss,se,Ms,te,Fs,k,Qt,is,Xt,Zt,ns,ea,sa,Ns,C,Cs,Fe,ta,Ds,ae,zs,Ne,aa,Ws,re,Rs,oe,Ce,er,Hs,P,D,ps,le,ra,ms,oa,Os,De,la,Bs,ze,ia,Us,ie,Gs,We,na,Ys,ne,Vs,Re,pa,Js,q,z,fs,pe,ma,us,fa,Ks,j,ua,me,ca,ha,fe,cs,da,_a,Qs,ue,Xs,A,W,hs,ce,ga,ds,va,Zs,E,$a,he,ba,wa,de,ya,ka,et,R,ja,_e,Ea,Ta,st,ge,tt,I,H,_s,ve,xa,gs,Pa,at,O,qa,He,Aa,Ia,rt,B,La,Oe,Sa,Ma,ot,$e,lt,Be,Fa,it,be,nt,T,Na,Ue,Ca,Da,Ge,za,Wa,pt,we,mt,Ye,Ra,ft,U,Ha,Ve,Oa,Ba,ut,ye,ct,L,G,vs,ke,Ua,$s,Ga,ht,Y,Ya,bs,Va,Ja,dt,je,_t,Ee,Je,sr,gt,Te,vt,V,Ka,ws,Qa,Xa,$t,xe,bt,Pe,wt,Ke,Za,yt;return J=new Ae({}),K=new po({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/image_captioning.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/image_captioning.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/image_captioning.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/image_captioning.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/image_captioning.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/image_captioning.ipynb"}]}}),Q=new $({props:{code:`pip install transformers datasets evaluate -q
pip install jiwer -q`,highlighted:`pip install transformers datasets evaluate -q
pip install jiwer -q`}}),X=new $({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),Z=new Ae({}),se=new $({props:{code:`from datasets import load_dataset

ds = load_dataset("lambdalabs/pokemon-blip-captions")
ds`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

ds = load_dataset(<span class="hljs-string">&quot;lambdalabs/pokemon-blip-captions&quot;</span>)
ds`}}),te=new $({props:{code:`DatasetDict({
    train: Dataset({
        features: ['image', 'text'],
        num_rows: 833
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;image&#x27;</span>, <span class="hljs-string">&#x27;text&#x27;</span>],
        num_rows: 833
    })
})`}}),C=new no({props:{$$slots:{default:[mo]},$$scope:{ctx:ys}}}),ae=new $({props:{code:`ds = ds["train"].train_test_split(test_size=0.1)
train_ds = ds["train"]
test_ds = ds["test"]`,highlighted:`ds = ds[<span class="hljs-string">&quot;train&quot;</span>].train_test_split(test_size=<span class="hljs-number">0.1</span>)
train_ds = ds[<span class="hljs-string">&quot;train&quot;</span>]
test_ds = ds[<span class="hljs-string">&quot;test&quot;</span>]`}}),re=new $({props:{code:`from textwrap import wrap
import matplotlib.pyplot as plt
import numpy as np


def plot_images(images, captions):
    plt.figure(figsize=(20, 20))
    for i in range(len(images)):
        ax = plt.subplot(1, len(images), i + 1)
        caption = captions[i]
        caption = "\\n".join(wrap(caption, 12))
        plt.title(caption)
        plt.imshow(images[i])
        plt.axis("off")


sample_images_to_visualize = [np.array(train_ds[i]["image"]) for i in range(5)]
sample_captions = [train_ds[i]["text"] for i in range(5)]
plot_images(sample_images_to_visualize, sample_captions)`,highlighted:`<span class="hljs-keyword">from</span> textwrap <span class="hljs-keyword">import</span> wrap
<span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np


<span class="hljs-keyword">def</span> <span class="hljs-title function_">plot_images</span>(<span class="hljs-params">images, captions</span>):
    plt.figure(figsize=(<span class="hljs-number">20</span>, <span class="hljs-number">20</span>))
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(images)):
        ax = plt.subplot(<span class="hljs-number">1</span>, <span class="hljs-built_in">len</span>(images), i + <span class="hljs-number">1</span>)
        caption = captions[i]
        caption = <span class="hljs-string">&quot;\\n&quot;</span>.join(wrap(caption, <span class="hljs-number">12</span>))
        plt.title(caption)
        plt.imshow(images[i])
        plt.axis(<span class="hljs-string">&quot;off&quot;</span>)


sample_images_to_visualize = [np.array(train_ds[i][<span class="hljs-string">&quot;image&quot;</span>]) <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">5</span>)]
sample_captions = [train_ds[i][<span class="hljs-string">&quot;text&quot;</span>] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">5</span>)]
plot_images(sample_images_to_visualize, sample_captions)`}}),le=new Ae({}),ie=new $({props:{code:`from transformers import AutoProcessor

checkpoint = "microsoft/git-base"
processor = AutoProcessor.from_pretrained(checkpoint)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

checkpoint = <span class="hljs-string">&quot;microsoft/git-base&quot;</span>
processor = AutoProcessor.from_pretrained(checkpoint)`}}),ne=new $({props:{code:`def transforms(example_batch):
    images = [x for x in example_batch["image"]]
    captions = [x for x in example_batch["text"]]
    inputs = processor(images=images, text=captions, padding="max_length")
    inputs.update({"labels": inputs["input_ids"]})
    return inputs


train_ds.set_transform(transforms)
test_ds.set_transform(transforms)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">example_batch</span>):
    images = [x <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> example_batch[<span class="hljs-string">&quot;image&quot;</span>]]
    captions = [x <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> example_batch[<span class="hljs-string">&quot;text&quot;</span>]]
    inputs = processor(images=images, text=captions, padding=<span class="hljs-string">&quot;max_length&quot;</span>)
    inputs.update({<span class="hljs-string">&quot;labels&quot;</span>: inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]})
    <span class="hljs-keyword">return</span> inputs


train_ds.set_transform(transforms)
test_ds.set_transform(transforms)`}}),pe=new Ae({}),ue=new $({props:{code:`from transformers import AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(checkpoint)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(checkpoint)`}}),ce=new Ae({}),ge=new $({props:{code:`from evaluate import load
import torch

wer = load("wer")


def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predicted = logits.argmax(-1)
    decoded_labels = processor.batch_decode(labels, skip_special_tokens=True)
    decoded_predictions = processor.batch_decode(predicted, skip_special_tokens=True)
    wer_score = wer.compute(predictions=decoded_predictions, references=decoded_labels)
    return {"wer_score": wer_score}`,highlighted:`<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> load
<span class="hljs-keyword">import</span> torch

wer = load(<span class="hljs-string">&quot;wer&quot;</span>)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">eval_pred</span>):
    logits, labels = eval_pred
    predicted = logits.argmax(-<span class="hljs-number">1</span>)
    decoded_labels = processor.batch_decode(labels, skip_special_tokens=<span class="hljs-literal">True</span>)
    decoded_predictions = processor.batch_decode(predicted, skip_special_tokens=<span class="hljs-literal">True</span>)
    wer_score = wer.compute(predictions=decoded_predictions, references=decoded_labels)
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;wer_score&quot;</span>: wer_score}`}}),ve=new Ae({}),$e=new $({props:{code:`from transformers import TrainingArguments, Trainer

model_name = checkpoint.split("/")[1]

training_args = TrainingArguments(
    output_dir=f"{model_name}-pokemon",
    learning_rate=5e-5,
    num_train_epochs=50,
    fp16=True,
    per_device_train_batch_size=32,
    per_device_eval_batch_size=32,
    gradient_accumulation_steps=2,
    save_total_limit=3,
    evaluation_strategy="steps",
    eval_steps=50,
    save_strategy="steps",
    save_steps=50,
    logging_steps=50,
    remove_unused_columns=False,
    push_to_hub=True,
    label_names=["labels"],
    load_best_model_at_end=True,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, Trainer

model_name = checkpoint.split(<span class="hljs-string">&quot;/&quot;</span>)[<span class="hljs-number">1</span>]

training_args = TrainingArguments(
    output_dir=<span class="hljs-string">f&quot;<span class="hljs-subst">{model_name}</span>-pokemon&quot;</span>,
    learning_rate=<span class="hljs-number">5e-5</span>,
    num_train_epochs=<span class="hljs-number">50</span>,
    fp16=<span class="hljs-literal">True</span>,
    per_device_train_batch_size=<span class="hljs-number">32</span>,
    per_device_eval_batch_size=<span class="hljs-number">32</span>,
    gradient_accumulation_steps=<span class="hljs-number">2</span>,
    save_total_limit=<span class="hljs-number">3</span>,
    evaluation_strategy=<span class="hljs-string">&quot;steps&quot;</span>,
    eval_steps=<span class="hljs-number">50</span>,
    save_strategy=<span class="hljs-string">&quot;steps&quot;</span>,
    save_steps=<span class="hljs-number">50</span>,
    logging_steps=<span class="hljs-number">50</span>,
    remove_unused_columns=<span class="hljs-literal">False</span>,
    push_to_hub=<span class="hljs-literal">True</span>,
    label_names=[<span class="hljs-string">&quot;labels&quot;</span>],
    load_best_model_at_end=<span class="hljs-literal">True</span>,
)`}}),be=new $({props:{code:`trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_ds,
    eval_dataset=test_ds,
    compute_metrics=compute_metrics,
)`,highlighted:`trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_ds,
    eval_dataset=test_ds,
    compute_metrics=compute_metrics,
)`}}),we=new $({props:{code:"trainer.train()",highlighted:"trainer.train()"}}),ye=new $({props:{code:"trainer.push_to_hub()",highlighted:"trainer.push_to_hub()"}}),ke=new Ae({}),je=new $({props:{code:`from PIL import Image
import requests

url = "https://huggingface.co/datasets/sayakpaul/sample-datasets/resolve/main/pokemon.png"
image = Image.open(requests.get(url, stream=True).raw)
image`,highlighted:`<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">import</span> requests

url = <span class="hljs-string">&quot;https://huggingface.co/datasets/sayakpaul/sample-datasets/resolve/main/pokemon.png&quot;</span>
image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)
image`}}),Te=new $({props:{code:`device = "cuda" if torch.cuda.is_available() else "cpu"

inputs = processor(images=image, return_tensors="pt").to(device)
pixel_values = inputs.pixel_values`,highlighted:`device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>

inputs = processor(images=image, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)
pixel_values = inputs.pixel_values`}}),xe=new $({props:{code:`generated_ids = model.generate(pixel_values=pixel_values, max_length=50)
generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(generated_caption)`,highlighted:`generated_ids = model.generate(pixel_values=pixel_values, max_length=<span class="hljs-number">50</span>)
generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(generated_caption)`}}),Pe=new $({props:{code:"a drawing of a pink and blue pokemon",highlighted:"a drawing of a pink and blue pokemon"}}),{c(){b=o("meta"),S=m(),w=o("h1"),y=o("a"),ss=o("span"),c(J.$$.fragment),Ct=m(),ts=o("span"),Dt=n("Image captioning"),ks=m(),c(K.$$.fragment),js=m(),Ie=o("p"),zt=n(`Image captioning is the task of predicting a caption for a given image. Common real world applications of it include
aiding visually impaired people that can help them navigate through different situations. Therefore, image captioning
helps to improve content accessibility for people by describing images to them.`),Es=m(),Le=o("p"),Wt=n("This guide will show you how to:"),Ts=m(),M=o("ul"),as=o("li"),Rt=n("Fine-tune an image captioning model."),Ht=m(),rs=o("li"),Ot=n("Use the fine-tuned model for inference."),xs=m(),Se=o("p"),Bt=n("Before you begin, make sure you have all the necessary libraries installed:"),Ps=m(),c(Q.$$.fragment),qs=m(),Me=o("p"),Ut=n("We encourage you to log in to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to log in:"),As=m(),c(X.$$.fragment),Is=m(),x=o("h2"),F=o("a"),os=o("span"),c(Z.$$.fragment),Gt=m(),ls=o("span"),Yt=n("Load the Pok\xE9mon BLIP captions dataset"),Ls=m(),N=o("p"),Vt=n(`Use the \u{1F917} Dataset library to load a dataset that consists of {image-caption} pairs. To create your own image captioning dataset
in PyTorch, you can follow `),ee=o("a"),Jt=n("this notebook"),Kt=n("."),Ss=m(),c(se.$$.fragment),Ms=m(),c(te.$$.fragment),Fs=m(),k=o("p"),Qt=n("The dataset has two features, "),is=o("code"),Xt=n("image"),Zt=n(" and "),ns=o("code"),ea=n("text"),sa=n("."),Ns=m(),c(C.$$.fragment),Cs=m(),Fe=o("p"),ta=n("Split the dataset\u2019s train split into a train and test set with the [~datasets.Dataset.train_test_split] method:"),Ds=m(),c(ae.$$.fragment),zs=m(),Ne=o("p"),aa=n("Let\u2019s visualize a couple of samples from the training set."),Ws=m(),c(re.$$.fragment),Rs=m(),oe=o("div"),Ce=o("img"),Hs=m(),P=o("h2"),D=o("a"),ps=o("span"),c(le.$$.fragment),ra=m(),ms=o("span"),oa=n("Preprocess the dataset"),Os=m(),De=o("p"),la=n("Since the dataset has two modalities (image and text), the pre-processing pipeline will preprocess images and the captions."),Bs=m(),ze=o("p"),ia=n("To do so, load the processor class associated with the model you are about to fine-tune."),Us=m(),c(ie.$$.fragment),Gs=m(),We=o("p"),na=n("The processor will internally pre-process the image (which includes resizing, and pixel scaling) and tokenize the caption."),Ys=m(),c(ne.$$.fragment),Vs=m(),Re=o("p"),pa=n("With the dataset ready, you can now set up the model for fine-tuning."),Js=m(),q=o("h2"),z=o("a"),fs=o("span"),c(pe.$$.fragment),ma=m(),us=o("span"),fa=n("Load a base model"),Ks=m(),j=o("p"),ua=n("Load the "),me=o("a"),ca=n("\u201Cmicrosoft/git-base\u201D"),ha=n(" into a "),fe=o("a"),cs=o("code"),da=n("AutoModelForCausalLM"),_a=n(" object."),Qs=m(),c(ue.$$.fragment),Xs=m(),A=o("h2"),W=o("a"),hs=o("span"),c(ce.$$.fragment),ga=m(),ds=o("span"),va=n("Evaluate"),Zs=m(),E=o("p"),$a=n("Image captioning models are typically evaluated with the "),he=o("a"),ba=n("Rouge Score"),wa=n(" or "),de=o("a"),ya=n("Word Error Rate"),ka=n(". For this guide, you will use the Word Error Rate (WER)."),et=m(),R=o("p"),ja=n("We use the \u{1F917} Evaluate library to do so. For potential limitations and other gotchas of the WER, refer to "),_e=o("a"),Ea=n("this guide"),Ta=n("."),st=m(),c(ge.$$.fragment),tt=m(),I=o("h2"),H=o("a"),_s=o("span"),c(ve.$$.fragment),xa=m(),gs=o("span"),Pa=n("Train!"),at=m(),O=o("p"),qa=n("Now, you are ready to start fine-tuning the model. You will use the \u{1F917} "),He=o("a"),Aa=n("Trainer"),Ia=n(" for this."),rt=m(),B=o("p"),La=n("First, define the training arguments using "),Oe=o("a"),Sa=n("TrainingArguments"),Ma=n("."),ot=m(),c($e.$$.fragment),lt=m(),Be=o("p"),Fa=n("Then pass them along with the datasets and the model to \u{1F917} Trainer."),it=m(),c(be.$$.fragment),nt=m(),T=o("p"),Na=n("To start training, simply call "),Ue=o("a"),Ca=n("train()"),Da=n(" on the "),Ge=o("a"),za=n("Trainer"),Wa=n(" object."),pt=m(),c(we.$$.fragment),mt=m(),Ye=o("p"),Ra=n("You should see the training loss drop smoothly as training progresses."),ft=m(),U=o("p"),Ha=n("Once training is completed, share your model to the Hub with the "),Ve=o("a"),Oa=n("push_to_hub()"),Ba=n(" method so everyone can use your model:"),ut=m(),c(ye.$$.fragment),ct=m(),L=o("h2"),G=o("a"),vs=o("span"),c(ke.$$.fragment),Ua=m(),$s=o("span"),Ga=n("Inference"),ht=m(),Y=o("p"),Ya=n("Take a sample image from "),bs=o("code"),Va=n("test_ds"),Ja=n(" to test the model."),dt=m(),c(je.$$.fragment),_t=m(),Ee=o("div"),Je=o("img"),gt=n(`
    
Prepare image for the model.

	`),c(Te.$$.fragment),vt=m(),V=o("p"),Ka=n("Call "),ws=o("code"),Qa=n("generate"),Xa=n(" and decode the predictions."),$t=m(),c(xe.$$.fragment),bt=m(),c(Pe.$$.fragment),wt=m(),Ke=o("p"),Za=n("Looks like the fine-tuned model generated a pretty good caption!"),this.h()},l(e){const t=lo('[data-svelte="svelte-1phssyn"]',document.head);b=l(t,"META",{name:!0,content:!0}),t.forEach(s),S=f(e),w=l(e,"H1",{class:!0});var qe=i(w);y=l(qe,"A",{id:!0,class:!0,href:!0});var tr=i(y);ss=l(tr,"SPAN",{});var ar=i(ss);h(J.$$.fragment,ar),ar.forEach(s),tr.forEach(s),Ct=f(qe),ts=l(qe,"SPAN",{});var rr=i(ts);Dt=p(rr,"Image captioning"),rr.forEach(s),qe.forEach(s),ks=f(e),h(K.$$.fragment,e),js=f(e),Ie=l(e,"P",{});var or=i(Ie);zt=p(or,`Image captioning is the task of predicting a caption for a given image. Common real world applications of it include
aiding visually impaired people that can help them navigate through different situations. Therefore, image captioning
helps to improve content accessibility for people by describing images to them.`),or.forEach(s),Es=f(e),Le=l(e,"P",{});var lr=i(Le);Wt=p(lr,"This guide will show you how to:"),lr.forEach(s),Ts=f(e),M=l(e,"UL",{});var kt=i(M);as=l(kt,"LI",{});var ir=i(as);Rt=p(ir,"Fine-tune an image captioning model."),ir.forEach(s),Ht=f(kt),rs=l(kt,"LI",{});var nr=i(rs);Ot=p(nr,"Use the fine-tuned model for inference."),nr.forEach(s),kt.forEach(s),xs=f(e),Se=l(e,"P",{});var pr=i(Se);Bt=p(pr,"Before you begin, make sure you have all the necessary libraries installed:"),pr.forEach(s),Ps=f(e),h(Q.$$.fragment,e),qs=f(e),Me=l(e,"P",{});var mr=i(Me);Ut=p(mr,"We encourage you to log in to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to log in:"),mr.forEach(s),As=f(e),h(X.$$.fragment,e),Is=f(e),x=l(e,"H2",{class:!0});var jt=i(x);F=l(jt,"A",{id:!0,class:!0,href:!0});var fr=i(F);os=l(fr,"SPAN",{});var ur=i(os);h(Z.$$.fragment,ur),ur.forEach(s),fr.forEach(s),Gt=f(jt),ls=l(jt,"SPAN",{});var cr=i(ls);Yt=p(cr,"Load the Pok\xE9mon BLIP captions dataset"),cr.forEach(s),jt.forEach(s),Ls=f(e),N=l(e,"P",{});var Et=i(N);Vt=p(Et,`Use the \u{1F917} Dataset library to load a dataset that consists of {image-caption} pairs. To create your own image captioning dataset
in PyTorch, you can follow `),ee=l(Et,"A",{href:!0,rel:!0});var hr=i(ee);Jt=p(hr,"this notebook"),hr.forEach(s),Kt=p(Et,"."),Et.forEach(s),Ss=f(e),h(se.$$.fragment,e),Ms=f(e),h(te.$$.fragment,e),Fs=f(e),k=l(e,"P",{});var Qe=i(k);Qt=p(Qe,"The dataset has two features, "),is=l(Qe,"CODE",{});var dr=i(is);Xt=p(dr,"image"),dr.forEach(s),Zt=p(Qe," and "),ns=l(Qe,"CODE",{});var _r=i(ns);ea=p(_r,"text"),_r.forEach(s),sa=p(Qe,"."),Qe.forEach(s),Ns=f(e),h(C.$$.fragment,e),Cs=f(e),Fe=l(e,"P",{});var gr=i(Fe);ta=p(gr,"Split the dataset\u2019s train split into a train and test set with the [~datasets.Dataset.train_test_split] method:"),gr.forEach(s),Ds=f(e),h(ae.$$.fragment,e),zs=f(e),Ne=l(e,"P",{});var vr=i(Ne);aa=p(vr,"Let\u2019s visualize a couple of samples from the training set."),vr.forEach(s),Ws=f(e),h(re.$$.fragment,e),Rs=f(e),oe=l(e,"DIV",{class:!0});var $r=i(oe);Ce=l($r,"IMG",{src:!0,alt:!0}),$r.forEach(s),Hs=f(e),P=l(e,"H2",{class:!0});var Tt=i(P);D=l(Tt,"A",{id:!0,class:!0,href:!0});var br=i(D);ps=l(br,"SPAN",{});var wr=i(ps);h(le.$$.fragment,wr),wr.forEach(s),br.forEach(s),ra=f(Tt),ms=l(Tt,"SPAN",{});var yr=i(ms);oa=p(yr,"Preprocess the dataset"),yr.forEach(s),Tt.forEach(s),Os=f(e),De=l(e,"P",{});var kr=i(De);la=p(kr,"Since the dataset has two modalities (image and text), the pre-processing pipeline will preprocess images and the captions."),kr.forEach(s),Bs=f(e),ze=l(e,"P",{});var jr=i(ze);ia=p(jr,"To do so, load the processor class associated with the model you are about to fine-tune."),jr.forEach(s),Us=f(e),h(ie.$$.fragment,e),Gs=f(e),We=l(e,"P",{});var Er=i(We);na=p(Er,"The processor will internally pre-process the image (which includes resizing, and pixel scaling) and tokenize the caption."),Er.forEach(s),Ys=f(e),h(ne.$$.fragment,e),Vs=f(e),Re=l(e,"P",{});var Tr=i(Re);pa=p(Tr,"With the dataset ready, you can now set up the model for fine-tuning."),Tr.forEach(s),Js=f(e),q=l(e,"H2",{class:!0});var xt=i(q);z=l(xt,"A",{id:!0,class:!0,href:!0});var xr=i(z);fs=l(xr,"SPAN",{});var Pr=i(fs);h(pe.$$.fragment,Pr),Pr.forEach(s),xr.forEach(s),ma=f(xt),us=l(xt,"SPAN",{});var qr=i(us);fa=p(qr,"Load a base model"),qr.forEach(s),xt.forEach(s),Ks=f(e),j=l(e,"P",{});var Xe=i(j);ua=p(Xe,"Load the "),me=l(Xe,"A",{href:!0,rel:!0});var Ar=i(me);ca=p(Ar,"\u201Cmicrosoft/git-base\u201D"),Ar.forEach(s),ha=p(Xe," into a "),fe=l(Xe,"A",{href:!0,rel:!0});var Ir=i(fe);cs=l(Ir,"CODE",{});var Lr=i(cs);da=p(Lr,"AutoModelForCausalLM"),Lr.forEach(s),Ir.forEach(s),_a=p(Xe," object."),Xe.forEach(s),Qs=f(e),h(ue.$$.fragment,e),Xs=f(e),A=l(e,"H2",{class:!0});var Pt=i(A);W=l(Pt,"A",{id:!0,class:!0,href:!0});var Sr=i(W);hs=l(Sr,"SPAN",{});var Mr=i(hs);h(ce.$$.fragment,Mr),Mr.forEach(s),Sr.forEach(s),ga=f(Pt),ds=l(Pt,"SPAN",{});var Fr=i(ds);va=p(Fr,"Evaluate"),Fr.forEach(s),Pt.forEach(s),Zs=f(e),E=l(e,"P",{});var Ze=i(E);$a=p(Ze,"Image captioning models are typically evaluated with the "),he=l(Ze,"A",{href:!0,rel:!0});var Nr=i(he);ba=p(Nr,"Rouge Score"),Nr.forEach(s),wa=p(Ze," or "),de=l(Ze,"A",{href:!0,rel:!0});var Cr=i(de);ya=p(Cr,"Word Error Rate"),Cr.forEach(s),ka=p(Ze,". For this guide, you will use the Word Error Rate (WER)."),Ze.forEach(s),et=f(e),R=l(e,"P",{});var qt=i(R);ja=p(qt,"We use the \u{1F917} Evaluate library to do so. For potential limitations and other gotchas of the WER, refer to "),_e=l(qt,"A",{href:!0,rel:!0});var Dr=i(_e);Ea=p(Dr,"this guide"),Dr.forEach(s),Ta=p(qt,"."),qt.forEach(s),st=f(e),h(ge.$$.fragment,e),tt=f(e),I=l(e,"H2",{class:!0});var At=i(I);H=l(At,"A",{id:!0,class:!0,href:!0});var zr=i(H);_s=l(zr,"SPAN",{});var Wr=i(_s);h(ve.$$.fragment,Wr),Wr.forEach(s),zr.forEach(s),xa=f(At),gs=l(At,"SPAN",{});var Rr=i(gs);Pa=p(Rr,"Train!"),Rr.forEach(s),At.forEach(s),at=f(e),O=l(e,"P",{});var It=i(O);qa=p(It,"Now, you are ready to start fine-tuning the model. You will use the \u{1F917} "),He=l(It,"A",{href:!0});var Hr=i(He);Aa=p(Hr,"Trainer"),Hr.forEach(s),Ia=p(It," for this."),It.forEach(s),rt=f(e),B=l(e,"P",{});var Lt=i(B);La=p(Lt,"First, define the training arguments using "),Oe=l(Lt,"A",{href:!0});var Or=i(Oe);Sa=p(Or,"TrainingArguments"),Or.forEach(s),Ma=p(Lt,"."),Lt.forEach(s),ot=f(e),h($e.$$.fragment,e),lt=f(e),Be=l(e,"P",{});var Br=i(Be);Fa=p(Br,"Then pass them along with the datasets and the model to \u{1F917} Trainer."),Br.forEach(s),it=f(e),h(be.$$.fragment,e),nt=f(e),T=l(e,"P",{});var es=i(T);Na=p(es,"To start training, simply call "),Ue=l(es,"A",{href:!0});var Ur=i(Ue);Ca=p(Ur,"train()"),Ur.forEach(s),Da=p(es," on the "),Ge=l(es,"A",{href:!0});var Gr=i(Ge);za=p(Gr,"Trainer"),Gr.forEach(s),Wa=p(es," object."),es.forEach(s),pt=f(e),h(we.$$.fragment,e),mt=f(e),Ye=l(e,"P",{});var Yr=i(Ye);Ra=p(Yr,"You should see the training loss drop smoothly as training progresses."),Yr.forEach(s),ft=f(e),U=l(e,"P",{});var St=i(U);Ha=p(St,"Once training is completed, share your model to the Hub with the "),Ve=l(St,"A",{href:!0});var Vr=i(Ve);Oa=p(Vr,"push_to_hub()"),Vr.forEach(s),Ba=p(St," method so everyone can use your model:"),St.forEach(s),ut=f(e),h(ye.$$.fragment,e),ct=f(e),L=l(e,"H2",{class:!0});var Mt=i(L);G=l(Mt,"A",{id:!0,class:!0,href:!0});var Jr=i(G);vs=l(Jr,"SPAN",{});var Kr=i(vs);h(ke.$$.fragment,Kr),Kr.forEach(s),Jr.forEach(s),Ua=f(Mt),$s=l(Mt,"SPAN",{});var Qr=i($s);Ga=p(Qr,"Inference"),Qr.forEach(s),Mt.forEach(s),ht=f(e),Y=l(e,"P",{});var Ft=i(Y);Ya=p(Ft,"Take a sample image from "),bs=l(Ft,"CODE",{});var Xr=i(bs);Va=p(Xr,"test_ds"),Xr.forEach(s),Ja=p(Ft," to test the model."),Ft.forEach(s),dt=f(e),h(je.$$.fragment,e),_t=f(e),Ee=l(e,"DIV",{class:!0});var Zr=i(Ee);Je=l(Zr,"IMG",{src:!0,alt:!0}),Zr.forEach(s),gt=p(e,`
    
Prepare image for the model.

	`),h(Te.$$.fragment,e),vt=f(e),V=l(e,"P",{});var Nt=i(V);Ka=p(Nt,"Call "),ws=l(Nt,"CODE",{});var eo=i(ws);Qa=p(eo,"generate"),eo.forEach(s),Xa=p(Nt," and decode the predictions."),Nt.forEach(s),$t=f(e),h(xe.$$.fragment,e),bt=f(e),h(Pe.$$.fragment,e),wt=f(e),Ke=l(e,"P",{});var so=i(Ke);Za=p(so,"Looks like the fine-tuned model generated a pretty good caption!"),so.forEach(s),this.h()},h(){u(b,"name","hf:doc:metadata"),u(b,"content",JSON.stringify(uo)),u(y,"id","image-captioning"),u(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(y,"href","#image-captioning"),u(w,"class","relative group"),u(F,"id","load-the-pokmon-blip-captions-dataset"),u(F,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(F,"href","#load-the-pokmon-blip-captions-dataset"),u(x,"class","relative group"),u(ee,"href","https://github.com/NielsRogge/Transformers-Tutorials/blob/master/GIT/Fine_tune_GIT_on_an_image_captioning_dataset.ipynb"),u(ee,"rel","nofollow"),to(Ce.src,er="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/sample_training_images_image_cap.png")||u(Ce,"src",er),u(Ce,"alt","Sample training images"),u(oe,"class","flex justify-center"),u(D,"id","preprocess-the-dataset"),u(D,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(D,"href","#preprocess-the-dataset"),u(P,"class","relative group"),u(z,"id","load-a-base-model"),u(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(z,"href","#load-a-base-model"),u(q,"class","relative group"),u(me,"href","https://huggingface.co/microsoft/git-base"),u(me,"rel","nofollow"),u(fe,"href","https://huggingface.co/docs/transformers/model_doc/auto#transformers.AutoModelForCausalLM"),u(fe,"rel","nofollow"),u(W,"id","evaluate"),u(W,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(W,"href","#evaluate"),u(A,"class","relative group"),u(he,"href","https://huggingface.co/spaces/evaluate-metric/rouge"),u(he,"rel","nofollow"),u(de,"href","https://huggingface.co/spaces/evaluate-metric/wer"),u(de,"rel","nofollow"),u(_e,"href","https://huggingface.co/spaces/evaluate-metric/wer"),u(_e,"rel","nofollow"),u(H,"id","train"),u(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(H,"href","#train"),u(I,"class","relative group"),u(He,"href","/docs/transformers/v4.27.1/en/main_classes/trainer#transformers.Trainer"),u(Oe,"href","/docs/transformers/v4.27.1/en/main_classes/trainer#transformers.TrainingArguments"),u(Ue,"href","/docs/transformers/v4.27.1/en/main_classes/trainer#transformers.Trainer.train"),u(Ge,"href","/docs/transformers/v4.27.1/en/main_classes/trainer#transformers.Trainer"),u(Ve,"href","/docs/transformers/v4.27.1/en/main_classes/trainer#transformers.Trainer.push_to_hub"),u(G,"id","inference"),u(G,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(G,"href","#inference"),u(L,"class","relative group"),to(Je.src,sr="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/test_image_image_cap.png")||u(Je,"src",sr),u(Je,"alt","Test image"),u(Ee,"class","flex justify-center")},m(e,t){a(document.head,b),r(e,S,t),r(e,w,t),a(w,y),a(y,ss),d(J,ss,null),a(w,Ct),a(w,ts),a(ts,Dt),r(e,ks,t),d(K,e,t),r(e,js,t),r(e,Ie,t),a(Ie,zt),r(e,Es,t),r(e,Le,t),a(Le,Wt),r(e,Ts,t),r(e,M,t),a(M,as),a(as,Rt),a(M,Ht),a(M,rs),a(rs,Ot),r(e,xs,t),r(e,Se,t),a(Se,Bt),r(e,Ps,t),d(Q,e,t),r(e,qs,t),r(e,Me,t),a(Me,Ut),r(e,As,t),d(X,e,t),r(e,Is,t),r(e,x,t),a(x,F),a(F,os),d(Z,os,null),a(x,Gt),a(x,ls),a(ls,Yt),r(e,Ls,t),r(e,N,t),a(N,Vt),a(N,ee),a(ee,Jt),a(N,Kt),r(e,Ss,t),d(se,e,t),r(e,Ms,t),d(te,e,t),r(e,Fs,t),r(e,k,t),a(k,Qt),a(k,is),a(is,Xt),a(k,Zt),a(k,ns),a(ns,ea),a(k,sa),r(e,Ns,t),d(C,e,t),r(e,Cs,t),r(e,Fe,t),a(Fe,ta),r(e,Ds,t),d(ae,e,t),r(e,zs,t),r(e,Ne,t),a(Ne,aa),r(e,Ws,t),d(re,e,t),r(e,Rs,t),r(e,oe,t),a(oe,Ce),r(e,Hs,t),r(e,P,t),a(P,D),a(D,ps),d(le,ps,null),a(P,ra),a(P,ms),a(ms,oa),r(e,Os,t),r(e,De,t),a(De,la),r(e,Bs,t),r(e,ze,t),a(ze,ia),r(e,Us,t),d(ie,e,t),r(e,Gs,t),r(e,We,t),a(We,na),r(e,Ys,t),d(ne,e,t),r(e,Vs,t),r(e,Re,t),a(Re,pa),r(e,Js,t),r(e,q,t),a(q,z),a(z,fs),d(pe,fs,null),a(q,ma),a(q,us),a(us,fa),r(e,Ks,t),r(e,j,t),a(j,ua),a(j,me),a(me,ca),a(j,ha),a(j,fe),a(fe,cs),a(cs,da),a(j,_a),r(e,Qs,t),d(ue,e,t),r(e,Xs,t),r(e,A,t),a(A,W),a(W,hs),d(ce,hs,null),a(A,ga),a(A,ds),a(ds,va),r(e,Zs,t),r(e,E,t),a(E,$a),a(E,he),a(he,ba),a(E,wa),a(E,de),a(de,ya),a(E,ka),r(e,et,t),r(e,R,t),a(R,ja),a(R,_e),a(_e,Ea),a(R,Ta),r(e,st,t),d(ge,e,t),r(e,tt,t),r(e,I,t),a(I,H),a(H,_s),d(ve,_s,null),a(I,xa),a(I,gs),a(gs,Pa),r(e,at,t),r(e,O,t),a(O,qa),a(O,He),a(He,Aa),a(O,Ia),r(e,rt,t),r(e,B,t),a(B,La),a(B,Oe),a(Oe,Sa),a(B,Ma),r(e,ot,t),d($e,e,t),r(e,lt,t),r(e,Be,t),a(Be,Fa),r(e,it,t),d(be,e,t),r(e,nt,t),r(e,T,t),a(T,Na),a(T,Ue),a(Ue,Ca),a(T,Da),a(T,Ge),a(Ge,za),a(T,Wa),r(e,pt,t),d(we,e,t),r(e,mt,t),r(e,Ye,t),a(Ye,Ra),r(e,ft,t),r(e,U,t),a(U,Ha),a(U,Ve),a(Ve,Oa),a(U,Ba),r(e,ut,t),d(ye,e,t),r(e,ct,t),r(e,L,t),a(L,G),a(G,vs),d(ke,vs,null),a(L,Ua),a(L,$s),a($s,Ga),r(e,ht,t),r(e,Y,t),a(Y,Ya),a(Y,bs),a(bs,Va),a(Y,Ja),r(e,dt,t),d(je,e,t),r(e,_t,t),r(e,Ee,t),a(Ee,Je),r(e,gt,t),d(Te,e,t),r(e,vt,t),r(e,V,t),a(V,Ka),a(V,ws),a(ws,Qa),a(V,Xa),r(e,$t,t),d(xe,e,t),r(e,bt,t),d(Pe,e,t),r(e,wt,t),r(e,Ke,t),a(Ke,Za),yt=!0},p(e,[t]){const qe={};t&2&&(qe.$$scope={dirty:t,ctx:e}),C.$set(qe)},i(e){yt||(_(J.$$.fragment,e),_(K.$$.fragment,e),_(Q.$$.fragment,e),_(X.$$.fragment,e),_(Z.$$.fragment,e),_(se.$$.fragment,e),_(te.$$.fragment,e),_(C.$$.fragment,e),_(ae.$$.fragment,e),_(re.$$.fragment,e),_(le.$$.fragment,e),_(ie.$$.fragment,e),_(ne.$$.fragment,e),_(pe.$$.fragment,e),_(ue.$$.fragment,e),_(ce.$$.fragment,e),_(ge.$$.fragment,e),_(ve.$$.fragment,e),_($e.$$.fragment,e),_(be.$$.fragment,e),_(we.$$.fragment,e),_(ye.$$.fragment,e),_(ke.$$.fragment,e),_(je.$$.fragment,e),_(Te.$$.fragment,e),_(xe.$$.fragment,e),_(Pe.$$.fragment,e),yt=!0)},o(e){g(J.$$.fragment,e),g(K.$$.fragment,e),g(Q.$$.fragment,e),g(X.$$.fragment,e),g(Z.$$.fragment,e),g(se.$$.fragment,e),g(te.$$.fragment,e),g(C.$$.fragment,e),g(ae.$$.fragment,e),g(re.$$.fragment,e),g(le.$$.fragment,e),g(ie.$$.fragment,e),g(ne.$$.fragment,e),g(pe.$$.fragment,e),g(ue.$$.fragment,e),g(ce.$$.fragment,e),g(ge.$$.fragment,e),g(ve.$$.fragment,e),g($e.$$.fragment,e),g(be.$$.fragment,e),g(we.$$.fragment,e),g(ye.$$.fragment,e),g(ke.$$.fragment,e),g(je.$$.fragment,e),g(Te.$$.fragment,e),g(xe.$$.fragment,e),g(Pe.$$.fragment,e),yt=!1},d(e){s(b),e&&s(S),e&&s(w),v(J),e&&s(ks),v(K,e),e&&s(js),e&&s(Ie),e&&s(Es),e&&s(Le),e&&s(Ts),e&&s(M),e&&s(xs),e&&s(Se),e&&s(Ps),v(Q,e),e&&s(qs),e&&s(Me),e&&s(As),v(X,e),e&&s(Is),e&&s(x),v(Z),e&&s(Ls),e&&s(N),e&&s(Ss),v(se,e),e&&s(Ms),v(te,e),e&&s(Fs),e&&s(k),e&&s(Ns),v(C,e),e&&s(Cs),e&&s(Fe),e&&s(Ds),v(ae,e),e&&s(zs),e&&s(Ne),e&&s(Ws),v(re,e),e&&s(Rs),e&&s(oe),e&&s(Hs),e&&s(P),v(le),e&&s(Os),e&&s(De),e&&s(Bs),e&&s(ze),e&&s(Us),v(ie,e),e&&s(Gs),e&&s(We),e&&s(Ys),v(ne,e),e&&s(Vs),e&&s(Re),e&&s(Js),e&&s(q),v(pe),e&&s(Ks),e&&s(j),e&&s(Qs),v(ue,e),e&&s(Xs),e&&s(A),v(ce),e&&s(Zs),e&&s(E),e&&s(et),e&&s(R),e&&s(st),v(ge,e),e&&s(tt),e&&s(I),v(ve),e&&s(at),e&&s(O),e&&s(rt),e&&s(B),e&&s(ot),v($e,e),e&&s(lt),e&&s(Be),e&&s(it),v(be,e),e&&s(nt),e&&s(T),e&&s(pt),v(we,e),e&&s(mt),e&&s(Ye),e&&s(ft),e&&s(U),e&&s(ut),v(ye,e),e&&s(ct),e&&s(L),v(ke),e&&s(ht),e&&s(Y),e&&s(dt),v(je,e),e&&s(_t),e&&s(Ee),e&&s(gt),v(Te,e),e&&s(vt),e&&s(V),e&&s($t),v(xe,e),e&&s(bt),v(Pe,e),e&&s(wt),e&&s(Ke)}}}const uo={local:"image-captioning",sections:[{local:"load-the-pokmon-blip-captions-dataset",title:"Load the Pok\xE9mon BLIP captions dataset"},{local:"preprocess-the-dataset",title:"Preprocess the dataset"},{local:"load-a-base-model",title:"Load a base model"},{local:"evaluate",title:"Evaluate"},{local:"train",title:"Train!"},{local:"inference",title:"Inference"}],title:"Image captioning"};function co(ys){return io(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class bo extends ao{constructor(b){super();ro(this,b,co,fo,oo,{})}}export{bo as default,uo as metadata};
