import{S as kt,i as $t,s as At,e as n,k as h,w as J,t,M as qt,c as p,d as s,m,a as o,x as w,h as l,b as c,G as a,g as i,y as T,L as Xt,q as d,o as j,B as U,v as Wt}from"../chunks/vendor-hf-doc-builder.js";import{I as Ia}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as q}from"../chunks/CodeBlock-hf-doc-builder.js";function Yt(Qs){let g,Se,E,X,je,Q,ba,Ue,_a,Qe,_,Ca,ce,va,ga,he,Ea,ka,Fe,k,W,Ie,F,$a,be,Aa,Ge,M,me,qa,Xa,G,Wa,Ya,R,Ba,Za,H,Va,Na,z,Sa,Qa,Re,ye,Fa,He,D,ze,$,Y,_e,x,Ga,Ce,Ra,De,ue,Ha,xe,B,za,P,Da,xa,Pe,O,Oe,Z,Pa,L,Oa,La,Le,K,Ke,u,Ka,ve,es,as,ge,ss,ts,Ee,ls,rs,ke,ns,ps,ee,os,is,ea,ae,aa,V,cs,se,hs,ms,sa,te,ta,N,ys,le,us,Ms,la,re,ra,C,fs,$e,Js,ws,Me,Ts,ds,na,ne,pa,v,js,fe,Us,Is,Ae,bs,_s,oa,pe,ia,y,Cs,qe,vs,gs,Xe,Es,ks,We,$s,As,Ye,qs,Xs,Be,Ws,Ys,Ze,Bs,Zs,ca,Je,Vs,ha,oe,ma,A,S,Ve,ie,Ns,Ne,Ss,ya,ua;return Q=new Ia({}),F=new Ia({}),D=new q({props:{code:"cGlwJTIwaW5zdGFsbCUyMG9wdHVuYSUyRnNpZ29wdCUyRndhbmRiJTJGcmF5JTVCdHVuZSU1RCUyMA==",highlighted:"pip install optuna/sigopt/wandb/ray[tune] "}}),x=new Ia({}),O=new q({props:{code:"ZGVmJTIwc2lnb3B0X2hwX3NwYWNlKHRyaWFsKSUzQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMmJvdW5kcyUyMiUzQSUyMCU3QiUyMm1pbiUyMiUzQSUyMDFlLTYlMkMlMjAlMjJtYXglMjIlM0ElMjAxZS00JTdEJTJDJTIwJTIybmFtZSUyMiUzQSUyMCUyMmxlYXJuaW5nX3JhdGUlMjIlMkMlMjAlMjJ0eXBlJTIyJTNBJTIwJTIyZG91YmxlJTIyJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY2F0ZWdvcmljYWxfdmFsdWVzJTIyJTNBJTIwJTVCJTIyMTYlMjIlMkMlMjAlMjIzMiUyMiUyQyUyMCUyMjY0JTIyJTJDJTIwJTIyMTI4JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybmFtZSUyMiUzQSUyMCUyMnBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnR5cGUlMjIlM0ElMjAlMjJjYXRlZ29yaWNhbCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">sigopt_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> [
<span class="hljs-meta">... </span>        {<span class="hljs-string">&quot;bounds&quot;</span>: {<span class="hljs-string">&quot;min&quot;</span>: <span class="hljs-number">1e-6</span>, <span class="hljs-string">&quot;max&quot;</span>: <span class="hljs-number">1e-4</span>}, <span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;learning_rate&quot;</span>, <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;double&quot;</span>},
<span class="hljs-meta">... </span>        {
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;categorical_values&quot;</span>: [<span class="hljs-string">&quot;16&quot;</span>, <span class="hljs-string">&quot;32&quot;</span>, <span class="hljs-string">&quot;64&quot;</span>, <span class="hljs-string">&quot;128&quot;</span>],
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;categorical&quot;</span>,
<span class="hljs-meta">... </span>        },
<span class="hljs-meta">... </span>    ]`}}),K=new q({props:{code:"ZGVmJTIwb3B0dW5hX2hwX3NwYWNlKHRyaWFsKSUzQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmxlYXJuaW5nX3JhdGUlMjIlM0ElMjB0cmlhbC5zdWdnZXN0X2Zsb2F0KCUyMmxlYXJuaW5nX3JhdGUlMjIlMkMlMjAxZS02JTJDJTIwMWUtNCUyQyUyMGxvZyUzRFRydWUpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIyJTNBJTIwdHJpYWwuc3VnZ2VzdF9jYXRlZ29yaWNhbCglMjJwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlMjIlMkMlMjAlNUIxNiUyQyUyMDMyJTJDJTIwNjQlMkMlMjAxMjglNUQpJTJDJTBBJTIwJTIwJTIwJTIwJTdE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">optuna_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;learning_rate&quot;</span>: trial.suggest_float(<span class="hljs-string">&quot;learning_rate&quot;</span>, <span class="hljs-number">1e-6</span>, <span class="hljs-number">1e-4</span>, log=<span class="hljs-literal">True</span>),
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>: trial.suggest_categorical(<span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>, [<span class="hljs-number">16</span>, <span class="hljs-number">32</span>, <span class="hljs-number">64</span>, <span class="hljs-number">128</span>]),
<span class="hljs-meta">... </span>    }`}}),ae=new q({props:{code:"YmVzdF90cmlhbHMlMjAlM0QlMjB0cmFpbmVyLmh5cGVycGFyYW1ldGVyX3NlYXJjaCglMEElMjAlMjAlMjAlMjBkaXJlY3Rpb24lM0QlNUIlMjJtaW5pbWl6ZSUyMiUyQyUyMCUyMm1heGltaXplJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwYmFja2VuZCUzRCUyMm9wdHVuYSUyMiUyQyUwQSUyMCUyMCUyMCUyMGhwX3NwYWNlJTNEb3B0dW5hX2hwX3NwYWNlJTJDJTBBJTIwJTIwJTIwJTIwbl90cmlhbHMlM0QyMCUyQyUwQSUyMCUyMCUyMCUyMGNvbXB1dGVfb2JqZWN0aXZlJTNEY29tcHV0ZV9vYmplY3RpdmUlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>best_trials = trainer.hyperparameter_search(
<span class="hljs-meta">... </span>    direction=[<span class="hljs-string">&quot;minimize&quot;</span>, <span class="hljs-string">&quot;maximize&quot;</span>],
<span class="hljs-meta">... </span>    backend=<span class="hljs-string">&quot;optuna&quot;</span>,
<span class="hljs-meta">... </span>    hp_space=optuna_hp_space,
<span class="hljs-meta">... </span>    n_trials=<span class="hljs-number">20</span>,
<span class="hljs-meta">... </span>    compute_objective=compute_objective,
<span class="hljs-meta">... </span>)`}}),te=new q({props:{code:"ZGVmJTIwcmF5X2hwX3NwYWNlKHRyaWFsKSUzQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmxlYXJuaW5nX3JhdGUlMjIlM0ElMjB0dW5lLmxvZ3VuaWZvcm0oMWUtNiUyQyUyMDFlLTQpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIyJTNBJTIwdHVuZS5jaG9pY2UoJTVCMTYlMkMlMjAzMiUyQyUyMDY0JTJDJTIwMTI4JTVEKSUyQyUwQSUyMCUyMCUyMCUyMCU3RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">ray_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;learning_rate&quot;</span>: tune.loguniform(<span class="hljs-number">1e-6</span>, <span class="hljs-number">1e-4</span>),
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>: tune.choice([<span class="hljs-number">16</span>, <span class="hljs-number">32</span>, <span class="hljs-number">64</span>, <span class="hljs-number">128</span>]),
<span class="hljs-meta">... </span>    }`}}),re=new q({props:{code:"ZGVmJTIwd2FuZGJfaHBfc3BhY2UodHJpYWwpJTNBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybWV0aG9kJTIyJTNBJTIwJTIycmFuZG9tJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybWV0cmljJTIyJTNBJTIwJTdCJTIybmFtZSUyMiUzQSUyMCUyMm9iamVjdGl2ZSUyMiUyQyUyMCUyMmdvYWwlMjIlM0ElMjAlMjJtaW5pbWl6ZSUyMiU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnBhcmFtZXRlcnMlMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJsZWFybmluZ19yYXRlJTIyJTNBJTIwJTdCJTIyZGlzdHJpYnV0aW9uJTIyJTNBJTIwJTIydW5pZm9ybSUyMiUyQyUyMCUyMm1pbiUyMiUzQSUyMDFlLTYlMkMlMjAlMjJtYXglMjIlM0ElMjAxZS00JTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIyJTNBJTIwJTdCJTIydmFsdWVzJTIyJTNBJTIwJTVCMTYlMkMlMjAzMiUyQyUyMDY0JTJDJTIwMTI4JTVEJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTdE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">wandb_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;method&quot;</span>: <span class="hljs-string">&quot;random&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;metric&quot;</span>: {<span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;objective&quot;</span>, <span class="hljs-string">&quot;goal&quot;</span>: <span class="hljs-string">&quot;minimize&quot;</span>},
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;parameters&quot;</span>: {
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;learning_rate&quot;</span>: {<span class="hljs-string">&quot;distribution&quot;</span>: <span class="hljs-string">&quot;uniform&quot;</span>, <span class="hljs-string">&quot;min&quot;</span>: <span class="hljs-number">1e-6</span>, <span class="hljs-string">&quot;max&quot;</span>: <span class="hljs-number">1e-4</span>},
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>: {<span class="hljs-string">&quot;values&quot;</span>: [<span class="hljs-number">16</span>, <span class="hljs-number">32</span>, <span class="hljs-number">64</span>, <span class="hljs-number">128</span>]},
<span class="hljs-meta">... </span>        },
<span class="hljs-meta">... </span>    }`}}),ne=new q({props:{code:"ZGVmJTIwbW9kZWxfaW5pdCh0cmlhbCklM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBBdXRvTW9kZWxGb3JTZXF1ZW5jZUNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbF9hcmdzLm1vZGVsX25hbWVfb3JfcGF0aCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZyb21fdGYlM0Rib29sKCUyMi5ja3B0JTIyJTIwaW4lMjBtb2RlbF9hcmdzLm1vZGVsX25hbWVfb3JfcGF0aCklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb25maWclM0Rjb25maWclMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjYWNoZV9kaXIlM0Rtb2RlbF9hcmdzLmNhY2hlX2RpciUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldmlzaW9uJTNEbW9kZWxfYXJncy5tb2RlbF9yZXZpc2lvbiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHVzZV9hdXRoX3Rva2VuJTNEVHJ1ZSUyMGlmJTIwbW9kZWxfYXJncy51c2VfYXV0aF90b2tlbiUyMGVsc2UlMjBOb25lJTJDJTBBJTIwJTIwJTIwJTIwKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">model_init</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> AutoModelForSequenceClassification.from_pretrained(
<span class="hljs-meta">... </span>        model_args.model_name_or_path,
<span class="hljs-meta">... </span>        from_tf=<span class="hljs-built_in">bool</span>(<span class="hljs-string">&quot;.ckpt&quot;</span> <span class="hljs-keyword">in</span> model_args.model_name_or_path),
<span class="hljs-meta">... </span>        config=config,
<span class="hljs-meta">... </span>        cache_dir=model_args.cache_dir,
<span class="hljs-meta">... </span>        revision=model_args.model_revision,
<span class="hljs-meta">... </span>        use_auth_token=<span class="hljs-literal">True</span> <span class="hljs-keyword">if</span> model_args.use_auth_token <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span>,
<span class="hljs-meta">... </span>    )`}}),pe=new q({props:{code:"dHJhaW5lciUyMCUzRCUyMFRyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0ROb25lJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEc21hbGxfdHJhaW5fZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfZGF0YXNldCUzRHNtYWxsX2V2YWxfZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMGNvbXB1dGVfbWV0cmljcyUzRGNvbXB1dGVfbWV0cmljcyUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX2luaXQlM0Rtb2RlbF9pbml0JTJDJTBBJTIwJTIwJTIwJTIwZGF0YV9jb2xsYXRvciUzRGRhdGFfY29sbGF0b3IlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=<span class="hljs-literal">None</span>,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=small_train_dataset,
<span class="hljs-meta">... </span>    eval_dataset=small_eval_dataset,
<span class="hljs-meta">... </span>    compute_metrics=compute_metrics,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    model_init=model_init,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)`}}),oe=new q({props:{code:"YmVzdF90cmlhbCUyMCUzRCUyMHRyYWluZXIuaHlwZXJwYXJhbWV0ZXJfc2VhcmNoKCUwQSUyMCUyMCUyMCUyMGRpcmVjdGlvbiUzRCUyMm1heGltaXplJTIyJTJDJTBBJTIwJTIwJTIwJTIwYmFja2VuZCUzRCUyMm9wdHVuYSUyMiUyQyUwQSUyMCUyMCUyMCUyMGhwX3NwYWNlJTNEb3B0dW5hX2hwX3NwYWNlJTJDJTBBJTIwJTIwJTIwJTIwbl90cmlhbHMlM0QyMCUyQyUwQSUyMCUyMCUyMCUyMGNvbXB1dGVfb2JqZWN0aXZlJTNEY29tcHV0ZV9vYmplY3RpdmUlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>best_trial = trainer.hyperparameter_search(
<span class="hljs-meta">... </span>    direction=<span class="hljs-string">&quot;maximize&quot;</span>,
<span class="hljs-meta">... </span>    backend=<span class="hljs-string">&quot;optuna&quot;</span>,
<span class="hljs-meta">... </span>    hp_space=optuna_hp_space,
<span class="hljs-meta">... </span>    n_trials=<span class="hljs-number">20</span>,
<span class="hljs-meta">... </span>    compute_objective=compute_objective,
<span class="hljs-meta">... </span>)`}}),ie=new Ia({}),{c(){g=n("meta"),Se=h(),E=n("h1"),X=n("a"),je=n("span"),J(Q.$$.fragment),ba=h(),Ue=n("span"),_a=t("Hyperparameter Search using Trainer API"),Qe=h(),_=n("p"),Ca=t("\u{1F917} Transformers provides a "),ce=n("a"),va=t("Trainer"),ga=t(" class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),he=n("a"),Ea=t("Trainer"),ka=t(" provides API for hyperparameter search. This doc shows how to enable it in example."),Fe=h(),k=n("h2"),W=n("a"),Ie=n("span"),J(F.$$.fragment),$a=h(),be=n("span"),Aa=t("Hyperparameter Search backend"),Ge=h(),M=n("p"),me=n("a"),qa=t("Trainer"),Xa=t(` supports four hyperparameter search backends currently:
`),G=n("a"),Wa=t("optuna"),Ya=t(", "),R=n("a"),Ba=t("sigopt"),Za=t(", "),H=n("a"),Va=t("raytune"),Na=t(" and "),z=n("a"),Sa=t("wandb"),Qa=t("."),Re=h(),ye=n("p"),Fa=t("you should install them before using them as the hyperparameter search backend"),He=h(),J(D.$$.fragment),ze=h(),$=n("h2"),Y=n("a"),_e=n("span"),J(x.$$.fragment),Ga=h(),Ce=n("span"),Ra=t("How to enable Hyperparameter search in example"),De=h(),ue=n("p"),Ha=t("Define the hyperparameter search space, different backends need different format."),xe=h(),B=n("p"),za=t("For sigopt, see sigopt "),P=n("a"),Da=t("object_parameter"),xa=t(", it\u2019s like following:"),Pe=h(),J(O.$$.fragment),Oe=h(),Z=n("p"),Pa=t("For optuna, see optuna "),L=n("a"),Oa=t("object_parameter"),La=t(", it\u2019s like following:"),Le=h(),J(K.$$.fragment),Ke=h(),u=n("p"),Ka=t("Optuna provides multi-objective HPO. You can pass "),ve=n("code"),es=t("direction"),as=t(" in "),ge=n("code"),ss=t("hyperparameter_search"),ts=t(" and define your own compute_objective to return multiple objective values. The Pareto Front ("),Ee=n("code"),ls=t("List[BestRun]"),rs=t(") will be returned in hyperparameter_search, you should refer to the test case "),ke=n("code"),ns=t("TrainerHyperParameterMultiObjectOptunaIntegrationTest"),ps=t(" in "),ee=n("a"),os=t("test_trainer"),is=t(". It\u2019s like following"),ea=h(),J(ae.$$.fragment),aa=h(),V=n("p"),cs=t("For raytune, see raytune "),se=n("a"),hs=t("object_parameter"),ms=t(", it\u2019s like following:"),sa=h(),J(te.$$.fragment),ta=h(),N=n("p"),ys=t("For wandb, see wandb "),le=n("a"),us=t("object_parameter"),Ms=t(", it\u2019s like following:"),la=h(),J(re.$$.fragment),ra=h(),C=n("p"),fs=t("Define a "),$e=n("code"),Js=t("model_init"),ws=t(" function and pass it to the "),Me=n("a"),Ts=t("Trainer"),ds=t(", as an example:"),na=h(),J(ne.$$.fragment),pa=h(),v=n("p"),js=t("Create a "),fe=n("a"),Us=t("Trainer"),Is=t(" with your "),Ae=n("code"),bs=t("model_init"),_s=t(" function, training arguments, training and test datasets, and evaluation function:"),oa=h(),J(pe.$$.fragment),ia=h(),y=n("p"),Cs=t("Call hyperparameter search, get the best trial parameters, backend could be "),qe=n("code"),vs=t('"optuna"'),gs=t("/"),Xe=n("code"),Es=t('"sigopt"'),ks=t("/"),We=n("code"),$s=t('"wandb"'),As=t("/"),Ye=n("code"),qs=t('"ray"'),Xs=t(". direction can be"),Be=n("code"),Ws=t('"minimize"'),Ys=t(" or "),Ze=n("code"),Bs=t('"maximize"'),Zs=t(", which indicates whether to optimize greater or lower objective."),ca=h(),Je=n("p"),Vs=t("You could define your own compute_objective function, if not defined, the default compute_objective will be called, and the sum of eval metric like f1 is returned as objective value."),ha=h(),J(oe.$$.fragment),ma=h(),A=n("h2"),S=n("a"),Ve=n("span"),J(ie.$$.fragment),Ns=h(),Ne=n("span"),Ss=t("Hyperparameter search For DDP finetune"),ya=t(`

Currently, Hyperparameter search for DDP is enabled for optuna and sigopt. Only the rank-zero process will generate the search trial and pass the argument to other ranks.`),this.h()},l(e){const r=qt('[data-svelte="svelte-1phssyn"]',document.head);g=p(r,"META",{name:!0,content:!0}),r.forEach(s),Se=m(e),E=p(e,"H1",{class:!0});var Ma=o(E);X=p(Ma,"A",{id:!0,class:!0,href:!0});var Fs=o(X);je=p(Fs,"SPAN",{});var Gs=o(je);w(Q.$$.fragment,Gs),Gs.forEach(s),Fs.forEach(s),ba=m(Ma),Ue=p(Ma,"SPAN",{});var Rs=o(Ue);_a=l(Rs,"Hyperparameter Search using Trainer API"),Rs.forEach(s),Ma.forEach(s),Qe=m(e),_=p(e,"P",{});var we=o(_);Ca=l(we,"\u{1F917} Transformers provides a "),ce=p(we,"A",{href:!0});var Hs=o(ce);va=l(Hs,"Trainer"),Hs.forEach(s),ga=l(we," class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),he=p(we,"A",{href:!0});var zs=o(he);Ea=l(zs,"Trainer"),zs.forEach(s),ka=l(we," provides API for hyperparameter search. This doc shows how to enable it in example."),we.forEach(s),Fe=m(e),k=p(e,"H2",{class:!0});var fa=o(k);W=p(fa,"A",{id:!0,class:!0,href:!0});var Ds=o(W);Ie=p(Ds,"SPAN",{});var xs=o(Ie);w(F.$$.fragment,xs),xs.forEach(s),Ds.forEach(s),$a=m(fa),be=p(fa,"SPAN",{});var Ps=o(be);Aa=l(Ps,"Hyperparameter Search backend"),Ps.forEach(s),fa.forEach(s),Ge=m(e),M=p(e,"P",{});var b=o(M);me=p(b,"A",{href:!0});var Os=o(me);qa=l(Os,"Trainer"),Os.forEach(s),Xa=l(b,` supports four hyperparameter search backends currently:
`),G=p(b,"A",{href:!0,rel:!0});var Ls=o(G);Wa=l(Ls,"optuna"),Ls.forEach(s),Ya=l(b,", "),R=p(b,"A",{href:!0,rel:!0});var Ks=o(R);Ba=l(Ks,"sigopt"),Ks.forEach(s),Za=l(b,", "),H=p(b,"A",{href:!0,rel:!0});var et=o(H);Va=l(et,"raytune"),et.forEach(s),Na=l(b," and "),z=p(b,"A",{href:!0,rel:!0});var at=o(z);Sa=l(at,"wandb"),at.forEach(s),Qa=l(b,"."),b.forEach(s),Re=m(e),ye=p(e,"P",{});var st=o(ye);Fa=l(st,"you should install them before using them as the hyperparameter search backend"),st.forEach(s),He=m(e),w(D.$$.fragment,e),ze=m(e),$=p(e,"H2",{class:!0});var Ja=o($);Y=p(Ja,"A",{id:!0,class:!0,href:!0});var tt=o(Y);_e=p(tt,"SPAN",{});var lt=o(_e);w(x.$$.fragment,lt),lt.forEach(s),tt.forEach(s),Ga=m(Ja),Ce=p(Ja,"SPAN",{});var rt=o(Ce);Ra=l(rt,"How to enable Hyperparameter search in example"),rt.forEach(s),Ja.forEach(s),De=m(e),ue=p(e,"P",{});var nt=o(ue);Ha=l(nt,"Define the hyperparameter search space, different backends need different format."),nt.forEach(s),xe=m(e),B=p(e,"P",{});var wa=o(B);za=l(wa,"For sigopt, see sigopt "),P=p(wa,"A",{href:!0,rel:!0});var pt=o(P);Da=l(pt,"object_parameter"),pt.forEach(s),xa=l(wa,", it\u2019s like following:"),wa.forEach(s),Pe=m(e),w(O.$$.fragment,e),Oe=m(e),Z=p(e,"P",{});var Ta=o(Z);Pa=l(Ta,"For optuna, see optuna "),L=p(Ta,"A",{href:!0,rel:!0});var ot=o(L);Oa=l(ot,"object_parameter"),ot.forEach(s),La=l(Ta,", it\u2019s like following:"),Ta.forEach(s),Le=m(e),w(K.$$.fragment,e),Ke=m(e),u=p(e,"P",{});var I=o(u);Ka=l(I,"Optuna provides multi-objective HPO. You can pass "),ve=p(I,"CODE",{});var it=o(ve);es=l(it,"direction"),it.forEach(s),as=l(I," in "),ge=p(I,"CODE",{});var ct=o(ge);ss=l(ct,"hyperparameter_search"),ct.forEach(s),ts=l(I," and define your own compute_objective to return multiple objective values. The Pareto Front ("),Ee=p(I,"CODE",{});var ht=o(Ee);ls=l(ht,"List[BestRun]"),ht.forEach(s),rs=l(I,") will be returned in hyperparameter_search, you should refer to the test case "),ke=p(I,"CODE",{});var mt=o(ke);ns=l(mt,"TrainerHyperParameterMultiObjectOptunaIntegrationTest"),mt.forEach(s),ps=l(I," in "),ee=p(I,"A",{href:!0,rel:!0});var yt=o(ee);os=l(yt,"test_trainer"),yt.forEach(s),is=l(I,". It\u2019s like following"),I.forEach(s),ea=m(e),w(ae.$$.fragment,e),aa=m(e),V=p(e,"P",{});var da=o(V);cs=l(da,"For raytune, see raytune "),se=p(da,"A",{href:!0,rel:!0});var ut=o(se);hs=l(ut,"object_parameter"),ut.forEach(s),ms=l(da,", it\u2019s like following:"),da.forEach(s),sa=m(e),w(te.$$.fragment,e),ta=m(e),N=p(e,"P",{});var ja=o(N);ys=l(ja,"For wandb, see wandb "),le=p(ja,"A",{href:!0,rel:!0});var Mt=o(le);us=l(Mt,"object_parameter"),Mt.forEach(s),Ms=l(ja,", it\u2019s like following:"),ja.forEach(s),la=m(e),w(re.$$.fragment,e),ra=m(e),C=p(e,"P",{});var Te=o(C);fs=l(Te,"Define a "),$e=p(Te,"CODE",{});var ft=o($e);Js=l(ft,"model_init"),ft.forEach(s),ws=l(Te," function and pass it to the "),Me=p(Te,"A",{href:!0});var Jt=o(Me);Ts=l(Jt,"Trainer"),Jt.forEach(s),ds=l(Te,", as an example:"),Te.forEach(s),na=m(e),w(ne.$$.fragment,e),pa=m(e),v=p(e,"P",{});var de=o(v);js=l(de,"Create a "),fe=p(de,"A",{href:!0});var wt=o(fe);Us=l(wt,"Trainer"),wt.forEach(s),Is=l(de," with your "),Ae=p(de,"CODE",{});var Tt=o(Ae);bs=l(Tt,"model_init"),Tt.forEach(s),_s=l(de," function, training arguments, training and test datasets, and evaluation function:"),de.forEach(s),oa=m(e),w(pe.$$.fragment,e),ia=m(e),y=p(e,"P",{});var f=o(y);Cs=l(f,"Call hyperparameter search, get the best trial parameters, backend could be "),qe=p(f,"CODE",{});var dt=o(qe);vs=l(dt,'"optuna"'),dt.forEach(s),gs=l(f,"/"),Xe=p(f,"CODE",{});var jt=o(Xe);Es=l(jt,'"sigopt"'),jt.forEach(s),ks=l(f,"/"),We=p(f,"CODE",{});var Ut=o(We);$s=l(Ut,'"wandb"'),Ut.forEach(s),As=l(f,"/"),Ye=p(f,"CODE",{});var It=o(Ye);qs=l(It,'"ray"'),It.forEach(s),Xs=l(f,". direction can be"),Be=p(f,"CODE",{});var bt=o(Be);Ws=l(bt,'"minimize"'),bt.forEach(s),Ys=l(f," or "),Ze=p(f,"CODE",{});var _t=o(Ze);Bs=l(_t,'"maximize"'),_t.forEach(s),Zs=l(f,", which indicates whether to optimize greater or lower objective."),f.forEach(s),ca=m(e),Je=p(e,"P",{});var Ct=o(Je);Vs=l(Ct,"You could define your own compute_objective function, if not defined, the default compute_objective will be called, and the sum of eval metric like f1 is returned as objective value."),Ct.forEach(s),ha=m(e),w(oe.$$.fragment,e),ma=m(e),A=p(e,"H2",{class:!0});var Ua=o(A);S=p(Ua,"A",{id:!0,class:!0,href:!0});var vt=o(S);Ve=p(vt,"SPAN",{});var gt=o(Ve);w(ie.$$.fragment,gt),gt.forEach(s),vt.forEach(s),Ns=m(Ua),Ne=p(Ua,"SPAN",{});var Et=o(Ne);Ss=l(Et,"Hyperparameter search For DDP finetune"),Et.forEach(s),Ua.forEach(s),ya=l(e,`

Currently, Hyperparameter search for DDP is enabled for optuna and sigopt. Only the rank-zero process will generate the search trial and pass the argument to other ranks.`),this.h()},h(){c(g,"name","hf:doc:metadata"),c(g,"content",JSON.stringify(Bt)),c(X,"id","hyperparameter-search-using-trainer-api"),c(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(X,"href","#hyperparameter-search-using-trainer-api"),c(E,"class","relative group"),c(ce,"href","/docs/transformers/main/en/main_classes/trainer#transformers.Trainer"),c(he,"href","/docs/transformers/main/en/main_classes/trainer#transformers.Trainer"),c(W,"id","hyperparameter-search-backend"),c(W,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(W,"href","#hyperparameter-search-backend"),c(k,"class","relative group"),c(me,"href","/docs/transformers/main/en/main_classes/trainer#transformers.Trainer"),c(G,"href","https://optuna.org/"),c(G,"rel","nofollow"),c(R,"href","https://sigopt.com/"),c(R,"rel","nofollow"),c(H,"href","https://docs.ray.io/en/latest/tune/index.html"),c(H,"rel","nofollow"),c(z,"href","https://wandb.ai/site/sweeps"),c(z,"rel","nofollow"),c(Y,"id","how-to-enable-hyperparameter-search-in-example"),c(Y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Y,"href","#how-to-enable-hyperparameter-search-in-example"),c($,"class","relative group"),c(P,"href","https://docs.sigopt.com/ai-module-api-references/api_reference/objects/object_parameter"),c(P,"rel","nofollow"),c(L,"href","https://optuna.readthedocs.io/en/stable/tutorial/10_key_features/002_configurations.html#sphx-glr-tutorial-10-key-features-002-configurations-py"),c(L,"rel","nofollow"),c(ee,"href","https://github.com/huggingface/transformers/blob/main/tests/trainer/test_trainer.py"),c(ee,"rel","nofollow"),c(se,"href","https://docs.ray.io/en/latest/tune/api/search_space.html"),c(se,"rel","nofollow"),c(le,"href","https://docs.wandb.ai/guides/sweeps/configuration"),c(le,"rel","nofollow"),c(Me,"href","/docs/transformers/main/en/main_classes/trainer#transformers.Trainer"),c(fe,"href","/docs/transformers/main/en/main_classes/trainer#transformers.Trainer"),c(S,"id","hyperparameter-search-for-ddp-finetune"),c(S,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(S,"href","#hyperparameter-search-for-ddp-finetune"),c(A,"class","relative group")},m(e,r){a(document.head,g),i(e,Se,r),i(e,E,r),a(E,X),a(X,je),T(Q,je,null),a(E,ba),a(E,Ue),a(Ue,_a),i(e,Qe,r),i(e,_,r),a(_,Ca),a(_,ce),a(ce,va),a(_,ga),a(_,he),a(he,Ea),a(_,ka),i(e,Fe,r),i(e,k,r),a(k,W),a(W,Ie),T(F,Ie,null),a(k,$a),a(k,be),a(be,Aa),i(e,Ge,r),i(e,M,r),a(M,me),a(me,qa),a(M,Xa),a(M,G),a(G,Wa),a(M,Ya),a(M,R),a(R,Ba),a(M,Za),a(M,H),a(H,Va),a(M,Na),a(M,z),a(z,Sa),a(M,Qa),i(e,Re,r),i(e,ye,r),a(ye,Fa),i(e,He,r),T(D,e,r),i(e,ze,r),i(e,$,r),a($,Y),a(Y,_e),T(x,_e,null),a($,Ga),a($,Ce),a(Ce,Ra),i(e,De,r),i(e,ue,r),a(ue,Ha),i(e,xe,r),i(e,B,r),a(B,za),a(B,P),a(P,Da),a(B,xa),i(e,Pe,r),T(O,e,r),i(e,Oe,r),i(e,Z,r),a(Z,Pa),a(Z,L),a(L,Oa),a(Z,La),i(e,Le,r),T(K,e,r),i(e,Ke,r),i(e,u,r),a(u,Ka),a(u,ve),a(ve,es),a(u,as),a(u,ge),a(ge,ss),a(u,ts),a(u,Ee),a(Ee,ls),a(u,rs),a(u,ke),a(ke,ns),a(u,ps),a(u,ee),a(ee,os),a(u,is),i(e,ea,r),T(ae,e,r),i(e,aa,r),i(e,V,r),a(V,cs),a(V,se),a(se,hs),a(V,ms),i(e,sa,r),T(te,e,r),i(e,ta,r),i(e,N,r),a(N,ys),a(N,le),a(le,us),a(N,Ms),i(e,la,r),T(re,e,r),i(e,ra,r),i(e,C,r),a(C,fs),a(C,$e),a($e,Js),a(C,ws),a(C,Me),a(Me,Ts),a(C,ds),i(e,na,r),T(ne,e,r),i(e,pa,r),i(e,v,r),a(v,js),a(v,fe),a(fe,Us),a(v,Is),a(v,Ae),a(Ae,bs),a(v,_s),i(e,oa,r),T(pe,e,r),i(e,ia,r),i(e,y,r),a(y,Cs),a(y,qe),a(qe,vs),a(y,gs),a(y,Xe),a(Xe,Es),a(y,ks),a(y,We),a(We,$s),a(y,As),a(y,Ye),a(Ye,qs),a(y,Xs),a(y,Be),a(Be,Ws),a(y,Ys),a(y,Ze),a(Ze,Bs),a(y,Zs),i(e,ca,r),i(e,Je,r),a(Je,Vs),i(e,ha,r),T(oe,e,r),i(e,ma,r),i(e,A,r),a(A,S),a(S,Ve),T(ie,Ve,null),a(A,Ns),a(A,Ne),a(Ne,Ss),i(e,ya,r),ua=!0},p:Xt,i(e){ua||(d(Q.$$.fragment,e),d(F.$$.fragment,e),d(D.$$.fragment,e),d(x.$$.fragment,e),d(O.$$.fragment,e),d(K.$$.fragment,e),d(ae.$$.fragment,e),d(te.$$.fragment,e),d(re.$$.fragment,e),d(ne.$$.fragment,e),d(pe.$$.fragment,e),d(oe.$$.fragment,e),d(ie.$$.fragment,e),ua=!0)},o(e){j(Q.$$.fragment,e),j(F.$$.fragment,e),j(D.$$.fragment,e),j(x.$$.fragment,e),j(O.$$.fragment,e),j(K.$$.fragment,e),j(ae.$$.fragment,e),j(te.$$.fragment,e),j(re.$$.fragment,e),j(ne.$$.fragment,e),j(pe.$$.fragment,e),j(oe.$$.fragment,e),j(ie.$$.fragment,e),ua=!1},d(e){s(g),e&&s(Se),e&&s(E),U(Q),e&&s(Qe),e&&s(_),e&&s(Fe),e&&s(k),U(F),e&&s(Ge),e&&s(M),e&&s(Re),e&&s(ye),e&&s(He),U(D,e),e&&s(ze),e&&s($),U(x),e&&s(De),e&&s(ue),e&&s(xe),e&&s(B),e&&s(Pe),U(O,e),e&&s(Oe),e&&s(Z),e&&s(Le),U(K,e),e&&s(Ke),e&&s(u),e&&s(ea),U(ae,e),e&&s(aa),e&&s(V),e&&s(sa),U(te,e),e&&s(ta),e&&s(N),e&&s(la),U(re,e),e&&s(ra),e&&s(C),e&&s(na),U(ne,e),e&&s(pa),e&&s(v),e&&s(oa),U(pe,e),e&&s(ia),e&&s(y),e&&s(ca),e&&s(Je),e&&s(ha),U(oe,e),e&&s(ma),e&&s(A),U(ie),e&&s(ya)}}}const Bt={local:"hyperparameter-search-using-trainer-api",sections:[{local:"hyperparameter-search-backend",title:"Hyperparameter Search backend"},{local:"how-to-enable-hyperparameter-search-in-example",title:"How to enable Hyperparameter search in example"},{local:"hyperparameter-search-for-ddp-finetune",title:"Hyperparameter search For DDP finetune"}],title:"Hyperparameter Search using Trainer API"};function Zt(Qs){return Wt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Qt extends kt{constructor(g){super();$t(this,g,Zt,Yt,At,{})}}export{Qt as default,Bt as metadata};
