import{S as lt,i as rt,s as nt,e as n,k as h,w as f,t,M as pt,c as p,d as e,m,a as o,x as J,h as l,b as c,G as s,g as i,y as w,L as ot,q as T,o as d,B as j,v as it}from"../chunks/vendor-hf-doc-builder.js";import{I as ys}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as S}from"../chunks/CodeBlock-hf-doc-builder.js";function ct(je){let _,qa,v,$,Ja,V,us,wa,Ms,Xa,I,fs,na,Js,ws,pa,Ts,ds,Wa,g,A,Ta,N,js,da,Us,Za,u,oa,Is,bs,Q,Cs,_s,F,vs,gs,R,ks,Es,G,$s,As,Ba,ia,qs,Ya,z,Sa,k,q,ja,H,Xs,Ua,Ws,Va,ca,Zs,Na,X,Bs,x,Ys,Ss,Qa,D,Fa,W,Vs,P,Ns,Qs,Ra,O,Ga,Z,Fs,K,Rs,Gs,za,L,Ha,B,zs,aa,Hs,xs,xa,sa,Da,b,Ds,Ia,Ps,Os,ha,Ks,Ls,Pa,ea,Oa,C,ae,ma,se,ee,ba,te,le,Ka,ta,La,y,re,Ca,ne,pe,_a,oe,ie,va,ce,he,ga,me,ye,ka,ue,Me,Ea,fe,Je,as,ya,we,ss,la,es,E,Y,$a,ra,Te,Aa,de,ts,ls;return V=new ys({}),N=new ys({}),z=new S({props:{code:"cGlwJTIwaW5zdGFsbCUyMG9wdHVuYSUyRnNpZ29wdCUyRndhbmRiJTJGcmF5JTVCdHVuZSU1RCUyMA==",highlighted:"pip install optuna/sigopt/wandb/ray[tune] "}}),H=new ys({}),D=new S({props:{code:"ZGVmJTIwc2lnb3B0X2hwX3NwYWNlKHRyaWFsKSUzQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUyMmJvdW5kcyUyMiUzQSUyMCU3QiUyMm1pbiUyMiUzQSUyMDFlLTYlMkMlMjAlMjJtYXglMjIlM0ElMjAxZS00JTdEJTJDJTIwJTIybmFtZSUyMiUzQSUyMCUyMmxlYXJuaW5nX3JhdGUlMjIlMkMlMjAlMjJ0eXBlJTIyJTNBJTIwJTIyZG91YmxlJTIyJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY2F0ZWdvcmljYWxfdmFsdWVzJTIyJTNBJTIwJTVCJTIyMTYlMjIlMkMlMjAlMjIzMiUyMiUyQyUyMCUyMjY0JTIyJTJDJTIwJTIyMTI4JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybmFtZSUyMiUzQSUyMCUyMnBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnR5cGUlMjIlM0ElMjAlMjJjYXRlZ29yaWNhbCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">sigopt_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> [
<span class="hljs-meta">... </span>        {<span class="hljs-string">&quot;bounds&quot;</span>: {<span class="hljs-string">&quot;min&quot;</span>: <span class="hljs-number">1e-6</span>, <span class="hljs-string">&quot;max&quot;</span>: <span class="hljs-number">1e-4</span>}, <span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;learning_rate&quot;</span>, <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;double&quot;</span>},
<span class="hljs-meta">... </span>        {
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;categorical_values&quot;</span>: [<span class="hljs-string">&quot;16&quot;</span>, <span class="hljs-string">&quot;32&quot;</span>, <span class="hljs-string">&quot;64&quot;</span>, <span class="hljs-string">&quot;128&quot;</span>],
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;categorical&quot;</span>,
<span class="hljs-meta">... </span>        },
<span class="hljs-meta">... </span>    ]`}}),O=new S({props:{code:"ZGVmJTIwb3B0dW5hX2hwX3NwYWNlKHRyaWFsKSUzQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmxlYXJuaW5nX3JhdGUlMjIlM0ElMjB0cmlhbC5zdWdnZXN0X2Zsb2F0KCUyMmxlYXJuaW5nX3JhdGUlMjIlMkMlMjAxZS02JTJDJTIwMWUtNCUyQyUyMGxvZyUzRFRydWUpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIyJTNBJTIwdHJpYWwuc3VnZ2VzdF9jYXRlZ29yaWNhbCglMjJwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlMjIlMkMlMjAlNUIxNiUyQyUyMDMyJTJDJTIwNjQlMkMlMjAxMjglNUQpJTJDJTBBJTIwJTIwJTIwJTIwJTdE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">optuna_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;learning_rate&quot;</span>: trial.suggest_float(<span class="hljs-string">&quot;learning_rate&quot;</span>, <span class="hljs-number">1e-6</span>, <span class="hljs-number">1e-4</span>, log=<span class="hljs-literal">True</span>),
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>: trial.suggest_categorical(<span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>, [<span class="hljs-number">16</span>, <span class="hljs-number">32</span>, <span class="hljs-number">64</span>, <span class="hljs-number">128</span>]),
<span class="hljs-meta">... </span>    }`}}),L=new S({props:{code:"ZGVmJTIwcmF5X2hwX3NwYWNlKHRyaWFsKSUzQSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmxlYXJuaW5nX3JhdGUlMjIlM0ElMjB0dW5lLmxvZ3VuaWZvcm0oMWUtNiUyQyUyMDFlLTQpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIyJTNBJTIwdHVuZS5jaG9pY2UoJTVCMTYlMkMlMjAzMiUyQyUyMDY0JTJDJTIwMTI4JTVEKSUyQyUwQSUyMCUyMCUyMCUyMCU3RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">ray_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;learning_rate&quot;</span>: tune.loguniform(<span class="hljs-number">1e-6</span>, <span class="hljs-number">1e-4</span>),
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>: tune.choice([<span class="hljs-number">16</span>, <span class="hljs-number">32</span>, <span class="hljs-number">64</span>, <span class="hljs-number">128</span>]),
<span class="hljs-meta">... </span>    }`}}),sa=new S({props:{code:"ZGVmJTIwd2FuZGJfaHBfc3BhY2UodHJpYWwpJTNBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybWV0aG9kJTIyJTNBJTIwJTIycmFuZG9tJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybWV0cmljJTIyJTNBJTIwJTdCJTIybmFtZSUyMiUzQSUyMCUyMm9iamVjdGl2ZSUyMiUyQyUyMCUyMmdvYWwlMjIlM0ElMjAlMjJtaW5pbWl6ZSUyMiU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnBhcmFtZXRlcnMlMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJsZWFybmluZ19yYXRlJTIyJTNBJTIwJTdCJTIyZGlzdHJpYnV0aW9uJTIyJTNBJTIwJTIydW5pZm9ybSUyMiUyQyUyMCUyMm1pbiUyMiUzQSUyMDFlLTYlMkMlMjAlMjJtYXglMjIlM0ElMjAxZS00JTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIyJTNBJTIwJTdCJTIydmFsdWVzJTIyJTNBJTIwJTVCMTYlMkMlMjAzMiUyQyUyMDY0JTJDJTIwMTI4JTVEJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTdE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">wandb_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;method&quot;</span>: <span class="hljs-string">&quot;random&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;metric&quot;</span>: {<span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;objective&quot;</span>, <span class="hljs-string">&quot;goal&quot;</span>: <span class="hljs-string">&quot;minimize&quot;</span>},
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;parameters&quot;</span>: {
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;learning_rate&quot;</span>: {<span class="hljs-string">&quot;distribution&quot;</span>: <span class="hljs-string">&quot;uniform&quot;</span>, <span class="hljs-string">&quot;min&quot;</span>: <span class="hljs-number">1e-6</span>, <span class="hljs-string">&quot;max&quot;</span>: <span class="hljs-number">1e-4</span>},
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>: {<span class="hljs-string">&quot;values&quot;</span>: [<span class="hljs-number">16</span>, <span class="hljs-number">32</span>, <span class="hljs-number">64</span>, <span class="hljs-number">128</span>]},
<span class="hljs-meta">... </span>        },
<span class="hljs-meta">... </span>    }`}}),ea=new S({props:{code:"ZGVmJTIwbW9kZWxfaW5pdCh0cmlhbCklM0ElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBBdXRvTW9kZWxGb3JTZXF1ZW5jZUNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbF9hcmdzLm1vZGVsX25hbWVfb3JfcGF0aCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZyb21fdGYlM0Rib29sKCUyMi5ja3B0JTIyJTIwaW4lMjBtb2RlbF9hcmdzLm1vZGVsX25hbWVfb3JfcGF0aCklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb25maWclM0Rjb25maWclMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjYWNoZV9kaXIlM0Rtb2RlbF9hcmdzLmNhY2hlX2RpciUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldmlzaW9uJTNEbW9kZWxfYXJncy5tb2RlbF9yZXZpc2lvbiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHVzZV9hdXRoX3Rva2VuJTNEVHJ1ZSUyMGlmJTIwbW9kZWxfYXJncy51c2VfYXV0aF90b2tlbiUyMGVsc2UlMjBOb25lJTJDJTBBJTIwJTIwJTIwJTIwKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">model_init</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> AutoModelForSequenceClassification.from_pretrained(
<span class="hljs-meta">... </span>        model_args.model_name_or_path,
<span class="hljs-meta">... </span>        from_tf=<span class="hljs-built_in">bool</span>(<span class="hljs-string">&quot;.ckpt&quot;</span> <span class="hljs-keyword">in</span> model_args.model_name_or_path),
<span class="hljs-meta">... </span>        config=config,
<span class="hljs-meta">... </span>        cache_dir=model_args.cache_dir,
<span class="hljs-meta">... </span>        revision=model_args.model_revision,
<span class="hljs-meta">... </span>        use_auth_token=<span class="hljs-literal">True</span> <span class="hljs-keyword">if</span> model_args.use_auth_token <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span>,
<span class="hljs-meta">... </span>    )`}}),ta=new S({props:{code:"dHJhaW5lciUyMCUzRCUyMFRyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0ROb25lJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEc21hbGxfdHJhaW5fZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfZGF0YXNldCUzRHNtYWxsX2V2YWxfZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMGNvbXB1dGVfbWV0cmljcyUzRGNvbXB1dGVfbWV0cmljcyUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX2luaXQlM0Rtb2RlbF9pbml0JTJDJTBBJTIwJTIwJTIwJTIwZGF0YV9jb2xsYXRvciUzRGRhdGFfY29sbGF0b3IlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=<span class="hljs-literal">None</span>,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=small_train_dataset,
<span class="hljs-meta">... </span>    eval_dataset=small_eval_dataset,
<span class="hljs-meta">... </span>    compute_metrics=compute_metrics,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    model_init=model_init,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)`}}),la=new S({props:{code:"YmVzdF90cmlhbCUyMCUzRCUyMHRyYWluZXIuaHlwZXJwYXJhbWV0ZXJfc2VhcmNoKCUwQSUyMCUyMCUyMCUyMGRpcmVjdGlvbiUzRCUyMm1heGltaXplJTIyJTJDJTBBJTIwJTIwJTIwJTIwYmFja2VuZCUzRCUyMm9wdHVuYSUyMiUyQyUwQSUyMCUyMCUyMCUyMGhwX3NwYWNlJTNEb3B0dW5hX2hwX3NwYWNlJTJDJTBBJTIwJTIwJTIwJTIwbl90cmlhbHMlM0QyMCUyQyUwQSUyMCUyMCUyMCUyMGNvbXB1dGVfb2JqZWN0aXZlJTNEY29tcHV0ZV9vYmplY3RpdmUlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>best_trial = trainer.hyperparameter_search(
<span class="hljs-meta">... </span>    direction=<span class="hljs-string">&quot;maximize&quot;</span>,
<span class="hljs-meta">... </span>    backend=<span class="hljs-string">&quot;optuna&quot;</span>,
<span class="hljs-meta">... </span>    hp_space=optuna_hp_space,
<span class="hljs-meta">... </span>    n_trials=<span class="hljs-number">20</span>,
<span class="hljs-meta">... </span>    compute_objective=compute_objective,
<span class="hljs-meta">... </span>)`}}),ra=new ys({}),{c(){_=n("meta"),qa=h(),v=n("h1"),$=n("a"),Ja=n("span"),f(V.$$.fragment),us=h(),wa=n("span"),Ms=t("Hyperparameter Search using Trainer API"),Xa=h(),I=n("p"),fs=t("\u{1F917} Transformers provides a "),na=n("a"),Js=t("Trainer"),ws=t(" class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),pa=n("a"),Ts=t("Trainer"),ds=t(" provides API for hyperparameter search. This doc shows how to enable it in example."),Wa=h(),g=n("h2"),A=n("a"),Ta=n("span"),f(N.$$.fragment),js=h(),da=n("span"),Us=t("Hyperparameter Search backend"),Za=h(),u=n("p"),oa=n("a"),Is=t("Trainer"),bs=t(` supports four hyperparameter search backends currently:
`),Q=n("a"),Cs=t("optuna"),_s=t(", "),F=n("a"),vs=t("sigopt"),gs=t(", "),R=n("a"),ks=t("raytune"),Es=t(" and "),G=n("a"),$s=t("wandb"),As=t("."),Ba=h(),ia=n("p"),qs=t("you should install them before using them as the hyperparameter search backend"),Ya=h(),f(z.$$.fragment),Sa=h(),k=n("h2"),q=n("a"),ja=n("span"),f(H.$$.fragment),Xs=h(),Ua=n("span"),Ws=t("How to enable Hyperparameter search in example"),Va=h(),ca=n("p"),Zs=t("Define the hyperparameter search space, different backends need different format."),Na=h(),X=n("p"),Bs=t("For sigopt, see sigopt "),x=n("a"),Ys=t("object_parameter"),Ss=t(", it\u2019s like following:"),Qa=h(),f(D.$$.fragment),Fa=h(),W=n("p"),Vs=t("For optuna, see optuna "),P=n("a"),Ns=t("object_parameter"),Qs=t(", it\u2019s like following:"),Ra=h(),f(O.$$.fragment),Ga=h(),Z=n("p"),Fs=t("For raytune, see raytune "),K=n("a"),Rs=t("object_parameter"),Gs=t(", it\u2019s like following:"),za=h(),f(L.$$.fragment),Ha=h(),B=n("p"),zs=t("For wandb, see wandb "),aa=n("a"),Hs=t("object_parameter"),xs=t(", it\u2019s like following:"),xa=h(),f(sa.$$.fragment),Da=h(),b=n("p"),Ds=t("Define a "),Ia=n("code"),Ps=t("model_init"),Os=t(" function and pass it to the "),ha=n("a"),Ks=t("Trainer"),Ls=t(", as an example:"),Pa=h(),f(ea.$$.fragment),Oa=h(),C=n("p"),ae=t("Create a "),ma=n("a"),se=t("Trainer"),ee=t(" with your "),ba=n("code"),te=t("model_init"),le=t(" function, training arguments, training and test datasets, and evaluation function:"),Ka=h(),f(ta.$$.fragment),La=h(),y=n("p"),re=t("Call hyperparameter search, get the best trial parameters, backend could be "),Ca=n("code"),ne=t('"optuna"'),pe=t("/"),_a=n("code"),oe=t('"sigopt"'),ie=t("/"),va=n("code"),ce=t('"wandb"'),he=t("/"),ga=n("code"),me=t('"ray"'),ye=t(". direction can be"),ka=n("code"),ue=t('"minimize"'),Me=t(" or "),Ea=n("code"),fe=t('"maximize"'),Je=t(", which indicates whether to optimize greater or lower objective."),as=h(),ya=n("p"),we=t("You could define your own compute_objective function, if not defined, the default compute_objective will be called, and the sum of eval metric like f1 is returned as objective value."),ss=h(),f(la.$$.fragment),es=h(),E=n("h2"),Y=n("a"),$a=n("span"),f(ra.$$.fragment),Te=h(),Aa=n("span"),de=t("Hyperparameter search For DDP finetune"),ts=t(`

Currently, Hyperparameter search for DDP is enabled for optuna and sigopt. Only the rank-zero process will generate the search trial and pass the argument to other ranks.`),this.h()},l(a){const r=pt('[data-svelte="svelte-1phssyn"]',document.head);_=p(r,"META",{name:!0,content:!0}),r.forEach(e),qa=m(a),v=p(a,"H1",{class:!0});var rs=o(v);$=p(rs,"A",{id:!0,class:!0,href:!0});var Ue=o($);Ja=p(Ue,"SPAN",{});var Ie=o(Ja);J(V.$$.fragment,Ie),Ie.forEach(e),Ue.forEach(e),us=m(rs),wa=p(rs,"SPAN",{});var be=o(wa);Ms=l(be,"Hyperparameter Search using Trainer API"),be.forEach(e),rs.forEach(e),Xa=m(a),I=p(a,"P",{});var ua=o(I);fs=l(ua,"\u{1F917} Transformers provides a "),na=p(ua,"A",{href:!0});var Ce=o(na);Js=l(Ce,"Trainer"),Ce.forEach(e),ws=l(ua," class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),pa=p(ua,"A",{href:!0});var _e=o(pa);Ts=l(_e,"Trainer"),_e.forEach(e),ds=l(ua," provides API for hyperparameter search. This doc shows how to enable it in example."),ua.forEach(e),Wa=m(a),g=p(a,"H2",{class:!0});var ns=o(g);A=p(ns,"A",{id:!0,class:!0,href:!0});var ve=o(A);Ta=p(ve,"SPAN",{});var ge=o(Ta);J(N.$$.fragment,ge),ge.forEach(e),ve.forEach(e),js=m(ns),da=p(ns,"SPAN",{});var ke=o(da);Us=l(ke,"Hyperparameter Search backend"),ke.forEach(e),ns.forEach(e),Za=m(a),u=p(a,"P",{});var U=o(u);oa=p(U,"A",{href:!0});var Ee=o(oa);Is=l(Ee,"Trainer"),Ee.forEach(e),bs=l(U,` supports four hyperparameter search backends currently:
`),Q=p(U,"A",{href:!0,rel:!0});var $e=o(Q);Cs=l($e,"optuna"),$e.forEach(e),_s=l(U,", "),F=p(U,"A",{href:!0,rel:!0});var Ae=o(F);vs=l(Ae,"sigopt"),Ae.forEach(e),gs=l(U,", "),R=p(U,"A",{href:!0,rel:!0});var qe=o(R);ks=l(qe,"raytune"),qe.forEach(e),Es=l(U," and "),G=p(U,"A",{href:!0,rel:!0});var Xe=o(G);$s=l(Xe,"wandb"),Xe.forEach(e),As=l(U,"."),U.forEach(e),Ba=m(a),ia=p(a,"P",{});var We=o(ia);qs=l(We,"you should install them before using them as the hyperparameter search backend"),We.forEach(e),Ya=m(a),J(z.$$.fragment,a),Sa=m(a),k=p(a,"H2",{class:!0});var ps=o(k);q=p(ps,"A",{id:!0,class:!0,href:!0});var Ze=o(q);ja=p(Ze,"SPAN",{});var Be=o(ja);J(H.$$.fragment,Be),Be.forEach(e),Ze.forEach(e),Xs=m(ps),Ua=p(ps,"SPAN",{});var Ye=o(Ua);Ws=l(Ye,"How to enable Hyperparameter search in example"),Ye.forEach(e),ps.forEach(e),Va=m(a),ca=p(a,"P",{});var Se=o(ca);Zs=l(Se,"Define the hyperparameter search space, different backends need different format."),Se.forEach(e),Na=m(a),X=p(a,"P",{});var os=o(X);Bs=l(os,"For sigopt, see sigopt "),x=p(os,"A",{href:!0,rel:!0});var Ve=o(x);Ys=l(Ve,"object_parameter"),Ve.forEach(e),Ss=l(os,", it\u2019s like following:"),os.forEach(e),Qa=m(a),J(D.$$.fragment,a),Fa=m(a),W=p(a,"P",{});var is=o(W);Vs=l(is,"For optuna, see optuna "),P=p(is,"A",{href:!0,rel:!0});var Ne=o(P);Ns=l(Ne,"object_parameter"),Ne.forEach(e),Qs=l(is,", it\u2019s like following:"),is.forEach(e),Ra=m(a),J(O.$$.fragment,a),Ga=m(a),Z=p(a,"P",{});var cs=o(Z);Fs=l(cs,"For raytune, see raytune "),K=p(cs,"A",{href:!0,rel:!0});var Qe=o(K);Rs=l(Qe,"object_parameter"),Qe.forEach(e),Gs=l(cs,", it\u2019s like following:"),cs.forEach(e),za=m(a),J(L.$$.fragment,a),Ha=m(a),B=p(a,"P",{});var hs=o(B);zs=l(hs,"For wandb, see wandb "),aa=p(hs,"A",{href:!0,rel:!0});var Fe=o(aa);Hs=l(Fe,"object_parameter"),Fe.forEach(e),xs=l(hs,", it\u2019s like following:"),hs.forEach(e),xa=m(a),J(sa.$$.fragment,a),Da=m(a),b=p(a,"P",{});var Ma=o(b);Ds=l(Ma,"Define a "),Ia=p(Ma,"CODE",{});var Re=o(Ia);Ps=l(Re,"model_init"),Re.forEach(e),Os=l(Ma," function and pass it to the "),ha=p(Ma,"A",{href:!0});var Ge=o(ha);Ks=l(Ge,"Trainer"),Ge.forEach(e),Ls=l(Ma,", as an example:"),Ma.forEach(e),Pa=m(a),J(ea.$$.fragment,a),Oa=m(a),C=p(a,"P",{});var fa=o(C);ae=l(fa,"Create a "),ma=p(fa,"A",{href:!0});var ze=o(ma);se=l(ze,"Trainer"),ze.forEach(e),ee=l(fa," with your "),ba=p(fa,"CODE",{});var He=o(ba);te=l(He,"model_init"),He.forEach(e),le=l(fa," function, training arguments, training and test datasets, and evaluation function:"),fa.forEach(e),Ka=m(a),J(ta.$$.fragment,a),La=m(a),y=p(a,"P",{});var M=o(y);re=l(M,"Call hyperparameter search, get the best trial parameters, backend could be "),Ca=p(M,"CODE",{});var xe=o(Ca);ne=l(xe,'"optuna"'),xe.forEach(e),pe=l(M,"/"),_a=p(M,"CODE",{});var De=o(_a);oe=l(De,'"sigopt"'),De.forEach(e),ie=l(M,"/"),va=p(M,"CODE",{});var Pe=o(va);ce=l(Pe,'"wandb"'),Pe.forEach(e),he=l(M,"/"),ga=p(M,"CODE",{});var Oe=o(ga);me=l(Oe,'"ray"'),Oe.forEach(e),ye=l(M,". direction can be"),ka=p(M,"CODE",{});var Ke=o(ka);ue=l(Ke,'"minimize"'),Ke.forEach(e),Me=l(M," or "),Ea=p(M,"CODE",{});var Le=o(Ea);fe=l(Le,'"maximize"'),Le.forEach(e),Je=l(M,", which indicates whether to optimize greater or lower objective."),M.forEach(e),as=m(a),ya=p(a,"P",{});var at=o(ya);we=l(at,"You could define your own compute_objective function, if not defined, the default compute_objective will be called, and the sum of eval metric like f1 is returned as objective value."),at.forEach(e),ss=m(a),J(la.$$.fragment,a),es=m(a),E=p(a,"H2",{class:!0});var ms=o(E);Y=p(ms,"A",{id:!0,class:!0,href:!0});var st=o(Y);$a=p(st,"SPAN",{});var et=o($a);J(ra.$$.fragment,et),et.forEach(e),st.forEach(e),Te=m(ms),Aa=p(ms,"SPAN",{});var tt=o(Aa);de=l(tt,"Hyperparameter search For DDP finetune"),tt.forEach(e),ms.forEach(e),ts=l(a,`

Currently, Hyperparameter search for DDP is enabled for optuna and sigopt. Only the rank-zero process will generate the search trial and pass the argument to other ranks.`),this.h()},h(){c(_,"name","hf:doc:metadata"),c(_,"content",JSON.stringify(ht)),c($,"id","hyperparameter-search-using-trainer-api"),c($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c($,"href","#hyperparameter-search-using-trainer-api"),c(v,"class","relative group"),c(na,"href","/docs/transformers/v4.32.0/en/main_classes/trainer#transformers.Trainer"),c(pa,"href","/docs/transformers/v4.32.0/en/main_classes/trainer#transformers.Trainer"),c(A,"id","hyperparameter-search-backend"),c(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(A,"href","#hyperparameter-search-backend"),c(g,"class","relative group"),c(oa,"href","/docs/transformers/v4.32.0/en/main_classes/trainer#transformers.Trainer"),c(Q,"href","https://optuna.org/"),c(Q,"rel","nofollow"),c(F,"href","https://sigopt.com/"),c(F,"rel","nofollow"),c(R,"href","https://docs.ray.io/en/latest/tune/index.html"),c(R,"rel","nofollow"),c(G,"href","https://wandb.ai/site/sweeps"),c(G,"rel","nofollow"),c(q,"id","how-to-enable-hyperparameter-search-in-example"),c(q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(q,"href","#how-to-enable-hyperparameter-search-in-example"),c(k,"class","relative group"),c(x,"href","https://docs.sigopt.com/ai-module-api-references/api_reference/objects/object_parameter"),c(x,"rel","nofollow"),c(P,"href","https://optuna.readthedocs.io/en/stable/tutorial/10_key_features/002_configurations.html#sphx-glr-tutorial-10-key-features-002-configurations-py"),c(P,"rel","nofollow"),c(K,"href","https://docs.ray.io/en/latest/tune/api/search_space.html"),c(K,"rel","nofollow"),c(aa,"href","https://docs.wandb.ai/guides/sweeps/configuration"),c(aa,"rel","nofollow"),c(ha,"href","/docs/transformers/v4.32.0/en/main_classes/trainer#transformers.Trainer"),c(ma,"href","/docs/transformers/v4.32.0/en/main_classes/trainer#transformers.Trainer"),c(Y,"id","hyperparameter-search-for-ddp-finetune"),c(Y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Y,"href","#hyperparameter-search-for-ddp-finetune"),c(E,"class","relative group")},m(a,r){s(document.head,_),i(a,qa,r),i(a,v,r),s(v,$),s($,Ja),w(V,Ja,null),s(v,us),s(v,wa),s(wa,Ms),i(a,Xa,r),i(a,I,r),s(I,fs),s(I,na),s(na,Js),s(I,ws),s(I,pa),s(pa,Ts),s(I,ds),i(a,Wa,r),i(a,g,r),s(g,A),s(A,Ta),w(N,Ta,null),s(g,js),s(g,da),s(da,Us),i(a,Za,r),i(a,u,r),s(u,oa),s(oa,Is),s(u,bs),s(u,Q),s(Q,Cs),s(u,_s),s(u,F),s(F,vs),s(u,gs),s(u,R),s(R,ks),s(u,Es),s(u,G),s(G,$s),s(u,As),i(a,Ba,r),i(a,ia,r),s(ia,qs),i(a,Ya,r),w(z,a,r),i(a,Sa,r),i(a,k,r),s(k,q),s(q,ja),w(H,ja,null),s(k,Xs),s(k,Ua),s(Ua,Ws),i(a,Va,r),i(a,ca,r),s(ca,Zs),i(a,Na,r),i(a,X,r),s(X,Bs),s(X,x),s(x,Ys),s(X,Ss),i(a,Qa,r),w(D,a,r),i(a,Fa,r),i(a,W,r),s(W,Vs),s(W,P),s(P,Ns),s(W,Qs),i(a,Ra,r),w(O,a,r),i(a,Ga,r),i(a,Z,r),s(Z,Fs),s(Z,K),s(K,Rs),s(Z,Gs),i(a,za,r),w(L,a,r),i(a,Ha,r),i(a,B,r),s(B,zs),s(B,aa),s(aa,Hs),s(B,xs),i(a,xa,r),w(sa,a,r),i(a,Da,r),i(a,b,r),s(b,Ds),s(b,Ia),s(Ia,Ps),s(b,Os),s(b,ha),s(ha,Ks),s(b,Ls),i(a,Pa,r),w(ea,a,r),i(a,Oa,r),i(a,C,r),s(C,ae),s(C,ma),s(ma,se),s(C,ee),s(C,ba),s(ba,te),s(C,le),i(a,Ka,r),w(ta,a,r),i(a,La,r),i(a,y,r),s(y,re),s(y,Ca),s(Ca,ne),s(y,pe),s(y,_a),s(_a,oe),s(y,ie),s(y,va),s(va,ce),s(y,he),s(y,ga),s(ga,me),s(y,ye),s(y,ka),s(ka,ue),s(y,Me),s(y,Ea),s(Ea,fe),s(y,Je),i(a,as,r),i(a,ya,r),s(ya,we),i(a,ss,r),w(la,a,r),i(a,es,r),i(a,E,r),s(E,Y),s(Y,$a),w(ra,$a,null),s(E,Te),s(E,Aa),s(Aa,de),i(a,ts,r),ls=!0},p:ot,i(a){ls||(T(V.$$.fragment,a),T(N.$$.fragment,a),T(z.$$.fragment,a),T(H.$$.fragment,a),T(D.$$.fragment,a),T(O.$$.fragment,a),T(L.$$.fragment,a),T(sa.$$.fragment,a),T(ea.$$.fragment,a),T(ta.$$.fragment,a),T(la.$$.fragment,a),T(ra.$$.fragment,a),ls=!0)},o(a){d(V.$$.fragment,a),d(N.$$.fragment,a),d(z.$$.fragment,a),d(H.$$.fragment,a),d(D.$$.fragment,a),d(O.$$.fragment,a),d(L.$$.fragment,a),d(sa.$$.fragment,a),d(ea.$$.fragment,a),d(ta.$$.fragment,a),d(la.$$.fragment,a),d(ra.$$.fragment,a),ls=!1},d(a){e(_),a&&e(qa),a&&e(v),j(V),a&&e(Xa),a&&e(I),a&&e(Wa),a&&e(g),j(N),a&&e(Za),a&&e(u),a&&e(Ba),a&&e(ia),a&&e(Ya),j(z,a),a&&e(Sa),a&&e(k),j(H),a&&e(Va),a&&e(ca),a&&e(Na),a&&e(X),a&&e(Qa),j(D,a),a&&e(Fa),a&&e(W),a&&e(Ra),j(O,a),a&&e(Ga),a&&e(Z),a&&e(za),j(L,a),a&&e(Ha),a&&e(B),a&&e(xa),j(sa,a),a&&e(Da),a&&e(b),a&&e(Pa),j(ea,a),a&&e(Oa),a&&e(C),a&&e(Ka),j(ta,a),a&&e(La),a&&e(y),a&&e(as),a&&e(ya),a&&e(ss),j(la,a),a&&e(es),a&&e(E),j(ra),a&&e(ts)}}}const ht={local:"hyperparameter-search-using-trainer-api",sections:[{local:"hyperparameter-search-backend",title:"Hyperparameter Search backend"},{local:"how-to-enable-hyperparameter-search-in-example",title:"How to enable Hyperparameter search in example"},{local:"hyperparameter-search-for-ddp-finetune",title:"Hyperparameter search For DDP finetune"}],title:"Hyperparameter Search using Trainer API"};function mt(je){return it(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ft extends lt{constructor(_){super();rt(this,_,mt,ct,nt,{})}}export{ft as default,ht as metadata};
