import{s as pt,n as it,o as ct}from"../chunks/scheduler.bdbef820.js";import{S as rt,i as ot,g as p,s as l,r,A as dt,h as i,f as a,c as n,j as nt,u as o,x as c,k as J,y as mt,a as e,v as d,d as m,t as h,w as u}from"../chunks/index.c0aea24a.js";import{C as g}from"../chunks/CodeBlock.6ccca92e.js";import{H as Cs,E as ht}from"../chunks/EditOnGithub.725ee0c1.js";function ut(Vs){let M,ts,O,as,U,es,X,Qs='<a href="https://github.com/facebookresearch/faiss" rel="nofollow">FAISS</a> and <a href="https://www.elastic.co/elasticsearch/" rel="nofollow">Elasticsearch</a> enables searching for examples in a dataset. This can be useful when you want to retrieve specific examples from a dataset that are relevant to your NLP task. For example, if you are working on an Open Domain Question Answering task, you may want to only return examples that are relevant to answering your question.',ls,k,zs="This guide will show you how to build an index for your dataset that will allow you to search it.",ns,W,ps,q,Es='FAISS retrieves documents based on the similarity of their vector representations. In this example, you will generate the vector representations with the <a href="https://huggingface.co/transformers/model_doc/dpr.html" rel="nofollow">DPR</a> model.',is,R,Ys="<li>Download the DPR model from 🤗 Transformers:</li>",cs,$,rs,j,Bs="<li>Load your dataset and compute the vector representations:</li>",os,F,ds,y,Hs='<li>Create the index with <a href="/docs/datasets/v3.0.2/en/package_reference/main_classes#datasets.Dataset.add_faiss_index">Dataset.add_faiss_index()</a>:</li>',ms,N,hs,f,Ls='<li>Now you can query your dataset with the <code>embeddings</code> index. Load the DPR Question Encoder, and search for a question with <a href="/docs/datasets/v3.0.2/en/package_reference/main_classes#datasets.Dataset.get_nearest_examples">Dataset.get_nearest_examples()</a>:</li>',us,v,gs,T,Ds='<li>You can access the index with <a href="/docs/datasets/v3.0.2/en/package_reference/main_classes#datasets.Dataset.get_index">Dataset.get_index()</a> and use it for special operations, e.g. query it using <code>range_search</code>:</li>',Js,G,Ms,w,Ss='<li>When you are done querying, save the index on disk with <a href="/docs/datasets/v3.0.2/en/package_reference/main_classes#datasets.Dataset.save_faiss_index">Dataset.save_faiss_index()</a>:</li>',js,C,ys,b,As='<li>Reload it at a later time with <a href="/docs/datasets/v3.0.2/en/package_reference/main_classes#datasets.Dataset.load_faiss_index">Dataset.load_faiss_index()</a>:</li>',fs,V,Ts,Q,ws,z,Ps="Unlike FAISS, Elasticsearch retrieves documents based on exact matches.",bs,E,Ks='Start Elasticsearch on your machine, or see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html" rel="nofollow">Elasticsearch installation guide</a> if you don’t already have it installed.',xs,Y,Os="<li>Load the dataset you want to index:</li>",_s,B,Zs,x,st='<li>Build the index with <a href="/docs/datasets/v3.0.2/en/package_reference/main_classes#datasets.Dataset.add_elasticsearch_index">Dataset.add_elasticsearch_index()</a>:</li>',Is,H,Us,_,tt='<li>Then you can query the <code>context</code> index with <a href="/docs/datasets/v3.0.2/en/package_reference/main_classes#datasets.Dataset.get_nearest_examples">Dataset.get_nearest_examples()</a>:</li>',Xs,L,ks,Z,at="<li>If you want to reuse the index, define the <code>es_index_name</code> parameter when you build the index:</li>",Ws,D,qs,I,et='<li>Reload it later with the index name when you call <a href="/docs/datasets/v3.0.2/en/package_reference/main_classes#datasets.Dataset.load_elasticsearch_index">Dataset.load_elasticsearch_index()</a>:</li>',Rs,S,$s,A,lt="For more advanced Elasticsearch usage, you can specify your own configuration with custom settings:",Fs,P,Ns,K,vs,ss,Gs;return U=new Cs({props:{title:"Search index",local:"search-index",headingTag:"h1"}}),W=new Cs({props:{title:"FAISS",local:"faiss",headingTag:"h2"}}),$=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERQUkNvbnRleHRFbmNvZGVyJTJDJTIwRFBSQ29udGV4dEVuY29kZXJUb2tlbml6ZXIlMEFpbXBvcnQlMjB0b3JjaCUwQXRvcmNoLnNldF9ncmFkX2VuYWJsZWQoRmFsc2UpJTBBY3R4X2VuY29kZXIlMjAlM0QlMjBEUFJDb250ZXh0RW5jb2Rlci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZkcHItY3R4X2VuY29kZXItc2luZ2xlLW5xLWJhc2UlMjIpJTBBY3R4X3Rva2VuaXplciUyMCUzRCUyMERQUkNvbnRleHRFbmNvZGVyVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRmRwci1jdHhfZW5jb2Rlci1zaW5nbGUtbnEtYmFzZSUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DPRContextEncoder, DPRContextEncoderTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span>torch.set_grad_enabled(<span class="hljs-literal">False</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ctx_encoder = DPRContextEncoder.from_pretrained(<span class="hljs-string">&quot;facebook/dpr-ctx_encoder-single-nq-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ctx_tokenizer = DPRContextEncoderTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/dpr-ctx_encoder-single-nq-base&quot;</span>)`,wrap:!1}}),F=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ2NyaW1lX2FuZF9wdW5pc2gnJTJDJTIwc3BsaXQlM0QndHJhaW4lNUIlM0ExMDAlNUQnKSUwQWRzX3dpdGhfZW1iZWRkaW5ncyUyMCUzRCUyMGRzLm1hcChsYW1iZGElMjBleGFtcGxlJTNBJTIwJTdCJ2VtYmVkZGluZ3MnJTNBJTIwY3R4X2VuY29kZXIoKipjdHhfdG9rZW5pemVyKGV4YW1wbGUlNUIlMjJsaW5lJTIyJTVEJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikpJTVCMCU1RCU1QjAlNUQubnVtcHkoKSU3RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_dataset(<span class="hljs-string">&#x27;crime_and_punish&#x27;</span>, split=<span class="hljs-string">&#x27;train[:100]&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds_with_embeddings = ds.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> example: {<span class="hljs-string">&#x27;embeddings&#x27;</span>: ctx_encoder(**ctx_tokenizer(example[<span class="hljs-string">&quot;line&quot;</span>], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>))[<span class="hljs-number">0</span>][<span class="hljs-number">0</span>].numpy()})`,wrap:!1}}),N=new g({props:{code:"ZHNfd2l0aF9lbWJlZGRpbmdzLmFkZF9mYWlzc19pbmRleChjb2x1bW4lM0QnZW1iZWRkaW5ncycp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>ds_with_embeddings.add_faiss_index(column=<span class="hljs-string">&#x27;embeddings&#x27;</span>)',wrap:!1}}),v=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERQUlF1ZXN0aW9uRW5jb2RlciUyQyUyMERQUlF1ZXN0aW9uRW5jb2RlclRva2VuaXplciUwQXFfZW5jb2RlciUyMCUzRCUyMERQUlF1ZXN0aW9uRW5jb2Rlci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZkcHItcXVlc3Rpb25fZW5jb2Rlci1zaW5nbGUtbnEtYmFzZSUyMiklMEFxX3Rva2VuaXplciUyMCUzRCUyMERQUlF1ZXN0aW9uRW5jb2RlclRva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZkcHItcXVlc3Rpb25fZW5jb2Rlci1zaW5nbGUtbnEtYmFzZSUyMiklMEElMEFxdWVzdGlvbiUyMCUzRCUyMCUyMklzJTIwaXQlMjBzZXJpb3VzJTIwJTNGJTIyJTBBcXVlc3Rpb25fZW1iZWRkaW5nJTIwJTNEJTIwcV9lbmNvZGVyKCoqcV90b2tlbml6ZXIocXVlc3Rpb24lMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSklNUIwJTVEJTVCMCU1RC5udW1weSgpJTBBc2NvcmVzJTJDJTIwcmV0cmlldmVkX2V4YW1wbGVzJTIwJTNEJTIwZHNfd2l0aF9lbWJlZGRpbmdzLmdldF9uZWFyZXN0X2V4YW1wbGVzKCdlbWJlZGRpbmdzJyUyQyUyMHF1ZXN0aW9uX2VtYmVkZGluZyUyQyUyMGslM0QxMCklMEFyZXRyaWV2ZWRfZXhhbXBsZXMlNUIlMjJsaW5lJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DPRQuestionEncoder, DPRQuestionEncoderTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span>q_encoder = DPRQuestionEncoder.from_pretrained(<span class="hljs-string">&quot;facebook/dpr-question_encoder-single-nq-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>q_tokenizer = DPRQuestionEncoderTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/dpr-question_encoder-single-nq-base&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>question = <span class="hljs-string">&quot;Is it serious ?&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>question_embedding = q_encoder(**q_tokenizer(question, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>))[<span class="hljs-number">0</span>][<span class="hljs-number">0</span>].numpy()
<span class="hljs-meta">&gt;&gt;&gt; </span>scores, retrieved_examples = ds_with_embeddings.get_nearest_examples(<span class="hljs-string">&#x27;embeddings&#x27;</span>, question_embedding, k=<span class="hljs-number">10</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>retrieved_examples[<span class="hljs-string">&quot;line&quot;</span>][<span class="hljs-number">0</span>]
<span class="hljs-string">&#x27;_that_ serious? It is not serious at all. It’s simply a fantasy to amuse\\r\\n&#x27;</span>`,wrap:!1}}),G=new g({props:{code:"ZmFpc3NfaW5kZXglMjAlM0QlMjBkc193aXRoX2VtYmVkZGluZ3MuZ2V0X2luZGV4KCdlbWJlZGRpbmdzJykuZmFpc3NfaW5kZXglMEFsaW1pdHMlMkMlMjBkaXN0YW5jZXMlMkMlMjBpbmRpY2VzJTIwJTNEJTIwZmFpc3NfaW5kZXgucmFuZ2Vfc2VhcmNoKHglM0RxdWVzdGlvbl9lbWJlZGRpbmcucmVzaGFwZSgxJTJDJTIwLTEpJTJDJTIwdGhyZXNoJTNEMC45NSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>faiss_index = ds_with_embeddings.get_index(<span class="hljs-string">&#x27;embeddings&#x27;</span>).faiss_index
<span class="hljs-meta">&gt;&gt;&gt; </span>limits, distances, indices = faiss_index.range_search(x=question_embedding.reshape(<span class="hljs-number">1</span>, -<span class="hljs-number">1</span>), thresh=<span class="hljs-number">0.95</span>)`,wrap:!1}}),C=new g({props:{code:"ZHNfd2l0aF9lbWJlZGRpbmdzLnNhdmVfZmFpc3NfaW5kZXgoJ2VtYmVkZGluZ3MnJTJDJTIwJ215X2luZGV4LmZhaXNzJyk=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>ds_with_embeddings.save_faiss_index(<span class="hljs-string">&#x27;embeddings&#x27;</span>, <span class="hljs-string">&#x27;my_index.faiss&#x27;</span>)',wrap:!1}}),V=new g({props:{code:"ZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ2NyaW1lX2FuZF9wdW5pc2gnJTJDJTIwc3BsaXQlM0QndHJhaW4lNUIlM0ExMDAlNUQnKSUwQWRzLmxvYWRfZmFpc3NfaW5kZXgoJ2VtYmVkZGluZ3MnJTJDJTIwJ215X2luZGV4LmZhaXNzJyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_dataset(<span class="hljs-string">&#x27;crime_and_punish&#x27;</span>, split=<span class="hljs-string">&#x27;train[:100]&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds.load_faiss_index(<span class="hljs-string">&#x27;embeddings&#x27;</span>, <span class="hljs-string">&#x27;my_index.faiss&#x27;</span>)`,wrap:!1}}),Q=new Cs({props:{title:"Elasticsearch",local:"elasticsearch",headingTag:"h2"}}),B=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBc3F1YWQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ3NxdWFkJyUyQyUyMHNwbGl0JTNEJ3ZhbGlkYXRpb24nKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>squad = load_dataset(<span class="hljs-string">&#x27;squad&#x27;</span>, split=<span class="hljs-string">&#x27;validation&#x27;</span>)`,wrap:!1}}),H=new g({props:{code:"c3F1YWQuYWRkX2VsYXN0aWNzZWFyY2hfaW5kZXgoJTIyY29udGV4dCUyMiUyQyUyMGhvc3QlM0QlMjJsb2NhbGhvc3QlMjIlMkMlMjBwb3J0JTNEJTIyOTIwMCUyMik=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>squad.add_elasticsearch_index(<span class="hljs-string">&quot;context&quot;</span>, host=<span class="hljs-string">&quot;localhost&quot;</span>, port=<span class="hljs-string">&quot;9200&quot;</span>)',wrap:!1}}),L=new g({props:{code:"cXVlcnklMjAlM0QlMjAlMjJtYWNoaW5lJTIyJTBBc2NvcmVzJTJDJTIwcmV0cmlldmVkX2V4YW1wbGVzJTIwJTNEJTIwc3F1YWQuZ2V0X25lYXJlc3RfZXhhbXBsZXMoJTIyY29udGV4dCUyMiUyQyUyMHF1ZXJ5JTJDJTIwayUzRDEwKSUwQXJldHJpZXZlZF9leGFtcGxlcyU1QiUyMnRpdGxlJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>query = <span class="hljs-string">&quot;machine&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>scores, retrieved_examples = squad.get_nearest_examples(<span class="hljs-string">&quot;context&quot;</span>, query, k=<span class="hljs-number">10</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>retrieved_examples[<span class="hljs-string">&quot;title&quot;</span>][<span class="hljs-number">0</span>]
<span class="hljs-string">&#x27;Computational_complexity_theory&#x27;</span>`,wrap:!1}}),D=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBc3F1YWQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ3NxdWFkJyUyQyUyMHNwbGl0JTNEJ3ZhbGlkYXRpb24nKSUwQXNxdWFkLmFkZF9lbGFzdGljc2VhcmNoX2luZGV4KCUyMmNvbnRleHQlMjIlMkMlMjBob3N0JTNEJTIybG9jYWxob3N0JTIyJTJDJTIwcG9ydCUzRCUyMjkyMDAlMjIlMkMlMjBlc19pbmRleF9uYW1lJTNEJTIyaGZfc3F1YWRfdmFsX2NvbnRleHQlMjIpJTBBc3F1YWQuZ2V0X2luZGV4KCUyMmNvbnRleHQlMjIpLmVzX2luZGV4X25hbWU=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>squad = load_dataset(<span class="hljs-string">&#x27;squad&#x27;</span>, split=<span class="hljs-string">&#x27;validation&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>squad.add_elasticsearch_index(<span class="hljs-string">&quot;context&quot;</span>, host=<span class="hljs-string">&quot;localhost&quot;</span>, port=<span class="hljs-string">&quot;9200&quot;</span>, es_index_name=<span class="hljs-string">&quot;hf_squad_val_context&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>squad.get_index(<span class="hljs-string">&quot;context&quot;</span>).es_index_name
hf_squad_val_context`,wrap:!1}}),S=new g({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBc3F1YWQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJ3NxdWFkJyUyQyUyMHNwbGl0JTNEJ3ZhbGlkYXRpb24nKSUwQXNxdWFkLmxvYWRfZWxhc3RpY3NlYXJjaF9pbmRleCglMjJjb250ZXh0JTIyJTJDJTIwaG9zdCUzRCUyMmxvY2FsaG9zdCUyMiUyQyUyMHBvcnQlM0QlMjI5MjAwJTIyJTJDJTIwZXNfaW5kZXhfbmFtZSUzRCUyMmhmX3NxdWFkX3ZhbF9jb250ZXh0JTIyKSUwQXF1ZXJ5JTIwJTNEJTIwJTIybWFjaGluZSUyMiUwQXNjb3JlcyUyQyUyMHJldHJpZXZlZF9leGFtcGxlcyUyMCUzRCUyMHNxdWFkLmdldF9uZWFyZXN0X2V4YW1wbGVzKCUyMmNvbnRleHQlMjIlMkMlMjBxdWVyeSUyQyUyMGslM0QxMCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>squad = load_dataset(<span class="hljs-string">&#x27;squad&#x27;</span>, split=<span class="hljs-string">&#x27;validation&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>squad.load_elasticsearch_index(<span class="hljs-string">&quot;context&quot;</span>, host=<span class="hljs-string">&quot;localhost&quot;</span>, port=<span class="hljs-string">&quot;9200&quot;</span>, es_index_name=<span class="hljs-string">&quot;hf_squad_val_context&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>query = <span class="hljs-string">&quot;machine&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>scores, retrieved_examples = squad.get_nearest_examples(<span class="hljs-string">&quot;context&quot;</span>, query, k=<span class="hljs-number">10</span>)`,wrap:!1}}),P=new g({props:{code:"aW1wb3J0JTIwZWxhc3RpY3NlYXJjaCUyMGFzJTIwZXMlMEFpbXBvcnQlMjBlbGFzdGljc2VhcmNoLmhlbHBlcnMlMEFmcm9tJTIwZWxhc3RpY3NlYXJjaCUyMGltcG9ydCUyMEVsYXN0aWNzZWFyY2glMEFlc19jbGllbnQlMjAlM0QlMjBFbGFzdGljc2VhcmNoKCU1QiU3QiUyMmhvc3QlMjIlM0ElMjAlMjJsb2NhbGhvc3QlMjIlMkMlMjAlMjJwb3J0JTIyJTNBJTIwJTIyOTIwMCUyMiU3RCU1RCklMjAlMjAlMjMlMjBkZWZhdWx0JTIwY2xpZW50JTBBZXNfY29uZmlnJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIyc2V0dGluZ3MlMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJudW1iZXJfb2Zfc2hhcmRzJTIyJTNBJTIwMSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmFuYWx5c2lzJTIyJTNBJTIwJTdCJTIyYW5hbHl6ZXIlMjIlM0ElMjAlN0IlMjJzdG9wX3N0YW5kYXJkJTIyJTNBJTIwJTdCJTIydHlwZSUyMiUzQSUyMCUyMnN0YW5kYXJkJTIyJTJDJTIwJTIyJTIwc3RvcHdvcmRzJTIyJTNBJTIwJTIyX2VuZ2xpc2hfJTIyJTdEJTdEJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIybWFwcGluZ3MlMjIlM0ElMjAlN0IlMjJwcm9wZXJ0aWVzJTIyJTNBJTIwJTdCJTIydGV4dCUyMiUzQSUyMCU3QiUyMnR5cGUlMjIlM0ElMjAlMjJ0ZXh0JTIyJTJDJTIwJTIyYW5hbHl6ZXIlMjIlM0ElMjAlMjJzdGFuZGFyZCUyMiUyQyUyMCUyMnNpbWlsYXJpdHklMjIlM0ElMjAlMjJCTTI1JTIyJTdEJTdEJTdEJTJDJTBBJTdEJTIwJTIwJTIzJTIwZGVmYXVsdCUyMGNvbmZpZyUwQWVzX2luZGV4X25hbWUlMjAlM0QlMjAlMjJoZl9zcXVhZF9jb250ZXh0JTIyJTIwJTIwJTIzJTIwbmFtZSUyMG9mJTIwdGhlJTIwaW5kZXglMjBpbiUyMEVsYXN0aWNzZWFyY2glMEFzcXVhZC5hZGRfZWxhc3RpY3NlYXJjaF9pbmRleCglMjJjb250ZXh0JTIyJTJDJTIwZXNfY2xpZW50JTNEZXNfY2xpZW50JTJDJTIwZXNfY29uZmlnJTNEZXNfY29uZmlnJTJDJTIwZXNfaW5kZXhfbmFtZSUzRGVzX2luZGV4X25hbWUp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> elasticsearch <span class="hljs-keyword">as</span> es
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> elasticsearch.helpers
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> elasticsearch <span class="hljs-keyword">import</span> Elasticsearch
<span class="hljs-meta">&gt;&gt;&gt; </span>es_client = Elasticsearch([{<span class="hljs-string">&quot;host&quot;</span>: <span class="hljs-string">&quot;localhost&quot;</span>, <span class="hljs-string">&quot;port&quot;</span>: <span class="hljs-string">&quot;9200&quot;</span>}])  <span class="hljs-comment"># default client</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>es_config = {
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;settings&quot;</span>: {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;number_of_shards&quot;</span>: <span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;analysis&quot;</span>: {<span class="hljs-string">&quot;analyzer&quot;</span>: {<span class="hljs-string">&quot;stop_standard&quot;</span>: {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;standard&quot;</span>, <span class="hljs-string">&quot; stopwords&quot;</span>: <span class="hljs-string">&quot;_english_&quot;</span>}}},
<span class="hljs-meta">... </span>    },
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;mappings&quot;</span>: {<span class="hljs-string">&quot;properties&quot;</span>: {<span class="hljs-string">&quot;text&quot;</span>: {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;analyzer&quot;</span>: <span class="hljs-string">&quot;standard&quot;</span>, <span class="hljs-string">&quot;similarity&quot;</span>: <span class="hljs-string">&quot;BM25&quot;</span>}}},
<span class="hljs-meta">... </span>}  <span class="hljs-comment"># default config</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>es_index_name = <span class="hljs-string">&quot;hf_squad_context&quot;</span>  <span class="hljs-comment"># name of the index in Elasticsearch</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>squad.add_elasticsearch_index(<span class="hljs-string">&quot;context&quot;</span>, es_client=es_client, es_config=es_config, es_index_name=es_index_name)`,wrap:!1}}),K=new ht({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/faiss_es.mdx"}}),{c(){M=p("meta"),ts=l(),O=p("p"),as=l(),r(U.$$.fragment),es=l(),X=p("p"),X.innerHTML=Qs,ls=l(),k=p("p"),k.textContent=zs,ns=l(),r(W.$$.fragment),ps=l(),q=p("p"),q.innerHTML=Es,is=l(),R=p("ol"),R.innerHTML=Ys,cs=l(),r($.$$.fragment),rs=l(),j=p("ol"),j.innerHTML=Bs,os=l(),r(F.$$.fragment),ds=l(),y=p("ol"),y.innerHTML=Hs,ms=l(),r(N.$$.fragment),hs=l(),f=p("ol"),f.innerHTML=Ls,us=l(),r(v.$$.fragment),gs=l(),T=p("ol"),T.innerHTML=Ds,Js=l(),r(G.$$.fragment),Ms=l(),w=p("ol"),w.innerHTML=Ss,js=l(),r(C.$$.fragment),ys=l(),b=p("ol"),b.innerHTML=As,fs=l(),r(V.$$.fragment),Ts=l(),r(Q.$$.fragment),ws=l(),z=p("p"),z.textContent=Ps,bs=l(),E=p("p"),E.innerHTML=Ks,xs=l(),Y=p("ol"),Y.innerHTML=Os,_s=l(),r(B.$$.fragment),Zs=l(),x=p("ol"),x.innerHTML=st,Is=l(),r(H.$$.fragment),Us=l(),_=p("ol"),_.innerHTML=tt,Xs=l(),r(L.$$.fragment),ks=l(),Z=p("ol"),Z.innerHTML=at,Ws=l(),r(D.$$.fragment),qs=l(),I=p("ol"),I.innerHTML=et,Rs=l(),r(S.$$.fragment),$s=l(),A=p("p"),A.textContent=lt,Fs=l(),r(P.$$.fragment),Ns=l(),r(K.$$.fragment),vs=l(),ss=p("p"),this.h()},l(s){const t=dt("svelte-u9bgzb",document.head);M=i(t,"META",{name:!0,content:!0}),t.forEach(a),ts=n(s),O=i(s,"P",{}),nt(O).forEach(a),as=n(s),o(U.$$.fragment,s),es=n(s),X=i(s,"P",{"data-svelte-h":!0}),c(X)!=="svelte-1n7ea75"&&(X.innerHTML=Qs),ls=n(s),k=i(s,"P",{"data-svelte-h":!0}),c(k)!=="svelte-hdwpul"&&(k.textContent=zs),ns=n(s),o(W.$$.fragment,s),ps=n(s),q=i(s,"P",{"data-svelte-h":!0}),c(q)!=="svelte-se9cy2"&&(q.innerHTML=Es),is=n(s),R=i(s,"OL",{"data-svelte-h":!0}),c(R)!=="svelte-x1ghuo"&&(R.innerHTML=Ys),cs=n(s),o($.$$.fragment,s),rs=n(s),j=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(j)!=="svelte-1teerx9"&&(j.innerHTML=Bs),os=n(s),o(F.$$.fragment,s),ds=n(s),y=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(y)!=="svelte-1q25g0a"&&(y.innerHTML=Hs),ms=n(s),o(N.$$.fragment,s),hs=n(s),f=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(f)!=="svelte-1moeh5g"&&(f.innerHTML=Ls),us=n(s),o(v.$$.fragment,s),gs=n(s),T=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(T)!=="svelte-1uog0ta"&&(T.innerHTML=Ds),Js=n(s),o(G.$$.fragment,s),Ms=n(s),w=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(w)!=="svelte-1f0kmi5"&&(w.innerHTML=Ss),js=n(s),o(C.$$.fragment,s),ys=n(s),b=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(b)!=="svelte-12andu6"&&(b.innerHTML=As),fs=n(s),o(V.$$.fragment,s),Ts=n(s),o(Q.$$.fragment,s),ws=n(s),z=i(s,"P",{"data-svelte-h":!0}),c(z)!=="svelte-8ul8ma"&&(z.textContent=Ps),bs=n(s),E=i(s,"P",{"data-svelte-h":!0}),c(E)!=="svelte-195ztu1"&&(E.innerHTML=Ks),xs=n(s),Y=i(s,"OL",{"data-svelte-h":!0}),c(Y)!=="svelte-3fd7ny"&&(Y.innerHTML=Os),_s=n(s),o(B.$$.fragment,s),Zs=n(s),x=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(x)!=="svelte-m0pf1x"&&(x.innerHTML=st),Is=n(s),o(H.$$.fragment,s),Us=n(s),_=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(_)!=="svelte-1bmegd2"&&(_.innerHTML=tt),Xs=n(s),o(L.$$.fragment,s),ks=n(s),Z=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(Z)!=="svelte-1tchn11"&&(Z.innerHTML=at),Ws=n(s),o(D.$$.fragment,s),qs=n(s),I=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(I)!=="svelte-a6y65g"&&(I.innerHTML=et),Rs=n(s),o(S.$$.fragment,s),$s=n(s),A=i(s,"P",{"data-svelte-h":!0}),c(A)!=="svelte-1ajeg13"&&(A.textContent=lt),Fs=n(s),o(P.$$.fragment,s),Ns=n(s),o(K.$$.fragment,s),vs=n(s),ss=i(s,"P",{}),nt(ss).forEach(a),this.h()},h(){J(M,"name","hf:doc:metadata"),J(M,"content",gt),J(j,"start","2"),J(y,"start","3"),J(f,"start","4"),J(T,"start","5"),J(w,"start","6"),J(b,"start","7"),J(x,"start","2"),J(_,"start","3"),J(Z,"start","4"),J(I,"start","5")},m(s,t){mt(document.head,M),e(s,ts,t),e(s,O,t),e(s,as,t),d(U,s,t),e(s,es,t),e(s,X,t),e(s,ls,t),e(s,k,t),e(s,ns,t),d(W,s,t),e(s,ps,t),e(s,q,t),e(s,is,t),e(s,R,t),e(s,cs,t),d($,s,t),e(s,rs,t),e(s,j,t),e(s,os,t),d(F,s,t),e(s,ds,t),e(s,y,t),e(s,ms,t),d(N,s,t),e(s,hs,t),e(s,f,t),e(s,us,t),d(v,s,t),e(s,gs,t),e(s,T,t),e(s,Js,t),d(G,s,t),e(s,Ms,t),e(s,w,t),e(s,js,t),d(C,s,t),e(s,ys,t),e(s,b,t),e(s,fs,t),d(V,s,t),e(s,Ts,t),d(Q,s,t),e(s,ws,t),e(s,z,t),e(s,bs,t),e(s,E,t),e(s,xs,t),e(s,Y,t),e(s,_s,t),d(B,s,t),e(s,Zs,t),e(s,x,t),e(s,Is,t),d(H,s,t),e(s,Us,t),e(s,_,t),e(s,Xs,t),d(L,s,t),e(s,ks,t),e(s,Z,t),e(s,Ws,t),d(D,s,t),e(s,qs,t),e(s,I,t),e(s,Rs,t),d(S,s,t),e(s,$s,t),e(s,A,t),e(s,Fs,t),d(P,s,t),e(s,Ns,t),d(K,s,t),e(s,vs,t),e(s,ss,t),Gs=!0},p:it,i(s){Gs||(m(U.$$.fragment,s),m(W.$$.fragment,s),m($.$$.fragment,s),m(F.$$.fragment,s),m(N.$$.fragment,s),m(v.$$.fragment,s),m(G.$$.fragment,s),m(C.$$.fragment,s),m(V.$$.fragment,s),m(Q.$$.fragment,s),m(B.$$.fragment,s),m(H.$$.fragment,s),m(L.$$.fragment,s),m(D.$$.fragment,s),m(S.$$.fragment,s),m(P.$$.fragment,s),m(K.$$.fragment,s),Gs=!0)},o(s){h(U.$$.fragment,s),h(W.$$.fragment,s),h($.$$.fragment,s),h(F.$$.fragment,s),h(N.$$.fragment,s),h(v.$$.fragment,s),h(G.$$.fragment,s),h(C.$$.fragment,s),h(V.$$.fragment,s),h(Q.$$.fragment,s),h(B.$$.fragment,s),h(H.$$.fragment,s),h(L.$$.fragment,s),h(D.$$.fragment,s),h(S.$$.fragment,s),h(P.$$.fragment,s),h(K.$$.fragment,s),Gs=!1},d(s){s&&(a(ts),a(O),a(as),a(es),a(X),a(ls),a(k),a(ns),a(ps),a(q),a(is),a(R),a(cs),a(rs),a(j),a(os),a(ds),a(y),a(ms),a(hs),a(f),a(us),a(gs),a(T),a(Js),a(Ms),a(w),a(js),a(ys),a(b),a(fs),a(Ts),a(ws),a(z),a(bs),a(E),a(xs),a(Y),a(_s),a(Zs),a(x),a(Is),a(Us),a(_),a(Xs),a(ks),a(Z),a(Ws),a(qs),a(I),a(Rs),a($s),a(A),a(Fs),a(Ns),a(vs),a(ss)),a(M),u(U,s),u(W,s),u($,s),u(F,s),u(N,s),u(v,s),u(G,s),u(C,s),u(V,s),u(Q,s),u(B,s),u(H,s),u(L,s),u(D,s),u(S,s),u(P,s),u(K,s)}}}const gt='{"title":"Search index","local":"search-index","sections":[{"title":"FAISS","local":"faiss","sections":[],"depth":2},{"title":"Elasticsearch","local":"elasticsearch","sections":[],"depth":2}],"depth":1}';function Jt(Vs){return ct(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Tt extends rt{constructor(M){super(),ot(this,M,Jt,ut,pt,{})}}export{Tt as component};
