import{s as Qe,n as He,o as ze}from"../chunks/scheduler.1f027d5a.js";import{S as Le,i as De,e as m,s as a,c as i,h as Ke,a as c,d as s,b as n,f as Fe,g as M,j as T,k as Se,l as Pe,m as t,n as d,t as J,o as p,p as o}from"../chunks/index.5cb6ce42.js";import{C as Oe}from"../chunks/CopyLLMTxtMenu.c64d5249.js";import{D as el,C as r}from"../chunks/DocNotebookDropdown.adbd98a5.js";import{H as b,E as ll}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.903c2670.js";function sl(Ce){let w,K,L,P,u,O,j,ee,h,le,y,Be='Datasets on the Hugging Face Hub rely on parquet files. We can <a href="https://huggingface.co/docs/hub/en/datasets-duckdb" rel="nofollow">interact with these files using DuckDB</a> as a fast in-memory database system. One of DuckDB’s features is <a href="https://duckdb.org/docs/extensions/vss.html" rel="nofollow">vector similarity search</a> which can be used with or without an index.',se,I,te,f,ae,U,ne,g,$e="First, we need to create embeddings for the dataset to search over. We will use the <code>sentence-transformers</code> library to create embeddings for the dataset.",ie,Z,Me,A,Ee='Now, let’s load the <a href="https://huggingface.co/datasets/ai-blueprint/fineweb-bbc-news" rel="nofollow">ai-blueprint/fineweb-bbc-news</a> dataset from the Hub.',de,W,Je,_,Ve="We can now create embeddings for the dataset. Normally, we might want to chunk our data into smaller batches to avoid losing precision, but for this example, we will just create embeddings for the full text of the dataset.",pe,k,oe,C,Ge="We can now upload our dataset with embeddings back to the Hub.",me,B,ce,$,Te,E,Re="We can now perform vector search on the dataset using <code>duckdb</code>. When doing so, we can either use an index or not. Searching <strong>without</strong> an index is slower but more precise, whereas searching <strong>with</strong> an index is faster but less precise.",we,V,re,G,Ne="To search without an index, we can use the <code>duckdb</code> library to connect to the dataset and perform a vector search. This is a slow operation, but normally works quick enough for small datasets up to let’s say 100k rows. Meaning querying our dataset will be somewhat slower.",be,R,ue,N,je,Y,Ye="This approach creates a local copy of the dataset and uses this to create an index. This has some minor overhead but it will significantly speed up the search once you’ve created it.",he,X,ye,x,Xe="Now we can perform a vector search with the index, which return the results instantly.",Ie,q,fe,v,xe="The query reduces from 30 seconds to sub-second response times and does not require you to deploy a heavy-weight vector search engine, while storage is handled by the Hub.",Ue,F,ge,S,qe="We have seen how to perform vector search on the Hub using <code>duckdb</code>. For small datasets &lt;100k rows, we can perform vector search without an index using the Hub as a vector search backend, but for larger datasets, we should create an index with the <code>vss</code> extension while doing local search and using the Hub as a storage backend.",Ze,Q,Ae,H,ve='<li><a href="https://huggingface.co/docs/hub/en/datasets-duckdb" rel="nofollow">Vector Search on Hugging Face</a></li> <li><a href="https://duckdb.org/docs/extensions/vss.html" rel="nofollow">Vector Search Indexing with DuckDB</a></li>',We,z,_e,D,ke;return u=new el({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/cookbook/blob/main/notebooks/en/vector_search_with_hub_as_backend.ipynb"}]}}),j=new Oe({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),h=new b({props:{title:"Vector Search on Hugging Face with the Hub as Backend",local:"vector-search-on-hugging-face-with-the-hub-as-backend",headingTag:"h1"}}),I=new b({props:{title:"Install dependencies",local:"install-dependencies",headingTag:"h2"}}),f=new r({props:{code:"IXBpcCUyMGluc3RhbGwlMjBkYXRhc2V0cyUyMGR1Y2tkYiUyMHNlbnRlbmNlLXRyYW5zZm9ybWVycyUyMG1vZGVsMnZlYyUyMC1x",highlighted:"!pip install datasets duckdb sentence-transformers model2vec -q",wrap:!1}}),U=new b({props:{title:"Create embeddings for the dataset",local:"create-embeddings-for-the-dataset",headingTag:"h2"}}),Z=new r({props:{code:"ZnJvbSUyMHNlbnRlbmNlX3RyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNlbnRlbmNlVHJhbnNmb3JtZXIlMEFmcm9tJTIwc2VudGVuY2VfdHJhbnNmb3JtZXJzLm1vZGVscyUyMGltcG9ydCUyMFN0YXRpY0VtYmVkZGluZyUwQSUwQXN0YXRpY19lbWJlZGRpbmclMjAlM0QlMjBTdGF0aWNFbWJlZGRpbmcuZnJvbV9tb2RlbDJ2ZWMoJTIybWluaXNobGFiJTJGcG90aW9uLWJhc2UtOE0lMjIpJTBBbW9kZWwlMjAlM0QlMjBTZW50ZW5jZVRyYW5zZm9ybWVyKG1vZHVsZXMlM0QlNUJzdGF0aWNfZW1iZWRkaW5nJTVEKQ==",highlighted:`<span class="hljs-keyword">from</span> sentence_transformers <span class="hljs-keyword">import</span> SentenceTransformer
<span class="hljs-keyword">from</span> sentence_transformers.models <span class="hljs-keyword">import</span> StaticEmbedding

static_embedding = StaticEmbedding.from_model2vec(<span class="hljs-string">&quot;minishlab/potion-base-8M&quot;</span>)
model = SentenceTransformer(modules=[static_embedding])`,wrap:!1}}),W=new r({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyYWktYmx1ZXByaW50JTJGZmluZXdlYi1iYmMtbmV3cyUyMik=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

ds = load_dataset(<span class="hljs-string">&quot;ai-blueprint/fineweb-bbc-news&quot;</span>)`,wrap:!1}}),k=new r({props:{code:"ZGVmJTIwY3JlYXRlX2VtYmVkZGluZ3MoYmF0Y2gpJTNBJTBBJTIwJTIwJTIwJTIwZW1iZWRkaW5ncyUyMCUzRCUyMG1vZGVsLmVuY29kZShiYXRjaCU1QiUyMnRleHQlMjIlNUQlMkMlMjBjb252ZXJ0X3RvX251bXB5JTNEVHJ1ZSklMEElMjAlMjAlMjAlMjBiYXRjaCU1QiUyMmVtYmVkZGluZ3MlMjIlNUQlMjAlM0QlMjBlbWJlZGRpbmdzLnRvbGlzdCgpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwYmF0Y2glMEElMEFkcyUyMCUzRCUyMGRzLm1hcChjcmVhdGVfZW1iZWRkaW5ncyUyQyUyMGJhdGNoZWQlM0RUcnVlKQ==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">create_embeddings</span>(<span class="hljs-params">batch</span>):
    embeddings = model.encode(batch[<span class="hljs-string">&quot;text&quot;</span>], convert_to_numpy=<span class="hljs-literal">True</span>)
    batch[<span class="hljs-string">&quot;embeddings&quot;</span>] = embeddings.tolist()
    <span class="hljs-keyword">return</span> batch

ds = ds.<span class="hljs-built_in">map</span>(create_embeddings, batched=<span class="hljs-literal">True</span>)`,wrap:!1}}),B=new r({props:{code:"ZHMucHVzaF90b19odWIoJTIyYWktYmx1ZXByaW50JTJGZmluZXdlYi1iYmMtbmV3cy1lbWJlZGRpbmdzJTIyKQ==",highlighted:'ds.push_to_hub(<span class="hljs-string">&quot;ai-blueprint/fineweb-bbc-news-embeddings&quot;</span>)',wrap:!1}}),$=new b({props:{title:"Vector Search the Hugging Face Hub",local:"vector-search-the-hugging-face-hub",headingTag:"h2"}}),V=new b({props:{title:"Without an index",local:"without-an-index",headingTag:"h3"}}),R=new r({props:{code:"aW1wb3J0JTIwZHVja2RiJTBBZnJvbSUyMHR5cGluZyUyMGltcG9ydCUyMExpc3QlMEElMEFkZWYlMjBzaW1pbGFyaXR5X3NlYXJjaF93aXRob3V0X2R1Y2tkYl9pbmRleCglMEElMjAlMjAlMjAlMjBxdWVyeSUzQSUyMHN0ciUyQyUwQSUyMCUyMCUyMCUyMGslM0ElMjBpbnQlMjAlM0QlMjA1JTJDJTBBJTIwJTIwJTIwJTIwZGF0YXNldF9uYW1lJTNBJTIwc3RyJTIwJTNEJTIwJTIyYWktYmx1ZXByaW50JTJGZmluZXdlYi1iYmMtbmV3cy1lbWJlZGRpbmdzJTIyJTJDJTBBJTIwJTIwJTIwJTIwZW1iZWRkaW5nX2NvbHVtbiUzQSUyMHN0ciUyMCUzRCUyMCUyMmVtYmVkZGluZ3MlMjIlMkMlMEEpJTNBJTIwJTIwJTIwJTIwJTBBJTIwJTIwJTIwJTIwJTIzJTIwVXNlJTIwc2FtZSUyMG1vZGVsJTIwYXMlMjB1c2VkJTIwZm9yJTIwaW5kZXhpbmclMEElMjAlMjAlMjAlMjBxdWVyeV92ZWN0b3IlMjAlM0QlMjBtb2RlbC5lbmNvZGUocXVlcnkpJTBBJTIwJTIwJTIwJTIwZW1iZWRkaW5nX2RpbSUyMCUzRCUyMG1vZGVsLmdldF9zZW50ZW5jZV9lbWJlZGRpbmdfZGltZW5zaW9uKCklMEElMEElMjAlMjAlMjAlMjBzcWwlMjAlM0QlMjBmJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwU0VMRUNUJTIwJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFycmF5X2Nvc2luZV9kaXN0YW5jZSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0JlbWJlZGRpbmdfY29sdW1uJTdEJTNBJTNBZmxvYXQlNUIlN0JlbWJlZGRpbmdfZGltJTdEJTVEJTJDJTIwJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCcXVlcnlfdmVjdG9yLnRvbGlzdCgpJTdEJTNBJTNBZmxvYXQlNUIlN0JlbWJlZGRpbmdfZGltJTdEJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUyMGFzJTIwZGlzdGFuY2UlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBGUk9NJTIwJ2hmJTNBJTJGJTJGZGF0YXNldHMlMkYlN0JkYXRhc2V0X25hbWUlN0QlMkYqKiUyRioucGFycXVldCclMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBPUkRFUiUyMEJZJTIwZGlzdGFuY2UlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBMSU1JVCUyMCU3QmslN0QlMEElMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjByZXR1cm4lMjBkdWNrZGIuc3FsKHNxbCkudG9fZGYoKSUwQSUwQXNpbWlsYXJpdHlfc2VhcmNoX3dpdGhvdXRfZHVja2RiX2luZGV4KCUyMldoYXQlMjBpcyUyMHRoZSUyMGZ1dHVyZSUyMG9mJTIwQUklM0YlMjIp",highlighted:`<span class="hljs-keyword">import</span> duckdb
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> <span class="hljs-type">List</span>

<span class="hljs-keyword">def</span> <span class="hljs-title function_">similarity_search_without_duckdb_index</span>(<span class="hljs-params">
    query: <span class="hljs-built_in">str</span>,
    k: <span class="hljs-built_in">int</span> = <span class="hljs-number">5</span>,
    dataset_name: <span class="hljs-built_in">str</span> = <span class="hljs-string">&quot;ai-blueprint/fineweb-bbc-news-embeddings&quot;</span>,
    embedding_column: <span class="hljs-built_in">str</span> = <span class="hljs-string">&quot;embeddings&quot;</span>,
</span>):    
    <span class="hljs-comment"># Use same model as used for indexing</span>
    query_vector = model.encode(query)
    embedding_dim = model.get_sentence_embedding_dimension()

    sql = <span class="hljs-string">f&quot;&quot;&quot;
        SELECT 
            *,
            array_cosine_distance(
                <span class="hljs-subst">{embedding_column}</span>::float[<span class="hljs-subst">{embedding_dim}</span>], 
                <span class="hljs-subst">{query_vector.tolist()}</span>::float[<span class="hljs-subst">{embedding_dim}</span>]
            ) as distance
        FROM &#x27;hf://datasets/<span class="hljs-subst">{dataset_name}</span>/**/*.parquet&#x27;
        ORDER BY distance
        LIMIT <span class="hljs-subst">{k}</span>
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> duckdb.sql(sql).to_df()

similarity_search_without_duckdb_index(<span class="hljs-string">&quot;What is the future of AI?&quot;</span>)`,wrap:!1}}),N=new b({props:{title:"With an index",local:"with-an-index",headingTag:"h3"}}),X=new r({props:{code:"aW1wb3J0JTIwZHVja2RiJTBBJTBBZGVmJTIwX3NldHVwX3ZzcygpJTNBJTBBJTIwJTIwJTIwJTIwZHVja2RiLnNxbCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBxdWVyeSUzRCUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMElOU1RBTEwlMjB2c3MlM0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBMT0FEJTIwdnNzJTNCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwKSUwQWRlZiUyMF9kcm9wX3RhYmxlKHRhYmxlX25hbWUpJTNBJTBBJTIwJTIwJTIwJTIwZHVja2RiLnNxbCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBxdWVyeSUzRGYlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBEUk9QJTIwVEFCTEUlMjBJRiUyMEVYSVNUUyUyMCU3QnRhYmxlX25hbWUlN0QlM0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjApJTBBJTBBZGVmJTIwX2NyZWF0ZV90YWJsZShkYXRhc2V0X25hbWUlMkMlMjB0YWJsZV9uYW1lJTJDJTIwZW1iZWRkaW5nX2NvbHVtbiklM0ElMEElMjAlMjAlMjAlMjBkdWNrZGIuc3FsKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXJ5JTNEZiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMENSRUFURSUyMFRBQkxFJTIwJTdCdGFibGVfbmFtZSU3RCUyMEFTJTIwJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwU0VMRUNUJTIwKiUyQyUyMCU3QmVtYmVkZGluZ19jb2x1bW4lN0QlM0ElM0FmbG9hdCU1QiU3Qm1vZGVsLmdldF9zZW50ZW5jZV9lbWJlZGRpbmdfZGltZW5zaW9uKCklN0QlNUQlMjBhcyUyMCU3QmVtYmVkZGluZ19jb2x1bW4lN0RfZmxvYXQlMjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBGUk9NJTIwJ2hmJTNBJTJGJTJGZGF0YXNldHMlMkYlN0JkYXRhc2V0X25hbWUlN0QlMkYqKiUyRioucGFycXVldCclM0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjApJTBBJTBBZGVmJTIwX2NyZWF0ZV9pbmRleCh0YWJsZV9uYW1lJTJDJTIwZW1iZWRkaW5nX2NvbHVtbiklM0ElMEElMjAlMjAlMjAlMjBkdWNrZGIuc3FsKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXJ5JTNEZiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMENSRUFURSUyMElOREVYJTIwbXlfaG5zd19pbmRleCUyME9OJTIwJTdCdGFibGVfbmFtZSU3RCUyMFVTSU5HJTIwSE5TVyUyMCglN0JlbWJlZGRpbmdfY29sdW1uJTdEX2Zsb2F0KSUyMFdJVEglMjAobWV0cmljJTIwJTNEJTIwJ2Nvc2luZScpJTNCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwKSUwQSUwQWRlZiUyMGNyZWF0ZV9pbmRleChkYXRhc2V0X25hbWUlMkMlMjB0YWJsZV9uYW1lJTJDJTIwZW1iZWRkaW5nX2NvbHVtbiklM0ElMEElMjAlMjAlMjAlMjBfc2V0dXBfdnNzKCklMEElMjAlMjAlMjAlMjBfZHJvcF90YWJsZSh0YWJsZV9uYW1lKSUwQSUyMCUyMCUyMCUyMF9jcmVhdGVfdGFibGUoZGF0YXNldF9uYW1lJTJDJTIwdGFibGVfbmFtZSUyQyUyMGVtYmVkZGluZ19jb2x1bW4pJTBBJTIwJTIwJTIwJTIwX2NyZWF0ZV9pbmRleCh0YWJsZV9uYW1lJTJDJTIwZW1iZWRkaW5nX2NvbHVtbiklMEElMEFjcmVhdGVfaW5kZXgoJTBBJTIwJTIwJTIwJTIwZGF0YXNldF9uYW1lJTNEJTIyYWktYmx1ZXByaW50JTJGZmluZXdlYi1iYmMtbmV3cy1lbWJlZGRpbmdzJTIyJTJDJTBBJTIwJTIwJTIwJTIwdGFibGVfbmFtZSUzRCUyMmZpbmV3ZWJfYmJjX25ld3NfZW1iZWRkaW5ncyUyMiUyQyUwQSUyMCUyMCUyMCUyMGVtYmVkZGluZ19jb2x1bW4lM0QlMjJlbWJlZGRpbmdzJTIyJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> duckdb

<span class="hljs-keyword">def</span> <span class="hljs-title function_">_setup_vss</span>():
    duckdb.sql(
        query=<span class="hljs-string">&quot;&quot;&quot;
        INSTALL vss;
        LOAD vss;
        &quot;&quot;&quot;</span>
    )
<span class="hljs-keyword">def</span> <span class="hljs-title function_">_drop_table</span>(<span class="hljs-params">table_name</span>):
    duckdb.sql(
        query=<span class="hljs-string">f&quot;&quot;&quot;
        DROP TABLE IF EXISTS <span class="hljs-subst">{table_name}</span>;
        &quot;&quot;&quot;</span>
    )

<span class="hljs-keyword">def</span> <span class="hljs-title function_">_create_table</span>(<span class="hljs-params">dataset_name, table_name, embedding_column</span>):
    duckdb.sql(
        query=<span class="hljs-string">f&quot;&quot;&quot;
        CREATE TABLE <span class="hljs-subst">{table_name}</span> AS 
        SELECT *, <span class="hljs-subst">{embedding_column}</span>::float[<span class="hljs-subst">{model.get_sentence_embedding_dimension()}</span>] as <span class="hljs-subst">{embedding_column}</span>_float 
        FROM &#x27;hf://datasets/<span class="hljs-subst">{dataset_name}</span>/**/*.parquet&#x27;;
        &quot;&quot;&quot;</span>
    )

<span class="hljs-keyword">def</span> <span class="hljs-title function_">_create_index</span>(<span class="hljs-params">table_name, embedding_column</span>):
    duckdb.sql(
        query=<span class="hljs-string">f&quot;&quot;&quot;
        CREATE INDEX my_hnsw_index ON <span class="hljs-subst">{table_name}</span> USING HNSW (<span class="hljs-subst">{embedding_column}</span>_float) WITH (metric = &#x27;cosine&#x27;);
        &quot;&quot;&quot;</span>
    )

<span class="hljs-keyword">def</span> <span class="hljs-title function_">create_index</span>(<span class="hljs-params">dataset_name, table_name, embedding_column</span>):
    _setup_vss()
    _drop_table(table_name)
    _create_table(dataset_name, table_name, embedding_column)
    _create_index(table_name, embedding_column)

create_index(
    dataset_name=<span class="hljs-string">&quot;ai-blueprint/fineweb-bbc-news-embeddings&quot;</span>,
    table_name=<span class="hljs-string">&quot;fineweb_bbc_news_embeddings&quot;</span>,
    embedding_column=<span class="hljs-string">&quot;embeddings&quot;</span>
)`,wrap:!1}}),q=new r({props:{code:"ZGVmJTIwc2ltaWxhcml0eV9zZWFyY2hfd2l0aF9kdWNrZGJfaW5kZXgoJTBBJTIwJTIwJTIwJTIwcXVlcnklM0ElMjBzdHIlMkMlMEElMjAlMjAlMjAlMjBrJTNBJTIwaW50JTIwJTNEJTIwNSUyQyUwQSUyMCUyMCUyMCUyMHRhYmxlX25hbWUlM0ElMjBzdHIlMjAlM0QlMjAlMjJmaW5ld2ViX2JiY19uZXdzX2VtYmVkZGluZ3MlMjIlMkMlMEElMjAlMjAlMjAlMjBlbWJlZGRpbmdfY29sdW1uJTNBJTIwc3RyJTIwJTNEJTIwJTIyZW1iZWRkaW5ncyUyMiUwQSklM0ElMEElMjAlMjAlMjAlMjBlbWJlZGRpbmclMjAlM0QlMjBtb2RlbC5lbmNvZGUocXVlcnkpLnRvbGlzdCgpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwZHVja2RiLnNxbCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBxdWVyeSUzRGYlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBTRUxFQ1QlMjAqJTJDJTIwYXJyYXlfY29zaW5lX2Rpc3RhbmNlKCU3QmVtYmVkZGluZ19jb2x1bW4lN0RfZmxvYXQlMkMlMjAlN0JlbWJlZGRpbmclN0QlM0ElM0FGTE9BVCU1QiU3Qm1vZGVsLmdldF9zZW50ZW5jZV9lbWJlZGRpbmdfZGltZW5zaW9uKCklN0QlNUQpJTIwYXMlMjBkaXN0YW5jZSUyMCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMEZST00lMjAlN0J0YWJsZV9uYW1lJTdEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwT1JERVIlMjBCWSUyMGRpc3RhbmNlJTIwJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTElNSVQlMjAlN0JrJTdEJTNCJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwKS50b19kZigpJTBBJTBBc2ltaWxhcml0eV9zZWFyY2hfd2l0aF9kdWNrZGJfaW5kZXgoJTIyV2hhdCUyMGlzJTIwdGhlJTIwZnV0dXJlJTIwb2YlMjBBSSUzRiUyMik=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">similarity_search_with_duckdb_index</span>(<span class="hljs-params">
    query: <span class="hljs-built_in">str</span>,
    k: <span class="hljs-built_in">int</span> = <span class="hljs-number">5</span>,
    table_name: <span class="hljs-built_in">str</span> = <span class="hljs-string">&quot;fineweb_bbc_news_embeddings&quot;</span>,
    embedding_column: <span class="hljs-built_in">str</span> = <span class="hljs-string">&quot;embeddings&quot;</span>
</span>):
    embedding = model.encode(query).tolist()
    <span class="hljs-keyword">return</span> duckdb.sql(
        query=<span class="hljs-string">f&quot;&quot;&quot;
        SELECT *, array_cosine_distance(<span class="hljs-subst">{embedding_column}</span>_float, <span class="hljs-subst">{embedding}</span>::FLOAT[<span class="hljs-subst">{model.get_sentence_embedding_dimension()}</span>]) as distance 
        FROM <span class="hljs-subst">{table_name}</span>
        ORDER BY distance 
        LIMIT <span class="hljs-subst">{k}</span>;
    &quot;&quot;&quot;</span>
    ).to_df()

similarity_search_with_duckdb_index(<span class="hljs-string">&quot;What is the future of AI?&quot;</span>)`,wrap:!1}}),F=new b({props:{title:"Conclusion",local:"conclusion",headingTag:"h2"}}),Q=new b({props:{title:"Learn more",local:"learn-more",headingTag:"h2"}}),z=new ll({props:{source:"https://github.com/huggingface/cookbook/blob/main/notebooks/en/vector_search_with_hub_as_backend.md"}}),{c(){w=m("meta"),K=a(),L=m("p"),P=a(),i(u.$$.fragment),O=a(),i(j.$$.fragment),ee=a(),i(h.$$.fragment),le=a(),y=m("p"),y.innerHTML=Be,se=a(),i(I.$$.fragment),te=a(),i(f.$$.fragment),ae=a(),i(U.$$.fragment),ne=a(),g=m("p"),g.innerHTML=$e,ie=a(),i(Z.$$.fragment),Me=a(),A=m("p"),A.innerHTML=Ee,de=a(),i(W.$$.fragment),Je=a(),_=m("p"),_.textContent=Ve,pe=a(),i(k.$$.fragment),oe=a(),C=m("p"),C.textContent=Ge,me=a(),i(B.$$.fragment),ce=a(),i($.$$.fragment),Te=a(),E=m("p"),E.innerHTML=Re,we=a(),i(V.$$.fragment),re=a(),G=m("p"),G.innerHTML=Ne,be=a(),i(R.$$.fragment),ue=a(),i(N.$$.fragment),je=a(),Y=m("p"),Y.textContent=Ye,he=a(),i(X.$$.fragment),ye=a(),x=m("p"),x.textContent=Xe,Ie=a(),i(q.$$.fragment),fe=a(),v=m("p"),v.textContent=xe,Ue=a(),i(F.$$.fragment),ge=a(),S=m("p"),S.innerHTML=qe,Ze=a(),i(Q.$$.fragment),Ae=a(),H=m("ul"),H.innerHTML=ve,We=a(),i(z.$$.fragment),_e=a(),D=m("p"),this.h()},l(e){const l=Ke("svelte-u9bgzb",document.head);w=c(l,"META",{name:!0,content:!0}),l.forEach(s),K=n(e),L=c(e,"P",{}),Fe(L).forEach(s),P=n(e),M(u.$$.fragment,e),O=n(e),M(j.$$.fragment,e),ee=n(e),M(h.$$.fragment,e),le=n(e),y=c(e,"P",{"data-svelte-h":!0}),T(y)!=="svelte-1t5w2hp"&&(y.innerHTML=Be),se=n(e),M(I.$$.fragment,e),te=n(e),M(f.$$.fragment,e),ae=n(e),M(U.$$.fragment,e),ne=n(e),g=c(e,"P",{"data-svelte-h":!0}),T(g)!=="svelte-slwj9z"&&(g.innerHTML=$e),ie=n(e),M(Z.$$.fragment,e),Me=n(e),A=c(e,"P",{"data-svelte-h":!0}),T(A)!=="svelte-qzt0uj"&&(A.innerHTML=Ee),de=n(e),M(W.$$.fragment,e),Je=n(e),_=c(e,"P",{"data-svelte-h":!0}),T(_)!=="svelte-1633f25"&&(_.textContent=Ve),pe=n(e),M(k.$$.fragment,e),oe=n(e),C=c(e,"P",{"data-svelte-h":!0}),T(C)!=="svelte-gc9dd9"&&(C.textContent=Ge),me=n(e),M(B.$$.fragment,e),ce=n(e),M($.$$.fragment,e),Te=n(e),E=c(e,"P",{"data-svelte-h":!0}),T(E)!=="svelte-pmvn8c"&&(E.innerHTML=Re),we=n(e),M(V.$$.fragment,e),re=n(e),G=c(e,"P",{"data-svelte-h":!0}),T(G)!=="svelte-10x7dzt"&&(G.innerHTML=Ne),be=n(e),M(R.$$.fragment,e),ue=n(e),M(N.$$.fragment,e),je=n(e),Y=c(e,"P",{"data-svelte-h":!0}),T(Y)!=="svelte-e2ywn4"&&(Y.textContent=Ye),he=n(e),M(X.$$.fragment,e),ye=n(e),x=c(e,"P",{"data-svelte-h":!0}),T(x)!=="svelte-nopqs8"&&(x.textContent=Xe),Ie=n(e),M(q.$$.fragment,e),fe=n(e),v=c(e,"P",{"data-svelte-h":!0}),T(v)!=="svelte-x1ox1z"&&(v.textContent=xe),Ue=n(e),M(F.$$.fragment,e),ge=n(e),S=c(e,"P",{"data-svelte-h":!0}),T(S)!=="svelte-1y1h50d"&&(S.innerHTML=qe),Ze=n(e),M(Q.$$.fragment,e),Ae=n(e),H=c(e,"UL",{"data-svelte-h":!0}),T(H)!=="svelte-1aqcu0g"&&(H.innerHTML=ve),We=n(e),M(z.$$.fragment,e),_e=n(e),D=c(e,"P",{}),Fe(D).forEach(s),this.h()},h(){Se(w,"name","hf:doc:metadata"),Se(w,"content",tl)},m(e,l){Pe(document.head,w),t(e,K,l),t(e,L,l),t(e,P,l),d(u,e,l),t(e,O,l),d(j,e,l),t(e,ee,l),d(h,e,l),t(e,le,l),t(e,y,l),t(e,se,l),d(I,e,l),t(e,te,l),d(f,e,l),t(e,ae,l),d(U,e,l),t(e,ne,l),t(e,g,l),t(e,ie,l),d(Z,e,l),t(e,Me,l),t(e,A,l),t(e,de,l),d(W,e,l),t(e,Je,l),t(e,_,l),t(e,pe,l),d(k,e,l),t(e,oe,l),t(e,C,l),t(e,me,l),d(B,e,l),t(e,ce,l),d($,e,l),t(e,Te,l),t(e,E,l),t(e,we,l),d(V,e,l),t(e,re,l),t(e,G,l),t(e,be,l),d(R,e,l),t(e,ue,l),d(N,e,l),t(e,je,l),t(e,Y,l),t(e,he,l),d(X,e,l),t(e,ye,l),t(e,x,l),t(e,Ie,l),d(q,e,l),t(e,fe,l),t(e,v,l),t(e,Ue,l),d(F,e,l),t(e,ge,l),t(e,S,l),t(e,Ze,l),d(Q,e,l),t(e,Ae,l),t(e,H,l),t(e,We,l),d(z,e,l),t(e,_e,l),t(e,D,l),ke=!0},p:He,i(e){ke||(J(u.$$.fragment,e),J(j.$$.fragment,e),J(h.$$.fragment,e),J(I.$$.fragment,e),J(f.$$.fragment,e),J(U.$$.fragment,e),J(Z.$$.fragment,e),J(W.$$.fragment,e),J(k.$$.fragment,e),J(B.$$.fragment,e),J($.$$.fragment,e),J(V.$$.fragment,e),J(R.$$.fragment,e),J(N.$$.fragment,e),J(X.$$.fragment,e),J(q.$$.fragment,e),J(F.$$.fragment,e),J(Q.$$.fragment,e),J(z.$$.fragment,e),ke=!0)},o(e){p(u.$$.fragment,e),p(j.$$.fragment,e),p(h.$$.fragment,e),p(I.$$.fragment,e),p(f.$$.fragment,e),p(U.$$.fragment,e),p(Z.$$.fragment,e),p(W.$$.fragment,e),p(k.$$.fragment,e),p(B.$$.fragment,e),p($.$$.fragment,e),p(V.$$.fragment,e),p(R.$$.fragment,e),p(N.$$.fragment,e),p(X.$$.fragment,e),p(q.$$.fragment,e),p(F.$$.fragment,e),p(Q.$$.fragment,e),p(z.$$.fragment,e),ke=!1},d(e){e&&(s(K),s(L),s(P),s(O),s(ee),s(le),s(y),s(se),s(te),s(ae),s(ne),s(g),s(ie),s(Me),s(A),s(de),s(Je),s(_),s(pe),s(oe),s(C),s(me),s(ce),s(Te),s(E),s(we),s(re),s(G),s(be),s(ue),s(je),s(Y),s(he),s(ye),s(x),s(Ie),s(fe),s(v),s(Ue),s(ge),s(S),s(Ze),s(Ae),s(H),s(We),s(_e),s(D)),s(w),o(u,e),o(j,e),o(h,e),o(I,e),o(f,e),o(U,e),o(Z,e),o(W,e),o(k,e),o(B,e),o($,e),o(V,e),o(R,e),o(N,e),o(X,e),o(q,e),o(F,e),o(Q,e),o(z,e)}}}const tl='{"title":"Vector Search on Hugging Face with the Hub as Backend","local":"vector-search-on-hugging-face-with-the-hub-as-backend","sections":[{"title":"Install dependencies","local":"install-dependencies","sections":[],"depth":2},{"title":"Create embeddings for the dataset","local":"create-embeddings-for-the-dataset","sections":[],"depth":2},{"title":"Vector Search the Hugging Face Hub","local":"vector-search-the-hugging-face-hub","sections":[{"title":"Without an index","local":"without-an-index","sections":[],"depth":3},{"title":"With an index","local":"with-an-index","sections":[],"depth":3}],"depth":2},{"title":"Conclusion","local":"conclusion","sections":[],"depth":2},{"title":"Learn more","local":"learn-more","sections":[],"depth":2}],"depth":1}';function al(Ce){return ze(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class pl extends Le{constructor(w){super(),De(this,w,al,sl,Qe,{})}}export{pl as component};
