import{s as Ua,f as ya,n as ja,o as Ta}from"../chunks/scheduler.39a43cde.js";import{S as wa,i as pa,g as e,s as a,r as U,A as ia,h as y,f as M,c as n,j as Ja,u as j,x as J,k as Us,y as ca,a as t,v as T,d as w,t as p,w as i}from"../chunks/index.23ec2204.js";import{D as oa,C as c}from"../chunks/DocNotebookDropdown.21a75b74.js";import{H as o,E as ua}from"../chunks/EditOnGithub.f2d09226.js";function Ia(et){let u,ws,js,ps,d,is,r,cs,m,yt='<em>作者 <a href="https://huggingface.co/m-ric" rel="nofollow">Aymeric Roucher</a></em>',os,C,Jt="本 notebook 演示了如何评估你的 RAG（Retrieval Augmented Generation），通过构建一个合成评估数据集并使用 LLM-as-a-judge 来计算你系统的准确性。",us,b,Ut='对于 RAG 系统的介绍，你可以查看<a href="rag_zephyr_langchain">这个技术指南</a>!',Is,h,jt="RAG 系统很复杂: 这里有一个 RAG 流程图，我们用蓝色标注了系统增强的所有可能性：",ds,B,Tt,rs,Z,wt="实施上述任何改进都可能会带来巨大的性能提升；但如果无法监控对系统性能的影响，那么进行任何更改都是无用的！让我们看看如何评估我们的 RAG 系统。",ms,G,Cs,A,pt="由于有如此多的部分需要调整，这些部分对性能有很大影响，因此对 RAG 系统进行基准测试是至关重要的。",bs,Q,it="对于我们的评估流水线，我们将需要：",hs,W,ct="<li>一个带有问题-答案对的评估数据集（QA 对）</li> <li>一个评估器，用于计算我们的系统在上面的评估数据集上的准确性。</li>",Bs,V,ot="➡️ 结果发现，我们可以在整个过程中使用 LLMs 来帮助！",Zs,R,ut='<li>评估数据集将由 LLM 🤖 合成生成，并且问题将由其他 LLM 🤖 过滤掉</li> <li>然后，<a href="https://huggingface.co/papers/2306.05685" rel="nofollow">LLM-as-a-judge</a> 智能体 🤖 将在这个合成数据集上执行评估。</li>',Gs,X,It="<strong>让我们深入挖掘并开始构建我们的评估流水线！</strong> 首先，安装所需的模型依赖项。",As,g,Qs,f,Ws,N,Vs,k,Rs,E,Xs,v,gs,Y,fs,S,dt="我们首先构建一个问题和相关上下文的综合数据集。方法是先从我们的知识库中获取元素，并让 LLM 根据这些文档生成问题。",Ns,F,rt="然后，我们设置其他 LLM 智能体作为生成问答对的质置过滤器：每个智能体将作为一个特定缺陷的过滤器。",ks,_,Es,q,vs,x,Ys,z,mt='我们采用 <a href="https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1" rel="nofollow">Mixtral</a> 作为问答对的生成，因为他在各个排行榜上表现极佳，比如 <a href="https://huggingface.co/spaces/lmsys/chatbot-arena-leaderboard" rel="nofollow">Chatbot Arena</a>。',Ss,H,Fs,$,_s,L,Ct="现在让我们生成我们的问答对。",qs,D,bt="对于这个例子，我们只生成 10 个问答对，并从 Hub 加载其余的。",xs,K,ht="但是对于你的特定知识库，考虑到你想要获得至少约 100 个测试样本，并且考虑到我们稍后会用我们的批判智能体过滤掉大约一半的样本，你应该生成更多的样本，超过 200 个。",zs,P,Hs,O,$s,ll,Ls,sl,Bt="之前的智能体生成的问题可能存在许多缺陷：在验证这些问题之前，我们应该进行质量检查。",Ds,Ml,Zt='因此，我们构建了批判智能体，它们将根据以下几个标准对每个问题进行评分，这些标准在<a href="https://huggingface.co/papers/2312.10003" rel="nofollow">这篇论文</a>中给出：',Ks,tl,Gt="<li><strong>具体性（Groundedness）</strong>：问题是否可以从给定的上下文中得到回答？</li> <li><strong>相关性（Relevance）</strong>：问题对用户是否相关？例如，<code>&quot;transformers 4.29.1 发布的日期是什么？&quot;</code>对于 ML 用户来说并不相关。</li>",Ps,al,At=`我们注意到的一个最后的失败案例是，当一个函数是为生成问题的特定环境量身定做的，但本身难以理解，比如<code>&quot;这个指南中使用的函数的名称是什么？&quot;</code>。
我们也为这个标准构建了一个批判智能体：`,Os,nl,Qt="<li><strong>独立（Stand-alone）</strong>：对于一个具有领域知识/互联网访问权限的人来说，问题在没有任何上下文的情况下是否可以理解？与此相反的是，对于从特定博客文章生成的问题比如”这篇文章中使用的函数是什么？”</li>",lM,el,Wt="我们系统地用所有这些智能体对函数进行评分，每当任何一个智能体的分数太低时，我们就从我们的评估数据集中删除这个问题。",sM,yl,Vt="💡 <strong><em>当要求智能体输出分数时，我们首先要求它们产生其理由。这将帮助我们验证分数，但最重要的是，要求它首先输出理由给了模型更多的 token 来思考和详细阐述答案，然后再将其总结成一个单一的分数 token。</em></strong>",MM,Jl,Rt="我们现在构建并运行这些批判智能体。",tM,Ul,aM,jl,nM,Tl,Xt="现在让我们基于我们批判智能体的分数过滤掉不好的问题：",eM,wl,yM,pl,gt=`Evaluation dataset before filtering:
`,JM,il,ft="现在我们合成评估数据集已完成！我们可以在这个评估数据集上评估不同的 RAG 系统。",UM,cl,Nt="我们在这里只生成了少数几个问答对，以减少时间和成本。下面，让我们通过加载一个预先生成的数据集来进行下一部分：",jM,ol,TM,ul,wM,Il,pM,dl,kt="<li>在这一部分，<strong>我们将知识库中的文档分割成更小的片段</strong>：这些将是被检索器选取的片段，然后被阅读器 LLM 作为支持其答案的元素。</li> <li>目标是构建语义上相关的片段：不要太小，以免不足以支持答案，也不要太大，以免稀释单个内容。</li>",iM,rl,Et="文本分割有许多选项：",cM,ml,vt=`<li>每隔 <code>n</code> 个单词/字符分割，但这有可能割裂段落甚至句子</li> <li>在 <code>n</code> 个单词/字符后分割，但只在句子边界处</li> <li><strong>递归分割</strong> 尝试通过树状处理文档来保留更多文档结构，首先在最大单元（章节）上分割，然后递归地在更小单元（段落，句子）上分割。
要了解更多关于分块的信息，我建议你阅读由 Greg Kamradt 编写的<a href="https://github.com/FullStackRetrieval-com/RetrievalTutorials/blob/main/5_Levels_Of_Text_Splitting.ipynb" rel="nofollow">不错的教程</a> 。</li>`,oM,Cl,Yt='<a href="https://huggingface.co/spaces/m-ric/chunk_visualizer" rel="nofollow">这个 space</a> 让你可视化不同的分割选项是如何影响你得到的片段的流程。',uM,bl,St=`<p>在以下内容中，我们使用 Langchain 的 <code>RecursiveCharacterTextSplitter</code>。
💡 <em>为了在我们的文本分割器中测量片段长度，我们的长度函数将不是字符的数量，而是 token 化文本中的 token 数量：实际上，对于后续处理 token 的嵌入器来说，以 token 为单位测量长度更为相关，并且在经验上表现更好.</em></p>`,IM,hl,dM,Bl,rM,Zl,mM,Gl,Ft="<strong>检索器的作用类似于内部搜索引擎</strong>：给定用户查询，它从你的知识库中返回最相关的文档。",CM,Al,_t='<p>对于知识库，我们使用 Langchain 向量数据库，因为它提供了一个方便的 <a href="https://github.com/facebookresearch/faiss" rel="nofollow">FAISS</a> 索引，并允许我们在整个处理过程中保留文档元数据。</p>',bM,Ql,qt="🛠️ <strong>包含可选项：</strong>",hM,Wl,xt='<li>调整分块方法：<ul><li>片段(chunks)的大小</li> <li>方法：在不同的分隔符上分割，使用<a href="https://python.langchain.com/docs/modules/data_connection/document_transformers/semantic-chunker" rel="nofollow">语义分块</a>…</li></ul></li> <li>更改嵌入模型</li>',BM,Vl,ZM,Rl,GM,Xl,zt="在这一部分，<strong>LLM 阅读器读取检索到的文档以形成其答案。</strong>",AM,gl,Ht="🛠️ 为了改善结果，我们尝试了以下选项：",QM,fl,$t="<li>切换重排序开启或关闭的状态</li> <li>更改阅读器模型</li>",WM,Nl,VM,kl,RM,El,XM,vl,gM,Yl,Lt=`RAG 系统和评估数据集现在准备好了。最后一步是在这个评估数据集上判断 RAG 系统的输出。
为此，<strong>我们设置了一个裁判智能体</strong>。 ⚖️🤖`,fM,Sl,Dt='在<a href="https://docs.ragas.io/en/latest/concepts/metrics/index.html" rel="nofollow">不同的 RAG 评估指标</a>中，我们选择只关注忠实度，因为这是衡量我们系统性能的最佳的端到端指标。',NM,Fl,Kt='<p>我们使用 GPT4 作为评判者，因为它在实际应用中表现良好，但你也可以尝试其他模型，例如 <a href="https://huggingface.co/kaist-ai/prometheus-13b-v1.0" rel="nofollow">kaist-ai/prometheus-13b-v1.0</a> 或 <a href="https://huggingface.co/BAAI/JudgeLM-33B-v1.0" rel="nofollow">BAAI/JudgeLM-33B-v1.0</a>。</p>',kM,_l,Pt='💡 <em>在评估提示中，我们给出了每个指标的详细描述，采用 1-5 分的评分刻度，正如 <a href="https://huggingface.co/kaist-ai/prometheus-13b-v1.0" rel="nofollow">Prometheus 的提示模板</a> 所做的那样：这有助于模型精确地确定其指标。如果你给评判 LLM 一个模糊的评分刻度，那么不同示例之间的输出将不够一致。</em>',EM,ql,Ot="💡 <em>再次提示 LLM 在给出最终评分之前先输出其理由，这样它就有更多的 token 来帮助它正式化和详细阐述评判。</em>",vM,xl,YM,zl,SM,Hl,FM,$l,la="🚀 让我们运行一下测试和评估一下答案!👇",_M,Ll,qM,Dl,xM,Kl,zM,Pl,HM,Ol,$M,ls,LM,ss,sa='让我们加载通过调整这个 notebook 中可用的不同选项所获得的结果。关于这些选项为何有效或无效的更多细节，请参阅 <a href="advanced_rag">高级 RAG</a> 的 notebook。',DM,Ms,Ma="正如在下面的图表中所看到的，一些调整并没有带来任何改善，而有些则带来了巨大的性能提升。",KM,ts,ta="➡️ <strong><em>所以没有单一的好方法：在调整你的 RAG 系统时，应该尝试几种不同的方向。</em></strong>",PM,as,OM,ns,lt,I,aa,st,es,na="如上图所示，这些调整对性能的影响各不相同。尤其是调整片段大小，既简单又非常有影响力。",Mt,ys,ea="但这只是针对我们的情况：你的结果可能大不相同：现在你已经有了一个可靠的评估流水线，可以开始探索其他选项了！🗺️",tt,Js,at,Ts,nt;return d=new oa({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/cookbook/blob/main/notebooks/zh-CN/rag_evaluation.ipynb"}]}}),r=new o({props:{title:"RAG 评估",local:"rag-评估",headingTag:"h1"}}),G=new o({props:{title:"评估RAG性能",local:"评估rag性能",headingTag:"h3"}}),g=new c({props:{code:"IXBpcCUyMGluc3RhbGwlMjAtcSUyMHRvcmNoJTIwdHJhbnNmb3JtZXJzJTIwdHJhbnNmb3JtZXJzJTIwbGFuZ2NoYWluJTIwc2VudGVuY2UtdHJhbnNmb3JtZXJzJTIwdHFkbSUyMG9wZW5weXhsJTIwb3BlbmFpJTIwcGFuZGFzJTIwZGF0YXNldHMlMjBsYW5nY2hhaW4tY29tbXVuaXR5JTIwcmFnYXRvdWlsbGU=",highlighted:"!pip install -q torch transformers transformers langchain sentence-transformers tqdm openpyxl openai pandas datasets langchain-community ragatouille",wrap:!1}}),f=new c({props:{code:"JTI1cmVsb2FkX2V4dCUyMGF1dG9yZWxvYWQlMEElMjVhdXRvcmVsb2FkJTIwMg==",highlighted:`%reload_ext autoreload
%autoreload <span class="hljs-number">2</span>`,wrap:!1}}),N=new c({props:{code:"ZnJvbSUyMHRxZG0uYXV0byUyMGltcG9ydCUyMHRxZG0lMEFpbXBvcnQlMjBwYW5kYXMlMjBhcyUyMHBkJTBBZnJvbSUyMHR5cGluZyUyMGltcG9ydCUyME9wdGlvbmFsJTJDJTIwTGlzdCUyQyUyMFR1cGxlJTBBaW1wb3J0JTIwanNvbiUwQWltcG9ydCUyMGRhdGFzZXRzJTBBJTBBcGQuc2V0X29wdGlvbiglMjJkaXNwbGF5Lm1heF9jb2x3aWR0aCUyMiUyQyUyME5vbmUp",highlighted:`<span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> <span class="hljs-type">Optional</span>, <span class="hljs-type">List</span>, <span class="hljs-type">Tuple</span>
<span class="hljs-keyword">import</span> json
<span class="hljs-keyword">import</span> datasets

pd.set_option(<span class="hljs-string">&quot;display.max_colwidth&quot;</span>, <span class="hljs-literal">None</span>)`,wrap:!1}}),k=new c({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`,wrap:!1}}),E=new o({props:{title:"加载你的知识基础",local:"加载你的知识基础",headingTag:"h3"}}),v=new c({props:{code:"ZHMlMjAlM0QlMjBkYXRhc2V0cy5sb2FkX2RhdGFzZXQoJTIybS1yaWMlMkZodWdnaW5nZmFjZV9kb2MlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKQ==",highlighted:'ds = datasets.load_dataset(<span class="hljs-string">&quot;m-ric/huggingface_doc&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)',wrap:!1}}),Y=new o({props:{title:"1. 为评估构建合成数据集",local:"1-为评估构建合成数据集",headingTag:"h1"}}),_=new o({props:{title:"1.1. 准备源数据文档",local:"11-准备源数据文档",headingTag:"h3"}}),q=new c({props:{code:"ZnJvbSUyMGxhbmdjaGFpbi50ZXh0X3NwbGl0dGVyJTIwaW1wb3J0JTIwUmVjdXJzaXZlQ2hhcmFjdGVyVGV4dFNwbGl0dGVyJTBBZnJvbSUyMGxhbmdjaGFpbi5kb2NzdG9yZS5kb2N1bWVudCUyMGltcG9ydCUyMERvY3VtZW50JTIwYXMlMjBMYW5nY2hhaW5Eb2N1bWVudCUwQSUwQWxhbmdjaGFpbl9kb2NzJTIwJTNEJTIwJTVCTGFuZ2NoYWluRG9jdW1lbnQocGFnZV9jb250ZW50JTNEZG9jJTVCJTIydGV4dCUyMiU1RCUyQyUyMG1ldGFkYXRhJTNEJTdCJTIyc291cmNlJTIyJTNBJTIwZG9jJTVCJTIyc291cmNlJTIyJTVEJTdEKSUyMGZvciUyMGRvYyUyMGluJTIwdHFkbShkcyklNUQlMEElMEElMEF0ZXh0X3NwbGl0dGVyJTIwJTNEJTIwUmVjdXJzaXZlQ2hhcmFjdGVyVGV4dFNwbGl0dGVyKCUwQSUyMCUyMCUyMCUyMGNodW5rX3NpemUlM0QyMDAwJTJDJTBBJTIwJTIwJTIwJTIwY2h1bmtfb3ZlcmxhcCUzRDIwMCUyQyUwQSUyMCUyMCUyMCUyMGFkZF9zdGFydF9pbmRleCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBzZXBhcmF0b3JzJTNEJTVCJTIyJTVDbiU1Q24lMjIlMkMlMjAlMjIlNUNuJTIyJTJDJTIwJTIyLiUyMiUyQyUyMCUyMiUyMCUyMiUyQyUyMCUyMiUyMiU1RCUyQyUwQSklMEElMEFkb2NzX3Byb2Nlc3NlZCUyMCUzRCUyMCU1QiU1RCUwQWZvciUyMGRvYyUyMGluJTIwbGFuZ2NoYWluX2RvY3MlM0ElMEElMjAlMjAlMjAlMjBkb2NzX3Byb2Nlc3NlZCUyMCUyQiUzRCUyMHRleHRfc3BsaXR0ZXIuc3BsaXRfZG9jdW1lbnRzKCU1QmRvYyU1RCk=",highlighted:`<span class="hljs-keyword">from</span> langchain.text_splitter <span class="hljs-keyword">import</span> RecursiveCharacterTextSplitter
<span class="hljs-keyword">from</span> langchain.docstore.document <span class="hljs-keyword">import</span> Document <span class="hljs-keyword">as</span> LangchainDocument

langchain_docs = [LangchainDocument(page_content=doc[<span class="hljs-string">&quot;text&quot;</span>], metadata={<span class="hljs-string">&quot;source&quot;</span>: doc[<span class="hljs-string">&quot;source&quot;</span>]}) <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> tqdm(ds)]


text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=<span class="hljs-number">2000</span>,
    chunk_overlap=<span class="hljs-number">200</span>,
    add_start_index=<span class="hljs-literal">True</span>,
    separators=[<span class="hljs-string">&quot;\\n\\n&quot;</span>, <span class="hljs-string">&quot;\\n&quot;</span>, <span class="hljs-string">&quot;.&quot;</span>, <span class="hljs-string">&quot; &quot;</span>, <span class="hljs-string">&quot;&quot;</span>],
)

docs_processed = []
<span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> langchain_docs:
    docs_processed += text_splitter.split_documents([doc])`,wrap:!1}}),x=new o({props:{title:"1.2. 为问题生成设置智能体",local:"12-为问题生成设置智能体",headingTag:"h3"}}),H=new c({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEluZmVyZW5jZUNsaWVudCUwQSUwQSUwQXJlcG9faWQlMjAlM0QlMjAlMjJtaXN0cmFsYWklMkZNaXh0cmFsLTh4N0ItSW5zdHJ1Y3QtdjAuMSUyMiUwQSUwQWxsbV9jbGllbnQlMjAlM0QlMjBJbmZlcmVuY2VDbGllbnQoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0RyZXBvX2lkJTJDJTBBJTIwJTIwJTIwJTIwdGltZW91dCUzRDEyMCUyQyUwQSklMEElMEElMEFkZWYlMjBjYWxsX2xsbShpbmZlcmVuY2VfY2xpZW50JTNBJTIwSW5mZXJlbmNlQ2xpZW50JTJDJTIwcHJvbXB0JTNBJTIwc3RyKSUzQSUwQSUyMCUyMCUyMCUyMHJlc3BvbnNlJTIwJTNEJTIwaW5mZXJlbmNlX2NsaWVudC5wb3N0KCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGpzb24lM0QlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJpbnB1dHMlMjIlM0ElMjBwcm9tcHQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJwYXJhbWV0ZXJzJTIyJTNBJTIwJTdCJTIybWF4X25ld190b2tlbnMlMjIlM0ElMjAxMDAwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydGFzayUyMiUzQSUyMCUyMnRleHQtZ2VuZXJhdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBqc29uLmxvYWRzKHJlc3BvbnNlLmRlY29kZSgpKSU1QjAlNUQlNUIlMjJnZW5lcmF0ZWRfdGV4dCUyMiU1RCUwQSUwQSUwQWNhbGxfbGxtKGxsbV9jbGllbnQlMkMlMjAlMjJUaGlzJTIwaXMlMjBhJTIwdGVzdCUyMGNvbnRleHQlMjIp",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient


repo_id = <span class="hljs-string">&quot;mistralai/Mixtral-8x7B-Instruct-v0.1&quot;</span>

llm_client = InferenceClient(
    model=repo_id,
    timeout=<span class="hljs-number">120</span>,
)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">call_llm</span>(<span class="hljs-params">inference_client: InferenceClient, prompt: <span class="hljs-built_in">str</span></span>):
    response = inference_client.post(
        json={
            <span class="hljs-string">&quot;inputs&quot;</span>: prompt,
            <span class="hljs-string">&quot;parameters&quot;</span>: {<span class="hljs-string">&quot;max_new_tokens&quot;</span>: <span class="hljs-number">1000</span>},
            <span class="hljs-string">&quot;task&quot;</span>: <span class="hljs-string">&quot;text-generation&quot;</span>,
        },
    )
    <span class="hljs-keyword">return</span> json.loads(response.decode())[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>]


call_llm(llm_client, <span class="hljs-string">&quot;This is a test context&quot;</span>)`,wrap:!1}}),$=new c({props:{code:"UUFfZ2VuZXJhdGlvbl9wcm9tcHQlMjAlM0QlMjAlMjIlMjIlMjIlMEFZb3VyJTIwdGFzayUyMGlzJTIwdG8lMjB3cml0ZSUyMGElMjBmYWN0b2lkJTIwcXVlc3Rpb24lMjBhbmQlMjBhbiUyMGFuc3dlciUyMGdpdmVuJTIwYSUyMGNvbnRleHQuJTBBWW91ciUyMGZhY3RvaWQlMjBxdWVzdGlvbiUyMHNob3VsZCUyMGJlJTIwYW5zd2VyYWJsZSUyMHdpdGglMjBhJTIwc3BlY2lmaWMlMkMlMjBjb25jaXNlJTIwcGllY2UlMjBvZiUyMGZhY3R1YWwlMjBpbmZvcm1hdGlvbiUyMGZyb20lMjB0aGUlMjBjb250ZXh0LiUwQVlvdXIlMjBmYWN0b2lkJTIwcXVlc3Rpb24lMjBzaG91bGQlMjBiZSUyMGZvcm11bGF0ZWQlMjBpbiUyMHRoZSUyMHNhbWUlMjBzdHlsZSUyMGFzJTIwcXVlc3Rpb25zJTIwdXNlcnMlMjBjb3VsZCUyMGFzayUyMGluJTIwYSUyMHNlYXJjaCUyMGVuZ2luZS4lMEFUaGlzJTIwbWVhbnMlMjB0aGF0JTIweW91ciUyMGZhY3RvaWQlMjBxdWVzdGlvbiUyME1VU1QlMjBOT1QlMjBtZW50aW9uJTIwc29tZXRoaW5nJTIwbGlrZSUyMCUyMmFjY29yZGluZyUyMHRvJTIwdGhlJTIwcGFzc2FnZSUyMiUyMG9yJTIwJTIyY29udGV4dCUyMi4lMEElMEFQcm92aWRlJTIweW91ciUyMGFuc3dlciUyMGFzJTIwZm9sbG93cyUzQSUwQSUwQU91dHB1dCUzQSUzQSUzQSUwQUZhY3RvaWQlMjBxdWVzdGlvbiUzQSUyMCh5b3VyJTIwZmFjdG9pZCUyMHF1ZXN0aW9uKSUwQUFuc3dlciUzQSUyMCh5b3VyJTIwYW5zd2VyJTIwdG8lMjB0aGUlMjBmYWN0b2lkJTIwcXVlc3Rpb24pJTBBJTBBTm93JTIwaGVyZSUyMGlzJTIwdGhlJTIwY29udGV4dC4lMEElMEFDb250ZXh0JTNBJTIwJTdCY29udGV4dCU3RCU1Q24lMEFPdXRwdXQlM0ElM0ElM0ElMjIlMjIlMjI=",highlighted:`QA_generation_prompt = <span class="hljs-string">&quot;&quot;&quot;
Your task is to write a factoid question and an answer given a context.
Your factoid question should be answerable with a specific, concise piece of factual information from the context.
Your factoid question should be formulated in the same style as questions users could ask in a search engine.
This means that your factoid question MUST NOT mention something like &quot;according to the passage&quot; or &quot;context&quot;.

Provide your answer as follows:

Output:::
Factoid question: (your factoid question)
Answer: (your answer to the factoid question)

Now here is the context.

Context: {context}\\n
Output:::&quot;&quot;&quot;</span>`,wrap:!1}}),P=new c({props:{code:"aW1wb3J0JTIwcmFuZG9tJTBBJTBBTl9HRU5FUkFUSU9OUyUyMCUzRCUyMDEwJTIwJTIwJTIzJTIwV2UlMjBpbnRlbnRpb25hbGx5JTIwZ2VuZXJhdGUlMjBvbmx5JTIwMTAlMjBRQSUyMGNvdXBsZXMlMjBoZXJlJTIwZm9yJTIwY29zdCUyMGFuZCUyMHRpbWUlMjBjb25zaWRlcmF0aW9ucyUwQSUwQXByaW50KGYlMjJHZW5lcmF0aW5nJTIwJTdCTl9HRU5FUkFUSU9OUyU3RCUyMFFBJTIwY291cGxlcy4uLiUyMiklMEElMEFvdXRwdXRzJTIwJTNEJTIwJTVCJTVEJTBBZm9yJTIwc2FtcGxlZF9jb250ZXh0JTIwaW4lMjB0cWRtKHJhbmRvbS5zYW1wbGUoZG9jc19wcm9jZXNzZWQlMkMlMjBOX0dFTkVSQVRJT05TKSklM0ElMEElMjAlMjAlMjAlMjAlMjMlMjBHZW5lcmF0ZSUyMFFBJTIwY291cGxlJTBBJTIwJTIwJTIwJTIwb3V0cHV0X1FBX2NvdXBsZSUyMCUzRCUyMGNhbGxfbGxtKGxsbV9jbGllbnQlMkMlMjBRQV9nZW5lcmF0aW9uX3Byb21wdC5mb3JtYXQoY29udGV4dCUzRHNhbXBsZWRfY29udGV4dC5wYWdlX2NvbnRlbnQpKSUwQSUyMCUyMCUyMCUyMHRyeSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXN0aW9uJTIwJTNEJTIwb3V0cHV0X1FBX2NvdXBsZS5zcGxpdCglMjJGYWN0b2lkJTIwcXVlc3Rpb24lM0ElMjAlMjIpJTVCLTElNUQuc3BsaXQoJTIyQW5zd2VyJTNBJTIwJTIyKSU1QjAlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXIlMjAlM0QlMjBvdXRwdXRfUUFfY291cGxlLnNwbGl0KCUyMkFuc3dlciUzQSUyMCUyMiklNUItMSU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFzc2VydCUyMGxlbihhbnN3ZXIpJTIwJTNDJTIwMzAwJTJDJTIwJTIyQW5zd2VyJTIwaXMlMjB0b28lMjBsb25nJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cy5hcHBlbmQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGV4dCUyMiUzQSUyMHNhbXBsZWRfY29udGV4dC5wYWdlX2NvbnRlbnQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJxdWVzdGlvbiUyMiUzQSUyMHF1ZXN0aW9uJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYW5zd2VyJTIyJTNBJTIwYW5zd2VyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyc291cmNlX2RvYyUyMiUzQSUyMHNhbXBsZWRfY29udGV4dC5tZXRhZGF0YSU1QiUyMnNvdXJjZSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjBleGNlcHQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb250aW51ZQ==",highlighted:`<span class="hljs-keyword">import</span> random

N_GENERATIONS = <span class="hljs-number">10</span>  <span class="hljs-comment"># We intentionally generate only 10 QA couples here for cost and time considerations</span>

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Generating <span class="hljs-subst">{N_GENERATIONS}</span> QA couples...&quot;</span>)

outputs = []
<span class="hljs-keyword">for</span> sampled_context <span class="hljs-keyword">in</span> tqdm(random.sample(docs_processed, N_GENERATIONS)):
    <span class="hljs-comment"># Generate QA couple</span>
    output_QA_couple = call_llm(llm_client, QA_generation_prompt.<span class="hljs-built_in">format</span>(context=sampled_context.page_content))
    <span class="hljs-keyword">try</span>:
        question = output_QA_couple.split(<span class="hljs-string">&quot;Factoid question: &quot;</span>)[-<span class="hljs-number">1</span>].split(<span class="hljs-string">&quot;Answer: &quot;</span>)[<span class="hljs-number">0</span>]
        answer = output_QA_couple.split(<span class="hljs-string">&quot;Answer: &quot;</span>)[-<span class="hljs-number">1</span>]
        <span class="hljs-keyword">assert</span> <span class="hljs-built_in">len</span>(answer) &lt; <span class="hljs-number">300</span>, <span class="hljs-string">&quot;Answer is too long&quot;</span>
        outputs.append(
            {
                <span class="hljs-string">&quot;context&quot;</span>: sampled_context.page_content,
                <span class="hljs-string">&quot;question&quot;</span>: question,
                <span class="hljs-string">&quot;answer&quot;</span>: answer,
                <span class="hljs-string">&quot;source_doc&quot;</span>: sampled_context.metadata[<span class="hljs-string">&quot;source&quot;</span>],
            }
        )
    <span class="hljs-keyword">except</span>:
        <span class="hljs-keyword">continue</span>`,wrap:!1}}),O=new c({props:{code:"ZGlzcGxheShwZC5EYXRhRnJhbWUob3V0cHV0cykuaGVhZCgxKSk=",highlighted:'display(pd.DataFrame(outputs).head(<span class="hljs-number">1</span>))',wrap:!1}}),ll=new o({props:{title:"1.3. 设置批判智能体",local:"13-设置批判智能体",headingTag:"h3"}}),Ul=new c({props:{code:" ",highlighted:`question_groundedness_critique_prompt = <span class="hljs-string">&quot;&quot;&quot;
You will be given a context and a question.
Your task is to provide a &#x27;total rating&#x27; scoring how well one can answer the given question unambiguously with the given context.
Give your answer on a scale of 1 to 5, where 1 means that the question is not answerable at all given the context, and 5 means that the question is clearly and unambiguously answerable with the context.

Provide your answer as follows:

Answer:::
Evaluation: (your rationale for the rating, as a text)
Total rating: (your rating, as a number between 1 and 5)

You MUST provide values for &#x27;Evaluation:&#x27; and &#x27;Total rating:&#x27; in your answer.

Now here are the question and context.

Question: {question}\\n
Context: {context}\\n
Answer::: &quot;&quot;&quot;</span>

question_relevance_critique_prompt = <span class="hljs-string">&quot;&quot;&quot;
You will be given a question.
Your task is to provide a &#x27;total rating&#x27; representing how useful this question can be to machine learning developers building NLP applications with the Hugging Face ecosystem.
Give your answer on a scale of 1 to 5, where 1 means that the question is not useful at all, and 5 means that the question is extremely useful.

Provide your answer as follows:

Answer:::
Evaluation: (your rationale for the rating, as a text)
Total rating: (your rating, as a number between 1 and 5)

You MUST provide values for &#x27;Evaluation:&#x27; and &#x27;Total rating:&#x27; in your answer.

Now here is the question.

Question: {question}\\n
Answer::: &quot;&quot;&quot;</span>

question_standalone_critique_prompt = <span class="hljs-string">&quot;&quot;&quot;
You will be given a question.
Your task is to provide a &#x27;total rating&#x27; representing how context-independant this question is.
Give your answer on a scale of 1 to 5, where 1 means that the question depends on additional information to be understood, and 5 means that the question makes sense by itself.
For instance, if the question refers to a particular setting, like &#x27;in the context&#x27; or &#x27;in the document&#x27;, the rating must be 1.
The questions can contain obscure technical nouns or acronyms like Gradio, Hub, Hugging Face or Space and still be a 5: it must simply be clear to an operator with access to documentation what the question is about.

For instance, &quot;What is the name of the checkpoint from which the ViT model is imported?&quot; should receive a 1, since there is an implicit mention of a context, thus the question is not independant from the context.

Provide your answer as follows:

Answer:::
Evaluation: (your rationale for the rating, as a text)
Total rating: (your rating, as a number between 1 and 5)

You MUST provide values for &#x27;Evaluation:&#x27; and &#x27;Total rating:&#x27; in your answer.

Now here is the question.

Question: {question}\\n
Answer::: &quot;&quot;&quot;</span>`,wrap:!1}}),jl=new c({props:{code:"cHJpbnQoJTIyR2VuZXJhdGluZyUyMGNyaXRpcXVlJTIwZm9yJTIwZWFjaCUyMFFBJTIwY291cGxlLi4uJTIyKSUwQWZvciUyMG91dHB1dCUyMGluJTIwdHFkbShvdXRwdXRzKSUzQSUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25zJTIwJTNEJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZ3JvdW5kZWRuZXNzJTIyJTNBJTIwY2FsbF9sbG0oJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGxtX2NsaWVudCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXN0aW9uX2dyb3VuZGVkbmVzc19jcml0aXF1ZV9wcm9tcHQuZm9ybWF0KGNvbnRleHQlM0RvdXRwdXQlNUIlMjJjb250ZXh0JTIyJTVEJTJDJTIwcXVlc3Rpb24lM0RvdXRwdXQlNUIlMjJxdWVzdGlvbiUyMiU1RCklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycmVsZXZhbmNlJTIyJTNBJTIwY2FsbF9sbG0oJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGxtX2NsaWVudCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHF1ZXN0aW9uX3JlbGV2YW5jZV9jcml0aXF1ZV9wcm9tcHQuZm9ybWF0KHF1ZXN0aW9uJTNEb3V0cHV0JTVCJTIycXVlc3Rpb24lMjIlNUQpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YW5kYWxvbmUlMjIlM0ElMjBjYWxsX2xsbSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsbG1fY2xpZW50JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcXVlc3Rpb25fc3RhbmRhbG9uZV9jcml0aXF1ZV9wcm9tcHQuZm9ybWF0KHF1ZXN0aW9uJTNEb3V0cHV0JTVCJTIycXVlc3Rpb24lMjIlNUQpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUyQyUwQSUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCUyMCUyMHRyeSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMGNyaXRlcmlvbiUyQyUyMGV2YWx1YXRpb24lMjBpbiUyMGV2YWx1YXRpb25zLml0ZW1zKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzY29yZSUyQyUyMGV2YWwlMjAlM0QlMjAoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW50KGV2YWx1YXRpb24uc3BsaXQoJTIyVG90YWwlMjByYXRpbmclM0ElMjAlMjIpJTVCLTElNUQuc3RyaXAoKSklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBldmFsdWF0aW9uLnNwbGl0KCUyMlRvdGFsJTIwcmF0aW5nJTNBJTIwJTIyKSU1Qi0yJTVELnNwbGl0KCUyMkV2YWx1YXRpb24lM0ElMjAlMjIpJTVCMSU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXQudXBkYXRlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGYlMjIlN0Jjcml0ZXJpb24lN0Rfc2NvcmUlMjIlM0ElMjBzY29yZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGYlMjIlN0Jjcml0ZXJpb24lN0RfZXZhbCUyMiUzQSUyMGV2YWwlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwZXhjZXB0JTIwRXhjZXB0aW9uJTIwYXMlMjBlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY29udGludWU=",highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Generating critique for each QA couple...&quot;</span>)
<span class="hljs-keyword">for</span> output <span class="hljs-keyword">in</span> tqdm(outputs):
    evaluations = {
        <span class="hljs-string">&quot;groundedness&quot;</span>: call_llm(
            llm_client,
            question_groundedness_critique_prompt.<span class="hljs-built_in">format</span>(context=output[<span class="hljs-string">&quot;context&quot;</span>], question=output[<span class="hljs-string">&quot;question&quot;</span>]),
        ),
        <span class="hljs-string">&quot;relevance&quot;</span>: call_llm(
            llm_client,
            question_relevance_critique_prompt.<span class="hljs-built_in">format</span>(question=output[<span class="hljs-string">&quot;question&quot;</span>]),
        ),
        <span class="hljs-string">&quot;standalone&quot;</span>: call_llm(
            llm_client,
            question_standalone_critique_prompt.<span class="hljs-built_in">format</span>(question=output[<span class="hljs-string">&quot;question&quot;</span>]),
        ),
    }
    <span class="hljs-keyword">try</span>:
        <span class="hljs-keyword">for</span> criterion, evaluation <span class="hljs-keyword">in</span> evaluations.items():
            score, <span class="hljs-built_in">eval</span> = (
                <span class="hljs-built_in">int</span>(evaluation.split(<span class="hljs-string">&quot;Total rating: &quot;</span>)[-<span class="hljs-number">1</span>].strip()),
                evaluation.split(<span class="hljs-string">&quot;Total rating: &quot;</span>)[-<span class="hljs-number">2</span>].split(<span class="hljs-string">&quot;Evaluation: &quot;</span>)[<span class="hljs-number">1</span>],
            )
            output.update(
                {
                    <span class="hljs-string">f&quot;<span class="hljs-subst">{criterion}</span>_score&quot;</span>: score,
                    <span class="hljs-string">f&quot;<span class="hljs-subst">{criterion}</span>_eval&quot;</span>: <span class="hljs-built_in">eval</span>,
                }
            )
    <span class="hljs-keyword">except</span> Exception <span class="hljs-keyword">as</span> e:
        <span class="hljs-keyword">continue</span>`,wrap:!1}}),wl=new c({props:{code:"aW1wb3J0JTIwcGFuZGFzJTIwYXMlMjBwZCUwQSUwQXBkLnNldF9vcHRpb24oJTIyZGlzcGxheS5tYXhfY29sd2lkdGglMjIlMkMlMjBOb25lKSUwQSUwQWdlbmVyYXRlZF9xdWVzdGlvbnMlMjAlM0QlMjBwZC5EYXRhRnJhbWUuZnJvbV9kaWN0KG91dHB1dHMpJTBBJTBBcHJpbnQoJTIyRXZhbHVhdGlvbiUyMGRhdGFzZXQlMjBiZWZvcmUlMjBmaWx0ZXJpbmclM0ElMjIpJTBBZGlzcGxheSglMEElMjAlMjAlMjAlMjBnZW5lcmF0ZWRfcXVlc3Rpb25zJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycXVlc3Rpb24lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJhbnN3ZXIlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJncm91bmRlZG5lc3Nfc2NvcmUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZWxldmFuY2Vfc2NvcmUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFuZGFsb25lX3Njb3JlJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVEJTBBJTIwJTIwJTIwJTIwJTVEJTBBKSUwQWdlbmVyYXRlZF9xdWVzdGlvbnMlMjAlM0QlMjBnZW5lcmF0ZWRfcXVlc3Rpb25zLmxvYyU1QiUwQSUyMCUyMCUyMCUyMChnZW5lcmF0ZWRfcXVlc3Rpb25zJTVCJTIyZ3JvdW5kZWRuZXNzX3Njb3JlJTIyJTVEJTIwJTNFJTNEJTIwNCklMEElMjAlMjAlMjAlMjAlMjYlMjAoZ2VuZXJhdGVkX3F1ZXN0aW9ucyU1QiUyMnJlbGV2YW5jZV9zY29yZSUyMiU1RCUyMCUzRSUzRCUyMDQpJTBBJTIwJTIwJTIwJTIwJTI2JTIwKGdlbmVyYXRlZF9xdWVzdGlvbnMlNUIlMjJzdGFuZGFsb25lX3Njb3JlJTIyJTVEJTIwJTNFJTNEJTIwNCklMEElNUQlMEFwcmludCglMjIlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlMjIpJTBBcHJpbnQoJTIyRmluYWwlMjBldmFsdWF0aW9uJTIwZGF0YXNldCUzQSUyMiklMEFkaXNwbGF5KCUwQSUyMCUyMCUyMCUyMGdlbmVyYXRlZF9xdWVzdGlvbnMlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJxdWVzdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmFuc3dlciUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmdyb3VuZGVkbmVzc19zY29yZSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJlbGV2YW5jZV9zY29yZSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YW5kYWxvbmVfc2NvcmUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUQlMEElMjAlMjAlMjAlMjAlNUQlMEEpJTBBJTBBZXZhbF9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldHMuRGF0YXNldC5mcm9tX3BhbmRhcyhnZW5lcmF0ZWRfcXVlc3Rpb25zJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiUyQyUyMHByZXNlcnZlX2luZGV4JTNERmFsc2Up",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

<span class="hljs-meta">&gt;&gt;&gt; </span>pd.set_option(<span class="hljs-string">&quot;display.max_colwidth&quot;</span>, <span class="hljs-literal">None</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_questions = pd.DataFrame.from_dict(outputs)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Evaluation dataset before filtering:&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>display(
<span class="hljs-meta">... </span>    generated_questions[
<span class="hljs-meta">... </span>        [
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;question&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;answer&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;groundedness_score&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;relevance_score&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;standalone_score&quot;</span>,
<span class="hljs-meta">... </span>        ]
<span class="hljs-meta">... </span>    ]
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_questions = generated_questions.loc[
<span class="hljs-meta">... </span>    (generated_questions[<span class="hljs-string">&quot;groundedness_score&quot;</span>] &gt;= <span class="hljs-number">4</span>)
<span class="hljs-meta">... </span>    &amp; (generated_questions[<span class="hljs-string">&quot;relevance_score&quot;</span>] &gt;= <span class="hljs-number">4</span>)
<span class="hljs-meta">... </span>    &amp; (generated_questions[<span class="hljs-string">&quot;standalone_score&quot;</span>] &gt;= <span class="hljs-number">4</span>)
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;============================================&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Final evaluation dataset:&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>display(
<span class="hljs-meta">... </span>    generated_questions[
<span class="hljs-meta">... </span>        [
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;question&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;answer&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;groundedness_score&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;relevance_score&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;standalone_score&quot;</span>,
<span class="hljs-meta">... </span>        ]
<span class="hljs-meta">... </span>    ]
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>eval_dataset = datasets.Dataset.from_pandas(generated_questions, split=<span class="hljs-string">&quot;train&quot;</span>, preserve_index=<span class="hljs-literal">False</span>)`,wrap:!1}}),ol=new c({props:{code:"ZXZhbF9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldHMubG9hZF9kYXRhc2V0KCUyMm0tcmljJTJGaHVnZ2luZ2ZhY2VfZG9jX3FhX2V2YWwlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyKQ==",highlighted:'eval_dataset = datasets.load_dataset(<span class="hljs-string">&quot;m-ric/huggingface_doc_qa_eval&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)',wrap:!1}}),ul=new o({props:{title:"2. 构建我们的 RAG 系统",local:"2-构建我们的-rag-系统",headingTag:"h1"}}),Il=new o({props:{title:"2.1. 预处理文档来构建我们的向量数据库",local:"21-预处理文档来构建我们的向量数据库",headingTag:"h3"}}),hl=new c({props:{code:"ZnJvbSUyMGxhbmdjaGFpbi5kb2NzdG9yZS5kb2N1bWVudCUyMGltcG9ydCUyMERvY3VtZW50JTIwYXMlMjBMYW5nY2hhaW5Eb2N1bWVudCUwQSUwQVJBV19LTk9XTEVER0VfQkFTRSUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMExhbmdjaGFpbkRvY3VtZW50KHBhZ2VfY29udGVudCUzRGRvYyU1QiUyMnRleHQlMjIlNUQlMkMlMjBtZXRhZGF0YSUzRCU3QiUyMnNvdXJjZSUyMiUzQSUyMGRvYyU1QiUyMnNvdXJjZSUyMiU1RCU3RCklMjBmb3IlMjBkb2MlMjBpbiUyMHRxZG0oZHMpJTBBJTVE",highlighted:`<span class="hljs-keyword">from</span> langchain.docstore.document <span class="hljs-keyword">import</span> Document <span class="hljs-keyword">as</span> LangchainDocument

RAW_KNOWLEDGE_BASE = [
    LangchainDocument(page_content=doc[<span class="hljs-string">&quot;text&quot;</span>], metadata={<span class="hljs-string">&quot;source&quot;</span>: doc[<span class="hljs-string">&quot;source&quot;</span>]}) <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> tqdm(ds)
]`,wrap:!1}}),Bl=new c({props:{code:"ZnJvbSUyMGxhbmdjaGFpbi50ZXh0X3NwbGl0dGVyJTIwaW1wb3J0JTIwUmVjdXJzaXZlQ2hhcmFjdGVyVGV4dFNwbGl0dGVyJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEElMEFkZWYlMjBzcGxpdF9kb2N1bWVudHMoJTBBJTIwJTIwJTIwJTIwY2h1bmtfc2l6ZSUzQSUyMGludCUyQyUwQSUyMCUyMCUyMCUyMGtub3dsZWRnZV9iYXNlJTNBJTIwTGlzdCU1QkxhbmdjaGFpbkRvY3VtZW50JTVEJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVyX25hbWUlM0ElMjBzdHIlMkMlMEEpJTIwLSUzRSUyMExpc3QlNUJMYW5nY2hhaW5Eb2N1bWVudCU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMFNwbGl0JTIwZG9jdW1lbnRzJTIwaW50byUyMGNodW5rcyUyMG9mJTIwc2l6ZSUyMCU2MGNodW5rX3NpemUlNjAlMjBjaGFyYWN0ZXJzJTIwYW5kJTIwcmV0dXJuJTIwYSUyMGxpc3QlMjBvZiUyMGRvY3VtZW50cy4lMEElMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElMjAlMjAlMjAlMjB0ZXh0X3NwbGl0dGVyJTIwJTNEJTIwUmVjdXJzaXZlQ2hhcmFjdGVyVGV4dFNwbGl0dGVyLmZyb21faHVnZ2luZ2ZhY2VfdG9rZW5pemVyKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKHRva2VuaXplcl9uYW1lKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNodW5rX3NpemUlM0RjaHVua19zaXplJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY2h1bmtfb3ZlcmxhcCUzRGludChjaHVua19zaXplJTIwJTJGJTIwMTApJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWRkX3N0YXJ0X2luZGV4JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0cmlwX3doaXRlc3BhY2UlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VwYXJhdG9ycyUzRCU1QiUyMiU1Q24lNUNuJTIyJTJDJTIwJTIyJTVDbiUyMiUyQyUyMCUyMi4lMjIlMkMlMjAlMjIlMjAlMjIlMkMlMjAlMjIlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjApJTBBJTBBJTIwJTIwJTIwJTIwZG9jc19wcm9jZXNzZWQlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBkb2MlMjBpbiUyMGtub3dsZWRnZV9iYXNlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZG9jc19wcm9jZXNzZWQlMjAlMkIlM0QlMjB0ZXh0X3NwbGl0dGVyLnNwbGl0X2RvY3VtZW50cyglNUJkb2MlNUQpJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwUmVtb3ZlJTIwZHVwbGljYXRlcyUwQSUyMCUyMCUyMCUyMHVuaXF1ZV90ZXh0cyUyMCUzRCUyMCU3QiU3RCUwQSUyMCUyMCUyMCUyMGRvY3NfcHJvY2Vzc2VkX3VuaXF1ZSUyMCUzRCUyMCU1QiU1RCUwQSUyMCUyMCUyMCUyMGZvciUyMGRvYyUyMGluJTIwZG9jc19wcm9jZXNzZWQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMGRvYy5wYWdlX2NvbnRlbnQlMjBub3QlMjBpbiUyMHVuaXF1ZV90ZXh0cyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHVuaXF1ZV90ZXh0cyU1QmRvYy5wYWdlX2NvbnRlbnQlNUQlMjAlM0QlMjBUcnVlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZG9jc19wcm9jZXNzZWRfdW5pcXVlLmFwcGVuZChkb2MpJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwZG9jc19wcm9jZXNzZWRfdW5pcXVl",highlighted:`<span class="hljs-keyword">from</span> langchain.text_splitter <span class="hljs-keyword">import</span> RecursiveCharacterTextSplitter
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer


<span class="hljs-keyword">def</span> <span class="hljs-title function_">split_documents</span>(<span class="hljs-params">
    chunk_size: <span class="hljs-built_in">int</span>,
    knowledge_base: <span class="hljs-type">List</span>[LangchainDocument],
    tokenizer_name: <span class="hljs-built_in">str</span>,
</span>) -&gt; <span class="hljs-type">List</span>[LangchainDocument]:
    <span class="hljs-string">&quot;&quot;&quot;
    Split documents into chunks of size \`chunk_size\` characters and return a list of documents.
    &quot;&quot;&quot;</span>
    text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
        AutoTokenizer.from_pretrained(tokenizer_name),
        chunk_size=chunk_size,
        chunk_overlap=<span class="hljs-built_in">int</span>(chunk_size / <span class="hljs-number">10</span>),
        add_start_index=<span class="hljs-literal">True</span>,
        strip_whitespace=<span class="hljs-literal">True</span>,
        separators=[<span class="hljs-string">&quot;\\n\\n&quot;</span>, <span class="hljs-string">&quot;\\n&quot;</span>, <span class="hljs-string">&quot;.&quot;</span>, <span class="hljs-string">&quot; &quot;</span>, <span class="hljs-string">&quot;&quot;</span>],
    )

    docs_processed = []
    <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> knowledge_base:
        docs_processed += text_splitter.split_documents([doc])

    <span class="hljs-comment"># Remove duplicates</span>
    unique_texts = {}
    docs_processed_unique = []
    <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> docs_processed:
        <span class="hljs-keyword">if</span> doc.page_content <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> unique_texts:
            unique_texts[doc.page_content] = <span class="hljs-literal">True</span>
            docs_processed_unique.append(doc)

    <span class="hljs-keyword">return</span> docs_processed_unique`,wrap:!1}}),Zl=new o({props:{title:"2.2.  检索器 - 嵌入 🗂️",local:"22-检索器---嵌入-",headingTag:"h3"}}),Vl=new c({props:{code:"ZnJvbSUyMGxhbmdjaGFpbi52ZWN0b3JzdG9yZXMlMjBpbXBvcnQlMjBGQUlTUyUwQWZyb20lMjBsYW5nY2hhaW5fY29tbXVuaXR5LmVtYmVkZGluZ3MlMjBpbXBvcnQlMjBIdWdnaW5nRmFjZUVtYmVkZGluZ3MlMEFmcm9tJTIwbGFuZ2NoYWluX2NvbW11bml0eS52ZWN0b3JzdG9yZXMudXRpbHMlMjBpbXBvcnQlMjBEaXN0YW5jZVN0cmF0ZWd5JTBBaW1wb3J0JTIwb3MlMEElMEElMEFkZWYlMjBsb2FkX2VtYmVkZGluZ3MoJTBBJTIwJTIwJTIwJTIwbGFuZ2NoYWluX2RvY3MlM0ElMjBMaXN0JTVCTGFuZ2NoYWluRG9jdW1lbnQlNUQlMkMlMEElMjAlMjAlMjAlMjBjaHVua19zaXplJTNBJTIwaW50JTJDJTBBJTIwJTIwJTIwJTIwZW1iZWRkaW5nX21vZGVsX25hbWUlM0ElMjBPcHRpb25hbCU1QnN0ciU1RCUyMCUzRCUyMCUyMnRoZW5scGVyJTJGZ3RlLXNtYWxsJTIyJTJDJTBBKSUyMC0lM0UlMjBGQUlTUyUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMENyZWF0ZXMlMjBhJTIwRkFJU1MlMjBpbmRleCUyMGZyb20lMjB0aGUlMjBnaXZlbiUyMGVtYmVkZGluZyUyMG1vZGVsJTIwYW5kJTIwZG9jdW1lbnRzLiUyMExvYWRzJTIwdGhlJTIwaW5kZXglMjBkaXJlY3RseSUyMGlmJTIwaXQlMjBhbHJlYWR5JTIwZXhpc3RzLiUwQSUwQSUyMCUyMCUyMCUyMEFyZ3MlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYW5nY2hhaW5fZG9jcyUzQSUyMGxpc3QlMjBvZiUyMGRvY3VtZW50cyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNodW5rX3NpemUlM0ElMjBzaXplJTIwb2YlMjB0aGUlMjBjaHVua3MlMjB0byUyMHNwbGl0JTIwdGhlJTIwZG9jdW1lbnRzJTIwaW50byUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVtYmVkZGluZ19tb2RlbF9uYW1lJTNBJTIwbmFtZSUyMG9mJTIwdGhlJTIwZW1iZWRkaW5nJTIwbW9kZWwlMjB0byUyMHVzZSUwQSUwQSUyMCUyMCUyMCUyMFJldHVybnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBGQUlTUyUyMGluZGV4JTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwJTIzJTIwbG9hZCUyMGVtYmVkZGluZ19tb2RlbCUwQSUyMCUyMCUyMCUyMGVtYmVkZGluZ19tb2RlbCUyMCUzRCUyMEh1Z2dpbmdGYWNlRW1iZWRkaW5ncyglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbF9uYW1lJTNEZW1iZWRkaW5nX21vZGVsX25hbWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtdWx0aV9wcm9jZXNzJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1vZGVsX2t3YXJncyUzRCU3QiUyMmRldmljZSUyMiUzQSUyMCUyMmN1ZGElMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVfa3dhcmdzJTNEJTdCJTIybm9ybWFsaXplX2VtYmVkZGluZ3MlMjIlM0ElMjBUcnVlJTdEJTJDJTIwJTIwJTIzJTIwc2V0JTIwVHJ1ZSUyMHRvJTIwY29tcHV0ZSUyMGNvc2luZSUyMHNpbWlsYXJpdHklMEElMjAlMjAlMjAlMjApJTBBJTBBJTIwJTIwJTIwJTIwJTIzJTIwQ2hlY2slMjBpZiUyMGVtYmVkZGluZ3MlMjBhbHJlYWR5JTIwZXhpc3QlMjBvbiUyMGRpc2slMEElMjAlMjAlMjAlMjBpbmRleF9uYW1lJTIwJTNEJTIwZiUyMmluZGV4X2NodW5rJTNBJTdCY2h1bmtfc2l6ZSU3RF9lbWJlZGRpbmdzJTNBJTdCZW1iZWRkaW5nX21vZGVsX25hbWUucmVwbGFjZSgnJTJGJyUyQyUyMCd+JyklN0QlMjIlMEElMjAlMjAlMjAlMjBpbmRleF9mb2xkZXJfcGF0aCUyMCUzRCUyMGYlMjIuJTJGZGF0YSUyRmluZGV4ZXMlMkYlN0JpbmRleF9uYW1lJTdEJTJGJTIyJTBBJTIwJTIwJTIwJTIwaWYlMjBvcy5wYXRoLmlzZGlyKGluZGV4X2ZvbGRlcl9wYXRoKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMEZBSVNTLmxvYWRfbG9jYWwoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5kZXhfZm9sZGVyX3BhdGglMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbWJlZGRpbmdfbW9kZWwlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBkaXN0YW5jZV9zdHJhdGVneSUzRERpc3RhbmNlU3RyYXRlZ3kuQ09TSU5FJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludCglMjJJbmRleCUyMG5vdCUyMGZvdW5kJTJDJTIwZ2VuZXJhdGluZyUyMGl0Li4uJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGRvY3NfcHJvY2Vzc2VkJTIwJTNEJTIwc3BsaXRfZG9jdW1lbnRzKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNodW5rX3NpemUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYW5nY2hhaW5fZG9jcyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVtYmVkZGluZ19tb2RlbF9uYW1lJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGtub3dsZWRnZV9pbmRleCUyMCUzRCUyMEZBSVNTLmZyb21fZG9jdW1lbnRzKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGRvY3NfcHJvY2Vzc2VkJTJDJTIwZW1iZWRkaW5nX21vZGVsJTJDJTIwZGlzdGFuY2Vfc3RyYXRlZ3klM0REaXN0YW5jZVN0cmF0ZWd5LkNPU0lORSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBrbm93bGVkZ2VfaW5kZXguc2F2ZV9sb2NhbChpbmRleF9mb2xkZXJfcGF0aCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBrbm93bGVkZ2VfaW5kZXg=",highlighted:`<span class="hljs-keyword">from</span> langchain.vectorstores <span class="hljs-keyword">import</span> FAISS
<span class="hljs-keyword">from</span> langchain_community.embeddings <span class="hljs-keyword">import</span> HuggingFaceEmbeddings
<span class="hljs-keyword">from</span> langchain_community.vectorstores.utils <span class="hljs-keyword">import</span> DistanceStrategy
<span class="hljs-keyword">import</span> os


<span class="hljs-keyword">def</span> <span class="hljs-title function_">load_embeddings</span>(<span class="hljs-params">
    langchain_docs: <span class="hljs-type">List</span>[LangchainDocument],
    chunk_size: <span class="hljs-built_in">int</span>,
    embedding_model_name: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">str</span>] = <span class="hljs-string">&quot;thenlper/gte-small&quot;</span>,
</span>) -&gt; FAISS:
    <span class="hljs-string">&quot;&quot;&quot;
    Creates a FAISS index from the given embedding model and documents. Loads the index directly if it already exists.

    Args:
        langchain_docs: list of documents
        chunk_size: size of the chunks to split the documents into
        embedding_model_name: name of the embedding model to use

    Returns:
        FAISS index
    &quot;&quot;&quot;</span>
    <span class="hljs-comment"># load embedding_model</span>
    embedding_model = HuggingFaceEmbeddings(
        model_name=embedding_model_name,
        multi_process=<span class="hljs-literal">True</span>,
        model_kwargs={<span class="hljs-string">&quot;device&quot;</span>: <span class="hljs-string">&quot;cuda&quot;</span>},
        encode_kwargs={<span class="hljs-string">&quot;normalize_embeddings&quot;</span>: <span class="hljs-literal">True</span>},  <span class="hljs-comment"># set True to compute cosine similarity</span>
    )

    <span class="hljs-comment"># Check if embeddings already exist on disk</span>
    index_name = <span class="hljs-string">f&quot;index_chunk:<span class="hljs-subst">{chunk_size}</span>_embeddings:<span class="hljs-subst">{embedding_model_name.replace(<span class="hljs-string">&#x27;/&#x27;</span>, <span class="hljs-string">&#x27;~&#x27;</span>)}</span>&quot;</span>
    index_folder_path = <span class="hljs-string">f&quot;./data/indexes/<span class="hljs-subst">{index_name}</span>/&quot;</span>
    <span class="hljs-keyword">if</span> os.path.isdir(index_folder_path):
        <span class="hljs-keyword">return</span> FAISS.load_local(
            index_folder_path,
            embedding_model,
            distance_strategy=DistanceStrategy.COSINE,
        )

    <span class="hljs-keyword">else</span>:
        <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Index not found, generating it...&quot;</span>)
        docs_processed = split_documents(
            chunk_size,
            langchain_docs,
            embedding_model_name,
        )
        knowledge_index = FAISS.from_documents(
            docs_processed, embedding_model, distance_strategy=DistanceStrategy.COSINE
        )
        knowledge_index.save_local(index_folder_path)
        <span class="hljs-keyword">return</span> knowledge_index`,wrap:!1}}),Rl=new o({props:{title:"2.3. 阅读器 - LLM 💬",local:"23-阅读器---llm-",headingTag:"h3"}}),Nl=new c({props:{code:"UkFHX1BST01QVF9URU1QTEFURSUyMCUzRCUyMCUyMiUyMiUyMiUwQSUzQyU3Q3N5c3RlbSU3QyUzRSUwQVVzaW5nJTIwdGhlJTIwaW5mb3JtYXRpb24lMjBjb250YWluZWQlMjBpbiUyMHRoZSUyMGNvbnRleHQlMkMlMEFnaXZlJTIwYSUyMGNvbXByZWhlbnNpdmUlMjBhbnN3ZXIlMjB0byUyMHRoZSUyMHF1ZXN0aW9uLiUwQVJlc3BvbmQlMjBvbmx5JTIwdG8lMjB0aGUlMjBxdWVzdGlvbiUyMGFza2VkJTJDJTIwcmVzcG9uc2UlMjBzaG91bGQlMjBiZSUyMGNvbmNpc2UlMjBhbmQlMjByZWxldmFudCUyMHRvJTIwdGhlJTIwcXVlc3Rpb24uJTBBUHJvdmlkZSUyMHRoZSUyMG51bWJlciUyMG9mJTIwdGhlJTIwc291cmNlJTIwZG9jdW1lbnQlMjB3aGVuJTIwcmVsZXZhbnQuJTBBSWYlMjB0aGUlMjBhbnN3ZXIlMjBjYW5ub3QlMjBiZSUyMGRlZHVjZWQlMjBmcm9tJTIwdGhlJTIwY29udGV4dCUyQyUyMGRvJTIwbm90JTIwZ2l2ZSUyMGFuJTIwYW5zd2VyLiUzQyUyRnMlM0UlMEElM0MlN0N1c2VyJTdDJTNFJTBBQ29udGV4dCUzQSUwQSU3QmNvbnRleHQlN0QlMEEtLS0lMEFOb3clMjBoZXJlJTIwaXMlMjB0aGUlMjBxdWVzdGlvbiUyMHlvdSUyMG5lZWQlMjB0byUyMGFuc3dlci4lMEElMEFRdWVzdGlvbiUzQSUyMCU3QnF1ZXN0aW9uJTdEJTBBJTNDJTJGcyUzRSUwQSUzQyU3Q2Fzc2lzdGFudCU3QyUzRSUwQSUyMiUyMiUyMg==",highlighted:`RAG_PROMPT_TEMPLATE = <span class="hljs-string">&quot;&quot;&quot;
&lt;|system|&gt;
Using the information contained in the context,
give a comprehensive answer to the question.
Respond only to the question asked, response should be concise and relevant to the question.
Provide the number of the source document when relevant.
If the answer cannot be deduced from the context, do not give an answer.&lt;/s&gt;
&lt;|user|&gt;
Context:
{context}
---
Now here is the question you need to answer.

Question: {question}
&lt;/s&gt;
&lt;|assistant|&gt;
&quot;&quot;&quot;</span>`,wrap:!1}}),kl=new c({props:{code:"ZnJvbSUyMGxhbmdjaGFpbl9jb21tdW5pdHkubGxtcyUyMGltcG9ydCUyMEh1Z2dpbmdGYWNlSHViJTBBJTBBcmVwb19pZCUyMCUzRCUyMCUyMkh1Z2dpbmdGYWNlSDQlMkZ6ZXBoeXItN2ItYmV0YSUyMiUwQVJFQURFUl9NT0RFTF9OQU1FJTIwJTNEJTIwJTIyemVwaHlyLTdiLWJldGElMjIlMEFIRl9BUElfVE9LRU4lMjAlM0QlMjAlMjIlMjIlMEElMEFSRUFERVJfTExNJTIwJTNEJTIwSHVnZ2luZ0ZhY2VIdWIoJTBBJTIwJTIwJTIwJTIwcmVwb19pZCUzRHJlcG9faWQlMkMlMEElMjAlMjAlMjAlMjB0YXNrJTNEJTIydGV4dC1nZW5lcmF0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwaHVnZ2luZ2ZhY2VodWJfYXBpX3Rva2VuJTNESEZfQVBJX1RPS0VOJTJDJTBBJTIwJTIwJTIwJTIwbW9kZWxfa3dhcmdzJTNEJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybWF4X25ld190b2tlbnMlMjIlM0ElMjA1MTIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0b3BfayUyMiUzQSUyMDMwJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydGVtcGVyYXR1cmUlMjIlM0ElMjAwLjElMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZXBldGl0aW9uX3BlbmFsdHklMjIlM0ElMjAxLjAzJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> langchain_community.llms <span class="hljs-keyword">import</span> HuggingFaceHub

repo_id = <span class="hljs-string">&quot;HuggingFaceH4/zephyr-7b-beta&quot;</span>
READER_MODEL_NAME = <span class="hljs-string">&quot;zephyr-7b-beta&quot;</span>
HF_API_TOKEN = <span class="hljs-string">&quot;&quot;</span>

READER_LLM = HuggingFaceHub(
    repo_id=repo_id,
    task=<span class="hljs-string">&quot;text-generation&quot;</span>,
    huggingfacehub_api_token=HF_API_TOKEN,
    model_kwargs={
        <span class="hljs-string">&quot;max_new_tokens&quot;</span>: <span class="hljs-number">512</span>,
        <span class="hljs-string">&quot;top_k&quot;</span>: <span class="hljs-number">30</span>,
        <span class="hljs-string">&quot;temperature&quot;</span>: <span class="hljs-number">0.1</span>,
        <span class="hljs-string">&quot;repetition_penalty&quot;</span>: <span class="hljs-number">1.03</span>,
    },
)`,wrap:!1}}),El=new c({props:{code:"ZnJvbSUyMHJhZ2F0b3VpbGxlJTIwaW1wb3J0JTIwUkFHUHJldHJhaW5lZE1vZGVsJTBBZnJvbSUyMGxhbmdjaGFpbl9jb3JlLnZlY3RvcnN0b3JlcyUyMGltcG9ydCUyMFZlY3RvclN0b3JlJTBBZnJvbSUyMGxhbmdjaGFpbl9jb3JlLmxhbmd1YWdlX21vZGVscy5sbG1zJTIwaW1wb3J0JTIwTExNJTBBJTBBJTBBZGVmJTIwYW5zd2VyX3dpdGhfcmFnKCUwQSUyMCUyMCUyMCUyMHF1ZXN0aW9uJTNBJTIwc3RyJTJDJTBBJTIwJTIwJTIwJTIwbGxtJTNBJTIwTExNJTJDJTBBJTIwJTIwJTIwJTIwa25vd2xlZGdlX2luZGV4JTNBJTIwVmVjdG9yU3RvcmUlMkMlMEElMjAlMjAlMjAlMjByZXJhbmtlciUzQSUyME9wdGlvbmFsJTVCUkFHUHJldHJhaW5lZE1vZGVsJTVEJTIwJTNEJTIwTm9uZSUyQyUwQSUyMCUyMCUyMCUyMG51bV9yZXRyaWV2ZWRfZG9jcyUzQSUyMGludCUyMCUzRCUyMDMwJTJDJTBBJTIwJTIwJTIwJTIwbnVtX2RvY3NfZmluYWwlM0ElMjBpbnQlMjAlM0QlMjA3JTJDJTBBKSUyMC0lM0UlMjBUdXBsZSU1QnN0ciUyQyUyMExpc3QlNUJMYW5nY2hhaW5Eb2N1bWVudCU1RCU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMkFuc3dlciUyMGElMjBxdWVzdGlvbiUyMHVzaW5nJTIwUkFHJTIwd2l0aCUyMHRoZSUyMGdpdmVuJTIwa25vd2xlZGdlJTIwaW5kZXguJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwJTIzJTIwR2F0aGVyJTIwZG9jdW1lbnRzJTIwd2l0aCUyMHJldHJpZXZlciUwQSUyMCUyMCUyMCUyMHJlbGV2YW50X2RvY3MlMjAlM0QlMjBrbm93bGVkZ2VfaW5kZXguc2ltaWxhcml0eV9zZWFyY2gocXVlcnklM0RxdWVzdGlvbiUyQyUyMGslM0RudW1fcmV0cmlldmVkX2RvY3MpJTBBJTIwJTIwJTIwJTIwcmVsZXZhbnRfZG9jcyUyMCUzRCUyMCU1QmRvYy5wYWdlX2NvbnRlbnQlMjBmb3IlMjBkb2MlMjBpbiUyMHJlbGV2YW50X2RvY3MlNUQlMjAlMjAlMjMlMjBrZWVwJTIwb25seSUyMHRoZSUyMHRleHQlMEElMEElMjAlMjAlMjAlMjAlMjMlMjBPcHRpb25hbGx5JTIwcmVyYW5rJTIwcmVzdWx0cyUwQSUyMCUyMCUyMCUyMGlmJTIwcmVyYW5rZXIlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZWxldmFudF9kb2NzJTIwJTNEJTIwcmVyYW5rZXIucmVyYW5rKHF1ZXN0aW9uJTJDJTIwcmVsZXZhbnRfZG9jcyUyQyUyMGslM0RudW1fZG9jc19maW5hbCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZWxldmFudF9kb2NzJTIwJTNEJTIwJTVCZG9jJTVCJTIyY29udGVudCUyMiU1RCUyMGZvciUyMGRvYyUyMGluJTIwcmVsZXZhbnRfZG9jcyU1RCUwQSUwQSUyMCUyMCUyMCUyMHJlbGV2YW50X2RvY3MlMjAlM0QlMjByZWxldmFudF9kb2NzJTVCJTNBbnVtX2RvY3NfZmluYWwlNUQlMEElMEElMjAlMjAlMjAlMjAlMjMlMjBCdWlsZCUyMHRoZSUyMGZpbmFsJTIwcHJvbXB0JTBBJTIwJTIwJTIwJTIwY29udGV4dCUyMCUzRCUyMCUyMiU1Q25FeHRyYWN0ZWQlMjBkb2N1bWVudHMlM0ElNUNuJTIyJTBBJTIwJTIwJTIwJTIwY29udGV4dCUyMCUyQiUzRCUyMCUyMiUyMi5qb2luKCU1QmYlMjJEb2N1bWVudCUyMCU3QnN0cihpKSU3RCUzQSUzQSUzQSU1Q24lMjIlMjAlMkIlMjBkb2MlMjBmb3IlMjBpJTJDJTIwZG9jJTIwaW4lMjBlbnVtZXJhdGUocmVsZXZhbnRfZG9jcyklNUQpJTBBJTBBJTIwJTIwJTIwJTIwZmluYWxfcHJvbXB0JTIwJTNEJTIwUkFHX1BST01QVF9URU1QTEFURS5mb3JtYXQocXVlc3Rpb24lM0RxdWVzdGlvbiUyQyUyMGNvbnRleHQlM0Rjb250ZXh0KSUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyMFJlZGFjdCUyMGFuJTIwYW5zd2VyJTBBJTIwJTIwJTIwJTIwYW5zd2VyJTIwJTNEJTIwbGxtKGZpbmFsX3Byb21wdCklMEElMEElMjAlMjAlMjAlMjByZXR1cm4lMjBhbnN3ZXIlMkMlMjByZWxldmFudF9kb2Nz",highlighted:`<span class="hljs-keyword">from</span> ragatouille <span class="hljs-keyword">import</span> RAGPretrainedModel
<span class="hljs-keyword">from</span> langchain_core.vectorstores <span class="hljs-keyword">import</span> VectorStore
<span class="hljs-keyword">from</span> langchain_core.language_models.llms <span class="hljs-keyword">import</span> LLM


<span class="hljs-keyword">def</span> <span class="hljs-title function_">answer_with_rag</span>(<span class="hljs-params">
    question: <span class="hljs-built_in">str</span>,
    llm: LLM,
    knowledge_index: VectorStore,
    reranker: <span class="hljs-type">Optional</span>[RAGPretrainedModel] = <span class="hljs-literal">None</span>,
    num_retrieved_docs: <span class="hljs-built_in">int</span> = <span class="hljs-number">30</span>,
    num_docs_final: <span class="hljs-built_in">int</span> = <span class="hljs-number">7</span>,
</span>) -&gt; <span class="hljs-type">Tuple</span>[<span class="hljs-built_in">str</span>, <span class="hljs-type">List</span>[LangchainDocument]]:
    <span class="hljs-string">&quot;&quot;&quot;Answer a question using RAG with the given knowledge index.&quot;&quot;&quot;</span>
    <span class="hljs-comment"># Gather documents with retriever</span>
    relevant_docs = knowledge_index.similarity_search(query=question, k=num_retrieved_docs)
    relevant_docs = [doc.page_content <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> relevant_docs]  <span class="hljs-comment"># keep only the text</span>

    <span class="hljs-comment"># Optionally rerank results</span>
    <span class="hljs-keyword">if</span> reranker:
        relevant_docs = reranker.rerank(question, relevant_docs, k=num_docs_final)
        relevant_docs = [doc[<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> relevant_docs]

    relevant_docs = relevant_docs[:num_docs_final]

    <span class="hljs-comment"># Build the final prompt</span>
    context = <span class="hljs-string">&quot;\\nExtracted documents:\\n&quot;</span>
    context += <span class="hljs-string">&quot;&quot;</span>.join([<span class="hljs-string">f&quot;Document <span class="hljs-subst">{<span class="hljs-built_in">str</span>(i)}</span>:::\\n&quot;</span> + doc <span class="hljs-keyword">for</span> i, doc <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(relevant_docs)])

    final_prompt = RAG_PROMPT_TEMPLATE.<span class="hljs-built_in">format</span>(question=question, context=context)

    <span class="hljs-comment"># Redact an answer</span>
    answer = llm(final_prompt)

    <span class="hljs-keyword">return</span> answer, relevant_docs`,wrap:!1}}),vl=new o({props:{title:"3. 对 RAG 系统进行基准测试",local:"3-对-rag-系统进行基准测试",headingTag:"h1"}}),xl=new c({props:{code:"ZnJvbSUyMGxhbmdjaGFpbl9jb3JlLmxhbmd1YWdlX21vZGVscyUyMGltcG9ydCUyMEJhc2VDaGF0TW9kZWwlMEElMEElMEFkZWYlMjBydW5fcmFnX3Rlc3RzKCUwQSUyMCUyMCUyMCUyMGV2YWxfZGF0YXNldCUzQSUyMGRhdGFzZXRzLkRhdGFzZXQlMkMlMEElMjAlMjAlMjAlMjBsbG0lMkMlMEElMjAlMjAlMjAlMjBrbm93bGVkZ2VfaW5kZXglM0ElMjBWZWN0b3JTdG9yZSUyQyUwQSUyMCUyMCUyMCUyMG91dHB1dF9maWxlJTNBJTIwc3RyJTJDJTBBJTIwJTIwJTIwJTIwcmVyYW5rZXIlM0ElMjBPcHRpb25hbCU1QlJBR1ByZXRyYWluZWRNb2RlbCU1RCUyMCUzRCUyME5vbmUlMkMlMEElMjAlMjAlMjAlMjB2ZXJib3NlJTNBJTIwT3B0aW9uYWwlNUJib29sJTVEJTIwJTNEJTIwVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHRlc3Rfc2V0dGluZ3MlM0ElMjBPcHRpb25hbCU1QnN0ciU1RCUyMCUzRCUyME5vbmUlMkMlMjAlMjAlMjMlMjBUbyUyMGRvY3VtZW50JTIwdGhlJTIwdGVzdCUyMHNldHRpbmdzJTIwdXNlZCUwQSklM0ElMEElMjAlMjAlMjAlMjAlMjIlMjIlMjJSdW5zJTIwUkFHJTIwdGVzdHMlMjBvbiUyMHRoZSUyMGdpdmVuJTIwZGF0YXNldCUyMGFuZCUyMHNhdmVzJTIwdGhlJTIwcmVzdWx0cyUyMHRvJTIwdGhlJTIwZ2l2ZW4lMjBvdXRwdXQlMjBmaWxlLiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMHRyeSUzQSUyMCUyMCUyMyUyMGxvYWQlMjBwcmV2aW91cyUyMGdlbmVyYXRpb25zJTIwaWYlMjB0aGV5JTIwZXhpc3QlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3aXRoJTIwb3BlbihvdXRwdXRfZmlsZSUyQyUyMCUyMnIlMjIpJTIwYXMlMjBmJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMGpzb24ubG9hZChmKSUwQSUyMCUyMCUyMCUyMGV4Y2VwdCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjAlNUIlNUQlMEElMEElMjAlMjAlMjAlMjBmb3IlMjBleGFtcGxlJTIwaW4lMjB0cWRtKGV2YWxfZGF0YXNldCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBxdWVzdGlvbiUyMCUzRCUyMGV4YW1wbGUlNUIlMjJxdWVzdGlvbiUyMiU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwcXVlc3Rpb24lMjBpbiUyMCU1Qm91dHB1dCU1QiUyMnF1ZXN0aW9uJTIyJTVEJTIwZm9yJTIwb3V0cHV0JTIwaW4lMjBvdXRwdXRzJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY29udGludWUlMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXIlMkMlMjByZWxldmFudF9kb2NzJTIwJTNEJTIwYW5zd2VyX3dpdGhfcmFnKHF1ZXN0aW9uJTJDJTIwbGxtJTJDJTIwa25vd2xlZGdlX2luZGV4JTJDJTIwcmVyYW5rZXIlM0RyZXJhbmtlciklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHZlcmJvc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludCglMjIlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlM0QlMjIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZiUyMlF1ZXN0aW9uJTNBJTIwJTdCcXVlc3Rpb24lN0QlMjIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZiUyMkFuc3dlciUzQSUyMCU3QmFuc3dlciU3RCUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludChmJ1RydWUlMjBhbnN3ZXIlM0ElMjAlN0JleGFtcGxlJTVCJTIyYW5zd2VyJTIyJTVEJTdEJyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXN1bHQlMjAlM0QlMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJxdWVzdGlvbiUyMiUzQSUyMHF1ZXN0aW9uJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydHJ1ZV9hbnN3ZXIlMjIlM0ElMjBleGFtcGxlJTVCJTIyYW5zd2VyJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyc291cmNlX2RvYyUyMiUzQSUyMGV4YW1wbGUlNUIlMjJzb3VyY2VfZG9jJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZ2VuZXJhdGVkX2Fuc3dlciUyMiUzQSUyMGFuc3dlciUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJldHJpZXZlZF9kb2NzJTIyJTNBJTIwJTVCZG9jJTIwZm9yJTIwZG9jJTIwaW4lMjByZWxldmFudF9kb2NzJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjB0ZXN0X3NldHRpbmdzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0JTVCJTIydGVzdF9zZXR0aW5ncyUyMiU1RCUyMCUzRCUyMHRlc3Rfc2V0dGluZ3MlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzLmFwcGVuZChyZXN1bHQpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2l0aCUyMG9wZW4ob3V0cHV0X2ZpbGUlMkMlMjAlMjJ3JTIyKSUyMGFzJTIwZiUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGpzb24uZHVtcChvdXRwdXRzJTJDJTIwZik=",highlighted:`<span class="hljs-keyword">from</span> langchain_core.language_models <span class="hljs-keyword">import</span> BaseChatModel


<span class="hljs-keyword">def</span> <span class="hljs-title function_">run_rag_tests</span>(<span class="hljs-params">
    eval_dataset: datasets.Dataset,
    llm,
    knowledge_index: VectorStore,
    output_file: <span class="hljs-built_in">str</span>,
    reranker: <span class="hljs-type">Optional</span>[RAGPretrainedModel] = <span class="hljs-literal">None</span>,
    verbose: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">bool</span>] = <span class="hljs-literal">True</span>,
    test_settings: <span class="hljs-type">Optional</span>[<span class="hljs-built_in">str</span>] = <span class="hljs-literal">None</span>,  <span class="hljs-comment"># To document the test settings used</span>
</span>):
    <span class="hljs-string">&quot;&quot;&quot;Runs RAG tests on the given dataset and saves the results to the given output file.&quot;&quot;&quot;</span>
    <span class="hljs-keyword">try</span>:  <span class="hljs-comment"># load previous generations if they exist</span>
        <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(output_file, <span class="hljs-string">&quot;r&quot;</span>) <span class="hljs-keyword">as</span> f:
            outputs = json.load(f)
    <span class="hljs-keyword">except</span>:
        outputs = []

    <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> tqdm(eval_dataset):
        question = example[<span class="hljs-string">&quot;question&quot;</span>]
        <span class="hljs-keyword">if</span> question <span class="hljs-keyword">in</span> [output[<span class="hljs-string">&quot;question&quot;</span>] <span class="hljs-keyword">for</span> output <span class="hljs-keyword">in</span> outputs]:
            <span class="hljs-keyword">continue</span>

        answer, relevant_docs = answer_with_rag(question, llm, knowledge_index, reranker=reranker)
        <span class="hljs-keyword">if</span> verbose:
            <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=======================================================&quot;</span>)
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Question: <span class="hljs-subst">{question}</span>&quot;</span>)
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Answer: <span class="hljs-subst">{answer}</span>&quot;</span>)
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&#x27;True answer: <span class="hljs-subst">{example[<span class="hljs-string">&quot;answer&quot;</span>]}</span>&#x27;</span>)
        result = {
            <span class="hljs-string">&quot;question&quot;</span>: question,
            <span class="hljs-string">&quot;true_answer&quot;</span>: example[<span class="hljs-string">&quot;answer&quot;</span>],
            <span class="hljs-string">&quot;source_doc&quot;</span>: example[<span class="hljs-string">&quot;source_doc&quot;</span>],
            <span class="hljs-string">&quot;generated_answer&quot;</span>: answer,
            <span class="hljs-string">&quot;retrieved_docs&quot;</span>: [doc <span class="hljs-keyword">for</span> doc <span class="hljs-keyword">in</span> relevant_docs],
        }
        <span class="hljs-keyword">if</span> test_settings:
            result[<span class="hljs-string">&quot;test_settings&quot;</span>] = test_settings
        outputs.append(result)

        <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(output_file, <span class="hljs-string">&quot;w&quot;</span>) <span class="hljs-keyword">as</span> f:
            json.dump(outputs, f)`,wrap:!1}}),zl=new c({props:{code:"RVZBTFVBVElPTl9QUk9NUFQlMjAlM0QlMjAlMjIlMjIlMjIlMjMlMjMlMjNUYXNrJTIwRGVzY3JpcHRpb24lM0ElMEFBbiUyMGluc3RydWN0aW9uJTIwKG1pZ2h0JTIwaW5jbHVkZSUyMGFuJTIwSW5wdXQlMjBpbnNpZGUlMjBpdCklMkMlMjBhJTIwcmVzcG9uc2UlMjB0byUyMGV2YWx1YXRlJTJDJTIwYSUyMHJlZmVyZW5jZSUyMGFuc3dlciUyMHRoYXQlMjBnZXRzJTIwYSUyMHNjb3JlJTIwb2YlMjA1JTJDJTIwYW5kJTIwYSUyMHNjb3JlJTIwcnVicmljJTIwcmVwcmVzZW50aW5nJTIwYSUyMGV2YWx1YXRpb24lMjBjcml0ZXJpYSUyMGFyZSUyMGdpdmVuLiUwQTEuJTIwV3JpdGUlMjBhJTIwZGV0YWlsZWQlMjBmZWVkYmFjayUyMHRoYXQlMjBhc3Nlc3MlMjB0aGUlMjBxdWFsaXR5JTIwb2YlMjB0aGUlMjByZXNwb25zZSUyMHN0cmljdGx5JTIwYmFzZWQlMjBvbiUyMHRoZSUyMGdpdmVuJTIwc2NvcmUlMjBydWJyaWMlMkMlMjBub3QlMjBldmFsdWF0aW5nJTIwaW4lMjBnZW5lcmFsLiUwQTIuJTIwQWZ0ZXIlMjB3cml0aW5nJTIwYSUyMGZlZWRiYWNrJTJDJTIwd3JpdGUlMjBhJTIwc2NvcmUlMjB0aGF0JTIwaXMlMjBhbiUyMGludGVnZXIlMjBiZXR3ZWVuJTIwMSUyMGFuZCUyMDUuJTIwWW91JTIwc2hvdWxkJTIwcmVmZXIlMjB0byUyMHRoZSUyMHNjb3JlJTIwcnVicmljLiUwQTMuJTIwVGhlJTIwb3V0cHV0JTIwZm9ybWF0JTIwc2hvdWxkJTIwbG9vayUyMGFzJTIwZm9sbG93cyUzQSUyMCU1QyUyMkZlZWRiYWNrJTNBJTIwJTdCJTdCd3JpdGUlMjBhJTIwZmVlZGJhY2slMjBmb3IlMjBjcml0ZXJpYSU3RCU3RCUyMCU1QlJFU1VMVCU1RCUyMCU3QiU3QmFuJTIwaW50ZWdlciUyMG51bWJlciUyMGJldHdlZW4lMjAxJTIwYW5kJTIwNSU3RCU3RCU1QyUyMiUwQTQuJTIwUGxlYXNlJTIwZG8lMjBub3QlMjBnZW5lcmF0ZSUyMGFueSUyMG90aGVyJTIwb3BlbmluZyUyQyUyMGNsb3NpbmclMkMlMjBhbmQlMjBleHBsYW5hdGlvbnMuJTIwQmUlMjBzdXJlJTIwdG8lMjBpbmNsdWRlJTIwJTVCUkVTVUxUJTVEJTIwaW4lMjB5b3VyJTIwb3V0cHV0LiUwQSUwQSUyMyUyMyUyM1RoZSUyMGluc3RydWN0aW9uJTIwdG8lMjBldmFsdWF0ZSUzQSUwQSU3Qmluc3RydWN0aW9uJTdEJTBBJTBBJTIzJTIzJTIzUmVzcG9uc2UlMjB0byUyMGV2YWx1YXRlJTNBJTBBJTdCcmVzcG9uc2UlN0QlMEElMEElMjMlMjMlMjNSZWZlcmVuY2UlMjBBbnN3ZXIlMjAoU2NvcmUlMjA1KSUzQSUwQSU3QnJlZmVyZW5jZV9hbnN3ZXIlN0QlMEElMEElMjMlMjMlMjNTY29yZSUyMFJ1YnJpY3MlM0ElMEElNUJJcyUyMHRoZSUyMHJlc3BvbnNlJTIwY29ycmVjdCUyQyUyMGFjY3VyYXRlJTJDJTIwYW5kJTIwZmFjdHVhbCUyMGJhc2VkJTIwb24lMjB0aGUlMjByZWZlcmVuY2UlMjBhbnN3ZXIlM0YlNUQlMEFTY29yZSUyMDElM0ElMjBUaGUlMjByZXNwb25zZSUyMGlzJTIwY29tcGxldGVseSUyMGluY29ycmVjdCUyQyUyMGluYWNjdXJhdGUlMkMlMjBhbmQlMkZvciUyMG5vdCUyMGZhY3R1YWwuJTBBU2NvcmUlMjAyJTNBJTIwVGhlJTIwcmVzcG9uc2UlMjBpcyUyMG1vc3RseSUyMGluY29ycmVjdCUyQyUyMGluYWNjdXJhdGUlMkMlMjBhbmQlMkZvciUyMG5vdCUyMGZhY3R1YWwuJTBBU2NvcmUlMjAzJTNBJTIwVGhlJTIwcmVzcG9uc2UlMjBpcyUyMHNvbWV3aGF0JTIwY29ycmVjdCUyQyUyMGFjY3VyYXRlJTJDJTIwYW5kJTJGb3IlMjBmYWN0dWFsLiUwQVNjb3JlJTIwNCUzQSUyMFRoZSUyMHJlc3BvbnNlJTIwaXMlMjBtb3N0bHklMjBjb3JyZWN0JTJDJTIwYWNjdXJhdGUlMkMlMjBhbmQlMjBmYWN0dWFsLiUwQVNjb3JlJTIwNSUzQSUyMFRoZSUyMHJlc3BvbnNlJTIwaXMlMjBjb21wbGV0ZWx5JTIwY29ycmVjdCUyQyUyMGFjY3VyYXRlJTJDJTIwYW5kJTIwZmFjdHVhbC4lMEElMEElMjMlMjMlMjNGZWVkYmFjayUzQSUyMiUyMiUyMiUwQSUwQWZyb20lMjBsYW5nY2hhaW4ucHJvbXB0cy5jaGF0JTIwaW1wb3J0JTIwKCUwQSUyMCUyMCUyMCUyMENoYXRQcm9tcHRUZW1wbGF0ZSUyQyUwQSUyMCUyMCUyMCUyMEh1bWFuTWVzc2FnZVByb21wdFRlbXBsYXRlJTJDJTBBKSUwQWZyb20lMjBsYW5nY2hhaW4uc2NoZW1hJTIwaW1wb3J0JTIwU3lzdGVtTWVzc2FnZSUwQSUwQSUwQWV2YWx1YXRpb25fcHJvbXB0X3RlbXBsYXRlJTIwJTNEJTIwQ2hhdFByb21wdFRlbXBsYXRlLmZyb21fbWVzc2FnZXMoJTBBJTIwJTIwJTIwJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwU3lzdGVtTWVzc2FnZShjb250ZW50JTNEJTIyWW91JTIwYXJlJTIwYSUyMGZhaXIlMjBldmFsdWF0b3IlMjBsYW5ndWFnZSUyMG1vZGVsLiUyMiklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBIdW1hbk1lc3NhZ2VQcm9tcHRUZW1wbGF0ZS5mcm9tX3RlbXBsYXRlKEVWQUxVQVRJT05fUFJPTVBUKSUyQyUwQSUyMCUyMCUyMCUyMCU1RCUwQSk=",highlighted:`EVALUATION_PROMPT = <span class="hljs-string">&quot;&quot;&quot;###Task Description:
An instruction (might include an Input inside it), a response to evaluate, a reference answer that gets a score of 5, and a score rubric representing a evaluation criteria are given.
1. Write a detailed feedback that assess the quality of the response strictly based on the given score rubric, not evaluating in general.
2. After writing a feedback, write a score that is an integer between 1 and 5. You should refer to the score rubric.
3. The output format should look as follows: \\&quot;Feedback: {{write a feedback for criteria}} [RESULT] {{an integer number between 1 and 5}}\\&quot;
4. Please do not generate any other opening, closing, and explanations. Be sure to include [RESULT] in your output.

###The instruction to evaluate:
{instruction}

###Response to evaluate:
{response}

###Reference Answer (Score 5):
{reference_answer}

###Score Rubrics:
[Is the response correct, accurate, and factual based on the reference answer?]
Score 1: The response is completely incorrect, inaccurate, and/or not factual.
Score 2: The response is mostly incorrect, inaccurate, and/or not factual.
Score 3: The response is somewhat correct, accurate, and/or factual.
Score 4: The response is mostly correct, accurate, and factual.
Score 5: The response is completely correct, accurate, and factual.

###Feedback:&quot;&quot;&quot;</span>

<span class="hljs-keyword">from</span> langchain.prompts.chat <span class="hljs-keyword">import</span> (
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
)
<span class="hljs-keyword">from</span> langchain.schema <span class="hljs-keyword">import</span> SystemMessage


evaluation_prompt_template = ChatPromptTemplate.from_messages(
    [
        SystemMessage(content=<span class="hljs-string">&quot;You are a fair evaluator language model.&quot;</span>),
        HumanMessagePromptTemplate.from_template(EVALUATION_PROMPT),
    ]
)`,wrap:!1}}),Hl=new c({props:{code:"ZnJvbSUyMGxhbmdjaGFpbi5jaGF0X21vZGVscyUyMGltcG9ydCUyMENoYXRPcGVuQUklMEElMEFPUEVOQUlfQVBJX0tFWSUyMCUzRCUyMCUyMiUyMiUwQSUwQWV2YWxfY2hhdF9tb2RlbCUyMCUzRCUyMENoYXRPcGVuQUkobW9kZWwlM0QlMjJncHQtNC0xMTA2LXByZXZpZXclMjIlMkMlMjB0ZW1wZXJhdHVyZSUzRDAlMkMlMjBvcGVuYWlfYXBpX2tleSUzRE9QRU5BSV9BUElfS0VZKSUwQWV2YWx1YXRvcl9uYW1lJTIwJTNEJTIwJTIyR1BUNCUyMiUwQSUwQSUwQWRlZiUyMGV2YWx1YXRlX2Fuc3dlcnMoJTBBJTIwJTIwJTIwJTIwYW5zd2VyX3BhdGglM0ElMjBzdHIlMkMlMEElMjAlMjAlMjAlMjBldmFsX2NoYXRfbW9kZWwlMkMlMEElMjAlMjAlMjAlMjBldmFsdWF0b3JfbmFtZSUzQSUyMHN0ciUyQyUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25fcHJvbXB0X3RlbXBsYXRlJTNBJTIwQ2hhdFByb21wdFRlbXBsYXRlJTJDJTBBKSUyMC0lM0UlMjBOb25lJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyRXZhbHVhdGVzJTIwZ2VuZXJhdGVkJTIwYW5zd2Vycy4lMjBNb2RpZmllcyUyMHRoZSUyMGdpdmVuJTIwYW5zd2VyJTIwZmlsZSUyMGluJTIwcGxhY2UlMjBmb3IlMjBiZXR0ZXIlMjBjaGVja3BvaW50aW5nLiUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMGFuc3dlcnMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBpZiUyMG9zLnBhdGguaXNmaWxlKGFuc3dlcl9wYXRoKSUzQSUyMCUyMCUyMyUyMGxvYWQlMjBwcmV2aW91cyUyMGdlbmVyYXRpb25zJTIwaWYlMjB0aGV5JTIwZXhpc3QlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhbnN3ZXJzJTIwJTNEJTIwanNvbi5sb2FkKG9wZW4oYW5zd2VyX3BhdGglMkMlMjAlMjJyJTIyKSklMEElMEElMjAlMjAlMjAlMjBmb3IlMjBleHBlcmltZW50JTIwaW4lMjB0cWRtKGFuc3dlcnMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBmJTIyZXZhbF9zY29yZV8lN0JldmFsdWF0b3JfbmFtZSU3RCUyMiUyMGluJTIwZXhwZXJpbWVudCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbnRpbnVlJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXZhbF9wcm9tcHQlMjAlM0QlMjBldmFsdWF0aW9uX3Byb21wdF90ZW1wbGF0ZS5mb3JtYXRfbWVzc2FnZXMoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5zdHJ1Y3Rpb24lM0RleHBlcmltZW50JTVCJTIycXVlc3Rpb24lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXNwb25zZSUzRGV4cGVyaW1lbnQlNUIlMjJnZW5lcmF0ZWRfYW5zd2VyJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVmZXJlbmNlX2Fuc3dlciUzRGV4cGVyaW1lbnQlNUIlMjJ0cnVlX2Fuc3dlciUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBldmFsX3Jlc3VsdCUyMCUzRCUyMGV2YWxfY2hhdF9tb2RlbC5pbnZva2UoZXZhbF9wcm9tcHQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZmVlZGJhY2slMkMlMjBzY29yZSUyMCUzRCUyMCU1Qml0ZW0uc3RyaXAoKSUyMGZvciUyMGl0ZW0lMjBpbiUyMGV2YWxfcmVzdWx0LmNvbnRlbnQuc3BsaXQoJTIyJTVCUkVTVUxUJTVEJTIyKSU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV4cGVyaW1lbnQlNUJmJTIyZXZhbF9zY29yZV8lN0JldmFsdWF0b3JfbmFtZSU3RCUyMiU1RCUyMCUzRCUyMHNjb3JlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXhwZXJpbWVudCU1QmYlMjJldmFsX2ZlZWRiYWNrXyU3QmV2YWx1YXRvcl9uYW1lJTdEJTIyJTVEJTIwJTNEJTIwZmVlZGJhY2slMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3aXRoJTIwb3BlbihhbnN3ZXJfcGF0aCUyQyUyMCUyMnclMjIpJTIwYXMlMjBmJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwanNvbi5kdW1wKGFuc3dlcnMlMkMlMjBmKQ==",highlighted:`<span class="hljs-keyword">from</span> langchain.chat_models <span class="hljs-keyword">import</span> ChatOpenAI

OPENAI_API_KEY = <span class="hljs-string">&quot;&quot;</span>

eval_chat_model = ChatOpenAI(model=<span class="hljs-string">&quot;gpt-4-1106-preview&quot;</span>, temperature=<span class="hljs-number">0</span>, openai_api_key=OPENAI_API_KEY)
evaluator_name = <span class="hljs-string">&quot;GPT4&quot;</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">evaluate_answers</span>(<span class="hljs-params">
    answer_path: <span class="hljs-built_in">str</span>,
    eval_chat_model,
    evaluator_name: <span class="hljs-built_in">str</span>,
    evaluation_prompt_template: ChatPromptTemplate,
</span>) -&gt; <span class="hljs-literal">None</span>:
    <span class="hljs-string">&quot;&quot;&quot;Evaluates generated answers. Modifies the given answer file in place for better checkpointing.&quot;&quot;&quot;</span>
    answers = []
    <span class="hljs-keyword">if</span> os.path.isfile(answer_path):  <span class="hljs-comment"># load previous generations if they exist</span>
        answers = json.load(<span class="hljs-built_in">open</span>(answer_path, <span class="hljs-string">&quot;r&quot;</span>))

    <span class="hljs-keyword">for</span> experiment <span class="hljs-keyword">in</span> tqdm(answers):
        <span class="hljs-keyword">if</span> <span class="hljs-string">f&quot;eval_score_<span class="hljs-subst">{evaluator_name}</span>&quot;</span> <span class="hljs-keyword">in</span> experiment:
            <span class="hljs-keyword">continue</span>

        eval_prompt = evaluation_prompt_template.format_messages(
            instruction=experiment[<span class="hljs-string">&quot;question&quot;</span>],
            response=experiment[<span class="hljs-string">&quot;generated_answer&quot;</span>],
            reference_answer=experiment[<span class="hljs-string">&quot;true_answer&quot;</span>],
        )
        eval_result = eval_chat_model.invoke(eval_prompt)
        feedback, score = [item.strip() <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> eval_result.content.split(<span class="hljs-string">&quot;[RESULT]&quot;</span>)]
        experiment[<span class="hljs-string">f&quot;eval_score_<span class="hljs-subst">{evaluator_name}</span>&quot;</span>] = score
        experiment[<span class="hljs-string">f&quot;eval_feedback_<span class="hljs-subst">{evaluator_name}</span>&quot;</span>] = feedback

        <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(answer_path, <span class="hljs-string">&quot;w&quot;</span>) <span class="hljs-keyword">as</span> f:
            json.dump(answers, f)`,wrap:!1}}),Ll=new c({props:{code:"aWYlMjBub3QlMjBvcy5wYXRoLmV4aXN0cyglMjIuJTJGb3V0cHV0JTIyKSUzQSUwQSUyMCUyMCUyMCUyMG9zLm1rZGlyKCUyMi4lMkZvdXRwdXQlMjIpJTBBJTBBZm9yJTIwY2h1bmtfc2l6ZSUyMGluJTIwJTVCMjAwJTVEJTNBJTIwJTIwJTIzJTIwQWRkJTIwb3RoZXIlMjBjaHVuayUyMHNpemVzJTIwKGluJTIwdG9rZW5zKSUyMGFzJTIwbmVlZGVkJTBBJTIwJTIwJTIwJTIwZm9yJTIwZW1iZWRkaW5ncyUyMGluJTIwJTVCJTIydGhlbmxwZXIlMkZndGUtc21hbGwlMjIlNUQlM0ElMjAlMjAlMjMlMjBBZGQlMjBvdGhlciUyMGVtYmVkZGluZ3MlMjBhcyUyMG5lZWRlZCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMHJlcmFuayUyMGluJTIwJTVCVHJ1ZSUyQyUyMEZhbHNlJTVEJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2V0dGluZ3NfbmFtZSUyMCUzRCUyMGYlMjJjaHVuayUzQSU3QmNodW5rX3NpemUlN0RfZW1iZWRkaW5ncyUzQSU3QmVtYmVkZGluZ3MucmVwbGFjZSgnJTJGJyUyQyUyMCd+JyklN0RfcmVyYW5rJTNBJTdCcmVyYW5rJTdEX3JlYWRlci1tb2RlbCUzQSU3QlJFQURFUl9NT0RFTF9OQU1FJTdEJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0X2ZpbGVfbmFtZSUyMCUzRCUyMGYlMjIuJTJGb3V0cHV0JTJGcmFnXyU3QnNldHRpbmdzX25hbWUlN0QuanNvbiUyMiUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KGYlMjJSdW5uaW5nJTIwZXZhbHVhdGlvbiUyMGZvciUyMCU3QnNldHRpbmdzX25hbWUlN0QlM0ElMjIpJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoJTIyTG9hZGluZyUyMGtub3dsZWRnZSUyMGJhc2UlMjBlbWJlZGRpbmdzLi4uJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGtub3dsZWRnZV9pbmRleCUyMCUzRCUyMGxvYWRfZW1iZWRkaW5ncyglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBSQVdfS05PV0xFREdFX0JBU0UlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjaHVua19zaXplJTNEY2h1bmtfc2l6ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVtYmVkZGluZ19tb2RlbF9uYW1lJTNEZW1iZWRkaW5ncyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludCglMjJSdW5uaW5nJTIwUkFHLi4uJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlcmFua2VyJTIwJTNEJTIwUkFHUHJldHJhaW5lZE1vZGVsLmZyb21fcHJldHJhaW5lZCglMjJjb2xiZXJ0LWlyJTJGY29sYmVydHYyLjAlMjIpJTIwaWYlMjByZXJhbmslMjBlbHNlJTIwTm9uZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJ1bl9yYWdfdGVzdHMoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEZXZhbF9kYXRhc2V0JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGxtJTNEUkVBREVSX0xMTSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGtub3dsZWRnZV9pbmRleCUzRGtub3dsZWRnZV9pbmRleCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dF9maWxlJTNEb3V0cHV0X2ZpbGVfbmFtZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlcmFua2VyJTNEcmVyYW5rZXIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB2ZXJib3NlJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0ZXN0X3NldHRpbmdzJTNEc2V0dGluZ3NfbmFtZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludCglMjJSdW5uaW5nJTIwZXZhbHVhdGlvbi4uLiUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBldmFsdWF0ZV9hbnN3ZXJzKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dF9maWxlX25hbWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBldmFsX2NoYXRfbW9kZWwlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBldmFsdWF0b3JfbmFtZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGV2YWx1YXRpb25fcHJvbXB0X3RlbXBsYXRlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKQ==",highlighted:`<span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> os.path.exists(<span class="hljs-string">&quot;./output&quot;</span>):
    os.mkdir(<span class="hljs-string">&quot;./output&quot;</span>)

<span class="hljs-keyword">for</span> chunk_size <span class="hljs-keyword">in</span> [<span class="hljs-number">200</span>]:  <span class="hljs-comment"># Add other chunk sizes (in tokens) as needed</span>
    <span class="hljs-keyword">for</span> embeddings <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;thenlper/gte-small&quot;</span>]:  <span class="hljs-comment"># Add other embeddings as needed</span>
        <span class="hljs-keyword">for</span> rerank <span class="hljs-keyword">in</span> [<span class="hljs-literal">True</span>, <span class="hljs-literal">False</span>]:
            settings_name = <span class="hljs-string">f&quot;chunk:<span class="hljs-subst">{chunk_size}</span>_embeddings:<span class="hljs-subst">{embeddings.replace(<span class="hljs-string">&#x27;/&#x27;</span>, <span class="hljs-string">&#x27;~&#x27;</span>)}</span>_rerank:<span class="hljs-subst">{rerank}</span>_reader-model:<span class="hljs-subst">{READER_MODEL_NAME}</span>&quot;</span>
            output_file_name = <span class="hljs-string">f&quot;./output/rag_<span class="hljs-subst">{settings_name}</span>.json&quot;</span>

            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Running evaluation for <span class="hljs-subst">{settings_name}</span>:&quot;</span>)

            <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Loading knowledge base embeddings...&quot;</span>)
            knowledge_index = load_embeddings(
                RAW_KNOWLEDGE_BASE,
                chunk_size=chunk_size,
                embedding_model_name=embeddings,
            )

            <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Running RAG...&quot;</span>)
            reranker = RAGPretrainedModel.from_pretrained(<span class="hljs-string">&quot;colbert-ir/colbertv2.0&quot;</span>) <span class="hljs-keyword">if</span> rerank <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span>
            run_rag_tests(
                eval_dataset=eval_dataset,
                llm=READER_LLM,
                knowledge_index=knowledge_index,
                output_file=output_file_name,
                reranker=reranker,
                verbose=<span class="hljs-literal">False</span>,
                test_settings=settings_name,
            )

            <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Running evaluation...&quot;</span>)
            evaluate_answers(
                output_file_name,
                eval_chat_model,
                evaluator_name,
                evaluation_prompt_template,
            )`,wrap:!1}}),Dl=new o({props:{title:"检查结果",local:"检查结果",headingTag:"h3"}}),Kl=new c({props:{code:"aW1wb3J0JTIwZ2xvYiUwQSUwQW91dHB1dHMlMjAlM0QlMjAlNUIlNUQlMEFmb3IlMjBmaWxlJTIwaW4lMjBnbG9iLmdsb2IoJTIyLiUyRm91dHB1dCUyRiouanNvbiUyMiklM0ElMEElMjAlMjAlMjAlMjBvdXRwdXQlMjAlM0QlMjBwZC5EYXRhRnJhbWUoanNvbi5sb2FkKG9wZW4oZmlsZSUyQyUyMCUyMnIlMjIpKSklMEElMjAlMjAlMjAlMjBvdXRwdXQlNUIlMjJzZXR0aW5ncyUyMiU1RCUyMCUzRCUyMGZpbGUlMEElMjAlMjAlMjAlMjBvdXRwdXRzLmFwcGVuZChvdXRwdXQpJTBBcmVzdWx0JTIwJTNEJTIwcGQuY29uY2F0KG91dHB1dHMp",highlighted:`<span class="hljs-keyword">import</span> glob

outputs = []
<span class="hljs-keyword">for</span> file <span class="hljs-keyword">in</span> glob.glob(<span class="hljs-string">&quot;./output/*.json&quot;</span>):
    output = pd.DataFrame(json.load(<span class="hljs-built_in">open</span>(file, <span class="hljs-string">&quot;r&quot;</span>)))
    output[<span class="hljs-string">&quot;settings&quot;</span>] = file
    outputs.append(output)
result = pd.concat(outputs)`,wrap:!1}}),Pl=new c({props:{code:"cmVzdWx0JTVCJTIyZXZhbF9zY29yZV9HUFQ0JTIyJTVEJTIwJTNEJTIwcmVzdWx0JTVCJTIyZXZhbF9zY29yZV9HUFQ0JTIyJTVELmFwcGx5KGxhbWJkYSUyMHglM0ElMjBpbnQoeCklMjBpZiUyMGlzaW5zdGFuY2UoeCUyQyUyMHN0ciklMjBlbHNlJTIwMSklMEFyZXN1bHQlNUIlMjJldmFsX3Njb3JlX0dQVDQlMjIlNUQlMjAlM0QlMjAocmVzdWx0JTVCJTIyZXZhbF9zY29yZV9HUFQ0JTIyJTVEJTIwLSUyMDEpJTIwJTJGJTIwNA==",highlighted:`result[<span class="hljs-string">&quot;eval_score_GPT4&quot;</span>] = result[<span class="hljs-string">&quot;eval_score_GPT4&quot;</span>].apply(<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">int</span>(x) <span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(x, <span class="hljs-built_in">str</span>) <span class="hljs-keyword">else</span> <span class="hljs-number">1</span>)
result[<span class="hljs-string">&quot;eval_score_GPT4&quot;</span>] = (result[<span class="hljs-string">&quot;eval_score_GPT4&quot;</span>] - <span class="hljs-number">1</span>) / <span class="hljs-number">4</span>`,wrap:!1}}),Ol=new c({props:{code:"YXZlcmFnZV9zY29yZXMlMjAlM0QlMjByZXN1bHQuZ3JvdXBieSglMjJzZXR0aW5ncyUyMiklNUIlMjJldmFsX3Njb3JlX0dQVDQlMjIlNUQubWVhbigpJTBBYXZlcmFnZV9zY29yZXMuc29ydF92YWx1ZXMoKQ==",highlighted:`average_scores = result.groupby(<span class="hljs-string">&quot;settings&quot;</span>)[<span class="hljs-string">&quot;eval_score_GPT4&quot;</span>].mean()
average_scores.sort_values()`,wrap:!1}}),ls=new o({props:{title:"结果示例",local:"结果示例",headingTag:"h2"}}),as=new c({props:{code:"aW1wb3J0JTIwcGxvdGx5LmV4cHJlc3MlMjBhcyUyMHB4JTBBJTBBc2NvcmVzJTIwJTNEJTIwZGF0YXNldHMubG9hZF9kYXRhc2V0KCUyMm0tcmljJTJGcmFnX3Njb3Jlc19jb29rYm9vayUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIpJTBBc2NvcmVzJTIwJTNEJTIwcGQuU2VyaWVzKHNjb3JlcyU1QiUyMnNjb3JlJTIyJTVEJTJDJTIwaW5kZXglM0RzY29yZXMlNUIlMjJzZXR0aW5ncyUyMiU1RCk=",highlighted:`<span class="hljs-keyword">import</span> plotly.express <span class="hljs-keyword">as</span> px

scores = datasets.load_dataset(<span class="hljs-string">&quot;m-ric/rag_scores_cookbook&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
scores = pd.Series(scores[<span class="hljs-string">&quot;score&quot;</span>], index=scores[<span class="hljs-string">&quot;settings&quot;</span>])`,wrap:!1}}),ns=new c({props:{code:"ZmlnJTIwJTNEJTIwcHguYmFyKCUwQSUyMCUyMCUyMCUyMHNjb3JlcyUyQyUwQSUyMCUyMCUyMCUyMGNvbG9yJTNEc2NvcmVzJTJDJTBBJTIwJTIwJTIwJTIwbGFiZWxzJTNEJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydmFsdWUlMjIlM0ElMjAlMjJBY2N1cmFjeSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnNldHRpbmdzJTIyJTNBJTIwJTIyQ29uZmlndXJhdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMGNvbG9yX2NvbnRpbnVvdXNfc2NhbGUlM0QlMjJibHVlcmVkJTIyJTJDJTBBKSUwQWZpZy51cGRhdGVfbGF5b3V0KCUwQSUyMCUyMCUyMCUyMHdpZHRoJTNEMTAwMCUyQyUwQSUyMCUyMCUyMCUyMGhlaWdodCUzRDYwMCUyQyUwQSUyMCUyMCUyMCUyMGJhcm1vZGUlM0QlMjJncm91cCUyMiUyQyUwQSUyMCUyMCUyMCUyMHlheGlzX3JhbmdlJTNEJTVCMCUyQyUyMDEwMCU1RCUyQyUwQSUyMCUyMCUyMCUyMHRpdGxlJTNEJTIyJTNDYiUzRUFjY3VyYWN5JTIwb2YlMjBkaWZmZXJlbnQlMjBSQUclMjBjb25maWd1cmF0aW9ucyUzQyUyRmIlM0UlMjIlMkMlMEElMjAlMjAlMjAlMjB4YXhpc190aXRsZSUzRCUyMlJBRyUyMHNldHRpbmdzJTIyJTJDJTBBJTIwJTIwJTIwJTIwZm9udCUzRGRpY3Qoc2l6ZSUzRDE1KSUyQyUwQSklMEFmaWcubGF5b3V0LnlheGlzLnRpY2tzdWZmaXglMjAlM0QlMjAlMjIlMjUlMjIlMEFmaWcudXBkYXRlX2NvbG9yYXhlcyhzaG93c2NhbGUlM0RGYWxzZSklMEFmaWcudXBkYXRlX3RyYWNlcyh0ZXh0dGVtcGxhdGUlM0QlMjIlMjUlN0J5JTNBLjFmJTdEJTIyJTJDJTIwdGV4dHBvc2l0aW9uJTNEJTIyb3V0c2lkZSUyMiklMEFmaWcuc2hvdygp",highlighted:`fig = px.bar(
    scores,
    color=scores,
    labels={
        <span class="hljs-string">&quot;value&quot;</span>: <span class="hljs-string">&quot;Accuracy&quot;</span>,
        <span class="hljs-string">&quot;settings&quot;</span>: <span class="hljs-string">&quot;Configuration&quot;</span>,
    },
    color_continuous_scale=<span class="hljs-string">&quot;bluered&quot;</span>,
)
fig.update_layout(
    width=<span class="hljs-number">1000</span>,
    height=<span class="hljs-number">600</span>,
    barmode=<span class="hljs-string">&quot;group&quot;</span>,
    yaxis_range=[<span class="hljs-number">0</span>, <span class="hljs-number">100</span>],
    title=<span class="hljs-string">&quot;&lt;b&gt;Accuracy of different RAG configurations&lt;/b&gt;&quot;</span>,
    xaxis_title=<span class="hljs-string">&quot;RAG settings&quot;</span>,
    font=<span class="hljs-built_in">dict</span>(size=<span class="hljs-number">15</span>),
)
fig.layout.yaxis.ticksuffix = <span class="hljs-string">&quot;%&quot;</span>
fig.update_coloraxes(showscale=<span class="hljs-literal">False</span>)
fig.update_traces(texttemplate=<span class="hljs-string">&quot;%{y:.1f}&quot;</span>, textposition=<span class="hljs-string">&quot;outside&quot;</span>)
fig.show()`,wrap:!1}}),Js=new ua({props:{source:"https://github.com/huggingface/cookbook/blob/main/notebooks/zh-CN/rag_evaluation.md"}}),{c(){u=e("meta"),ws=a(),js=e("p"),ps=a(),U(d.$$.fragment),is=a(),U(r.$$.fragment),cs=a(),m=e("p"),m.innerHTML=yt,os=a(),C=e("p"),C.textContent=Jt,us=a(),b=e("p"),b.innerHTML=Ut,Is=a(),h=e("p"),h.textContent=jt,ds=a(),B=e("img"),rs=a(),Z=e("p"),Z.textContent=wt,ms=a(),U(G.$$.fragment),Cs=a(),A=e("p"),A.textContent=pt,bs=a(),Q=e("p"),Q.textContent=it,hs=a(),W=e("ol"),W.innerHTML=ct,Bs=a(),V=e("p"),V.textContent=ot,Zs=a(),R=e("ol"),R.innerHTML=ut,Gs=a(),X=e("p"),X.innerHTML=It,As=a(),U(g.$$.fragment),Qs=a(),U(f.$$.fragment),Ws=a(),U(N.$$.fragment),Vs=a(),U(k.$$.fragment),Rs=a(),U(E.$$.fragment),Xs=a(),U(v.$$.fragment),gs=a(),U(Y.$$.fragment),fs=a(),S=e("p"),S.textContent=dt,Ns=a(),F=e("p"),F.textContent=rt,ks=a(),U(_.$$.fragment),Es=a(),U(q.$$.fragment),vs=a(),U(x.$$.fragment),Ys=a(),z=e("p"),z.innerHTML=mt,Ss=a(),U(H.$$.fragment),Fs=a(),U($.$$.fragment),_s=a(),L=e("p"),L.textContent=Ct,qs=a(),D=e("p"),D.textContent=bt,xs=a(),K=e("p"),K.textContent=ht,zs=a(),U(P.$$.fragment),Hs=a(),U(O.$$.fragment),$s=a(),U(ll.$$.fragment),Ls=a(),sl=e("p"),sl.textContent=Bt,Ds=a(),Ml=e("p"),Ml.innerHTML=Zt,Ks=a(),tl=e("ul"),tl.innerHTML=Gt,Ps=a(),al=e("p"),al.innerHTML=At,Os=a(),nl=e("ul"),nl.innerHTML=Qt,lM=a(),el=e("p"),el.textContent=Wt,sM=a(),yl=e("p"),yl.innerHTML=Vt,MM=a(),Jl=e("p"),Jl.textContent=Rt,tM=a(),U(Ul.$$.fragment),aM=a(),U(jl.$$.fragment),nM=a(),Tl=e("p"),Tl.textContent=Xt,eM=a(),U(wl.$$.fragment),yM=a(),pl=e("pre"),pl.textContent=gt,JM=a(),il=e("p"),il.textContent=ft,UM=a(),cl=e("p"),cl.textContent=Nt,jM=a(),U(ol.$$.fragment),TM=a(),U(ul.$$.fragment),wM=a(),U(Il.$$.fragment),pM=a(),dl=e("ul"),dl.innerHTML=kt,iM=a(),rl=e("p"),rl.textContent=Et,cM=a(),ml=e("ul"),ml.innerHTML=vt,oM=a(),Cl=e("p"),Cl.innerHTML=Yt,uM=a(),bl=e("blockquote"),bl.innerHTML=St,IM=a(),U(hl.$$.fragment),dM=a(),U(Bl.$$.fragment),rM=a(),U(Zl.$$.fragment),mM=a(),Gl=e("p"),Gl.innerHTML=Ft,CM=a(),Al=e("blockquote"),Al.innerHTML=_t,bM=a(),Ql=e("p"),Ql.innerHTML=qt,hM=a(),Wl=e("ul"),Wl.innerHTML=xt,BM=a(),U(Vl.$$.fragment),ZM=a(),U(Rl.$$.fragment),GM=a(),Xl=e("p"),Xl.innerHTML=zt,AM=a(),gl=e("p"),gl.textContent=Ht,QM=a(),fl=e("ul"),fl.innerHTML=$t,WM=a(),U(Nl.$$.fragment),VM=a(),U(kl.$$.fragment),RM=a(),U(El.$$.fragment),XM=a(),U(vl.$$.fragment),gM=a(),Yl=e("p"),Yl.innerHTML=Lt,fM=a(),Sl=e("p"),Sl.innerHTML=Dt,NM=a(),Fl=e("blockquote"),Fl.innerHTML=Kt,kM=a(),_l=e("p"),_l.innerHTML=Pt,EM=a(),ql=e("p"),ql.innerHTML=Ot,vM=a(),U(xl.$$.fragment),YM=a(),U(zl.$$.fragment),SM=a(),U(Hl.$$.fragment),FM=a(),$l=e("p"),$l.textContent=la,_M=a(),U(Ll.$$.fragment),qM=a(),U(Dl.$$.fragment),xM=a(),U(Kl.$$.fragment),zM=a(),U(Pl.$$.fragment),HM=a(),U(Ol.$$.fragment),$M=a(),U(ls.$$.fragment),LM=a(),ss=e("p"),ss.innerHTML=sa,DM=a(),Ms=e("p"),Ms.textContent=Ma,KM=a(),ts=e("p"),ts.innerHTML=ta,PM=a(),U(as.$$.fragment),OM=a(),U(ns.$$.fragment),lt=a(),I=e("img"),st=a(),es=e("p"),es.textContent=na,Mt=a(),ys=e("p"),ys.textContent=ea,tt=a(),U(Js.$$.fragment),at=a(),Ts=e("p"),this.h()},l(l){const s=ia("svelte-u9bgzb",document.head);u=y(s,"META",{name:!0,content:!0}),s.forEach(M),ws=n(l),js=y(l,"P",{}),Ja(js).forEach(M),ps=n(l),j(d.$$.fragment,l),is=n(l),j(r.$$.fragment,l),cs=n(l),m=y(l,"P",{"data-svelte-h":!0}),J(m)!=="svelte-11ntk8v"&&(m.innerHTML=yt),os=n(l),C=y(l,"P",{"data-svelte-h":!0}),J(C)!=="svelte-zw09hj"&&(C.textContent=Jt),us=n(l),b=y(l,"P",{"data-svelte-h":!0}),J(b)!=="svelte-1aes7ge"&&(b.innerHTML=Ut),Is=n(l),h=y(l,"P",{"data-svelte-h":!0}),J(h)!=="svelte-12rale4"&&(h.textContent=jt),ds=n(l),B=y(l,"IMG",{src:!0,height:!0}),rs=n(l),Z=y(l,"P",{"data-svelte-h":!0}),J(Z)!=="svelte-10yhsu2"&&(Z.textContent=wt),ms=n(l),j(G.$$.fragment,l),Cs=n(l),A=y(l,"P",{"data-svelte-h":!0}),J(A)!=="svelte-lx6lk2"&&(A.textContent=pt),bs=n(l),Q=y(l,"P",{"data-svelte-h":!0}),J(Q)!=="svelte-1ukr2wc"&&(Q.textContent=it),hs=n(l),W=y(l,"OL",{"data-svelte-h":!0}),J(W)!=="svelte-1mbcpc9"&&(W.innerHTML=ct),Bs=n(l),V=y(l,"P",{"data-svelte-h":!0}),J(V)!=="svelte-19a0j42"&&(V.textContent=ot),Zs=n(l),R=y(l,"OL",{"data-svelte-h":!0}),J(R)!=="svelte-1ebb9tz"&&(R.innerHTML=ut),Gs=n(l),X=y(l,"P",{"data-svelte-h":!0}),J(X)!=="svelte-1946r5u"&&(X.innerHTML=It),As=n(l),j(g.$$.fragment,l),Qs=n(l),j(f.$$.fragment,l),Ws=n(l),j(N.$$.fragment,l),Vs=n(l),j(k.$$.fragment,l),Rs=n(l),j(E.$$.fragment,l),Xs=n(l),j(v.$$.fragment,l),gs=n(l),j(Y.$$.fragment,l),fs=n(l),S=y(l,"P",{"data-svelte-h":!0}),J(S)!=="svelte-126kv91"&&(S.textContent=dt),Ns=n(l),F=y(l,"P",{"data-svelte-h":!0}),J(F)!=="svelte-4o7kqj"&&(F.textContent=rt),ks=n(l),j(_.$$.fragment,l),Es=n(l),j(q.$$.fragment,l),vs=n(l),j(x.$$.fragment,l),Ys=n(l),z=y(l,"P",{"data-svelte-h":!0}),J(z)!=="svelte-pq5chq"&&(z.innerHTML=mt),Ss=n(l),j(H.$$.fragment,l),Fs=n(l),j($.$$.fragment,l),_s=n(l),L=y(l,"P",{"data-svelte-h":!0}),J(L)!=="svelte-1b1cnkb"&&(L.textContent=Ct),qs=n(l),D=y(l,"P",{"data-svelte-h":!0}),J(D)!=="svelte-k9r3k0"&&(D.textContent=bt),xs=n(l),K=y(l,"P",{"data-svelte-h":!0}),J(K)!=="svelte-a6hv5k"&&(K.textContent=ht),zs=n(l),j(P.$$.fragment,l),Hs=n(l),j(O.$$.fragment,l),$s=n(l),j(ll.$$.fragment,l),Ls=n(l),sl=y(l,"P",{"data-svelte-h":!0}),J(sl)!=="svelte-cy38ww"&&(sl.textContent=Bt),Ds=n(l),Ml=y(l,"P",{"data-svelte-h":!0}),J(Ml)!=="svelte-1osbiot"&&(Ml.innerHTML=Zt),Ks=n(l),tl=y(l,"UL",{"data-svelte-h":!0}),J(tl)!=="svelte-14wv6gb"&&(tl.innerHTML=Gt),Ps=n(l),al=y(l,"P",{"data-svelte-h":!0}),J(al)!=="svelte-1fze0kb"&&(al.innerHTML=At),Os=n(l),nl=y(l,"UL",{"data-svelte-h":!0}),J(nl)!=="svelte-1yamk70"&&(nl.innerHTML=Qt),lM=n(l),el=y(l,"P",{"data-svelte-h":!0}),J(el)!=="svelte-djxjqi"&&(el.textContent=Wt),sM=n(l),yl=y(l,"P",{"data-svelte-h":!0}),J(yl)!=="svelte-crlkwa"&&(yl.innerHTML=Vt),MM=n(l),Jl=y(l,"P",{"data-svelte-h":!0}),J(Jl)!=="svelte-1g2jy8m"&&(Jl.textContent=Rt),tM=n(l),j(Ul.$$.fragment,l),aM=n(l),j(jl.$$.fragment,l),nM=n(l),Tl=y(l,"P",{"data-svelte-h":!0}),J(Tl)!=="svelte-cjndcy"&&(Tl.textContent=Xt),eM=n(l),j(wl.$$.fragment,l),yM=n(l),pl=y(l,"PRE",{"data-svelte-h":!0}),J(pl)!=="svelte-1k8l62p"&&(pl.textContent=gt),JM=n(l),il=y(l,"P",{"data-svelte-h":!0}),J(il)!=="svelte-q06ahl"&&(il.textContent=ft),UM=n(l),cl=y(l,"P",{"data-svelte-h":!0}),J(cl)!=="svelte-1kklb5b"&&(cl.textContent=Nt),jM=n(l),j(ol.$$.fragment,l),TM=n(l),j(ul.$$.fragment,l),wM=n(l),j(Il.$$.fragment,l),pM=n(l),dl=y(l,"UL",{"data-svelte-h":!0}),J(dl)!=="svelte-pqbs43"&&(dl.innerHTML=kt),iM=n(l),rl=y(l,"P",{"data-svelte-h":!0}),J(rl)!=="svelte-1965er0"&&(rl.textContent=Et),cM=n(l),ml=y(l,"UL",{"data-svelte-h":!0}),J(ml)!=="svelte-vakfil"&&(ml.innerHTML=vt),oM=n(l),Cl=y(l,"P",{"data-svelte-h":!0}),J(Cl)!=="svelte-1ln9tsn"&&(Cl.innerHTML=Yt),uM=n(l),bl=y(l,"BLOCKQUOTE",{"data-svelte-h":!0}),J(bl)!=="svelte-1b121lz"&&(bl.innerHTML=St),IM=n(l),j(hl.$$.fragment,l),dM=n(l),j(Bl.$$.fragment,l),rM=n(l),j(Zl.$$.fragment,l),mM=n(l),Gl=y(l,"P",{"data-svelte-h":!0}),J(Gl)!=="svelte-1movshy"&&(Gl.innerHTML=Ft),CM=n(l),Al=y(l,"BLOCKQUOTE",{"data-svelte-h":!0}),J(Al)!=="svelte-1l6mdl0"&&(Al.innerHTML=_t),bM=n(l),Ql=y(l,"P",{"data-svelte-h":!0}),J(Ql)!=="svelte-lzkfgi"&&(Ql.innerHTML=qt),hM=n(l),Wl=y(l,"UL",{"data-svelte-h":!0}),J(Wl)!=="svelte-1kyrqvw"&&(Wl.innerHTML=xt),BM=n(l),j(Vl.$$.fragment,l),ZM=n(l),j(Rl.$$.fragment,l),GM=n(l),Xl=y(l,"P",{"data-svelte-h":!0}),J(Xl)!=="svelte-10ha94n"&&(Xl.innerHTML=zt),AM=n(l),gl=y(l,"P",{"data-svelte-h":!0}),J(gl)!=="svelte-171gtnx"&&(gl.textContent=Ht),QM=n(l),fl=y(l,"UL",{"data-svelte-h":!0}),J(fl)!=="svelte-1f36qso"&&(fl.innerHTML=$t),WM=n(l),j(Nl.$$.fragment,l),VM=n(l),j(kl.$$.fragment,l),RM=n(l),j(El.$$.fragment,l),XM=n(l),j(vl.$$.fragment,l),gM=n(l),Yl=y(l,"P",{"data-svelte-h":!0}),J(Yl)!=="svelte-1sqpkjh"&&(Yl.innerHTML=Lt),fM=n(l),Sl=y(l,"P",{"data-svelte-h":!0}),J(Sl)!=="svelte-1mbklsh"&&(Sl.innerHTML=Dt),NM=n(l),Fl=y(l,"BLOCKQUOTE",{"data-svelte-h":!0}),J(Fl)!=="svelte-1gm4r4w"&&(Fl.innerHTML=Kt),kM=n(l),_l=y(l,"P",{"data-svelte-h":!0}),J(_l)!=="svelte-1tcmdhq"&&(_l.innerHTML=Pt),EM=n(l),ql=y(l,"P",{"data-svelte-h":!0}),J(ql)!=="svelte-t9ho3k"&&(ql.innerHTML=Ot),vM=n(l),j(xl.$$.fragment,l),YM=n(l),j(zl.$$.fragment,l),SM=n(l),j(Hl.$$.fragment,l),FM=n(l),$l=y(l,"P",{"data-svelte-h":!0}),J($l)!=="svelte-1ffahd2"&&($l.textContent=la),_M=n(l),j(Ll.$$.fragment,l),qM=n(l),j(Dl.$$.fragment,l),xM=n(l),j(Kl.$$.fragment,l),zM=n(l),j(Pl.$$.fragment,l),HM=n(l),j(Ol.$$.fragment,l),$M=n(l),j(ls.$$.fragment,l),LM=n(l),ss=y(l,"P",{"data-svelte-h":!0}),J(ss)!=="svelte-19tissg"&&(ss.innerHTML=sa),DM=n(l),Ms=y(l,"P",{"data-svelte-h":!0}),J(Ms)!=="svelte-l94cws"&&(Ms.textContent=Ma),KM=n(l),ts=y(l,"P",{"data-svelte-h":!0}),J(ts)!=="svelte-1wj10g3"&&(ts.innerHTML=ta),PM=n(l),j(as.$$.fragment,l),OM=n(l),j(ns.$$.fragment,l),lt=n(l),I=y(l,"IMG",{src:!0,height:!0,width:!0}),st=n(l),es=y(l,"P",{"data-svelte-h":!0}),J(es)!=="svelte-ds7a4o"&&(es.textContent=na),Mt=n(l),ys=y(l,"P",{"data-svelte-h":!0}),J(ys)!=="svelte-1dxb7bg"&&(ys.textContent=ea),tt=n(l),j(Js.$$.fragment,l),at=n(l),Ts=y(l,"P",{}),Ja(Ts).forEach(M),this.h()},h(){Us(u,"name","hf:doc:metadata"),Us(u,"content",da),ya(B.src,Tt="https://huggingface.co/datasets/huggingface/cookbook-images/resolve/main/RAG_workflow.png")||Us(B,"src",Tt),Us(B,"height","700"),ya(I.src,aa="https://huggingface.co/datasets/huggingface/cookbook-images/resolve/main/RAG_settings_accuracy.png")||Us(I,"src",aa),Us(I,"height","500"),Us(I,"width","800")},m(l,s){ca(document.head,u),t(l,ws,s),t(l,js,s),t(l,ps,s),T(d,l,s),t(l,is,s),T(r,l,s),t(l,cs,s),t(l,m,s),t(l,os,s),t(l,C,s),t(l,us,s),t(l,b,s),t(l,Is,s),t(l,h,s),t(l,ds,s),t(l,B,s),t(l,rs,s),t(l,Z,s),t(l,ms,s),T(G,l,s),t(l,Cs,s),t(l,A,s),t(l,bs,s),t(l,Q,s),t(l,hs,s),t(l,W,s),t(l,Bs,s),t(l,V,s),t(l,Zs,s),t(l,R,s),t(l,Gs,s),t(l,X,s),t(l,As,s),T(g,l,s),t(l,Qs,s),T(f,l,s),t(l,Ws,s),T(N,l,s),t(l,Vs,s),T(k,l,s),t(l,Rs,s),T(E,l,s),t(l,Xs,s),T(v,l,s),t(l,gs,s),T(Y,l,s),t(l,fs,s),t(l,S,s),t(l,Ns,s),t(l,F,s),t(l,ks,s),T(_,l,s),t(l,Es,s),T(q,l,s),t(l,vs,s),T(x,l,s),t(l,Ys,s),t(l,z,s),t(l,Ss,s),T(H,l,s),t(l,Fs,s),T($,l,s),t(l,_s,s),t(l,L,s),t(l,qs,s),t(l,D,s),t(l,xs,s),t(l,K,s),t(l,zs,s),T(P,l,s),t(l,Hs,s),T(O,l,s),t(l,$s,s),T(ll,l,s),t(l,Ls,s),t(l,sl,s),t(l,Ds,s),t(l,Ml,s),t(l,Ks,s),t(l,tl,s),t(l,Ps,s),t(l,al,s),t(l,Os,s),t(l,nl,s),t(l,lM,s),t(l,el,s),t(l,sM,s),t(l,yl,s),t(l,MM,s),t(l,Jl,s),t(l,tM,s),T(Ul,l,s),t(l,aM,s),T(jl,l,s),t(l,nM,s),t(l,Tl,s),t(l,eM,s),T(wl,l,s),t(l,yM,s),t(l,pl,s),t(l,JM,s),t(l,il,s),t(l,UM,s),t(l,cl,s),t(l,jM,s),T(ol,l,s),t(l,TM,s),T(ul,l,s),t(l,wM,s),T(Il,l,s),t(l,pM,s),t(l,dl,s),t(l,iM,s),t(l,rl,s),t(l,cM,s),t(l,ml,s),t(l,oM,s),t(l,Cl,s),t(l,uM,s),t(l,bl,s),t(l,IM,s),T(hl,l,s),t(l,dM,s),T(Bl,l,s),t(l,rM,s),T(Zl,l,s),t(l,mM,s),t(l,Gl,s),t(l,CM,s),t(l,Al,s),t(l,bM,s),t(l,Ql,s),t(l,hM,s),t(l,Wl,s),t(l,BM,s),T(Vl,l,s),t(l,ZM,s),T(Rl,l,s),t(l,GM,s),t(l,Xl,s),t(l,AM,s),t(l,gl,s),t(l,QM,s),t(l,fl,s),t(l,WM,s),T(Nl,l,s),t(l,VM,s),T(kl,l,s),t(l,RM,s),T(El,l,s),t(l,XM,s),T(vl,l,s),t(l,gM,s),t(l,Yl,s),t(l,fM,s),t(l,Sl,s),t(l,NM,s),t(l,Fl,s),t(l,kM,s),t(l,_l,s),t(l,EM,s),t(l,ql,s),t(l,vM,s),T(xl,l,s),t(l,YM,s),T(zl,l,s),t(l,SM,s),T(Hl,l,s),t(l,FM,s),t(l,$l,s),t(l,_M,s),T(Ll,l,s),t(l,qM,s),T(Dl,l,s),t(l,xM,s),T(Kl,l,s),t(l,zM,s),T(Pl,l,s),t(l,HM,s),T(Ol,l,s),t(l,$M,s),T(ls,l,s),t(l,LM,s),t(l,ss,s),t(l,DM,s),t(l,Ms,s),t(l,KM,s),t(l,ts,s),t(l,PM,s),T(as,l,s),t(l,OM,s),T(ns,l,s),t(l,lt,s),t(l,I,s),t(l,st,s),t(l,es,s),t(l,Mt,s),t(l,ys,s),t(l,tt,s),T(Js,l,s),t(l,at,s),t(l,Ts,s),nt=!0},p:ja,i(l){nt||(w(d.$$.fragment,l),w(r.$$.fragment,l),w(G.$$.fragment,l),w(g.$$.fragment,l),w(f.$$.fragment,l),w(N.$$.fragment,l),w(k.$$.fragment,l),w(E.$$.fragment,l),w(v.$$.fragment,l),w(Y.$$.fragment,l),w(_.$$.fragment,l),w(q.$$.fragment,l),w(x.$$.fragment,l),w(H.$$.fragment,l),w($.$$.fragment,l),w(P.$$.fragment,l),w(O.$$.fragment,l),w(ll.$$.fragment,l),w(Ul.$$.fragment,l),w(jl.$$.fragment,l),w(wl.$$.fragment,l),w(ol.$$.fragment,l),w(ul.$$.fragment,l),w(Il.$$.fragment,l),w(hl.$$.fragment,l),w(Bl.$$.fragment,l),w(Zl.$$.fragment,l),w(Vl.$$.fragment,l),w(Rl.$$.fragment,l),w(Nl.$$.fragment,l),w(kl.$$.fragment,l),w(El.$$.fragment,l),w(vl.$$.fragment,l),w(xl.$$.fragment,l),w(zl.$$.fragment,l),w(Hl.$$.fragment,l),w(Ll.$$.fragment,l),w(Dl.$$.fragment,l),w(Kl.$$.fragment,l),w(Pl.$$.fragment,l),w(Ol.$$.fragment,l),w(ls.$$.fragment,l),w(as.$$.fragment,l),w(ns.$$.fragment,l),w(Js.$$.fragment,l),nt=!0)},o(l){p(d.$$.fragment,l),p(r.$$.fragment,l),p(G.$$.fragment,l),p(g.$$.fragment,l),p(f.$$.fragment,l),p(N.$$.fragment,l),p(k.$$.fragment,l),p(E.$$.fragment,l),p(v.$$.fragment,l),p(Y.$$.fragment,l),p(_.$$.fragment,l),p(q.$$.fragment,l),p(x.$$.fragment,l),p(H.$$.fragment,l),p($.$$.fragment,l),p(P.$$.fragment,l),p(O.$$.fragment,l),p(ll.$$.fragment,l),p(Ul.$$.fragment,l),p(jl.$$.fragment,l),p(wl.$$.fragment,l),p(ol.$$.fragment,l),p(ul.$$.fragment,l),p(Il.$$.fragment,l),p(hl.$$.fragment,l),p(Bl.$$.fragment,l),p(Zl.$$.fragment,l),p(Vl.$$.fragment,l),p(Rl.$$.fragment,l),p(Nl.$$.fragment,l),p(kl.$$.fragment,l),p(El.$$.fragment,l),p(vl.$$.fragment,l),p(xl.$$.fragment,l),p(zl.$$.fragment,l),p(Hl.$$.fragment,l),p(Ll.$$.fragment,l),p(Dl.$$.fragment,l),p(Kl.$$.fragment,l),p(Pl.$$.fragment,l),p(Ol.$$.fragment,l),p(ls.$$.fragment,l),p(as.$$.fragment,l),p(ns.$$.fragment,l),p(Js.$$.fragment,l),nt=!1},d(l){l&&(M(ws),M(js),M(ps),M(is),M(cs),M(m),M(os),M(C),M(us),M(b),M(Is),M(h),M(ds),M(B),M(rs),M(Z),M(ms),M(Cs),M(A),M(bs),M(Q),M(hs),M(W),M(Bs),M(V),M(Zs),M(R),M(Gs),M(X),M(As),M(Qs),M(Ws),M(Vs),M(Rs),M(Xs),M(gs),M(fs),M(S),M(Ns),M(F),M(ks),M(Es),M(vs),M(Ys),M(z),M(Ss),M(Fs),M(_s),M(L),M(qs),M(D),M(xs),M(K),M(zs),M(Hs),M($s),M(Ls),M(sl),M(Ds),M(Ml),M(Ks),M(tl),M(Ps),M(al),M(Os),M(nl),M(lM),M(el),M(sM),M(yl),M(MM),M(Jl),M(tM),M(aM),M(nM),M(Tl),M(eM),M(yM),M(pl),M(JM),M(il),M(UM),M(cl),M(jM),M(TM),M(wM),M(pM),M(dl),M(iM),M(rl),M(cM),M(ml),M(oM),M(Cl),M(uM),M(bl),M(IM),M(dM),M(rM),M(mM),M(Gl),M(CM),M(Al),M(bM),M(Ql),M(hM),M(Wl),M(BM),M(ZM),M(GM),M(Xl),M(AM),M(gl),M(QM),M(fl),M(WM),M(VM),M(RM),M(XM),M(gM),M(Yl),M(fM),M(Sl),M(NM),M(Fl),M(kM),M(_l),M(EM),M(ql),M(vM),M(YM),M(SM),M(FM),M($l),M(_M),M(qM),M(xM),M(zM),M(HM),M($M),M(LM),M(ss),M(DM),M(Ms),M(KM),M(ts),M(PM),M(OM),M(lt),M(I),M(st),M(es),M(Mt),M(ys),M(tt),M(at),M(Ts)),M(u),i(d,l),i(r,l),i(G,l),i(g,l),i(f,l),i(N,l),i(k,l),i(E,l),i(v,l),i(Y,l),i(_,l),i(q,l),i(x,l),i(H,l),i($,l),i(P,l),i(O,l),i(ll,l),i(Ul,l),i(jl,l),i(wl,l),i(ol,l),i(ul,l),i(Il,l),i(hl,l),i(Bl,l),i(Zl,l),i(Vl,l),i(Rl,l),i(Nl,l),i(kl,l),i(El,l),i(vl,l),i(xl,l),i(zl,l),i(Hl,l),i(Ll,l),i(Dl,l),i(Kl,l),i(Pl,l),i(Ol,l),i(ls,l),i(as,l),i(ns,l),i(Js,l)}}}const da='{"title":"RAG 评估","local":"rag-评估","sections":[{"title":"评估RAG性能","local":"评估rag性能","sections":[],"depth":3},{"title":"加载你的知识基础","local":"加载你的知识基础","sections":[],"depth":3}],"depth":1}';function ra(et){return Ta(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ba extends wa{constructor(u){super(),pa(this,u,ra,Ia,Ua,{})}}export{Ba as component};
