"        data = graph.query(description_query, params={\"candidate\": entity})\n",
    "        return data[0][\"context\"]\n",
    "    except IndexError:\n",
    "        return \"No information was found\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdecc24b-8065-4755-98cc-9c6d093d4897",
   "metadata": {},
   "source": [
    "You can observe that we have defined the Cypher statement used to retrieve information.\n",
    "Therefore, we can avoid generating Cypher statements and use the LLM agent to only populate the input parameters.\n",
    "To provide additional information to an LLM agent about when to use the tool and their input parameters, we wrap the function as a tool."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f4cde772-0d05-475d-a2f0-b53e1669bd13",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Optional, Type\n",
    "\n",
    "from langchain_core.callbacks import (\n",
    "    AsyncCallbackManagerForToolRun,\n",
    "    CallbackManagerForToolRun,\n",
    ")\n",
    "from langchain_core.tools import BaseTool\n",
    "\n",
    "# Import things that are needed generically\n",
    "from pydantic import BaseModel, Field\n",
    "\n",
    "\n",
    "class InformationInput(BaseModel):\n",
    "    entity: str = Field(description=\"movie or a person mentioned in the question\")\n",
    "\n",
    "\n",
    "class InformationTool(BaseTool):\n",
    "    name = \"Information\"\n",
    "    description = (\n",
    "        \"useful for when you need to answer questions about various actors or movies\"\n",
    "    )\n",
    "    args_schema: Type[BaseModel] = InformationInput\n",
    "\n",
    "    def _run(\n",
    "        self,\n",
    "        entity: str,\n",
    "        run_manager: Optional[CallbackManagerForToolRun] = None,\n",
    "    ) -> str:\n",
    "        \"\"\"Use the tool.\"\"\"\n",
    "        return get_information(entity)\n",
    "\n",
    "    async def _arun(\n",
    "        self,\n",
    "        entity: str,\n",
    "        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,\n",
    "    ) -> str:\n",
    "        \"\"\"Use the tool asynchronously.\"\"\"\n",
    "        return get_information(entity)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff4820aa-2b57-4558-901f-6d984b326738",
   "metadata": {},
   "source": [
    "## OpenAI Agent\n",
    "\n",
    "LangChain expression language makes it very convenient to define an agent to interact with a graph database over the semantic layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6e959ac2-537d-4358-a43b-e3a47f68e1d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List, Tuple\n",
    "\n",
    "from langchain.agents import AgentExecutor\n",
    "from langchain.agents.format_scratchpad import format_to_openai_function_messages\n",
    "from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser\n",
    "from langchain_core.messages import AIMessage, HumanMessage\n",
    "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
    "from langchain_core.utils.function_calling import convert_to_openai_function\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "llm = ChatOpenAI(model=\"gpt-3.5-turbo\", temperature=0)\n",
    "tools = [InformationTool()]\n",
    "\n",
    "llm_with_tools = llm.bind(functions=[convert_to_openai_function(t) for t in tools])\n",
    "\n",
    "prompt = ChatPromptTemplate.from_messages(\n",
    "    [\n",
    "        (\n",
    "            \"system\",\n",
    "            \"You are a helpful assistant that finds information about movies \"\n",
    "            \" and recommends them. If tools require follow up questions, \"\n",
    "            \"make sure to ask the user for clarification. Make sure to include any \"\n",
    "            \"available options that need to be clarified in the follow up questions \"\n",
    "            \"Do only the things the user specifically requested. \",\n",
    "        ),\n",
    "        MessagesPlaceholder(variable_name=\"chat_history\"),\n",
    "        (\"user\", \"{input}\"),\n",
    "        MessagesPlaceholder(variable_name=\"agent_scratchpad\"),\n",
    "    ]\n",
    ")\n",
    "\n",
    "\n",
    "def _format_chat_history(chat_history: List[Tuple[str, str]]):\n",
    "    buffer = []\n",
    "    for human, ai in chat_history:\n",
    "        buffer.append(HumanMessage(content=human))\n",
    "        buffer.append(AIMessage(content=ai))\n",
    "    return buffer\n",
    "\n",
    "\n",
    "agent = (\n",
    "    {\n",
    "        \"input\": lambda x: x[\"input\"],\n",
    "        \"chat_history\": lambda x: _format_chat_history(x[\"chat_history\"])\n",
    "        if x.get(\"chat_history\")\n",
    "        else [],\n",
    "        \"agent_scratchpad\": lambda x: format_to_openai_function_messages(\n",
    "            x[\"intermediate_steps\"]\n",
    "        ),\n",
    "    }\n",
    "    | prompt\n",
    "    | llm_with_tools\n",
    "    | OpenAIFunctionsAgentOutputParser()\n",
    ")\n",
    "\n",
    "agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b0459833-fe84-4ebc-9823-a3a3ffd929e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m\n",
      "Invoking: `Information` with `{'entity': 'Casino'}`\n",
      "\n",
      "\n",
      "\u001b[0m\u001b[36;1m\u001b[1;3mtype:Movie\n",
      "title: Casino\n",
      "year: 1995-11-22\n",
      "ACTED_IN: Joe Pesci, Robert De Niro, Sharon Stone, James Woods\n",
      "\u001b[0m\u001b[32;1m\u001b[1;3mThe movie \"Casino\" starred Joe Pesci, Robert De Niro, Sharon Stone, and James Woods.\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'input': 'Who played in Casino?',\n",
       " 'output': 'The movie \"Casino\" starred Joe Pesci, Robert De Niro, Sharon Stone, and James Woods.'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agent_executor.invoke({\"input\": \"Who played in Casino?\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2759973-de8a-4624-8930-c90a21d6caa3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",


{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d9172545",
   "metadata": {},
   "source": [
    "# How to retrieve using multiple vectors per document\n",
    "\n",
    "It can often be useful to store multiple vectors per document. There are multiple use cases where this is beneficial. For example, we can embed multiple chunks of a document and associate those embeddings with the parent document, allowing retriever hits on the chunks to return the larger document.\n",
    "\n",
    "LangChain implements a base [MultiVectorRetriever](https://python.langchain.com/api_reference/langchain/retrievers/langchain.retrievers.multi_vector.MultiVectorRetriever.html), which simplifies this process. Much of the complexity lies in how to create the multiple vectors per document. This notebook covers some of the common ways to create those vectors and use the `MultiVectorRetriever`.\n",
    "\n",
    "The methods to create multiple vectors per document include:\n",
    "\n",
    "- Smaller chunks: split a document into smaller chunks, and embed those (this is [ParentDocumentRetriever](https://python.langchain.com/api_reference/langchain/retrievers/langchain.retrievers.parent_document_retriever.ParentDocumentRetriever.html)).\n",
    "- Summary: create a summary for each document, embed that along with (or instead of) the document.\n",
    "- Hypothetical questions: create hypothetical questions that each document would be appropriate to answer, embed those along with (or instead of) the document.\n",
    "\n",
    "Note that this also enables another method of adding embeddings - manually. This is useful because you can explicitly add questions or queries that should lead to a document being recovered, giving you more control.\n",
    "\n",
    "Below we walk through an example. First we instantiate some documents. We will index them in an (in-memory) [Chroma](/docs/integrations/providers/chroma/) vector store using [OpenAI](https://python.langchain.com/docs/integrations/text_embedding/openai/) embeddings, but any LangChain vector store or embeddings model will suffice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09cecd95-3499-465a-895a-944627ffb77f",
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install --upgrade --quiet  langchain-chroma langchain langchain-openai > /dev/null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "18c1421a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.storage import InMemoryByteStore\n",
    "from langchain_chroma import Chroma\n",
    "from langchain_community.document_loaders import TextLoader\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "\n",
    "loaders = [\n",
    "    TextLoader(\"paul_graham_essay.txt\"),\n",
    "    TextLoader(\"state_of_the_union.txt\"),\n",
    "]\n",
    "docs = []\n",
    "for loader in loaders:\n",
    "    docs.extend(loader.load())\n",
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=10000)\n",
    "docs = text_splitter.split_documents(docs)\n",
    "\n",
    "# The vectorstore to use to index the child chunks\n",
    "vectorstore = Chroma(\n",
    "    collection_name=\"full_documents\", embedding_function=OpenAIEmbeddings()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa17beda",
   "metadata": {},
   "source": [
    "## Smaller chunks\n",
    "\n",
    "Often times it can be useful to retrieve larger chunks of information, but embed smaller chunks. This allows for embeddings to capture the semantic meaning as closely as possible, but for as much context as possible to be passed downstream. Note that this is what the [ParentDocumentRetriever](https://python.langchain.com/api_reference/langchain/retrievers/langchain.retrievers.parent_document_retriever.ParentDocumentRetriever.html) does. Here we show what is going on under the hood.\n",
    "\n",
    "We will make a distinction between the vector store, which indexes embeddings of the (sub) documents, and the document store, which houses the \"parent\" documents and associates them with an identifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0e7b6b45",
   "metadata": {},
   "outputs": [],
   "source": [
    "import uuid\n",
    "\n",
    "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
    "\n",
    "# The storage layer for the parent documents\n",
    "store = InMemoryByteStore()\n",
    "id_key = \"doc_id\"\n",
    "\n",
    "# The retriever (empty to start)\n",
    "retriever = MultiVectorRetriever(\n",
    "    vectorstore=vectorstore,\n",
    "    byte_store=store,\n",
    "    id_key=id_key,\n",
    ")\n",
    "\n",
    "doc_ids = [str(uuid.uuid4()) for _ in docs]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4feded4-856a-4282-91c3-53aabc62e6ff",
   "metadata": {},
   "source": [
    "We next generate the \"sub\" documents by splitting the original documents. Note that we store the document identifier in the `metadata` of the corresponding [Document](https://python.langchain.com/api_reference/core/documents/langchain_core.documents.base.Document.html) object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5d23247d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The splitter to use to create smaller chunks\n",
    "child_text_splitter = RecursiveCharacterTextSplitter(chunk_size=400)\n",
    "\n",
    "sub_docs = []\n",
    "for i, doc in enumerate(docs):\n",
    "    _id = doc_ids[i]\n",
    "    _sub_docs = child_text_splitter.split_documents([doc])\n",
    "    for _doc in _sub_docs:\n",
    "        _doc.metadata[id_key] = _id\n",
    "    sub_docs.extend(_sub_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e0634f8-90d5-4250-981a-5257c8a6d455",
   "metadata": {},
   "source": [
    "Finally, we index the documents in our vector store and document store:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "92ed5861",
   "metadata": {},
   "outputs": [],
   "source": [
    "retriever.vectorstore.add_documents(sub_docs)\n",
    "retriever.docstore.mset(list(zip(doc_ids, docs)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14c48c6d-850c-4317-9b6e-1ade92f2f710",
   "metadata": {},
   "source": [
    "The vector store alone will retrieve small chunks:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8afed60c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content='Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \\n\\nOne of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court.', metadata={'doc_id': '064eca46-a4c4-4789-8e3b-583f9597e54f', 'source': 'state_of_the_union.txt'})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retriever.vectorstore.similarity_search(\"justice breyer\")[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "717097c7-61d9-4306-8625-ef8f1940c127",
   "metadata": {},
   "source": [
    "Whereas the retriever will return the larger parent document:"
   ]
  },
  {


]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retrieved_docs = retriever.invoke(\"justice breyer\")\n",
    "\n",
    "len(retrieved_docs[0].page_content)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "097a5396",
   "metadata": {},
   "source": [
    "## Hypothetical Queries\n",
    "\n",
    "An LLM can also be used to generate a list of hypothetical questions that could be asked of a particular document, which might bear close semantic similarity to relevant queries in a [RAG](/docs/tutorials/rag) application. These questions can then be embedded and associated with the documents to improve retrieval.\n",
    "\n",
    "Below, we use the [with_structured_output](/docs/how_to/structured_output/) method to structure the LLM output into a list of strings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "03d85234-c33a-4a43-861d-47328e1ec2ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "\n",
    "from pydantic import BaseModel, Field\n",
    "\n",
    "\n",
    "class HypotheticalQuestions(BaseModel):\n",
    "    \"\"\"Generate hypothetical questions.\"\"\"\n",
    "\n",
    "    questions: List[str] = Field(..., description=\"List of questions\")\n",
    "\n",
    "\n",
    "chain = (\n",
    "    {\"doc\": lambda x: x.page_content}\n",
    "    # Only asking for 3 hypothetical questions, but this could be adjusted\n",
    "    | ChatPromptTemplate.from_template(\n",
    "        \"Generate a list of exactly 3 hypothetical questions that the below document could be used to answer:\\n\\n{doc}\"\n",
    "    )\n",
    "    | ChatOpenAI(max_retries=0, model=\"gpt-4o\").with_structured_output(\n",
    "        HypotheticalQuestions\n",
    "    )\n",
    "    | (lambda x: x.questions)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6dddc40f-62af-413c-b944-f94a5e1f2f4e",
   "metadata": {},
   "source": [
    "Invoking the chain on a single document demonstrates that it outputs a list of questions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "11d30554",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"What impact did the IBM 1401 have on the author's early programming experiences?\",\n",
       " \"How did the transition from using the IBM 1401 to microcomputers influence the author's programming journey?\",\n",
       " \"What role did Lisp play in shaping the author's understanding and approach to AI?\"]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chain.invoke(docs[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dcffc572-7b20-4b77-857a-90ec360a8f7e",
   "metadata": {},
   "source": [
    "We can batch then batch the chain over all documents and assemble our vector store and document store as before:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b2cd6e75",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Batch chain over documents to generate hypothetical questions\n",
    "hypothetical_questions = chain.batch(docs, {\"max_concurrency\": 5})\n",
    "\n",
    "\n",
    "# The vectorstore to use to index the child chunks\n",
    "vectorstore = Chroma(\n",
    "    collection_name=\"hypo-questions\", embedding_function=OpenAIEmbeddings()\n",
    ")\n",
    "# The storage layer for the parent documents\n",
    "store = InMemoryByteStore()\n",
    "id_key = \"doc_id\"\n",
    "# The retriever (empty to start)\n",
    "retriever = MultiVectorRetriever(\n",
    "    vectorstore=vectorstore,\n",
    "    byte_store=store,\n",
    "    id_key=id_key,\n",
    ")\n",
    "doc_ids = [str(uuid.uuid4()) for _ in docs]\n",
    "\n",
    "\n",
    "# Generate Document objects from hypothetical questions\n",
    "question_docs = []\n",
    "for i, question_list in enumerate(hypothetical_questions):\n",
    "    question_docs.extend(\n",
    "        [Document(page_content=s, metadata={id_key: doc_ids[i]}) for s in question_list]\n",
    "    )\n",
    "\n",
    "\n",
    "retriever.vectorstore.add_documents(question_docs)\n",
    "retriever.docstore.mset(list(zip(doc_ids, docs)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75cba8ab-a06f-4545-85fc-cf49d0204b5e",
   "metadata": {},
   "source": [
    "Note that querying the underlying vector store will retrieve hypothetical questions that are semantically similar to the input query:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7b442b90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='What might be the potential benefits of nominating Circuit Court of Appeals Judge Ketanji Brown Jackson to the United States Supreme Court?', metadata={'doc_id': '43292b74-d1b8-4200-8a8b-ea0cb57fbcdb'}),\n",
       " Document(page_content='How might the Bipartisan Infrastructure Law impact the economic competition between the U.S. and China?', metadata={'doc_id': '66174780-d00c-4166-9791-f0069846e734'}),\n",
       " Document(page_content='What factors led to the creation of Y Combinator?', metadata={'doc_id': '72003c4e-4cc9-4f09-a787-0b541a65b38c'}),\n",
       " Document(page_content='How did the ability to publish essays online change the landscape for writers and thinkers?', metadata={'doc_id': 'e8d2c648-f245-4bcc-b8d3-14e64a164b64'})]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_docs = retriever.vectorstore.similarity_search(\"justice breyer\")\n",
    "\n",
    "sub_docs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63c32e43-5f4a-463b-a0c2-2101986f70e6",
   "metadata": {},
   "source": [
    "And invoking the retriever will return the corresponding document:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7594b24e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9194"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retrieved_docs = retriever.invoke(\"justice breyer\")\n",
    "len(retrieved_docs[0].page_content)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}


# How to partially format prompt templates

This guide assumes familiarity with the following concepts:

Like partially binding arguments to a function, it can make sense to "partial" a prompt template - e.g. pass in a subset of the required values, as to create a new prompt template which expects only the remaining subset of values.

1. Partial formatting with string values.

2. Partial formatting with functions that return string values.

In the examples below, we go over the motivations for both use cases as well as how to do it in LangChain.

One common use case for wanting to partial a prompt template is if you get access to some of the variables in a prompt before others. For example, suppose you have a prompt template that requires two variables, `foo` and `baz`. If you get the `foo` value early on in your chain, but the `baz` value later, it can be inconvenient to pass both variables all the way through the chain. Instead, you can partial the prompt template with the `foo` value, and then pass the partialed prompt template along and just use that. Below is an example of doing this:

stdout

foobaz

from langchain_core.prompts import PromptTemplate

You can also just initialize the prompt with the partialed variables.

foobaz

template="{foo}{bar}", input_variables=["bar"], partial_variables={"foo": "foo"}

The other common use is to partial with a function. The use case for this is when you have a variable you know that you always want to fetch in a common way. A prime example of this is with date or time. Imagine you have a prompt which you always want to have the current date. You can't hard code it in the prompt, and passing it along with the other input variables is inconvenient. In this case, it's handy to be able to partial the prompt with a function that always returns the current date.

Tell me a funny joke about the day 04/21/2024, 19:43:57

template="Tell me a {adjective} joke about the day {date}",

You can also just initialize the prompt with the partialed variables, which often makes more sense in this workflow.

Tell me a funny joke about the day 04/21/2024, 19:43:57

You've now learned how to partially apply variables to your prompt templates.

Next, check out the other how-to guides on prompt templates in this section, like [adding few-shot examples to your prompt templates](/docs/how_to/few_shot_examples_chat).


# How to create custom callback handlers

LangChain has some built-in callback handlers, but you will often want to create your own handlers with custom logic.

To create a custom callback handler, we need to determine the [event(s)](https://python.langchain.com/api_reference/core/callbacks/langchain_core.callbacks.base.BaseCallbackHandler.html#langchain-core-callbacks-base-basecallbackhandler) we want our callback handler to handle as well as what we want our callback handler to do when the event is triggered. Then all we need to do is attach the callback handler to the object, for example via [the constructor](/docs/how_to/callbacks_constructor) or [at runtime](/docs/how_to/callbacks_runtime).

In the example below, we'll implement streaming with a custom handler.

In our custom callback handler `MyCustomHandler`, we implement the `on_llm_new_token` handler to print the token we have just received. We then attach our custom handler to the model object as a constructor callback.

%pip install -qU langchain langchain_anthropic

My custom handler, token: Here
My custom handler, token: 's
My custom handler, token:  a
My custom handler, token:  bear
My custom handler, token:  joke
My custom handler, token:  for
My custom handler, token:  you
My custom handler, token: :
My custom handler, token:
Why
My custom handler, token:  di
My custom handler, token: d the
My custom handler, token:  bear
My custom handler, token:  dissol
My custom handler, token: ve
My custom handler, token:  in
My custom handler, token:  water
My custom handler, token: ?
My custom handler, token:
Because
My custom handler, token:  it
My custom handler, token:  was
My custom handler, token:  a
My custom handler, token:  polar
My custom handler, token:  bear
My custom handler, token: !

from langchain_anthropic import ChatAnthropic

from langchain_core.callbacks import BaseCallbackHandler

from langchain_core.prompts import ChatPromptTemplate

def on_llm_new_token(self, token: str, **kwargs) -> None:

print(f"My custom handler, token: {token}")

prompt = ChatPromptTemplate.from_messages(["Tell me a joke about {animal}"])

# To enable streaming, we pass in `streaming=True` to the ChatModel constructor

# Additionally, we pass in our custom handler as a list to the callbacks parameter

response = chain.invoke({"animal": "bears"})

You can see [this reference page](https://python.langchain.com/api_reference/core/callbacks/langchain_core.callbacks.base.BaseCallbackHandler.html#langchain-core-callbacks-base-basecallbackhandler) for a list of events you can handle. Note that the `handle_chain_*` events run for most LCEL runnables.

You've now learned how to create your own custom callback handlers.

Next, check out the other how-to guides in this section, such as [how to attach callbacks to a runnable](/docs/how_to/callbacks_attach).


{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c95fcd15cd52c944",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "# How to split by HTML sections\n",
    "## Description and motivation\n",
    "Similar in concept to the [HTMLHeaderTextSplitter](/docs/how_to/HTML_header_metadata_splitter), the `HTMLSectionSplitter` is a \"structure-aware\" chunker that splits text at the element level and adds metadata for each header \"relevant\" to any given chunk.\n",
    "\n",
    "It can return chunks element by element or combine elements with the same metadata, with the objectives of (a) keeping related text grouped (more or less) semantically and (b) preserving context-rich information encoded in document structures.\n",
    "\n",
    "Use `xslt_path` to provide an absolute path to transform the HTML so that it can detect sections based on provided tags. The default is to use the `converting_to_header.xslt` file in the `data_connection/document_transformers` directory. This is for converting the html to a format/layout that is easier to detect sections. For example, `span` based on their font size can be converted to header tags to be detected as a section.\n",
    "\n",
    "## Usage examples\n",
    "### 1) How to split HTML strings:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-02T18:57:49.208965400Z",
     "start_time": "2023-10-02T18:57:48.899756Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Foo \\n Some intro text about Foo.', metadata={'Header 1': 'Foo'}),\n",
       " Document(page_content='Bar main section \\n Some intro text about Bar. \\n Bar subsection 1 \\n Some text about the first subtopic of Bar. \\n Bar subsection 2 \\n Some text about the second subtopic of Bar.', metadata={'Header 2': 'Bar main section'}),\n",
       " Document(page_content='Baz \\n Some text about Baz \\n \\n \\n Some concluding text about Foo', metadata={'Header 2': 'Baz'})]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_text_splitters import HTMLSectionSplitter\n",
    "\n",
    "html_string = \"\"\"\n",
    "    <!DOCTYPE html>\n",
    "    \n",
    "    \n",
    "
\n",
    "
Foo
\n",
    "
Some intro text about Foo.
\n",
    "
\n",
    "
Bar main section
\n",
    "
Some intro text about Bar.
\n",
    "
Bar subsection 1
\n",
    "
Some text about the first subtopic of Bar.
\n",
    "
Bar subsection 2
\n",
    "
Some text about the second subtopic of Bar.
\n",
    "
\n",
    "
\n",
    "
Baz
\n",
    "
Some text about Baz
\n",
    "
\n",
    "
\n",
    "
Some concluding text about Foo
\n",
    "
\n",
    "    \n",
    "    \n",
    "\"\"\"\n",
    "\n",
    "headers_to_split_on = [(\"h1\", \"Header 1\"), (\"h2\", \"Header 2\")]\n",
    "\n",
    "html_splitter = HTMLSectionSplitter(headers_to_split_on)\n",
    "html_header_splits = html_splitter.split_text(html_string)\n",
    "html_header_splits"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e29b4aade2a0070c",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "### 2) How to constrain chunk sizes:\n",
    "\n",
    "`HTMLSectionSplitter` can be used with other text splitters as part of a chunking pipeline. Internally, it uses the `RecursiveCharacterTextSplitter` when the section size is larger than the chunk size. It also considers the font size of the text to determine whether it is a section or not based on the determined font size threshold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6ada8ea093ea0475",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-02T18:57:51.016141300Z",
     "start_time": "2023-10-02T18:57:50.647495400Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Foo \\n Some intro text about Foo.', metadata={'Header 1': 'Foo'}),\n",
       " Document(page_content='Bar main section \\n Some intro text about Bar.', metadata={'Header 2': 'Bar main section'}),\n",
       " Document(page_content='Bar subsection 1 \\n Some text about the first subtopic of Bar.', metadata={'Header 3': 'Bar subsection 1'}),\n",
       " Document(page_content='Bar subsection 2 \\n Some text about the second subtopic of Bar.', metadata={'Header 3': 'Bar subsection 2'}),\n",
       " Document(page_content='Baz \\n Some text about Baz \\n \\n \\n Some concluding text about Foo', metadata={'Header 2': 'Baz'})]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "\n",
    "html_string = \"\"\"\n",
    "    <!DOCTYPE html>\n",
    "    \n",
    "    \n",
    "
\n",
    "
Foo
\n",
    "
Some intro text about Foo.
\n",
    "
\n",
    "
Bar main section
\n",
    "
Some intro text about Bar.
\n",
    "
Bar subsection 1
\n",
    "
Some text about the first subtopic of Bar.
\n",
    "
Bar subsection 2
\n",
    "
Some text about the second subtopic of Bar.
\n",
    "
\n",
    "
\n",
    "
Baz
\n",
    "
Some text about Baz
\n",
    "
\n",
    "
\n",
    "
Some concluding text about Foo
\n",
    "
\n",
    "    \n",
    "    \n",
    "\"\"\"\n",
    "\n",
    "headers_to_split_on = [\n",
    "    (\"h1\", \"Header 1\"),\n",
    "    (\"h2\", \"Header 2\"),\n",
    "    (\"h3\", \"Header 3\"),\n",
    "    (\"h4\", \"Header 4\"),\n",
    "]\n",
    "\n",
    "html_splitter = HTMLSectionSplitter(headers_to_split_on)\n",
    "\n",
    "html_header_splits = html_splitter.split_text(html_string)\n",
    "\n",
    "chunk_size = 500\n",
    "chunk_overlap = 30\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=chunk_size, chunk_overlap=chunk_overlap\n",
    ")\n",
    "\n",
    "# Split\n",


{
 "cells": [
  {
   "cell_type": "raw",
   "id": "e2596041-9b76-4e74-836f-e6235086bbf0",
   "metadata": {},
   "source": [
    "---\n",
    "sidebar_position: 1\n",
    "keywords: [RunnableParallel, RunnableMap, LCEL]\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b022ab74-794d-4c54-ad47-ff9549ddb9d2",
   "metadata": {},
   "source": [
    "# How to invoke runnables in parallel\n",
    "\n",
    ":::info Prerequisites\n",
    "\n",
    "This guide assumes familiarity with the following concepts:\n",
    "- [LangChain Expression Language (LCEL)](/docs/concepts/#langchain-expression-language)\n",
    "- [Chaining runnables](/docs/how_to/sequence)\n",
    "\n",
    ":::\n",
    "\n",
    "The [`RunnableParallel`](https://python.langchain.com/api_reference/core/runnables/langchain_core.runnables.base.RunnableParallel.html) primitive is essentially a dict whose values are runnables (or things that can be coerced to runnables, like functions). It runs all of its values in parallel, and each value is called with the overall input of the `RunnableParallel`. The final return value is a dict with the results of each value under its appropriate key.\n",
    "\n",
    "## Formatting with `RunnableParallels`\n",
    "\n",
    "`RunnableParallels` are useful for parallelizing operations, but can also be useful for manipulating the output of one Runnable to match the input format of the next Runnable in a sequence. You can use them to split or fork the chain so that multiple components can process the input in parallel. Later, other components can join or merge the results to synthesize a final response. This type of chain creates a computation graph that looks like the following:\n",
    "\n",
    "```text\n",
    "     Input\n",
    "      / \\\n",
    "     /   \\\n",
    " Branch1 Branch2\n",
    "     \\   /\n",
    "      \\ /\n",
    "      Combine\n",
    "```\n",
    "\n",
    "Below, the input to prompt is expected to be a map with keys `\"context\"` and `\"question\"`. The user input is just the question. So we need to get the context using our retriever and passthrough the user input under the `\"question\"` key.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2627ffd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# | output: false\n",
    "# | echo: false\n",
    "\n",
    "%pip install -qU langchain langchain_openai\n",
    "\n",
    "import os\n",
    "from getpass import getpass\n",
    "\n",
    "if \"OPENAI_API_KEY\" not in os.environ:\n",
    "    os.environ[\"OPENAI_API_KEY\"] = getpass()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "267d1460-53c1-4fdb-b2c3-b6a1eb7fccff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Harrison worked at Kensho.'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_community.vectorstores import FAISS\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "from langchain_core.runnables import RunnablePassthrough\n",
    "from langchain_openai import ChatOpenAI, OpenAIEmbeddings\n",
    "\n",
    "vectorstore = FAISS.from_texts(\n",
    "    [\"harrison worked at kensho\"], embedding=OpenAIEmbeddings()\n",
    ")\n",
    "retriever = vectorstore.as_retriever()\n",
    "template = \"\"\"Answer the question based only on the following context:\n",
    "{context}\n",
    "\n",
    "Question: {question}\n",
    "\"\"\"\n",
    "\n",
    "# The prompt expects input with keys for \"context\" and \"question\"\n",
    "prompt = ChatPromptTemplate.from_template(template)\n",
    "\n",
    "model = ChatOpenAI()\n",
    "\n",
    "retrieval_chain = (\n",
    "    {\"context\": retriever, \"question\": RunnablePassthrough()}\n",
    "    | prompt\n",
    "    | model\n",
    "    | StrOutputParser()\n",
    ")\n",
    "\n",
    "retrieval_chain.invoke(\"where did harrison work?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "392cd4c4-e7ed-4ab8-934d-f7a4eca55ee1",
   "metadata": {},
   "source": [
    ":::tip\n",
    "Note that when composing a RunnableParallel with another Runnable we don't even need to wrap our dictionary in the RunnableParallel class — the type conversion is handled for us. In the context of a chain, these are equivalent:\n",
    ":::\n",
    "\n",
    "```\n",
    "{\"context\": retriever, \"question\": RunnablePassthrough()}\n",
    "```\n",
    "\n",
    "```\n",
    "RunnableParallel({\"context\": retriever, \"question\": RunnablePassthrough()})\n",
    "```\n",
    "\n",
    "```\n",
    "RunnableParallel(context=retriever, question=RunnablePassthrough())\n",
    "```\n",
    "\n",
    "See the section on [coercion for more](/docs/how_to/sequence/#coercion)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c1b8baa-3a80-44f0-bb79-d22f79815d3d",
   "metadata": {},
   "source": [
    "## Using itemgetter as shorthand\n",
    "\n",
    "Note that you can use Python's `itemgetter` as shorthand to extract data from the map when combining with `RunnableParallel`. You can find more information about itemgetter in the [Python Documentation](https://docs.python.org/3/library/operator.html#operator.itemgetter). \n",
    "\n",
    "In the example below, we use itemgetter to extract specific keys from the map:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84fc49e1-2daf-4700-ae33-a0a6ed47d5f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Harrison ha lavorato a Kensho.'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from operator import itemgetter\n",
    "\n",
    "from langchain_community.vectorstores import FAISS\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "from langchain_core.runnables import RunnablePassthrough\n",
    "from langchain_openai import ChatOpenAI, OpenAIEmbeddings\n",
    "\n",
    "vectorstore = FAISS.from_texts(\n",
    "    [\"harrison worked at kensho\"], embedding=OpenAIEmbeddings()\n",
    ")\n",
    "retriever = vectorstore.as_retriever()\n",
    "\n",
    "template = \"\"\"Answer the question based only on the following context:\n",
    "{context}\n",
    "\n",
    "Question: {question}\n",
    "\n",
    "Answer in the following language: {language}\n",
    "\"\"\"\n",
    "prompt = ChatPromptTemplate.from_template(template)\n",
    "\n",
    "chain = (\n",
    "    {\n",
    "        \"context\": itemgetter(\"question\") | retriever,\n",
    "        \"question\": itemgetter(\"question\"),\n",
    "        \"language\": itemgetter(\"language\"),\n",
    "    }\n",
    "    | prompt\n",
    "    | model\n",
    "    | StrOutputParser()\n",
    ")\n",
    "\n",
    "chain.invoke({\"question\": \"where did harrison work\", \"language\": \"italian\"})"
   ]
  },
  {


- [How to: recursively split text](/docs/how_to/recursive_text_splitter)
- [How to: split by HTML headers](/docs/how_to/HTML_header_metadata_splitter)
- [How to: split by HTML sections](/docs/how_to/HTML_section_aware_splitter)
- [How to: split by character](/docs/how_to/character_text_splitter)
- [How to: split code](/docs/how_to/code_splitter)
- [How to: split Markdown by headers](/docs/how_to/markdown_header_metadata_splitter)
- [How to: recursively split JSON](/docs/how_to/recursive_json_splitter)
- [How to: split text into semantic chunks](/docs/how_to/semantic-chunker)
- [How to: split by tokens](/docs/how_to/split_by_token)

### Embedding models

[Embedding Models](/docs/concepts/#embedding-models) take a piece of text and create a numerical representation of it.

- [How to: embed text data](/docs/how_to/embed_text)
- [How to: cache embedding results](/docs/how_to/caching_embeddings)

### Vector stores

[Vector stores](/docs/concepts/#vector-stores) are databases that can efficiently store and retrieve embeddings.

- [How to: use a vector store to retrieve data](/docs/how_to/vectorstores)

### Retrievers

[Retrievers](/docs/concepts/#retrievers) are responsible for taking a query and returning relevant documents.

- [How to: use a vector store to retrieve data](/docs/how_to/vectorstore_retriever)
- [How to: generate multiple queries to retrieve data for](/docs/how_to/MultiQueryRetriever)
- [How to: use contextual compression to compress the data retrieved](/docs/how_to/contextual_compression)
- [How to: write a custom retriever class](/docs/how_to/custom_retriever)
- [How to: add similarity scores to retriever results](/docs/how_to/add_scores_retriever)
- [How to: combine the results from multiple retrievers](/docs/how_to/ensemble_retriever)
- [How to: reorder retrieved results to mitigate the "lost in the middle" effect](/docs/how_to/long_context_reorder)
- [How to: generate multiple embeddings per document](/docs/how_to/multi_vector)
- [How to: retrieve the whole document for a chunk](/docs/how_to/parent_document_retriever)
- [How to: generate metadata filters](/docs/how_to/self_query)
- [How to: create a time-weighted retriever](/docs/how_to/time_weighted_vectorstore)
- [How to: use hybrid vector and keyword retrieval](/docs/how_to/hybrid)

### Indexing

Indexing is the process of keeping your vectorstore in-sync with the underlying data source.

- [How to: reindex data to keep your vectorstore in-sync with the underlying data source](/docs/how_to/indexing)

### Tools

LangChain [Tools](/docs/concepts/#tools) contain a description of the tool (to pass to the language model) as well as the implementation of the function to call. Refer [here](/docs/integrations/tools/) for a list of pre-buit tools.

- [How to: create tools](/docs/how_to/custom_tools)
- [How to: use built-in tools and toolkits](/docs/how_to/tools_builtin)
- [How to: use chat models to call tools](/docs/how_to/tool_calling)
- [How to: pass tool outputs to chat models](/docs/how_to/tool_results_pass_to_model)
- [How to: pass run time values to tools](/docs/how_to/tool_runtime)
- [How to: add a human-in-the-loop for tools](/docs/how_to/tools_human)
- [How to: handle tool errors](/docs/how_to/tools_error)
- [How to: force models to call a tool](/docs/how_to/tool_choice)
- [How to: disable parallel tool calling](/docs/how_to/tool_calling_parallel)
- [How to: access the `RunnableConfig` from a tool](/docs/how_to/tool_configure)
- [How to: stream events from a tool](/docs/how_to/tool_stream_events)
- [How to: return artifacts from a tool](/docs/how_to/tool_artifacts/)
- [How to: convert Runnables to tools](/docs/how_to/convert_runnable_to_tool)
- [How to: add ad-hoc tool calling capability to models](/docs/how_to/tools_prompting)
- [How to: pass in runtime secrets](/docs/how_to/runnable_runtime_secrets)

### Multimodal

- [How to: pass multimodal data directly to models](/docs/how_to/multimodal_inputs/)
- [How to: use multimodal prompts](/docs/how_to/multimodal_prompts/)


{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4ef893cf-eac1-45e6-9eb6-72e9ca043200",
   "metadata": {},
   "source": [
    "# How to get your RAG application to return sources\n",
    "\n",
    "Often in Q&A applications it's important to show users the sources that were used to generate the answer. The simplest way to do this is for the chain to return the Documents that were retrieved in each generation.\n",
    "\n",
    "We'll work off of the Q&A app we built over the [LLM Powered Autonomous Agents](https://lilianweng.github.io/posts/2023-06-23-agent/) blog post by Lilian Weng in the [RAG tutorial](/docs/tutorials/rag).\n",
    "\n",
    "We will cover two approaches:\n",
    "\n",
    "1. Using the built-in [create_retrieval_chain](https://python.langchain.com/api_reference/langchain/chains/langchain.chains.retrieval.create_retrieval_chain.html), which returns sources by default;\n",
    "2. Using a simple [LCEL](/docs/concepts#langchain-expression-language-lcel) implementation, to show the operating principle.\n",
    "\n",
    "We will also show how to structure sources into the model response, such that a model can report what specific sources it used in generating its answer."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "487d8d79-5ee9-4aa4-9fdf-cd5f4303e099",
   "metadata": {},
   "source": [
    "## Setup\n",
    "\n",
    "### Dependencies\n",
    "\n",
    "We'll use OpenAI embeddings and a Chroma vector store in this walkthrough, but everything shown here works with any [Embeddings](/docs/concepts#embedding-models), [VectorStore](/docs/concepts#vectorstores) or [Retriever](/docs/concepts#retrievers). \n",
    "\n",
    "We'll use the following packages:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "28d272cd-4e31-40aa-bbb4-0be0a1f49a14",
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install --upgrade --quiet  langchain langchain-community langchainhub langchain-openai langchain-chroma beautifulsoup4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51ef48de-70b6-4f43-8e0b-ab9b84c9c02a",
   "metadata": {},
   "source": [
    "We need to set environment variable `OPENAI_API_KEY`, which can be done directly or loaded from a `.env` file like so:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "143787ca-d8e6-4dc9-8281-4374f4d71720",
   "metadata": {},
   "outputs": [],
   "source": [
    "import getpass\n",
    "import os\n",
    "\n",
    "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass()\n",
    "\n",
    "# import dotenv\n",
    "\n",
    "# dotenv.load_dotenv()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1665e740-ce01-4f09-b9ed-516db0bd326f",
   "metadata": {},
   "source": [
    "### LangSmith\n",
    "\n",
    "Many of the applications you build with LangChain will contain multiple steps with multiple invocations of LLM calls. As these applications get more and more complex, it becomes crucial to be able to inspect what exactly is going on inside your chain or agent. The best way to do this is with [LangSmith](https://smith.langchain.com).\n",
    "\n",
    "Note that LangSmith is not needed, but it is helpful. If you do want to use LangSmith, after you sign up at the link above, make sure to set your environment variables to start logging traces:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07411adb-3722-4f65-ab7f-8f6f57663d11",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
    "os.environ[\"LANGCHAIN_API_KEY\"] = getpass.getpass()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa6ba684-26cf-4860-904e-a4d51380c134",
   "metadata": {},
   "source": [
    "## Using `create_retrieval_chain`\n",
    "\n",
    "Let's first select a LLM:\n",
    "\n",
    "import ChatModelTabs from \"@theme/ChatModelTabs\";\n",
    "\n",
    "<ChatModelTabs customVarName=\"llm\" />\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5e7513b0-81e5-4477-8007-101e523f271c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# | output: false\n",
    "# | echo: false\n",
    "\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "llm = ChatOpenAI()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b1bdfd7-8acf-4655-834d-ba7463a80fef",
   "metadata": {},
   "source": [
    "Here is Q&A app with sources we built over the [LLM Powered Autonomous Agents](https://lilianweng.github.io/posts/2023-06-23-agent/) blog post by Lilian Weng in the [RAG tutorial](/docs/tutorials/rag):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24a69b8c-024e-4e34-b827-9c9de46512a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import bs4\n",
    "from langchain.chains import create_retrieval_chain\n",
    "from langchain.chains.combine_documents import create_stuff_documents_chain\n",
    "from langchain_chroma import Chroma\n",
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "\n",
    "# 1. Load, chunk and index the contents of the blog to create a retriever.\n",
    "loader = WebBaseLoader(\n",
    "    web_paths=(\"https://lilianweng.github.io/posts/2023-06-23-agent/\",),\n",
    "    bs_kwargs=dict(\n",
    "        parse_only=bs4.SoupStrainer(\n",
    "            class_=(\"post-content\", \"post-title\", \"post-header\")\n",
    "        )\n",
    "    ),\n",
    ")\n",
    "docs = loader.load()\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
    "splits = text_splitter.split_documents(docs)\n",
    "vectorstore = Chroma.from_documents(documents=splits, embedding=OpenAIEmbeddings())\n",
    "retriever = vectorstore.as_retriever()\n",
    "\n",
    "\n",
    "# 2. Incorporate the retriever into a question-answering chain.\n",
    "system_prompt = (\n",
    "    \"You are an assistant for question-answering tasks. \"\n",
    "    \"Use the following pieces of retrieved context to answer \"\n",
    "    \"the question. If you don't know the answer, say that you \"\n",
    "    \"don't know. Use three sentences maximum and keep the \"\n",
    "    \"answer concise.\"\n",
    "    \"\\n\\n\"\n",
    "    \"{context}\"\n",
    ")\n",
    "\n",
    "prompt = ChatPromptTemplate.from_messages(\n",
    "    [\n",
    "        (\"system\", system_prompt),\n",
    "        (\"human\", \"{input}\"),\n",
    "    ]\n",
    ")\n",
    "\n",


"  Document(metadata={'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, page_content='Fig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\\nThe system comprises of 4 stages:\\n(1) Task planning: LLM works as the brain and parses the user requests into multiple tasks. There are four attributes associated with each task: task type, ID, dependencies, and arguments. They use few-shot examples to guide LLM to do task parsing and planning.\\nInstruction:')],\n",
       " 'answer': 'Task decomposition is a technique used in artificial intelligence to break down complex tasks into smaller and more manageable subtasks. This approach helps agents or models to tackle difficult problems by dividing them into simpler steps, improving performance and interpretability. Different methods like Chain of Thought and Tree of Thoughts have been developed to enhance task decomposition in AI systems.'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_core.runnables import RunnablePassthrough\n",
    "\n",
    "\n",
    "def format_docs(docs):\n",
    "    return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
    "\n",
    "\n",
    "# This Runnable takes a dict with keys 'input' and 'context',\n",
    "# formats them into a prompt, and generates a response.\n",
    "rag_chain_from_docs = (\n",
    "    {\n",
    "        \"input\": lambda x: x[\"input\"],  # input query\n",
    "        \"context\": lambda x: format_docs(x[\"context\"]),  # context\n",
    "    }\n",
    "    | prompt  # format query and context into prompt\n",
    "    | llm  # generate response\n",
    "    | StrOutputParser()  # coerce to string\n",
    ")\n",
    "\n",
    "# Pass input query to retriever\n",
    "retrieve_docs = (lambda x: x[\"input\"]) | retriever\n",
    "\n",
    "# Below, we chain `.assign` calls. This takes a dict and successively\n",
    "# adds keys-- \"context\" and \"answer\"-- where the value for each key\n",
    "# is determined by a Runnable. The Runnable operates on all existing\n",
    "# keys in the dict.\n",
    "chain = RunnablePassthrough.assign(context=retrieve_docs).assign(\n",
    "    answer=rag_chain_from_docs\n",
    ")\n",
    "\n",
    "chain.invoke({\"input\": \"What is Task Decomposition\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b437da5d-ca09-4d15-9be2-c35e5a1ace77",
   "metadata": {},
   "source": [
    ":::tip\n",
    "\n",
    "Check out the [LangSmith trace](https://smith.langchain.com/public/1c055a3b-0236-4670-a3fb-023d418ba796/r)\n",
    "\n",
    ":::"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1c17797-d965-4fd2-b8d4-d386f25dd352",
   "metadata": {},
   "source": [
    "## Structure sources in model response\n",
    "\n",
    "Up to this point, we've simply propagated the documents returned from the retrieval step through to the final response. But this may not illustrate what subset of information the model relied on when generating its answer. Below, we show how to structure sources into the model response, allowing the model to report what specific context it relied on for its answer.\n",
    "\n",
    "Because the above LCEL implementation is composed of [Runnable](/docs/concepts/#runnable-interface) primitives, it is straightforward to extend. Below, we make a simple change:\n",
    "\n",
    "- We use the model's tool-calling features to generate [structured output](/docs/how_to/structured_output/), consisting of an answer and list of sources. The schema for the response is represented in the `AnswerWithSources` TypedDict, below.\n",
    "- We remove the `StrOutputParser()`, as we expect `dict` output in this scenario."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8f916b14-1b0a-4975-a62f-52f1353bde15",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "\n",
    "from langchain_core.runnables import RunnablePassthrough\n",
    "from typing_extensions import Annotated, TypedDict\n",
    "\n",
    "\n",
    "# Desired schema for response\n",
    "class AnswerWithSources(TypedDict):\n",
    "    \"\"\"An answer to the question, with sources.\"\"\"\n",
    "\n",
    "    answer: str\n",
    "    sources: Annotated[\n",
    "        List[str],\n",
    "        ...,\n",
    "        \"List of sources (author + year) used to answer the question\",\n",
    "    ]\n",
    "\n",
    "\n",
    "# Our rag_chain_from_docs has the following changes:\n",
    "# - add `.with_structured_output` to the LLM;\n",
    "# - remove the output parser\n",
    "rag_chain_from_docs = (\n",
    "    {\n",
    "        \"input\": lambda x: x[\"input\"],\n",
    "        \"context\": lambda x: format_docs(x[\"context\"]),\n",
    "    }\n",
    "    | prompt\n",
    "    | llm.with_structured_output(AnswerWithSources)\n",
    ")\n",
    "\n",
    "retrieve_docs = (lambda x: x[\"input\"]) | retriever\n",
    "\n",
    "chain = RunnablePassthrough.assign(context=retrieve_docs).assign(\n",
    "    answer=rag_chain_from_docs\n",
    ")\n",
    "\n",
    "response = chain.invoke({\"input\": \"What is Chain of Thought?\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7a8fc0c5-afb3-4012-a467-3951996a6850",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"answer\": \"Chain of Thought (CoT) is a prompting technique that enhances model performance on complex tasks by instructing the model to \\\"think step by step\\\" to decompose hard tasks into smaller and simpler steps. It transforms big tasks into multiple manageable tasks and sheds light on the interpretation of the model's thinking process.\",\n",
      "  \"sources\": [\n",
      "    \"Wei et al. 2022\"\n",
      "  ]\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "print(json.dumps(response[\"answer\"], indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7440f785-29c5-4c6b-9656-0d9d5efbac05",
   "metadata": {},
   "source": [
    ":::tip\n",
    "\n",
    "View [LangSmith trace](https://smith.langchain.com/public/0eeddf06-3a7b-4f27-974c-310ca8160f60/r)\n",
    "\n",
    ":::"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}


{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a05c860c",
   "metadata": {},
   "source": [
    "# How to split text by tokens \n",
    "\n",
    "Language models have a token limit. You should not exceed the token limit. When you split your text into chunks it is therefore a good idea to count the number of tokens. There are many tokenizers. When you count tokens in your text you should use the same tokenizer as used in the language model. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7683b36a",
   "metadata": {},
   "source": [
    "## tiktoken\n",
    "\n",
    ":::note\n",
    "[tiktoken](https://github.com/openai/tiktoken) is a fast `BPE` tokenizer created by `OpenAI`.\n",
    ":::\n",
    "\n",
    "\n",
    "We can use `tiktoken` to estimate tokens used. It will probably be more accurate for the OpenAI models.\n",
    "\n",
    "1. How the text is split: by character passed in.\n",
    "2. How the chunk size is measured: by `tiktoken` tokenizer.\n",
    "\n",
    "[CharacterTextSplitter](https://python.langchain.com/api_reference/text_splitters/character/langchain_text_splitters.character.CharacterTextSplitter.html), [RecursiveCharacterTextSplitter](https://python.langchain.com/api_reference/text_splitters/character/langchain_text_splitters.character.RecursiveCharacterTextSplitter.html), and [TokenTextSplitter](https://python.langchain.com/api_reference/langchain_text_splitters/base/langchain_text_splitters.base.TokenTextSplitter.html) can be used with `tiktoken` directly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c4ef83e-f43a-4658-ad1a-3952e0a5bbe7",
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install --upgrade --quiet langchain-text-splitters tiktoken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1ad2d0f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_text_splitters import CharacterTextSplitter\n",
    "\n",
    "# This is a long document we can split up.\n",
    "with open(\"state_of_the_union.txt\") as f:\n",
    "    state_of_the_union = f.read()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3ba1d8a",
   "metadata": {},
   "source": [
    "To split with a [CharacterTextSplitter](https://python.langchain.com/api_reference/text_splitters/character/langchain_text_splitters.character.CharacterTextSplitter.html) and then merge chunks with `tiktoken`, use its `.from_tiktoken_encoder()` method. Note that splits from this method can be larger than the chunk size measured by the `tiktoken` tokenizer.\n",
    "\n",
    "The `.from_tiktoken_encoder()` method takes either `encoding_name` as an argument (e.g. `cl100k_base`), or the `model_name` (e.g. `gpt-4`). All additional arguments like `chunk_size`, `chunk_overlap`, and `separators` are used to instantiate `CharacterTextSplitter`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "825f7c0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = CharacterTextSplitter.from_tiktoken_encoder(\n",
    "    encoding_name=\"cl100k_base\", chunk_size=100, chunk_overlap=0\n",
    ")\n",
    "texts = text_splitter.split_text(state_of_the_union)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ae35d165",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and the Cabinet. Justices of the Supreme Court. My fellow Americans.  \n",
      "\n",
      "Last year COVID-19 kept us apart. This year we are finally together again. \n",
      "\n",
      "Tonight, we meet as Democrats Republicans and Independents. But most importantly as Americans. \n",
      "\n",
      "With a duty to one another to the American people to the Constitution.\n"
     ]
    }
   ],
   "source": [
    "print(texts[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de5b6a6e",
   "metadata": {},
   "source": [
    "To implement a hard constraint on the chunk size, we can use `RecursiveCharacterTextSplitter.from_tiktoken_encoder`, where each split will be recursively split if it has a larger size:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0262a991",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
    "    model_name=\"gpt-4\",\n",
    "    chunk_size=100,\n",
    "    chunk_overlap=0,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04457e3a",
   "metadata": {},
   "source": [
    "We can also load a `TokenTextSplitter` splitter, which works with `tiktoken` directly and will ensure each split is smaller than chunk size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4454c70e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Madam Speaker, Madam Vice President, our\n"
     ]
    }
   ],
   "source": [
    "from langchain_text_splitters import TokenTextSplitter\n",
    "\n",
    "text_splitter = TokenTextSplitter(chunk_size=10, chunk_overlap=0)\n",
    "\n",
    "texts = text_splitter.split_text(state_of_the_union)\n",
    "print(texts[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bc155d0",
   "metadata": {},
   "source": [
    "Some written languages (e.g. Chinese and Japanese) have characters which encode to 2 or more tokens. Using the `TokenTextSplitter` directly can split the tokens for a character between two chunks causing malformed Unicode characters. Use `RecursiveCharacterTextSplitter.from_tiktoken_encoder` or `CharacterTextSplitter.from_tiktoken_encoder` to ensure chunks contain valid Unicode strings."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55f95f06",
   "metadata": {},
   "source": [
    "## spaCy\n",
    "\n",
    ":::note\n",
    "[spaCy](https://spacy.io/) is an open-source software library for advanced natural language processing, written in the programming languages Python and Cython.\n",
    ":::\n",
    "\n",
    "LangChain implements splitters based on the [spaCy tokenizer](https://spacy.io/api/tokenizer).\n",
    "\n",
    "1. How the text is split: by `spaCy` tokenizer.\n",
    "2. How the chunk size is measured: by number of characters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0b9242f-690c-4819-b35a-bb68187281ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install --upgrade --quiet  spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f1de7767",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is a long document we can split up.\n",
    "with open(\"state_of_the_union.txt\") as f:\n",
    "    state_of_the_union = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cef2b29e",
   "metadata": {},
   "outputs": [
    {


This guide assumes familiarity with the following concepts:

- [Custom callback handlers](/docs/how_to/custom_callbacks)

In many cases, it is advantageous to pass in handlers instead when running the object. When we pass through [`CallbackHandlers`](https://python.langchain.com/api_reference/core/callbacks/langchain_core.callbacks.base.BaseCallbackHandler.html#langchain-core-callbacks-base-basecallbackhandler) using the `callbacks` keyword arg when executing an run, those callbacks will be issued by all nested objects involved in the execution. For example, when a handler is passed through to an Agent, it will be used for all callbacks related to the agent and all the objects involved in the agent's execution, in this case, the Tools and LLM.

This prevents us from having to manually attach the handlers to each individual nested object. Here's an example:

%pip install -qU langchain langchain_anthropic

Chain RunnableSequence started

Chain ChatPromptTemplate started

Chain ended, outputs: messages=[HumanMessage(content='What is 1 + 2?')]

Chat model started

Chat model ended, response: generations=[[ChatGeneration(text='1 + 2 = 3', message=AIMessage(content='1 + 2 = 3', response_metadata={'id': 'msg_01D8Tt5FdtBk5gLTfBPm2tac', 'model': 'claude-3-sonnet-20240229', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': {'input_tokens': 16, 'output_tokens': 13}}, id='run-bb0dddd8-85f3-4e6b-8553-eaa79f859ef8-0'))]] llm_output={'id': 'msg_01D8Tt5FdtBk5gLTfBPm2tac', 'model': 'claude-3-sonnet-20240229', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': {'input_tokens': 16, 'output_tokens': 13}} run=None

Chain ended, outputs: content='1 + 2 = 3' response_metadata={'id': 'msg_01D8Tt5FdtBk5gLTfBPm2tac', 'model': 'claude-3-sonnet-20240229', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': {'input_tokens': 16, 'output_tokens': 13}} id='run-bb0dddd8-85f3-4e6b-8553-eaa79f859ef8-0'

AIMessage(content='1 + 2 = 3', response_metadata={'id': 'msg_01D8Tt5FdtBk5gLTfBPm2tac', 'model': 'claude-3-sonnet-20240229', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': {'input_tokens': 16, 'output_tokens': 13}}, id='run-bb0dddd8-85f3-4e6b-8553-eaa79f859ef8-0')

from langchain_anthropic import ChatAnthropic

from langchain_core.callbacks import BaseCallbackHandler

from langchain_core.messages import BaseMessage

from langchain_core.outputs import LLMResult

from langchain_core.prompts import ChatPromptTemplate

self, serialized: Dict[str, Any], messages: List[List[BaseMessage]], **kwargs

def on_llm_end(self, response: LLMResult, **kwargs) -> None:

print(f"Chat model ended, response: {response}")

self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs

def on_chain_end(self, outputs: Dict[str, Any], **kwargs) -> None:

print(f"Chain ended, outputs: {outputs}")

prompt = ChatPromptTemplate.from_template("What is 1 + {number}?")

chain.invoke({"number": "2"}, config={"callbacks": callbacks})

If there are already existing callbacks associated with a module, these will run in addition to any passed in at runtime.

You've now learned how to pass callbacks at runtime.

Next, check out the other how-to guides in this section, such as how to [pass callbacks into a module constructor](/docs/how_to/custom_callbacks).


{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "612eac0a",
   "metadata": {},
   "source": [
    "# How to do retrieval with contextual compression\n",
    "\n",
    "One challenge with retrieval is that usually you don't know the specific queries your document storage system will face when you ingest data into the system. This means that the information most relevant to a query may be buried in a document with a lot of irrelevant text. Passing that full document through your application can lead to more expensive LLM calls and poorer responses.\n",
    "\n",
    "Contextual compression is meant to fix this. The idea is simple: instead of immediately returning retrieved documents as-is, you can compress them using the context of the given query, so that only the relevant information is returned. “Compressing” here refers to both compressing the contents of an individual document and filtering out documents wholesale.\n",
    "\n",
    "To use the Contextual Compression Retriever, you'll need:\n",
    "\n",
    "- a base retriever\n",
    "- a Document Compressor\n",
    "\n",
    "The Contextual Compression Retriever passes queries to the base retriever, takes the initial documents and passes them through the Document Compressor. The Document Compressor takes a list of documents and shortens it by reducing the contents of documents or dropping documents altogether.\n",
    "\n",
    "## Get started"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e0029369",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function for printing docs\n",
    "\n",
    "\n",
    "def pretty_print_docs(docs):\n",
    "    print(\n",
    "        f\"\\n{'-' * 100}\\n\".join(\n",
    "            [f\"Document {i+1}:\\n\\n\" + d.page_content for i, d in enumerate(docs)]\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d2360fc",
   "metadata": {},
   "source": [
    "## Using a vanilla vector store retriever\n",
    "Let's start by initializing a simple vector store retriever and storing the 2023 State of the Union speech (in chunks). We can see that given an example question our retriever returns one or two relevant docs and a few irrelevant docs. And even the relevant docs have a lot of irrelevant information in them.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "25c26947-958d-4219-8ca0-daa3a51bd344",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document 1:\n",
      "\n",
      "Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \n",
      "\n",
      "Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \n",
      "\n",
      "One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \n",
      "\n",
      "And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 2:\n",
      "\n",
      "A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder. Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. \n",
      "\n",
      "And if we are to advance liberty and justice, we need to secure the Border and fix the immigration system. \n",
      "\n",
      "We can do both. At our border, we’ve installed new technology like cutting-edge scanners to better detect drug smuggling.  \n",
      "\n",
      "We’ve set up joint patrols with Mexico and Guatemala to catch more human traffickers.  \n",
      "\n",
      "We’re putting in place dedicated immigration judges so families fleeing persecution and violence can have their cases heard faster. \n",
      "\n",
      "We’re securing commitments and supporting partners in South and Central America to host more refugees and secure their own borders.\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 3:\n",
      "\n",
      "And for our LGBTQ+ Americans, let’s finally get the bipartisan Equality Act to my desk. The onslaught of state laws targeting transgender Americans and their families is wrong. \n",
      "\n",
      "As I said last year, especially to our younger transgender Americans, I will always have your back as your President, so you can be yourself and reach your God-given potential. \n",
      "\n",
      "While it often appears that we never agree, that isn’t true. I signed 80 bipartisan bills into law last year. From preventing government shutdowns to protecting Asian-Americans from still-too-common hate crimes to reforming military justice. \n",
      "\n",
      "And soon, we’ll strengthen the Violence Against Women Act that I first wrote three decades ago. It is important for us to show the nation that we can come together and do big things. \n",
      "\n",
      "So tonight I’m offering a Unity Agenda for the Nation. Four big things we can do together.  \n",
      "\n",
      "First, beat the opioid epidemic.\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 4:\n",
      "\n",
      "Tonight, I’m announcing a crackdown on these companies overcharging American businesses and consumers. \n",
      "\n",
      "And as Wall Street firms take over more nursing homes, quality in those homes has gone down and costs have gone up.  \n",
      "\n",
      "That ends on my watch. \n",
      "\n",
      "Medicare is going to set higher standards for nursing homes and make sure your loved ones get the care they deserve and expect. \n",
      "\n",
      "We’ll also cut costs and keep the economy going strong by giving workers a fair shot, provide more training and apprenticeships, hire them based on their skills not degrees. \n",
      "\n",
      "Let’s pass the Paycheck Fairness Act and paid leave.  \n",
      "\n",
      "Raise the minimum wage to $15 an hour and extend the Child Tax Credit, so no one has to raise a family in poverty. \n",
      "\n",
      "Let’s increase Pell Grants and increase our historic support of HBCUs, and invest in what Jill—our First Lady who teaches full-time—calls America’s best-kept secret: community colleges.\n"
     ]
    }
   ],
   "source": [
    "from langchain_community.document_loaders import TextLoader\n",
    "from langchain_community.vectorstores import FAISS\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "from langchain_text_splitters import CharacterTextSplitter\n",
    "\n",
    "documents = TextLoader(\"state_of_the_union.txt\").load()\n",
    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
    "texts = text_splitter.split_documents(documents)\n",
    "retriever = FAISS.from_documents(texts, OpenAIEmbeddings()).as_retriever()\n",
    "\n",
    "docs = retriever.invoke(\"What did the president say about Ketanji Brown Jackson\")\n",
    "pretty_print_docs(docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3473c553",
   "metadata": {},
   "source": [
    "## Adding contextual compression with an `LLMChainExtractor`\n",
    "Now let's wrap our base retriever with a `ContextualCompressionRetriever`. We'll add an `LLMChainExtractor`, which will iterate over the initially returned documents and extract from each only the content that is relevant to the query.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d83e3c63-bcde-43e9-998e-35bf2ebef49b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document 1:\n",
      "\n",


"\n",
      "We’re securing commitments and supporting partners in South and Central America to host more refugees and secure their own borders.\n"
     ]
    }
   ],
   "source": [
    "from langchain.retrievers.document_compressors import EmbeddingsFilter\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "\n",
    "embeddings = OpenAIEmbeddings()\n",
    "embeddings_filter = EmbeddingsFilter(embeddings=embeddings, similarity_threshold=0.76)\n",
    "compression_retriever = ContextualCompressionRetriever(\n",
    "    base_compressor=embeddings_filter, base_retriever=retriever\n",
    ")\n",
    "\n",
    "compressed_docs = compression_retriever.invoke(\n",
    "    \"What did the president say about Ketanji Jackson Brown\"\n",
    ")\n",
    "pretty_print_docs(compressed_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2074462b",
   "metadata": {},
   "source": [
    "## Stringing compressors and document transformers together\n",
    "Using the `DocumentCompressorPipeline` we can also easily combine multiple compressors in sequence. Along with compressors we can add `BaseDocumentTransformer`s to our pipeline, which don't perform any contextual compression but simply perform some transformation on a set of documents. For example `TextSplitter`s can be used as document transformers to split documents into smaller pieces, and the `EmbeddingsRedundantFilter` can be used to filter out redundant documents based on embedding similarity between documents.\n",
    "\n",
    "Below we create a compressor pipeline by first splitting our docs into smaller chunks, then removing redundant documents, and then filtering based on relevance to the query.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "617a1756",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.retrievers.document_compressors import DocumentCompressorPipeline\n",
    "from langchain_community.document_transformers import EmbeddingsRedundantFilter\n",
    "from langchain_text_splitters import CharacterTextSplitter\n",
    "\n",
    "splitter = CharacterTextSplitter(chunk_size=300, chunk_overlap=0, separator=\". \")\n",
    "redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings)\n",
    "relevant_filter = EmbeddingsFilter(embeddings=embeddings, similarity_threshold=0.76)\n",
    "pipeline_compressor = DocumentCompressorPipeline(\n",
    "    transformers=[splitter, redundant_filter, relevant_filter]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "40b9c1db-7ac2-4257-935a-b107da50bb43",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document 1:\n",
      "\n",
      "One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \n",
      "\n",
      "And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 2:\n",
      "\n",
      "As I said last year, especially to our younger transgender Americans, I will always have your back as your President, so you can be yourself and reach your God-given potential. \n",
      "\n",
      "While it often appears that we never agree, that isn’t true. I signed 80 bipartisan bills into law last year\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 3:\n",
      "\n",
      "A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Document 4:\n",
      "\n",
      "Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. \n",
      "\n",
      "And if we are to advance liberty and justice, we need to secure the Border and fix the immigration system. \n",
      "\n",
      "We can do both\n"
     ]
    }
   ],
   "source": [
    "compression_retriever = ContextualCompressionRetriever(\n",
    "    base_compressor=pipeline_compressor, base_retriever=retriever\n",
    ")\n",
    "\n",
    "compressed_docs = compression_retriever.invoke(\n",
    "    \"What did the president say about Ketanji Jackson Brown\"\n",
    ")\n",
    "pretty_print_docs(compressed_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78581dcb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}


"Params: {'model_name': 'CustomChatModel'}\n"
     ]
    }
   ],
   "source": [
    "llm = CustomLLM(n=5)\n",
    "print(llm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8cd49199",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This '"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm.invoke(\"This is a foobar thing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "511b3cb1-9c6f-49b6-9002-a2ec490632b0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'world'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "await llm.ainvoke(\"world\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d9d5bec2-d60a-4ebd-a97d-ac32c98ab02f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['woof ', 'meow ']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm.batch([\"woof woof woof\", \"meow meow meow\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fe246b29-7a93-4bef-8861-389445598c25",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['woof ', 'meow ']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "await llm.abatch([\"woof woof woof\", \"meow meow meow\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3a67c38f-b83b-4eb9-a231-441c55ee8c82",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "h|e|l|l|o|"
     ]
    }
   ],
   "source": [
    "async for token in llm.astream(\"hello\"):\n",
    "    print(token, end=\"|\", flush=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b62c282b-3a35-4529-aac4-2c2f0916790e",
   "metadata": {},
   "source": [
    "Let's confirm that in integrates nicely with other `LangChain` APIs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d5578e74-7fa8-4673-afee-7a59d442aaff",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain_core.prompts import ChatPromptTemplate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "672ff664-8673-4832-9f4f-335253880141",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "prompt = ChatPromptTemplate.from_messages(\n",
    "    [(\"system\", \"you are a bot\"), (\"human\", \"{input}\")]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c400538a-9146-4c93-9fac-293d8f9ca6bf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "llm = CustomLLM(n=7)\n",
    "chain = prompt | llm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "080964af-3e2d-4573-85cb-0d7cc58a6f42",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'event': 'on_chain_start', 'run_id': '05f24b4f-7ea3-4fb6-8417-3aa21633462f', 'name': 'RunnableSequence', 'tags': [], 'metadata': {}, 'data': {'input': {'input': 'hello there!'}}}\n",
      "{'event': 'on_prompt_start', 'name': 'ChatPromptTemplate', 'run_id': '7e996251-a926-4344-809e-c425a9846d21', 'tags': ['seq:step:1'], 'metadata': {}, 'data': {'input': {'input': 'hello there!'}}}\n",
      "{'event': 'on_prompt_end', 'name': 'ChatPromptTemplate', 'run_id': '7e996251-a926-4344-809e-c425a9846d21', 'tags': ['seq:step:1'], 'metadata': {}, 'data': {'input': {'input': 'hello there!'}, 'output': ChatPromptValue(messages=[SystemMessage(content='you are a bot'), HumanMessage(content='hello there!')])}}\n",
      "{'event': 'on_llm_start', 'name': 'CustomLLM', 'run_id': 'a8766beb-10f4-41de-8750-3ea7cf0ca7e2', 'tags': ['seq:step:2'], 'metadata': {}, 'data': {'input': {'prompts': ['System: you are a bot\\nHuman: hello there!']}}}\n",
      "{'event': 'on_llm_stream', 'name': 'CustomLLM', 'run_id': 'a8766beb-10f4-41de-8750-3ea7cf0ca7e2', 'tags': ['seq:step:2'], 'metadata': {}, 'data': {'chunk': 'S'}}\n",
      "{'event': 'on_chain_stream', 'run_id': '05f24b4f-7ea3-4fb6-8417-3aa21633462f', 'tags': [], 'metadata': {}, 'name': 'RunnableSequence', 'data': {'chunk': 'S'}}\n",
      "{'event': 'on_llm_stream', 'name': 'CustomLLM', 'run_id': 'a8766beb-10f4-41de-8750-3ea7cf0ca7e2', 'tags': ['seq:step:2'], 'metadata': {}, 'data': {'chunk': 'y'}}\n",
      "{'event': 'on_chain_stream', 'run_id': '05f24b4f-7ea3-4fb6-8417-3aa21633462f', 'tags': [], 'metadata': {}, 'name': 'RunnableSequence', 'data': {'chunk': 'y'}}\n"
     ]
    }
   ],
   "source": [
    "idx = 0\n",
    "async for event in chain.astream_events({\"input\": \"hello there!\"}, version=\"v1\"):\n",
    "    print(event)\n",
    "    idx += 1\n",
    "    if idx > 7:\n",
    "        # Truncate\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a85e848a-5316-4318-b770-3f8fd34f4231",
   "metadata": {},
   "source": [
    "## Contributing\n",
    "\n",
    "We appreciate all chat model integration contributions. \n",
    "\n",
    "Here's a checklist to help make sure your contribution gets added to LangChain:\n",
    "\n",
    "Documentation:\n",
    "\n",


{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fc0db1bc",
   "metadata": {},
   "source": [
    "# How to reorder retrieved results to mitigate the \"lost in the middle\" effect\n",
    "\n",
    "Substantial performance degradations in [RAG](/docs/tutorials/rag) applications have been [documented](https://arxiv.org/abs/2307.03172) as the number of retrieved documents grows (e.g., beyond ten). In brief: models are liable to miss relevant information in the middle of long contexts.\n",
    "\n",
    "By contrast, queries against vector stores will typically return documents in descending order of relevance (e.g., as measured by cosine similarity of [embeddings](/docs/concepts/#embedding-models)).\n",
    "\n",
    "To mitigate the [\"lost in the middle\"](https://arxiv.org/abs/2307.03172) effect, you can re-order documents after retrieval such that the most relevant documents are positioned at extrema (e.g., the first and last pieces of context), and the least relevant documents are positioned in the middle. In some cases this can help surface the most relevant information to LLMs.\n",
    "\n",
    "The [LongContextReorder](https://python.langchain.com/api_reference/community/document_transformers/langchain_community.document_transformers.long_context_reorder.LongContextReorder.html) document transformer implements this re-ordering procedure. Below we demonstrate an example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2074fdaa-edff-468a-970f-6f5f26e93d4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install --upgrade --quiet  sentence-transformers langchain-chroma langchain langchain-openai langchain-huggingface > /dev/null"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c97eaaf2-34b7-4770-9949-e1abc4ca5226",
   "metadata": {},
   "source": [
    "First we embed some artificial documents and index them in an (in-memory) [Chroma](/docs/integrations/providers/chroma/) vector store. We will use [Hugging Face](/docs/integrations/text_embedding/huggingfacehub/) embeddings, but any LangChain vector store or embeddings model will suffice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "49cbcd8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='This is a document about the Boston Celtics'),\n",
       " Document(page_content='The Celtics are my favourite team.'),\n",
       " Document(page_content='L. Kornet is one of the best Celtics players.'),\n",
       " Document(page_content='The Boston Celtics won the game by 20 points'),\n",
       " Document(page_content='Larry Bird was an iconic NBA player.'),\n",
       " Document(page_content='Elden Ring is one of the best games in the last 15 years.'),\n",
       " Document(page_content='Basquetball is a great sport.'),\n",
       " Document(page_content='I simply love going to the movies'),\n",
       " Document(page_content='Fly me to the moon is one of my favourite songs.'),\n",
       " Document(page_content='This is just a random text.')]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_chroma import Chroma\n",
    "from langchain_huggingface import HuggingFaceEmbeddings\n",
    "\n",
    "# Get embeddings.\n",
    "embeddings = HuggingFaceEmbeddings(model_name=\"all-MiniLM-L6-v2\")\n",
    "\n",
    "texts = [\n",
    "    \"Basquetball is a great sport.\",\n",
    "    \"Fly me to the moon is one of my favourite songs.\",\n",
    "    \"The Celtics are my favourite team.\",\n",
    "    \"This is a document about the Boston Celtics\",\n",
    "    \"I simply love going to the movies\",\n",
    "    \"The Boston Celtics won the game by 20 points\",\n",
    "    \"This is just a random text.\",\n",
    "    \"Elden Ring is one of the best games in the last 15 years.\",\n",
    "    \"L. Kornet is one of the best Celtics players.\",\n",
    "    \"Larry Bird was an iconic NBA player.\",\n",
    "]\n",
    "\n",
    "# Create a retriever\n",
    "retriever = Chroma.from_texts(texts, embedding=embeddings).as_retriever(\n",
    "    search_kwargs={\"k\": 10}\n",
    ")\n",
    "query = \"What can you tell me about the Celtics?\"\n",
    "\n",
    "# Get relevant documents ordered by relevance score\n",
    "docs = retriever.invoke(query)\n",
    "docs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "175d031a-43fa-42f4-93c4-2ba52c3c3ee5",
   "metadata": {},
   "source": [
    "Note that documents are returned in descending order of relevance to the query. The `LongContextReorder` document transformer will implement the re-ordering described above:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9a1181f2-a3dc-4614-9233-2196ab65939e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='The Celtics are my favourite team.'),\n",
       " Document(page_content='The Boston Celtics won the game by 20 points'),\n",
       " Document(page_content='Elden Ring is one of the best games in the last 15 years.'),\n",
       " Document(page_content='I simply love going to the movies'),\n",
       " Document(page_content='This is just a random text.'),\n",
       " Document(page_content='Fly me to the moon is one of my favourite songs.'),\n",
       " Document(page_content='Basquetball is a great sport.'),\n",
       " Document(page_content='Larry Bird was an iconic NBA player.'),\n",
       " Document(page_content='L. Kornet is one of the best Celtics players.'),\n",
       " Document(page_content='This is a document about the Boston Celtics')]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_community.document_transformers import LongContextReorder\n",
    "\n",
    "# Reorder the documents:\n",
    "# Less relevant document will be at the middle of the list and more\n",
    "# relevant elements at beginning / end.\n",
    "reordering = LongContextReorder()\n",
    "reordered_docs = reordering.transform_documents(docs)\n",
    "\n",
    "# Confirm that the 4 relevant documents are at beginning and end.\n",
    "reordered_docs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8d2ef0c-c397-4d8d-8118-3f7acf86d241",
   "metadata": {},
   "source": [
    "Below, we show how to incorporate the re-ordered documents into a simple question-answering chain:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8bbea705-d5b9-4ed5-9957-e12547283622",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "The Celtics are a professional basketball team and one of the most iconic franchises in the NBA. They are highly regarded and have a large fan base. The team has had many successful seasons and is often considered one of the top teams in the league. They have a strong history and have produced many great players, such as Larry Bird and L. Kornet. The team is based in Boston and is often referred to as the Boston Celtics.\n"
     ]
    }
   ],
   "source": [
    "from langchain.chains.combine_documents import create_stuff_documents_chain\n",
    "from langchain_core.prompts import PromptTemplate\n",
    "from langchain_openai import OpenAI\n",
    "\n",
    "llm = OpenAI()\n",
    "\n",
    "prompt_template = \"\"\"\n",


"Given these texts:\n",
    "-----\n",
    "{context}\n",
    "-----\n",
    "Please answer the following question:\n",
    "{query}\n",
    "\"\"\"\n",
    "\n",
    "prompt = PromptTemplate(\n",
    "    template=prompt_template,\n",
    "    input_variables=[\"context\", \"query\"],\n",
    ")\n",
    "\n",
    "# Create and invoke the chain:\n",
    "chain = create_stuff_documents_chain(llm, prompt)\n",
    "response = chain.invoke({\"context\": reordered_docs, \"query\": query})\n",
    "print(response)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}