template)    def _evaluate_agent_trajectory(        self,        *,        prediction: str,        input: str,        agent_trajectory: Sequence[Tuple[AgentAction, str]],        reference: Optional[str] = None,        **kwargs: Any,    ) -> dict:        vals = [            f"{i}: Action=[{action.tool}] returned observation = [{observation}]"            for i, (action, observation) in enumerate(agent_trajectory)        ]        trajectory = "\n".join(vals)        response = self.chain.run(dict(trajectory=trajectory, input=input), **kwargs)        decision = response.split("\n")[-1].strip()        score = 1 if decision == "Y" else 0        return {"score": score, "value": decision, "reasoning": response}The example above will return a score of 1 if the language model predicts that any of the actions were unnecessary, and it returns a score of 0 if all of them were predicted to be necessary.You can call this evaluator to grade the intermediate steps of your agent's trajectory.evaluator = StepNecessityEvaluator()evaluator.evaluate_agent_trajectory(    prediction="The answer is pi",    input="What is today?",    agent_trajectory=[        (            AgentAction(tool="ask", tool_input="What is


AgentAction(tool="ask", tool_input="What is today?", log=""),            "tomorrow's yesterday",        ),        (            AgentAction(tool="check_tv", tool_input="Watch tv for half hour", log=""),            "bzzz",        ),    ],)    {'score': 1, 'value': 'Y', 'reasoning': 'Y'}PreviousTrajectory EvaluatorsNextAgent TrajectoryCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright Â© 2023 LangChain, Inc.


Comparison Evaluators | ğŸ¦œï¸�ğŸ”— Langchain

Skip to main contentğŸ¦œï¸�ğŸ”— LangChainDocsUse casesIntegrationsAPILangSmithJS/TS DocsCTRLKGet startedIntroductionInstallationQuickstartModulesModel I/â€‹OData connectionChainsMemoryAgentsCallbacksModulesGuidesEvaluationString EvaluatorsComparison EvaluatorsCustom Pairwise EvaluatorPairwise Embedding DistancePairwise String ComparisonTrajectory EvaluatorsExamplesDebuggingDeploymentLangSmithModel ComparisonEcosystemAdditional resourcesGuidesEvaluationComparison EvaluatorsComparison EvaluatorsğŸ“„ï¸� Custom Pairwise EvaluatorYou can make your own pairwise string evaluators by inheriting from PairwiseStringEvaluator class and overwriting the evaluatestringpairs method (and the aevaluatestringpairs method if you want to use the evaluator asynchronously).ğŸ“„ï¸� Pairwise Embedding DistanceOne way to measure the similarity (or dissimilarity) between two predictions on a shared or similar input is to embed the predictions and compute a vector distance between the two embeddings.[1]ğŸ“„ï¸� Pairwise String ComparisonOften you will want to compare predictions of an LLM, Chain, or Agent for a given input. The StringComparison evaluators facilitate this so you can answer questions like:PreviousString DistanceNextCustom Pairwise EvaluatorCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright Â© 2023 LangChain, Inc.


Custom Pairwise Evaluator | ğŸ¦œï¸�ğŸ”— Langchain


Skip to main contentğŸ¦œï¸�ğŸ”— LangChainDocsUse casesIntegrationsAPILangSmithJS/TS DocsCTRLKGet startedIntroductionInstallationQuickstartModulesModel I/â€‹OData connectionChainsMemoryAgentsCallbacksModulesGuidesEvaluationString EvaluatorsComparison EvaluatorsCustom Pairwise EvaluatorPairwise Embedding DistancePairwise String ComparisonTrajectory EvaluatorsExamplesDebuggingDeploymentLangSmithModel ComparisonEcosystemAdditional resourcesGuidesEvaluationComparison EvaluatorsCustom Pairwise EvaluatorOn this pageCustom Pairwise EvaluatorYou can make your own pairwise string evaluators by inheriting from PairwiseStringEvaluator class and overwriting the _evaluate_string_pairs method (and the _aevaluate_string_pairs method if you want to use the evaluator asynchronously).In this example, you will make a simple custom evaluator that just returns whether the first prediction has more whitespace tokenized 'words' than the second.You can check out the reference docs for the PairwiseStringEvaluator interface for more info.from typing import Optional, Anyfrom langchain.evaluation import PairwiseStringEvaluatorclass LengthComparisonPairwiseEvalutor(PairwiseStringEvaluator):    """    Custom evaluator to compare two strings.    """    def _evaluate_string_pairs(        self,        *,        prediction: str,        prediction_b: str,        reference: Optional[str] = None,        input: Optional[str] = None,        **kwargs: Any,    ) -> dict:        score = int(len(prediction.split()) > len(prediction_b.split()))        return {"score": score}evaluator =


return {"score": score}evaluator = LengthComparisonPairwiseEvalutor()evaluator.evaluate_string_pairs(    prediction="The quick brown fox jumped over the lazy dog.",    prediction_b="The quick brown fox jumped over the dog.",)    {'score': 1}LLM-Based Exampleâ€‹That example was simple to illustrate the API, but it wasn't very useful in practice. Below, use an LLM with some custom instructions to form a simple preference scorer similar to the built-in PairwiseStringEvalChain. We will use ChatAnthropic for the evaluator chain.# %pip install anthropic# %env ANTHROPIC_API_KEY=YOUR_API_KEYfrom typing import Optional, Anyfrom langchain.evaluation import PairwiseStringEvaluatorfrom langchain.chat_models import ChatAnthropicfrom langchain.chains import LLMChainclass CustomPreferenceEvaluator(PairwiseStringEvaluator):    """    Custom evaluator to compare two strings using a custom LLMChain.    """    def __init__(self) -> None:        llm = ChatAnthropic(model="claude-2", temperature=0)        self.eval_chain = LLMChain.from_string(            llm,            """Which option is preferred? Do not take order into account. Evaluate based on accuracy and helpfulness. If neither is preferred, respond with C. Provide your reasoning, then finish with Preference: A/B/CInput: How do I get the path of the parent directory in python 3.8?Option A: You can use the following code:```pythonimport osos.path.dirname(os.path.dirname(os.path.abspath(__file__)))Option B: You can use the following code:from pathlib import


B: You can use the following code:from pathlib import PathPath(__file__).absolute().parentReasoning: Both options return the same result. However, since option B is more concise and easily understand, it is preferred.


Preference: BWhich option is preferred? Do not take order into account. Evaluate based on accuracy and helpfulness. If neither is preferred, respond with C. Provide your reasoning, then finish with Preference: A/B/C
Input: {input}
Option A: {prediction}
Option B: {prediction_b}
Reasoning:""",


)@propertydef requires_input(self) -> bool:    return True@propertydef requires_reference(self) -> bool:    return Falsedef _evaluate_string_pairs(    self,    *,    prediction: str,    prediction_b: str,    reference: Optional[str] = None,    input: Optional[str] = None,    **kwargs: Any,) -> dict:    result = self.eval_chain(        {            "input": input,            "prediction": prediction,            "prediction_b": prediction_b,            "stop": ["Which option is preferred?"],        },        **kwargs,    )    response_text = result["text"]    reasoning, preference = response_text.split("Preference:", maxsplit=1)    preference = preference.strip()    score = 1.0 if preference == "A" else (0.0 if preference == "B" else None)    return {"reasoning": reasoning.strip(), "value": preference, "score": score}```pythonevaluator = CustomPreferenceEvaluator()evaluator.evaluate_string_pairs(    input="How do I import from a relative directory?",    prediction="use importlib! importlib.import_module('.my_package', '.')",    prediction_b="from .sibling import foo",)    {'reasoning': 'Option B is preferred over option A for importing from a relative directory, because it is more straightforward and concise.\n\nOption A uses the importlib module, which allows importing a module by


straightforward and concise.\n\nOption A uses the importlib module, which allows importing a module by specifying the full name as a string. While this works, it is less clear compared to option B.\n\nOption B directly imports from the relative path using dot notation, which clearly shows that it is a relative import. This is the recommended way to do relative imports in Python.\n\nIn summary, option B is more accurate and helpful as it uses the standard Python relative import syntax.',     'value': 'B',     'score': 0.0}# Setting requires_input to return True adds additional validation to avoid returning a grade when insufficient data is provided to the chain.try:    evaluator.evaluate_string_pairs(        prediction="use importlib! importlib.import_module('.my_package', '.')",        prediction_b="from .sibling import foo",    )except ValueError as e:    print(e)    CustomPreferenceEvaluator requires an input string.PreviousComparison EvaluatorsNextPairwise Embedding DistanceLLM-Based ExampleCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright Â© 2023 LangChain, Inc.


Pairwise Embedding Distance | ğŸ¦œï¸�ğŸ”— Langchain


Skip to main contentğŸ¦œï¸�ğŸ”— LangChainDocsUse casesIntegrationsAPILangSmithJS/TS DocsCTRLKGet startedIntroductionInstallationQuickstartModulesModel I/â€‹OData connectionChainsMemoryAgentsCallbacksModulesGuidesEvaluationString EvaluatorsComparison EvaluatorsCustom Pairwise EvaluatorPairwise Embedding DistancePairwise String ComparisonTrajectory EvaluatorsExamplesDebuggingDeploymentLangSmithModel ComparisonEcosystemAdditional resourcesGuidesEvaluationComparison EvaluatorsPairwise Embedding DistanceOn this pagePairwise Embedding DistanceOne way to measure the similarity (or dissimilarity) between two predictions on a shared or similar input is to embed the predictions and compute a vector distance between the two embeddings.[1]You can load the pairwise_embedding_distance evaluator to do this.Note: This returns a distance score, meaning that the lower the number, the more similar the outputs are, according to their embedded representation.Check out the reference docs for the PairwiseEmbeddingDistanceEvalChain for more info.from langchain.evaluation import load_evaluatorevaluator = load_evaluator("pairwise_embedding_distance")evaluator.evaluate_string_pairs(    prediction="Seattle is hot in June", prediction_b="Seattle is cool in June.")    {'score': 0.0966466944859925}evaluator.evaluate_string_pairs(    prediction="Seattle is warm in June", prediction_b="Seattle is cool in June.")    {'score': 0.03761174337464557}Select the Distance Metricâ€‹By default, the evalutor uses cosine distance. You can choose a different distance metric if you'd like. from langchain.evaluation import EmbeddingDistancelist(EmbeddingDistance)    [<EmbeddingDistance.COSINE: 'cosine'>,     <EmbeddingDistance.EUCLIDEAN: 'euclidean'>,


<EmbeddingDistance.EUCLIDEAN: 'euclidean'>,     <EmbeddingDistance.MANHATTAN: 'manhattan'>,     <EmbeddingDistance.CHEBYSHEV: 'chebyshev'>,     <EmbeddingDistance.HAMMING: 'hamming'>]evaluator = load_evaluator(    "pairwise_embedding_distance", distance_metric=EmbeddingDistance.EUCLIDEAN)Select Embeddings to Useâ€‹The constructor uses OpenAI embeddings by default, but you can configure this however you want. Below, use huggingface local embeddingsfrom langchain.embeddings import HuggingFaceEmbeddingsembedding_model = HuggingFaceEmbeddings()hf_evaluator = load_evaluator("pairwise_embedding_distance", embeddings=embedding_model)hf_evaluator.evaluate_string_pairs(    prediction="Seattle is hot in June", prediction_b="Seattle is cool in June.")    {'score': 0.5486443280477362}hf_evaluator.evaluate_string_pairs(    prediction="Seattle is warm in June", prediction_b="Seattle is cool in June.")    {'score': 0.21018880025138598}1. Note: When it comes to semantic similarity, this often gives better results than older string distance metrics (such as those in the `PairwiseStringDistanceEvalChain`), though it tends to be less reliable than evaluators that use the LLM directly (such as the `PairwiseStringEvalChain`) PreviousCustom Pairwise EvaluatorNextPairwise String ComparisonSelect the Distance MetricSelect Embeddings to UseCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright Â© 2023 LangChain, Inc.


Pairwise String Comparison | ğŸ¦œï¸�ğŸ”— Langchain


Skip to main contentğŸ¦œï¸�ğŸ”— LangChainDocsUse casesIntegrationsAPILangSmithJS/TS DocsCTRLKGet startedIntroductionInstallationQuickstartModulesModel I/â€‹OData connectionChainsMemoryAgentsCallbacksModulesGuidesEvaluationString EvaluatorsComparison EvaluatorsCustom Pairwise EvaluatorPairwise Embedding DistancePairwise String ComparisonTrajectory EvaluatorsExamplesDebuggingDeploymentLangSmithModel ComparisonEcosystemAdditional resourcesGuidesEvaluationComparison EvaluatorsPairwise String ComparisonOn this pagePairwise String ComparisonOften you will want to compare predictions of an LLM, Chain, or Agent for a given input. The StringComparison evaluators facilitate this so you can answer questions like:Which LLM or prompt produces a preferred output for a given question?Which examples should I include for few-shot example selection?Which output is better to include for fintetuning?The simplest and often most reliable automated way to choose a preferred prediction for a given input is to use the pairwise_string evaluator.Check out the reference docs for the PairwiseStringEvalChain for more info.from langchain.evaluation import load_evaluatorevaluator = load_evaluator("pairwise_string", requires_reference=True)evaluator.evaluate_string_pairs(    prediction="there are three dogs",    prediction_b="4",    input="how many dogs are in the park?",    reference="four",)    {'reasoning': 'Response A provides an incorrect answer by stating there are three dogs in the park, while the reference answer indicates there are four. Response B, on the other hand, provides the correct answer, matching the reference answer. Although Response B is less detailed, it is accurate and directly answers the question. \n\nTherefore, the better response is [[B]].\n',     'value': 'B',     'score': 0}Without Referencesâ€‹When references aren't available, you can still predict the preferred response.


The results will reflect the evaluation model's preference, which is less reliable and may result


in preferences that are factually incorrect.from langchain.evaluation import load_evaluatorevaluator = load_evaluator("pairwise_string")evaluator.evaluate_string_pairs(    prediction="Addition is a mathematical operation.",    prediction_b="Addition is a mathematical operation that adds two numbers to create a third number, the 'sum'.",    input="What is addition?",)    {'reasoning': "Response A is accurate but lacks depth and detail. It simply states that addition is a mathematical operation without explaining what it does or how it works. \n\nResponse B, on the other hand, provides a more detailed explanation. It not only identifies addition as a mathematical operation, but also explains that it involves adding two numbers to create a third number, the 'sum'. This response is more helpful and informative, providing a clearer understanding of what addition is.\n\nTherefore, the better response is B.\n",     'value': 'B',     'score': 0}Customize the LLMâ€‹By default, the loader uses gpt-4 in the evaluation chain. You can customize this when loading.from langchain.chat_models import ChatAnthropicllm = ChatAnthropic(temperature=0)evaluator = load_evaluator("pairwise_string", llm=llm, requires_reference=True)evaluator.evaluate_string_pairs(    prediction="there are three dogs",    prediction_b="4",    input="how many dogs are in the park?",    reference="four",)    {'reasoning': 'Response A provides a specific number but is inaccurate based on the reference answer. Response B provides the correct number but lacks detail or explanation. Overall, Response B is more helpful and accurate in directly answering the question, despite lacking depth or creativity.\n\n[[B]]\n',


question, despite lacking depth or creativity.\n\n[[B]]\n',     'value': 'B',     'score': 0}Customize the Evaluation Promptâ€‹You can use your own custom evaluation prompt to add more task-specific instructions or to instruct the evaluator to score the output.*Note: If you use a prompt that expects generates a result in a unique format, you may also have to pass in a custom output parser (output_parser=your_parser()) instead of the default PairwiseStringResultOutputParserfrom langchain.prompts import PromptTemplateprompt_template = PromptTemplate.from_template(    """Given the input context, which is most similar to the reference label: A or B?Reason step by step and finally, respond with either [[A]] or [[B]] on its own line.DATA----input: {input}reference: {reference}A: {prediction}B: {prediction_b}---Reasoning:""")evaluator = load_evaluator(    "pairwise_string", prompt=prompt_template, requires_reference=True)# The prompt was assigned to the evaluatorprint(evaluator.prompt)    input_variables=['input', 'prediction', 'prediction_b', 'reference'] output_parser=None partial_variables={} template='Given the input context, which is most similar to the reference label: A or B?\nReason step by step and finally, respond with either [[A]] or [[B]] on its own line.\n\nDATA\n----\ninput: {input}\nreference: {reference}\nA: {prediction}\nB: {prediction_b}\n---\nReasoning:\n\n' template_format='f-string' validate_template=Trueevaluator.evaluate_string_pairs(    prediction="The dog that ate the ice cream was named fido.",    prediction_b="The dog's name is spot",    input="What is the name


prediction_b="The dog's name is spot",    input="What is the name of the dog that ate the ice cream?",    reference="The dog's name is fido",)    {'reasoning': "Option A is most similar to the reference label. Both the reference label and option A state that the dog's name is Fido. Option B, on the other hand, gives a different name for the dog. Therefore, option A is the most similar to the reference label. \n",     'value': 'A',     'score': 1}PreviousPairwise Embedding DistanceNextTrajectory EvaluatorsWithout ReferencesCustomize the LLMCustomize the Evaluation PromptCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright Â© 2023 LangChain, Inc.


Model Comparison | ğŸ¦œï¸�ğŸ”— Langchain


Skip to main contentğŸ¦œï¸�ğŸ”— LangChainDocsUse casesIntegrationsAPILangSmithJS/TS DocsCTRLKGet startedIntroductionInstallationQuickstartModulesModel I/â€‹OData connectionChainsMemoryAgentsCallbacksModulesGuidesEvaluationDebuggingDeploymentLangSmithModel ComparisonEcosystemAdditional resourcesGuidesModel ComparisonModel ComparisonConstructing your language model application will likely involved choosing between many different options of prompts, models, and even chains to use. When doing so, you will want to compare these different options on different inputs in an easy, flexible, and intuitive way. LangChain provides the concept of a ModelLaboratory to test out and try different models.from langchain import LLMChain, OpenAI, Cohere, HuggingFaceHub, PromptTemplatefrom langchain.model_laboratory import ModelLaboratoryllms = [    OpenAI(temperature=0),    Cohere(model="command-xlarge-20221108", max_tokens=20, temperature=0),    HuggingFaceHub(repo_id="google/flan-t5-xl", model_kwargs={"temperature": 1}),]model_lab = ModelLaboratory.from_llms(llms)model_lab.compare("What color is a flamingo?")    Input:    What color is a flamingo?        OpenAI    Params: {'model': 'text-davinci-002', 'temperature': 0.0, 'max_tokens': 256, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1, 'best_of': 1}            Flamingos are pink.        Cohere    Params: {'model': 'command-xlarge-20221108',


Cohere    Params: {'model': 'command-xlarge-20221108', 'max_tokens': 20, 'temperature': 0.0, 'k': 0, 'p': 1, 'frequency_penalty': 0, 'presence_penalty': 0}            Pink        HuggingFaceHub    Params: {'repo_id': 'google/flan-t5-xl', 'temperature': 1}    pink    prompt = PromptTemplate(    template="What is the capital of {state}?", input_variables=["state"])model_lab_with_prompt = ModelLaboratory.from_llms(llms, prompt=prompt)model_lab_with_prompt.compare("New York")    Input:    New York        OpenAI    Params: {'model': 'text-davinci-002', 'temperature': 0.0, 'max_tokens': 256, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1, 'best_of': 1}            The capital of New York is Albany.        Cohere    Params: {'model': 'command-xlarge-20221108', 'max_tokens': 20, 'temperature': 0.0, 'k': 0, 'p': 1, 'frequency_penalty': 0, 'presence_penalty': 0}            The capital of New York is Albany.        HuggingFaceHub    Params: {'repo_id': 'google/flan-t5-xl', 'temperature': 1}    st john s    from langchain


'temperature': 1}    st john s    from langchain import SelfAskWithSearchChain, SerpAPIWrapperopen_ai_llm = OpenAI(temperature=0)search = SerpAPIWrapper()self_ask_with_search_openai = SelfAskWithSearchChain(    llm=open_ai_llm, search_chain=search, verbose=True)cohere_llm = Cohere(temperature=0, model="command-xlarge-20221108")search = SerpAPIWrapper()self_ask_with_search_cohere = SelfAskWithSearchChain(    llm=cohere_llm, search_chain=search, verbose=True)chains = [self_ask_with_search_openai, self_ask_with_search_cohere]names = [str(open_ai_llm), str(cohere_llm)]model_lab = ModelLaboratory(chains, names=names)model_lab.compare("What is the hometown of the reigning men's U.S. Open champion?")    Input:    What is the hometown of the reigning men's U.S. Open champion?        OpenAI    Params: {'model': 'text-davinci-002', 'temperature': 0.0, 'max_tokens': 256, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'n': 1, 'best_of': 1}            > Entering new chain...    What is the hometown of the reigning men's U.S. Open champion?    Are follow up questions needed here: Yes.    Follow up: Who is the reigning men's U.S. Open champion?    Intermediate answer: Carlos Alcaraz.    Follow up: Where is Carlos Alcaraz from?    Intermediate answer: El


Follow up: Where is Carlos Alcaraz from?    Intermediate answer: El Palmar, Spain.    So the final answer is: El Palmar, Spain    > Finished chain.        So the final answer is: El Palmar, Spain        Cohere    Params: {'model': 'command-xlarge-20221108', 'max_tokens': 256, 'temperature': 0.0, 'k': 0, 'p': 1, 'frequency_penalty': 0, 'presence_penalty': 0}            > Entering new chain...    What is the hometown of the reigning men's U.S. Open champion?    Are follow up questions needed here: Yes.    Follow up: Who is the reigning men's U.S. Open champion?    Intermediate answer: Carlos Alcaraz.    So the final answer is:        Carlos Alcaraz    > Finished chain.        So the final answer is:        Carlos Alcaraz    PreviousLangSmith WalkthroughNextEcosystemCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright Â© 2023 LangChain, Inc.


LangSmith | ğŸ¦œï¸�ğŸ”— Langchain

Skip to main contentğŸ¦œï¸�ğŸ”— LangChainDocsUse casesIntegrationsAPILangSmithJS/TS DocsCTRLKGet startedIntroductionInstallationQuickstartModulesModel I/â€‹OData connectionChainsMemoryAgentsCallbacksModulesGuidesEvaluationDebuggingDeploymentLangSmithLangSmith WalkthroughModel ComparisonEcosystemAdditional resourcesGuidesLangSmithLangSmithLangSmith helps you trace and evaluate your language model applications and intelligent agents to help you
move from prototype to production.Check out the interactive walkthrough below to get started.For more information, please refer to the LangSmith documentationğŸ“„ï¸� LangSmith WalkthroughLangChain makes it easy to prototype LLM applications and Agents. However, delivering LLM applications to production can be deceptively difficult. You will likely have to heavily customize and iterate on your prompts, chains, and other components to create a high-quality product.PreviousTemplate reposNextLangSmith WalkthroughCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright Â© 2023 LangChain, Inc.


LangSmith Walkthrough | ğŸ¦œï¸�ğŸ”— Langchain


Skip to main contentğŸ¦œï¸�ğŸ”— LangChainDocsUse casesIntegrationsAPILangSmithJS/TS DocsCTRLKGet startedIntroductionInstallationQuickstartModulesModel I/â€‹OData connectionChainsMemoryAgentsCallbacksModulesGuidesEvaluationDebuggingDeploymentLangSmithLangSmith WalkthroughModel ComparisonEcosystemAdditional resourcesGuidesLangSmithLangSmith WalkthroughOn this pageLangSmith WalkthroughLangChain makes it easy to prototype LLM applications and Agents. However, delivering LLM applications to production can be deceptively difficult. You will likely have to heavily customize and iterate on your prompts, chains, and other components to create a high-quality product.To aid in this process, we've launched LangSmith, a unified platform for debugging, testing, and monitoring your LLM applications.When might this come in handy? You may find it useful when you want to:Quickly debug a new chain, agent, or set of toolsVisualize how components (chains, llms, retrievers, etc.) relate and are usedEvaluate different prompts and LLMs for a single componentRun a given chain several times over a dataset to ensure it consistently meets a quality barCapture usage traces and using LLMs or analytics pipelines to generate insightsPrerequisitesâ€‹Create a LangSmith account and create an API key (see bottom left corner). Familiarize yourself with the platform by looking through the docsNote LangSmith is in closed beta; we're in the process of rolling it out to more users. However, you can fill out the form on the website for expedited access.Now, let's get started!Log runs to LangSmithâ€‹First, configure your environment variables to tell LangChain to log traces. This is done by setting the LANGCHAIN_TRACING_V2 environment variable to true.


You can tell LangChain which project to log to by setting the LANGCHAIN_PROJECT environment variable (if this isn't set, runs will be logged to the default project). This will automatically create the project for you if it doesn't exist. You must also set the LANGCHAIN_ENDPOINT and LANGCHAIN_API_KEY environment variables.For more information on other ways to set up tracing, please reference the LangSmith documentationNOTE: You must also set your OPENAI_API_KEY and SERPAPI_API_KEY environment variables in order to run the following tutorial.NOTE: You can only access an API key when you first create it. Keep it somewhere safe.NOTE: You can also use a context manager in python to log traces usingfrom langchain.callbacks.manager import tracing_v2_enabledwith tracing_v2_enabled(project_name="My Project"):    agent.run("How many people live in canada as of 2023?")However, in this example, we will use environment variables.import osfrom uuid import uuid4unique_id = uuid4().hex[0:8]os.environ["LANGCHAIN_TRACING_V2"] = "true"os.environ["LANGCHAIN_PROJECT"] = f"Tracing Walkthrough - {unique_id}"os.environ["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com"os.environ["LANGCHAIN_API_KEY"] = ""  # Update to your API key# Used by the agent in this tutorial# os.environ["OPENAI_API_KEY"] = "<YOUR-OPENAI-API-KEY>"# os.environ["SERPAPI_API_KEY"] = "<YOUR-SERPAPI-API-KEY>"Create the langsmith client to interact with the APIfrom langsmith import Clientclient = Client()Create a LangChain component and log runs to the platform. In this example, we will create a ReAct-style agent with access to Search and Calculator as tools. However, LangSmith works regardless of which type of LangChain component you use (LLMs, Chat Models,


LangSmith works regardless of which type of LangChain component you use (LLMs, Chat Models, Tools, Retrievers, Agents are all supported).from langchain.chat_models import ChatOpenAIfrom langchain.agents import AgentType, initialize_agent, load_toolsllm = ChatOpenAI(temperature=0)tools = load_tools(["serpapi", "llm-math"], llm=llm)agent = initialize_agent(    tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=False)We are running the agent concurrently on multiple inputs to reduce latency. Runs get logged to LangSmith in the background so execution latency is unaffected.import asyncioinputs = [    "How many people live in canada as of 2023?",    "who is dua lipa's boyfriend? what is his age raised to the .43 power?",    "what is dua lipa's boyfriend age raised to the .43 power?",    "how far is it from paris to boston in miles",    "what was the total number of points scored in the 2023 super bowl? what is that number raised to the .23 power?",    "what was the total number of points scored in the 2023 super bowl raised to the .23 power?",    "how many more points were scored in the 2023 super bowl than in the 2022 super bowl?",    "what is 153 raised to .1312 power?",    "who is kendall jenner's boyfriend? what is his height (in inches) raised to .13 power?",    "what is 1213 divided by 4345?",]results = []async def arun(agent, input_example):    try:        return await agent.arun(input_example)    except Exception as e:


return await agent.arun(input_example)    except Exception as e:        # The agent sometimes makes mistakes! These will be captured by the tracing.        return efor input_example in inputs:    results.append(arun(agent, input_example))results = await asyncio.gather(*results)from langchain.callbacks.tracers.langchain import wait_for_all_tracers# Logs are submitted in a background thread to avoid blocking execution.# For the sake of this tutorial, we want to make sure# they've been submitted before moving on. This is also# useful for serverless deployments.wait_for_all_tracers()Assuming you've successfully set up your environment, your agent traces should show up in the Projects section in the app. Congrats!Evaluate another agent implementationâ€‹In addition to logging runs, LangSmith also allows you to test and evaluate your LLM applications.In this section, you will leverage LangSmith to create a benchmark dataset and run AI-assisted evaluators on an agent. You will do so in a few steps:Create a dataset from pre-existing run inputs and outputsInitialize a new agent to benchmarkConfigure evaluators to grade an agent's outputRun the agent over the dataset and evaluate the results1. Create a LangSmith datasetâ€‹Below, we use the LangSmith client to create a dataset from the agent runs you just logged above. You will use these later to measure performance for a new agent. This is simply taking the inputs and outputs of the runs and saving them as examples to a dataset. A dataset is a collection of examples, which are nothing more than input-output pairs you can use as test cases to your application.Note: this is a simple, walkthrough example. In a real-world setting, you'd ideally first validate the outputs before adding them to a benchmark dataset to be used for evaluating other agents.For more information on datasets, including how to


to a benchmark dataset to be used for evaluating other agents.For more information on datasets, including how to create them from CSVs or other files or how to create them in the platform, please refer to the LangSmith documentation.dataset_name = f"calculator-example-dataset-{unique_id}"dataset = client.create_dataset(    dataset_name, description="A calculator example dataset")runs = client.list_runs(    project_name=os.environ["LANGCHAIN_PROJECT"],    execution_order=1,  # Only return the top-level runs    error=False,  # Only runs that succeed)for run in runs:    client.create_example(inputs=run.inputs, outputs=run.outputs, dataset_id=dataset.id)2. Initialize a new agent to benchmarkâ€‹You can evaluate any LLM, chain, or agent. Since chains can have memory, we will pass in a chain_factory (aka a constructor ) function to initialize for each call.In this case, we will test an agent that uses OpenAI's function calling endpoints.from langchain.chat_models import ChatOpenAIfrom langchain.agents import AgentType, initialize_agent, load_toolsllm = ChatOpenAI(model="gpt-3.5-turbo-0613", temperature=0)tools = load_tools(["serpapi", "llm-math"], llm=llm)# Since chains can be stateful (e.g. they can have memory), we provide# a way to initialize a new chain for each row in the dataset. This is done# by passing in a factory function that returns a new chain for each row.def agent_factory():    return initialize_agent(tools, llm, agent=AgentType.OPENAI_FUNCTIONS, verbose=False)# If your chain is NOT stateful, your factory can return the object directly# to improve runtime performance. For example:# chain_factory = lambda: agent3. Configure


object directly# to improve runtime performance. For example:# chain_factory = lambda: agent3. Configure evaluationâ€‹Manually comparing the results of chains in the UI is effective, but it can be time consuming.


It can be helpful to use automated metrics and AI-assisted feedback to evaluate your component's performance.Below, we will create some pre-implemented run evaluators that do the following:Compare results against ground truth labels. (You used the debug outputs above for this)Measure semantic (dis)similarity using embedding distanceEvaluate 'aspects' of the agent's response in a reference-free manner using custom criteriaFor a longer discussion of how to select an appropriate evaluator for your use case and how to create your own


custom evaluators, please refer to the LangSmith documentation.from langchain.evaluation import EvaluatorTypefrom langchain.smith import RunEvalConfigevaluation_config = RunEvalConfig(    # Evaluators can either be an evaluator type (e.g., "qa", "criteria", "embedding_distance", etc.) or a configuration for that evaluator    evaluators=[        # Measures whether a QA response is "Correct", based on a reference answer        # You can also select via the raw string "qa"        EvaluatorType.QA,        # Measure the embedding distance between the output and the reference answer        # Equivalent to: EvalConfig.EmbeddingDistance(embeddings=OpenAIEmbeddings())        EvaluatorType.EMBEDDING_DISTANCE,        # Grade whether the output satisfies the stated criteria. You can select a default one such as "helpfulness" or provide your own.        RunEvalConfig.LabeledCriteria("helpfulness"),        # Both the Criteria and LabeledCriteria evaluators can be configured with a dictionary of custom criteria.        RunEvalConfig.Criteria(            {                "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?"            }        ),    ],    # You can add custom StringEvaluator or RunEvaluator objects here as well, which will automatically be    # applied to each prediction. Check out the docs for examples.


be    # applied to each prediction. Check out the docs for examples.    custom_evaluators=[],)4. Run the agent and evaluatorsâ€‹Use the arun_on_dataset (or synchronous run_on_dataset) function to evaluate your model. This will:Fetch example rows from the specified datasetRun your llm or chain on each example.Apply evalutors to the resulting run traces and corresponding reference examples to generate automated feedback.The results will be visible in the LangSmith app.from langchain.smith import (    arun_on_dataset,    run_on_dataset,  # Available if your chain doesn't support async calls.)chain_results = await arun_on_dataset(    client=client,    dataset_name=dataset_name,    llm_or_chain_factory=agent_factory,    evaluation=evaluation_config,    verbose=True,    tags=["testing-notebook"],  # Optional, adds a tag to the resulting chain runs)# Sometimes, the agent will error due to parsing issues, incompatible tool inputs, etc.# These are logged as warnings here and captured as errors in the tracing UI.    View the evaluation results for project '2023-07-17-11-25-20-AgentExecutor' at:    https://dev.smith.langchain.com/projects/p/1c9baec3-ae86-4fac-9e99-e1b9f8e7818c?eval=true    Processed examples: 1    Chain failed for example 5a2ac8da-8c2b-4d12-acb9-5c4b0f47fe8a. Error: LLMMathChain._evaluate("    age_of_Dua_Lipa_boyfriend ** 0.43    ") raised error:


** 0.43    ") raised error: 'age_of_Dua_Lipa_boyfriend'. Please try again with a valid numerical expression    Processed examples: 4    Chain failed for example 91439261-1c86-4198-868b-a6c1cc8a051b. Error: Too many arguments to single-input tool Calculator. Args: ['height ^ 0.13', {'height': 68}]    Processed examples: 9Review the test resultsâ€‹You can review the test results tracing UI below by navigating to the "Datasets & Testing" page and selecting the "calculator-example-dataset-*" dataset, clicking on the Test Runs tab, then inspecting the runs in the corresponding project. This will show the new runs and the feedback logged from the selected evaluators. Note that runs that error out will not have feedback.Exporting datasets and runsâ€‹LangSmith lets you export data to common formats such as CSV or JSONL directly in the web app. You can also use the client to fetch runs for further analysis, to store in your own database, or to share with others. Let's fetch the run traces from the evaluation run.runs = list(client.list_runs(dataset_name=dataset_name))runs[0]    Run(id=UUID('e39f310b-c5a8-4192-8a59-6a9498e1cb85'), name='AgentExecutor', start_time=datetime.datetime(2023, 7, 17, 18, 25, 30, 653872), run_type=<RunTypeEnum.chain: 'chain'>, end_time=datetime.datetime(2023, 7, 17, 18, 25, 35, 359642), extra={'runtime': {'library': 'langchain', 'runtime': 'python', 'platform':


extra={'runtime': {'library': 'langchain', 'runtime': 'python', 'platform': 'macOS-13.4.1-arm64-arm-64bit', 'sdk_version': '0.0.8', 'library_version': '0.0.231', 'runtime_version': '3.11.2'}, 'total_tokens': 512, 'prompt_tokens': 451, 'completion_tokens': 61}, error=None, serialized=None, events=[{'name': 'start', 'time': '2023-07-17T18:25:30.653872'}, {'name': 'end', 'time': '2023-07-17T18:25:35.359642'}], inputs={'input': 'what is 1213 divided by 4345?'}, outputs={'output': '1213 divided by 4345 is approximately 0.2792.'}, reference_example_id=UUID('a75cf754-4f73-46fd-b126-9bcd0695e463'), parent_run_id=None, tags=['openai-functions', 'testing-notebook'], execution_order=1, session_id=UUID('1c9baec3-ae86-4fac-9e99-e1b9f8e7818c'), child_run_ids=[UUID('40d0fdca-0b2b-47f4-a9da-f2b229aa4ed5'), UUID('cfa5130f-264c-4126-8950-ec1c4c31b800'), UUID('ba638a2f-2a57-45db-91e8-9a7a66a42c5a'), UUID('fcc29b5a-cdb7-4bcc-8194-47729bbdf5fb'),


UUID('a6f92bf5-cfba-4747-9336-370cb00c928a'), UUID('65312576-5a39-4250-b820-4dfae7d73945')], child_runs=None, feedback_stats={'correctness': {'n': 1, 'avg': 1.0, 'mode': 1}, 'helpfulness': {'n': 1, 'avg': 1.0, 'mode': 1}, 'fifth-grader-score': {'n': 1, 'avg': 1.0, 'mode': 1}, 'embedding_cosine_distance': {'n': 1, 'avg': 0.144522385071361, 'mode': 0.144522385071361}})client.read_project(project_id=runs[0].session_id).feedback_stats    {'correctness': {'n': 7, 'avg': 0.5714285714285714, 'mode': 1},     'helpfulness': {'n': 7, 'avg': 0.7142857142857143, 'mode': 1},     'fifth-grader-score': {'n': 7, 'avg': 0.7142857142857143, 'mode': 1},     'embedding_cosine_distance': {'n': 7,      'avg': 0.11462010799473926,      'mode': 0.0130477459560272}}Conclusionâ€‹Congratulations! You have succesfully traced and evaluated an agent using LangSmith!This was a quick guide to get started, but there are many more ways to use LangSmith to speed up your developer flow and produce better results.For more information on how you can get the most out of LangSmith, check out


produce better results.For more information on how you can get the most out of LangSmith, check out LangSmith documentation, and please reach out with questions, feature requests, or feedback at support@langchain.dev.PreviousLangSmithNextModel ComparisonPrerequisitesLog runs to LangSmithEvaluate another agent implementation1. Create a LangSmith dataset2. Initialize a new agent to benchmark3. Configure evaluation4. Run the agent and evaluatorsReview the test resultsExporting datasets and runsConclusionCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright Â© 2023 LangChain, Inc.


Deployment | ğŸ¦œï¸�ğŸ”— Langchain


Skip to main contentğŸ¦œï¸�ğŸ”— LangChainDocsUse casesIntegrationsAPILangSmithJS/TS DocsCTRLKGet startedIntroductionInstallationQuickstartModulesModel I/â€‹OData connectionChainsMemoryAgentsCallbacksModulesGuidesEvaluationDebuggingDeploymentTemplate reposLangSmithModel ComparisonEcosystemAdditional resourcesGuidesDeploymentOn this pageDeploymentIn today's fast-paced technological landscape, the use of Large Language Models (LLMs) is rapidly expanding. As a result, it's crucial for developers to understand how to effectively deploy these models in production environments. LLM interfaces typically fall into two categories:Case 1: Utilizing External LLM Providers (OpenAI, Anthropic, etc.)
In this scenario, most of the computational burden is handled by the LLM providers, while LangChain simplifies the implementation of business logic around these services. This approach includes features such as prompt templating, chat message generation, caching, vector embedding database creation, preprocessing, etc.Case 2: Self-hosted Open-Source Models


Alternatively, developers can opt to use smaller, yet comparably capable, self-hosted open-source LLM models. This approach can significantly decrease costs, latency, and privacy concerns associated with transferring data to external LLM providers.Regardless of the framework that forms the backbone of your product, deploying LLM applications comes with its own set of challenges. It's vital to understand the trade-offs and key considerations when evaluating serving frameworks.Outlineâ€‹This guide aims to provide a comprehensive overview of the requirements for deploying LLMs in a production setting, focusing on:Designing a Robust LLM Application ServiceMaintaining Cost-EfficiencyEnsuring Rapid IterationUnderstanding these components is crucial when assessing serving systems. LangChain integrates with several open-source projects designed to tackle these issues, providing a robust framework for productionizing your LLM applications. Some notable frameworks include:Ray ServeBentoMLOpenLLMModalJinaThese links will provide further information on each ecosystem, assisting you in finding the best fit for your LLM deployment needs.Designing a Robust LLM Application Serviceâ€‹When deploying an LLM service in production, it's imperative to provide a seamless user experience free from outages. Achieving 24/7 service availability involves creating and maintaining several sub-systems surrounding your application.Monitoringâ€‹Monitoring forms an integral part of any system running in a production environment. In the context of LLMs, it is essential to monitor both performance and quality metrics.Performance Metrics: These metrics provide insights into the efficiency and capacity of your model. Here are some key examples:Query per second (QPS): This measures the number of queries your model processes in a second, offering insights into its utilization.Latency: This metric quantifies the delay from when your client sends a request to when they receive a response.Tokens Per Second (TPS): This represents the number of tokens your model can generate in a second.Quality


Second (TPS): This represents the number of tokens your model can generate in a second.Quality Metrics: These metrics are typically customized according to the business use-case. For instance, how does the output of your system compare to a baseline, such as a previous version? Although these metrics can be calculated offline, you need to log the necessary data to use them later.Fault toleranceâ€‹Your application may encounter errors such as exceptions in your model inference or business logic code, causing failures and disrupting traffic. Other potential issues could arise from the machine running your application, such as unexpected hardware breakdowns or loss of spot-instances during high-demand periods. One way to mitigate these risks is by increasing redundancy through replica scaling and implementing recovery mechanisms for failed replicas. However, model replicas aren't the only potential points of failure. It's essential to build resilience against various failures that could occur at any point in your stack.Zero down time upgradeâ€‹System upgrades are often necessary but can result in service disruptions if not handled correctly. One way to prevent downtime during upgrades is by implementing a smooth transition process from the old version to the new one. Ideally, the new version of your LLM service is deployed, and traffic gradually shifts from the old to the new version, maintaining a constant QPS throughout the process.Load balancingâ€‹Load balancing, in simple terms, is a technique to distribute work evenly across multiple computers, servers, or other resources to optimize the utilization of the system, maximize throughput, minimize response time, and avoid overload of any single resource. Think of it as a traffic officer directing cars (requests) to different roads (servers) so that no single road becomes too congested.There are several strategies for load balancing. For example, one common method is the Round Robin strategy, where each request is sent to the next server in line, cycling back to the first when all servers have received a request. This works well when all servers are equally capable.


the first when all servers have received a request. This works well when all servers are equally capable. However, if some servers are more powerful than others, you might use a Weighted Round Robin or Least Connections strategy, where more requests are sent to the more powerful servers, or to those currently handling the fewest active requests. Let's imagine you're running a LLM chain. If your application becomes popular, you could have hundreds or even thousands of users asking questions at the same time. If one server gets too busy (high load), the load balancer would direct new requests to another server that is less busy. This way, all your users get a timely response and the system remains stable.Maintaining Cost-Efficiency and Scalabilityâ€‹Deploying LLM services can be costly, especially when you're handling a large volume of user interactions. Charges by LLM providers are usually based on tokens used, making a chat system inference on these models potentially expensive. However, several strategies can help manage these costs without compromising the quality of the service.Self-hosting modelsâ€‹Several smaller and open-source LLMs are emerging to tackle the issue of reliance on LLM providers. Self-hosting allows you to maintain similar quality to LLM provider models while managing costs. The challenge lies in building a reliable, high-performing LLM serving system on your own machines. Resource Management and Auto-Scalingâ€‹Computational logic within your application requires precise resource allocation. For instance, if part of your traffic is served by an OpenAI endpoint and another part by a self-hosted model, it's crucial to allocate suitable resources for each. Auto-scalingâ€”adjusting resource allocation based on trafficâ€”can significantly impact the cost of running your application. This strategy requires a balance between cost and responsiveness, ensuring neither resource over-provisioning nor compromised application responsiveness.Utilizing Spot Instancesâ€‹On platforms like AWS, spot instances


nor compromised application responsiveness.Utilizing Spot Instancesâ€‹On platforms like AWS, spot instances offer substantial cost savings, typically priced at about a third of on-demand instances. The trade-off is a higher crash rate, necessitating a robust fault-tolerance mechanism for effective use.Independent Scalingâ€‹When self-hosting your models, you should consider independent scaling. For example, if you have two translation models, one fine-tuned for French and another for Spanish, incoming requests might necessitate different scaling requirements for each.Batching requestsâ€‹In the context of Large Language Models, batching requests can enhance efficiency by better utilizing your GPU resources. GPUs are inherently parallel processors, designed to handle multiple tasks simultaneously. If you send individual requests to the model, the GPU might not be fully utilized as it's only working on a single task at a time. On the other hand, by batching requests together, you're allowing the GPU to work on multiple tasks at once, maximizing its utilization and improving inference speed. This not only leads to cost savings but can also improve the overall latency of your LLM service.In summary, managing costs while scaling your LLM services requires a strategic approach. Utilizing self-hosting models, managing resources effectively, employing auto-scaling, using spot instances, independently scaling models, and batching requests are key strategies to consider. Open-source libraries such as Ray Serve and BentoML are designed to deal with these complexities. Ensuring Rapid Iterationâ€‹The LLM landscape is evolving at an unprecedented pace, with new libraries and model architectures being introduced constantly. Consequently, it's crucial to avoid tying yourself to a solution specific to one particular framework. This is especially relevant in serving, where changes to your infrastructure can be time-consuming, expensive, and risky. Strive for infrastructure that is not locked into any specific machine learning library or framework, but instead offers a general-purpose, scalable serving layer. Here are some aspects where flexibility plays a key


but instead offers a general-purpose, scalable serving layer. Here are some aspects where flexibility plays a key role:Model compositionâ€‹Deploying systems like LangChain demands the ability to piece together different models and connect them via logic. Take the example of building a natural language input SQL query engine. Querying an LLM and obtaining the SQL command is only part of the system. You need to extract metadata from the connected database, construct a prompt for the LLM, run the SQL query on an engine, collect and feed back the response to the LLM as the query runs, and present the results to the user. This demonstrates the need to seamlessly integrate various complex components built in Python into a dynamic chain of logical blocks that can be served together.Cloud providersâ€‹Many hosted solutions are restricted to a single cloud provider, which can limit your options in today's multi-cloud world. Depending on where your other infrastructure components are built, you might prefer to stick with your chosen cloud provider.Infrastructure as Code (IaC)â€‹Rapid iteration also involves the ability to recreate your infrastructure quickly and reliably. This is where Infrastructure as Code (IaC) tools like Terraform, CloudFormation, or Kubernetes YAML files come into play. They allow you to define your infrastructure in code files, which can be version controlled and quickly deployed, enabling faster and more reliable iterations.CI/CDâ€‹In a fast-paced environment, implementing CI/CD pipelines can significantly speed up the iteration process. They help automate the testing and deployment of your LLM applications, reducing the risk of errors and enabling faster feedback and iteration.PreviousDebuggingNextTemplate reposOutlineDesigning a Robust LLM Application ServiceMonitoringFault toleranceZero down time upgradeLoad balancingMaintaining Cost-Efficiency and ScalabilitySelf-hosting modelsResource Management and Auto-ScalingUtilizing Spot InstancesIndependent ScalingBatching requestsEnsuring Rapid IterationModel compositionCloud providersInfrastructure as Code


Spot InstancesIndependent ScalingBatching requestsEnsuring Rapid IterationModel compositionCloud providersInfrastructure as Code (IaC)CI/CDCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright Â© 2023 LangChain, Inc.


Template repos | ğŸ¦œï¸�ğŸ”— Langchain


Skip to main contentğŸ¦œï¸�ğŸ”— LangChainDocsUse casesIntegrationsAPILangSmithJS/TS DocsCTRLKGet startedIntroductionInstallationQuickstartModulesModel I/â€‹OData connectionChainsMemoryAgentsCallbacksModulesGuidesEvaluationDebuggingDeploymentTemplate reposLangSmithModel ComparisonEcosystemAdditional resourcesGuidesDeploymentTemplate reposOn this pageTemplate reposSo, you've created a really cool chain - now what? How do you deploy it and make it easily shareable with the world?This section covers several options for that. Note that these options are meant for quick deployment of prototypes and demos, not for production systems. If you need help with the deployment of a production system, please contact us directly.What follows is a list of template GitHub repositories designed to be easily forked and modified to use your chain. This list is far from exhaustive, and we are EXTREMELY open to contributions here.Streamlitâ€‹This repo serves as a template for how to deploy a LangChain with Streamlit.
It implements a chatbot interface.
It also contains instructions for how to deploy this app on the Streamlit platform.Gradio (on Hugging Face)â€‹This repo serves as a template for how deploy a LangChain with Gradio.
It implements a chatbot interface, with a "Bring-Your-Own-Token" approach (nice for not wracking up big bills).
It also contains instructions for how to deploy this app on the Hugging Face platform.
This is heavily influenced by James Weaver's excellent examples.Chainlitâ€‹This repo is a cookbook explaining how to visualize and deploy LangChain agents with Chainlit.
You create ChatGPT-like UIs with Chainlit. Some of the key features include intermediary steps visualisation, element management & display (images, text, carousel, etc.) as well as cloud deployment.


Chainlit doc on the integration with LangChainBeamâ€‹This repo serves as a template for how deploy a LangChain with Beam.It implements a Question Answering app and contains instructions for deploying the app as a serverless REST API.Vercelâ€‹A minimal example on how to run LangChain on Vercel using Flask.FastAPI + Vercelâ€‹A minimal example on how to run LangChain on Vercel using FastAPI and LangCorn/Uvicorn.Kinstaâ€‹A minimal example on how to deploy LangChain to Kinsta using Flask.Fly.ioâ€‹A minimal example of how to deploy LangChain to Fly.io using Flask.Digitalocean App Platformâ€‹A minimal example on how to deploy LangChain to DigitalOcean App Platform.CI/CD Google Cloud Build + Dockerfile + Serverless Google Cloud Runâ€‹Boilerplate LangChain project on how to deploy to Google Cloud Run using Docker with Cloud Build CI/CD pipelineGoogle Cloud Runâ€‹A minimal example on how to deploy LangChain to Google Cloud Run.SteamShipâ€‹This repository contains LangChain adapters for Steamship, enabling LangChain developers to rapidly deploy their apps on Steamship. This includes: production-ready endpoints, horizontal scaling across dependencies, persistent storage of app state, multi-tenancy support, etc.Langchain-serveâ€‹This repository allows users to deploy any LangChain app as REST/WebSocket APIs or, as Slack Bots with ease. Benefit from the scalability and serverless architecture of Jina AI Cloud, or deploy on-premise with Kubernetes.BentoMLâ€‹This repository provides an example of how to deploy a LangChain application with BentoML. BentoML is a framework that enables the containerization of machine learning applications as standard OCI images. BentoML also allows for the automatic generation of OpenAPI and


learning applications as standard OCI images. BentoML also allows for the automatic generation of OpenAPI and gRPC endpoints. With BentoML, you can integrate models from all popular ML frameworks and deploy them as microservices running on the most optimal hardware and scaling independently.OpenLLMâ€‹OpenLLM is a platform for operating large language models (LLMs) in production. With OpenLLM, you can run inference with any open-source LLM, deploy to the cloud or on-premises, and build powerful AI apps. It supports a wide range of open-source LLMs, offers flexible APIs, and first-class support for LangChain and BentoML.


See OpenLLM's integration doc for usage with LangChain.Databuttonâ€‹These templates serve as examples of how to build, deploy, and share LangChain applications using Databutton. You can create user interfaces with Streamlit, automate tasks by scheduling Python code, and store files and data in the built-in store. Examples include a Chatbot interface with conversational memory, a Personal search engine, and a starter template for LangChain apps. Deploying and sharing is just one click away.PreviousDeploymentNextLangSmithStreamlitGradio (on Hugging Face)ChainlitBeamVercelFastAPI + VercelKinstaFly.ioDigitalocean App PlatformCI/CD Google Cloud Build + Dockerfile + Serverless Google Cloud RunGoogle Cloud RunSteamShipLangchain-serveBentoMLOpenLLMDatabuttonCommunityDiscordTwitterGitHubPythonJS/TSMoreHomepageBlogCopyright Â© 2023 LangChain, Inc.