generated_responses = await super()._agenerate(messages, stop, run_manager)
        request_end_time = datetime.datetime.now().timestamp()
        message_dicts, params = super()._create_message_dicts(messages, stop)
        for i, generation in enumerate(generated_responses.generations):
            response_dict, params = super()._create_message_dicts(
                [generation.message], stop
            )
            pl_request_id = await promptlayer_api_request_async(
                "langchain.PromptLayerChatOpenAI.async",
                "langchain",
                message_dicts,
                params,
                self.pl_tags,
                response_dict,
                request_start_time,
                request_end_time,
                get_api_key(),
                return_pl_id=self.return_pl_id,
            )
            if self.return_pl_id:
                if generation.generation_info is None or not isinstance(
                    generation.generation_info, dict
                ):
                    generation.generation_info = {}
                generation.generation_info["pl_request_id"] = pl_request_id
        return generated_responses
    @property
    def _llm_type(self) -> str:
        return "promptlayer-openai-chat"
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        return {
            **super()._identifying_params,
            "pl_tags": self.pl_tags,
            "return_pl_id": self.return_pl_id,
        }
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.chat_models.google_palm
"""Wrapper around Google's PaLM Chat API."""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional
from pydantic import BaseModel, root_validator
from tenacity import (
    before_sleep_log,
    retry,
    retry_if_exception_type,
    stop_after_attempt,
    wait_exponential,
)
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.chat_models.base import BaseChatModel
from langchain.schema import (
    AIMessage,
    BaseMessage,
    ChatGeneration,
    ChatMessage,
    ChatResult,
    HumanMessage,
    SystemMessage,
)
from langchain.utils import get_from_dict_or_env
if TYPE_CHECKING:
    import google.generativeai as genai
logger = logging.getLogger(__name__)
class ChatGooglePalmError(Exception):
    pass
def _truncate_at_stop_tokens(
    text: str,
    stop: Optional[List[str]],
) -> str:
    """Truncates text at the earliest stop token found."""
    if stop is None:
        return text
    for stop_token in stop:
        stop_token_idx = text.find(stop_token)
        if stop_token_idx != -1:
            text = text[:stop_token_idx]
    return text
def _response_to_result(
    response: genai.types.ChatResponse,
    stop: Optional[List[str]],
) -> ChatResult:
    """Converts a PaLM API response into a LangChain ChatResult."""
    if not response.candidates:
        raise ChatGooglePalmError("ChatResponse must have at least one candidate.")


raise ChatGooglePalmError("ChatResponse must have at least one candidate.")
    generations: List[ChatGeneration] = []
    for candidate in response.candidates:
        author = candidate.get("author")
        if author is None:
            raise ChatGooglePalmError(f"ChatResponse must have an author: {candidate}")
        content = _truncate_at_stop_tokens(candidate.get("content", ""), stop)
        if content is None:
            raise ChatGooglePalmError(f"ChatResponse must have a content: {candidate}")
        if author == "ai":
            generations.append(
                ChatGeneration(text=content, message=AIMessage(content=content))
            )
        elif author == "human":
            generations.append(
                ChatGeneration(
                    text=content,
                    message=HumanMessage(content=content),
                )
            )
        else:
            generations.append(
                ChatGeneration(
                    text=content,
                    message=ChatMessage(role=author, content=content),
                )
            )
    return ChatResult(generations=generations)
def _messages_to_prompt_dict(
    input_messages: List[BaseMessage],
) -> genai.types.MessagePromptDict:
    """Converts a list of LangChain messages into a PaLM API MessagePrompt structure."""
    import google.generativeai as genai
    context: str = ""
    examples: List[genai.types.MessageDict] = []
    messages: List[genai.types.MessageDict] = []
    remaining = list(enumerate(input_messages))
    while remaining:
        index, input_message = remaining.pop(0)
        if isinstance(input_message, SystemMessage):
            if index != 0:
                raise ChatGooglePalmError("System message must be first input message.")


raise ChatGooglePalmError("System message must be first input message.")
            context = input_message.content
        elif isinstance(input_message, HumanMessage) and input_message.example:
            if messages:
                raise ChatGooglePalmError(
                    "Message examples must come before other messages."
                )
            _, next_input_message = remaining.pop(0)
            if isinstance(next_input_message, AIMessage) and next_input_message.example:
                examples.extend(
                    [
                        genai.types.MessageDict(
                            author="human", content=input_message.content
                        ),
                        genai.types.MessageDict(
                            author="ai", content=next_input_message.content
                        ),
                    ]
                )
            else:
                raise ChatGooglePalmError(
                    "Human example message must be immediately followed by an "
                    " AI example response."
                )
        elif isinstance(input_message, AIMessage) and input_message.example:
            raise ChatGooglePalmError(
                "AI example message must be immediately preceded by a Human "
                "example message."
            )
        elif isinstance(input_message, AIMessage):
            messages.append(
                genai.types.MessageDict(author="ai", content=input_message.content)
            )
        elif isinstance(input_message, HumanMessage):
            messages.append(
                genai.types.MessageDict(author="human", content=input_message.content)
            )
        elif isinstance(input_message, ChatMessage):
            messages.append(
                genai.types.MessageDict(
                    author=input_message.role, content=input_message.content
                )
            )
        else:
            raise ChatGooglePalmError(
                "Messages without an explicit role not supported by PaLM API."
            )
    return genai.types.MessagePromptDict(
        context=context,
        examples=examples,


return genai.types.MessagePromptDict(
        context=context,
        examples=examples,
        messages=messages,
    )
def _create_retry_decorator() -> Callable[[Any], Any]:
    """Returns a tenacity retry decorator, preconfigured to handle PaLM exceptions"""
    import google.api_core.exceptions
    multiplier = 2
    min_seconds = 1
    max_seconds = 60
    max_retries = 10
    return retry(
        reraise=True,
        stop=stop_after_attempt(max_retries),
        wait=wait_exponential(multiplier=multiplier, min=min_seconds, max=max_seconds),
        retry=(
            retry_if_exception_type(google.api_core.exceptions.ResourceExhausted)
            | retry_if_exception_type(google.api_core.exceptions.ServiceUnavailable)
            | retry_if_exception_type(google.api_core.exceptions.GoogleAPIError)
        ),
        before_sleep=before_sleep_log(logger, logging.WARNING),
    )
def chat_with_retry(llm: ChatGooglePalm, **kwargs: Any) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator()
    @retry_decorator
    def _chat_with_retry(**kwargs: Any) -> Any:
        return llm.client.chat(**kwargs)
    return _chat_with_retry(**kwargs)
async def achat_with_retry(llm: ChatGooglePalm, **kwargs: Any) -> Any:
    """Use tenacity to retry the async completion call."""
    retry_decorator = _create_retry_decorator()
    @retry_decorator
    async def _achat_with_retry(**kwargs: Any) -> Any:
        # Use OpenAI's async api https://github.com/openai/openai-python#async-api
        return await llm.client.chat_async(**kwargs)


return await llm.client.chat_async(**kwargs)
    return await _achat_with_retry(**kwargs)
[docs]class ChatGooglePalm(BaseChatModel, BaseModel):
    """Wrapper around Google's PaLM Chat API.
    To use you must have the google.generativeai Python package installed and
    either:
        1. The ``GOOGLE_API_KEY``` environment varaible set with your API key, or
        2. Pass your API key using the google_api_key kwarg to the ChatGoogle
           constructor.
    Example:
        .. code-block:: python
            from langchain.chat_models import ChatGooglePalm
            chat = ChatGooglePalm()
    """
    client: Any  #: :meta private:
    model_name: str = "models/chat-bison-001"
    """Model name to use."""
    google_api_key: Optional[str] = None
    temperature: Optional[float] = None
    """Run inference with this temperature. Must by in the closed
       interval [0.0, 1.0]."""
    top_p: Optional[float] = None
    """Decode using nucleus sampling: consider the smallest set of tokens whose
       probability sum is at least top_p. Must be in the closed interval [0.0, 1.0]."""
    top_k: Optional[int] = None
    """Decode using top-k sampling: consider the set of top_k most probable tokens.
       Must be positive."""
    n: int = 1
    """Number of chat completions to generate for each prompt. Note that the API may
       not return the full n completions if duplicates are generated."""
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate api key, python package exists, temperature, top_p, and top_k."""


"""Validate api key, python package exists, temperature, top_p, and top_k."""
        google_api_key = get_from_dict_or_env(
            values, "google_api_key", "GOOGLE_API_KEY"
        )
        try:
            import google.generativeai as genai
            genai.configure(api_key=google_api_key)
        except ImportError:
            raise ChatGooglePalmError(
                "Could not import google.generativeai python package."
            )
        values["client"] = genai
        if values["temperature"] is not None and not 0 <= values["temperature"] <= 1:
            raise ValueError("temperature must be in the range [0.0, 1.0]")
        if values["top_p"] is not None and not 0 <= values["top_p"] <= 1:
            raise ValueError("top_p must be in the range [0.0, 1.0]")
        if values["top_k"] is not None and values["top_k"] <= 0:
            raise ValueError("top_k must be positive")
        return values
    def _generate(
        self,
        messages: List[BaseMessage],
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
    ) -> ChatResult:
        prompt = _messages_to_prompt_dict(messages)
        response: genai.types.ChatResponse = chat_with_retry(
            self,
            model=self.model_name,
            prompt=prompt,
            temperature=self.temperature,
            top_p=self.top_p,
            top_k=self.top_k,
            candidate_count=self.n,
        )
        return _response_to_result(response, stop)
    async def _agenerate(
        self,


async def _agenerate(
        self,
        messages: List[BaseMessage],
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
    ) -> ChatResult:
        prompt = _messages_to_prompt_dict(messages)
        response: genai.types.ChatResponse = await achat_with_retry(
            self,
            model=self.model_name,
            prompt=prompt,
            temperature=self.temperature,
            top_p=self.top_p,
            top_k=self.top_k,
            candidate_count=self.n,
        )
        return _response_to_result(response, stop)
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {
            "model_name": self.model_name,
            "temperature": self.temperature,
            "top_p": self.top_p,
            "top_k": self.top_k,
            "n": self.n,
        }
    @property
    def _llm_type(self) -> str:
        return "google-palm-chat"
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.chat_models.openai
"""OpenAI chat wrapper."""
from __future__ import annotations
import logging
import sys
from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union
from pydantic import Extra, Field, root_validator
from tenacity import (
    before_sleep_log,
    retry,
    retry_if_exception_type,
    stop_after_attempt,
    wait_exponential,
)
from langchain.callbacks.manager import (
    AsyncCallbackManagerForLLMRun,
    CallbackManagerForLLMRun,
)
from langchain.chat_models.base import BaseChatModel
from langchain.schema import (
    AIMessage,
    BaseMessage,
    ChatGeneration,
    ChatMessage,
    ChatResult,
    HumanMessage,
    SystemMessage,
)
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
def _create_retry_decorator(llm: ChatOpenAI) -> Callable[[Any], Any]:
    import openai
    min_seconds = 1
    max_seconds = 60
    # Wait 2^x * 1 second between each retry starting with
    # 4 seconds, then up to 10 seconds, then 10 seconds afterwards
    return retry(
        reraise=True,
        stop=stop_after_attempt(llm.max_retries),
        wait=wait_exponential(multiplier=1, min=min_seconds, max=max_seconds),
        retry=(
            retry_if_exception_type(openai.error.Timeout)
            | retry_if_exception_type(openai.error.APIError)
            | retry_if_exception_type(openai.error.APIConnectionError)
            | retry_if_exception_type(openai.error.RateLimitError)
            | retry_if_exception_type(openai.error.ServiceUnavailableError)
        ),


| retry_if_exception_type(openai.error.ServiceUnavailableError)
        ),
        before_sleep=before_sleep_log(logger, logging.WARNING),
    )
async def acompletion_with_retry(llm: ChatOpenAI, **kwargs: Any) -> Any:
    """Use tenacity to retry the async completion call."""
    retry_decorator = _create_retry_decorator(llm)
    @retry_decorator
    async def _completion_with_retry(**kwargs: Any) -> Any:
        # Use OpenAI's async api https://github.com/openai/openai-python#async-api
        return await llm.client.acreate(**kwargs)
    return await _completion_with_retry(**kwargs)
def _convert_dict_to_message(_dict: dict) -> BaseMessage:
    role = _dict["role"]
    if role == "user":
        return HumanMessage(content=_dict["content"])
    elif role == "assistant":
        return AIMessage(content=_dict["content"])
    elif role == "system":
        return SystemMessage(content=_dict["content"])
    else:
        return ChatMessage(content=_dict["content"], role=role)
def _convert_message_to_dict(message: BaseMessage) -> dict:
    if isinstance(message, ChatMessage):
        message_dict = {"role": message.role, "content": message.content}
    elif isinstance(message, HumanMessage):
        message_dict = {"role": "user", "content": message.content}
    elif isinstance(message, AIMessage):
        message_dict = {"role": "assistant", "content": message.content}
    elif isinstance(message, SystemMessage):
        message_dict = {"role": "system", "content": message.content}
    else:
        raise ValueError(f"Got unknown type {message}")
    if "name" in message.additional_kwargs:


if "name" in message.additional_kwargs:
        message_dict["name"] = message.additional_kwargs["name"]
    return message_dict
[docs]class ChatOpenAI(BaseChatModel):
    """Wrapper around OpenAI Chat large language models.
    To use, you should have the ``openai`` python package installed, and the
    environment variable ``OPENAI_API_KEY`` set with your API key.
    Any parameters that are valid to be passed to the openai.create call can be passed
    in, even if not explicitly saved on this class.
    Example:
        .. code-block:: python
            from langchain.chat_models import ChatOpenAI
            openai = ChatOpenAI(model_name="gpt-3.5-turbo")
    """
    client: Any  #: :meta private:
    model_name: str = Field(default="gpt-3.5-turbo", alias="model")
    """Model name to use."""
    temperature: float = 0.7
    """What sampling temperature to use."""
    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Holds any model parameters valid for `create` call not explicitly specified."""
    openai_api_key: Optional[str] = None
    """Base URL path for API requests,
    leave blank if not using a proxy or service emulator."""
    openai_api_base: Optional[str] = None
    openai_organization: Optional[str] = None
    request_timeout: Optional[Union[float, Tuple[float, float]]] = None
    """Timeout for requests to OpenAI completion API. Default is 600 seconds."""
    max_retries: int = 6
    """Maximum number of retries to make when generating."""
    streaming: bool = False
    """Whether to stream the results or not."""


streaming: bool = False
    """Whether to stream the results or not."""
    n: int = 1
    """Number of chat completions to generate for each prompt."""
    max_tokens: Optional[int] = None
    """Maximum number of tokens to generate."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.ignore
        allow_population_by_field_name = True
    @root_validator(pre=True)
    def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Build extra kwargs from additional params that were passed in."""
        all_required_field_names = cls.all_required_field_names()
        extra = values.get("model_kwargs", {})
        for field_name in list(values):
            if field_name in extra:
                raise ValueError(f"Found {field_name} supplied twice.")
            if field_name not in all_required_field_names:
                logger.warning(
                    f"""WARNING! {field_name} is not default parameter.
                    {field_name} was transferred to model_kwargs.
                    Please confirm that {field_name} is what you intended."""
                )
                extra[field_name] = values.pop(field_name)
        invalid_model_kwargs = all_required_field_names.intersection(extra.keys())
        if invalid_model_kwargs:
            raise ValueError(
                f"Parameters {invalid_model_kwargs} should be specified explicitly. "
                f"Instead they were passed in as part of `model_kwargs` parameter."
            )
        values["model_kwargs"] = extra
        return values
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        openai_api_key = get_from_dict_or_env(
            values, "openai_api_key", "OPENAI_API_KEY"


values, "openai_api_key", "OPENAI_API_KEY"
        )
        openai_organization = get_from_dict_or_env(
            values,
            "openai_organization",
            "OPENAI_ORGANIZATION",
            default="",
        )
        openai_api_base = get_from_dict_or_env(
            values,
            "openai_api_base",
            "OPENAI_API_BASE",
            default="",
        )
        try:
            import openai
        except ImportError:
            raise ValueError(
                "Could not import openai python package. "
                "Please install it with `pip install openai`."
            )
        openai.api_key = openai_api_key
        if openai_organization:
            openai.organization = openai_organization
        if openai_api_base:
            openai.api_base = openai_api_base
        try:
            values["client"] = openai.ChatCompletion
        except AttributeError:
            raise ValueError(
                "`openai` has no `ChatCompletion` attribute, this is likely "
                "due to an old version of the openai package. Try upgrading it "
                "with `pip install --upgrade openai`."
            )
        if values["n"] < 1:
            raise ValueError("n must be at least 1.")
        if values["n"] > 1 and values["streaming"]:
            raise ValueError("n must be 1 when streaming.")
        return values
    @property
    def _default_params(self) -> Dict[str, Any]:
        """Get the default parameters for calling OpenAI API."""
        return {
            "model": self.model_name,
            "request_timeout": self.request_timeout,
            "max_tokens": self.max_tokens,
            "stream": self.streaming,


"max_tokens": self.max_tokens,
            "stream": self.streaming,
            "n": self.n,
            "temperature": self.temperature,
            **self.model_kwargs,
        }
    def _create_retry_decorator(self) -> Callable[[Any], Any]:
        import openai
        min_seconds = 1
        max_seconds = 60
        # Wait 2^x * 1 second between each retry starting with
        # 4 seconds, then up to 10 seconds, then 10 seconds afterwards
        return retry(
            reraise=True,
            stop=stop_after_attempt(self.max_retries),
            wait=wait_exponential(multiplier=1, min=min_seconds, max=max_seconds),
            retry=(
                retry_if_exception_type(openai.error.Timeout)
                | retry_if_exception_type(openai.error.APIError)
                | retry_if_exception_type(openai.error.APIConnectionError)
                | retry_if_exception_type(openai.error.RateLimitError)
                | retry_if_exception_type(openai.error.ServiceUnavailableError)
            ),
            before_sleep=before_sleep_log(logger, logging.WARNING),
        )
[docs]    def completion_with_retry(self, **kwargs: Any) -> Any:
        """Use tenacity to retry the completion call."""
        retry_decorator = self._create_retry_decorator()
        @retry_decorator
        def _completion_with_retry(**kwargs: Any) -> Any:
            return self.client.create(**kwargs)
        return _completion_with_retry(**kwargs)
    def _combine_llm_outputs(self, llm_outputs: List[Optional[dict]]) -> dict:
        overall_token_usage: dict = {}
        for output in llm_outputs:
            if output is None:
                # Happens in streaming
                continue
            token_usage = output["token_usage"]


# Happens in streaming
                continue
            token_usage = output["token_usage"]
            for k, v in token_usage.items():
                if k in overall_token_usage:
                    overall_token_usage[k] += v
                else:
                    overall_token_usage[k] = v
        return {"token_usage": overall_token_usage, "model_name": self.model_name}
    def _generate(
        self,
        messages: List[BaseMessage],
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
    ) -> ChatResult:
        message_dicts, params = self._create_message_dicts(messages, stop)
        if self.streaming:
            inner_completion = ""
            role = "assistant"
            params["stream"] = True
            for stream_resp in self.completion_with_retry(
                messages=message_dicts, **params
            ):
                role = stream_resp["choices"][0]["delta"].get("role", role)
                token = stream_resp["choices"][0]["delta"].get("content", "")
                inner_completion += token
                if run_manager:
                    run_manager.on_llm_new_token(token)
            message = _convert_dict_to_message(
                {"content": inner_completion, "role": role}
            )
            return ChatResult(generations=[ChatGeneration(message=message)])
        response = self.completion_with_retry(messages=message_dicts, **params)
        return self._create_chat_result(response)
    def _create_message_dicts(
        self, messages: List[BaseMessage], stop: Optional[List[str]]
    ) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
        params: Dict[str, Any] = {**{"model": self.model_name}, **self._default_params}


if stop is not None:
            if "stop" in params:
                raise ValueError("`stop` found in both the input and default params.")
            params["stop"] = stop
        message_dicts = [_convert_message_to_dict(m) for m in messages]
        return message_dicts, params
    def _create_chat_result(self, response: Mapping[str, Any]) -> ChatResult:
        generations = []
        for res in response["choices"]:
            message = _convert_dict_to_message(res["message"])
            gen = ChatGeneration(message=message)
            generations.append(gen)
        llm_output = {"token_usage": response["usage"], "model_name": self.model_name}
        return ChatResult(generations=generations, llm_output=llm_output)
    async def _agenerate(
        self,
        messages: List[BaseMessage],
        stop: Optional[List[str]] = None,
        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
    ) -> ChatResult:
        message_dicts, params = self._create_message_dicts(messages, stop)
        if self.streaming:
            inner_completion = ""
            role = "assistant"
            params["stream"] = True
            async for stream_resp in await acompletion_with_retry(
                self, messages=message_dicts, **params
            ):
                role = stream_resp["choices"][0]["delta"].get("role", role)
                token = stream_resp["choices"][0]["delta"].get("content", "")
                inner_completion += token
                if run_manager:
                    await run_manager.on_llm_new_token(token)
            message = _convert_dict_to_message(
                {"content": inner_completion, "role": role}
            )
            return ChatResult(generations=[ChatGeneration(message=message)])
        else:


)
            return ChatResult(generations=[ChatGeneration(message=message)])
        else:
            response = await acompletion_with_retry(
                self, messages=message_dicts, **params
            )
            return self._create_chat_result(response)
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {**{"model_name": self.model_name}, **self._default_params}
    @property
    def _llm_type(self) -> str:
        """Return type of chat model."""
        return "openai-chat"
[docs]    def get_num_tokens(self, text: str) -> int:
        """Calculate num tokens with tiktoken package."""
        # tiktoken NOT supported for Python 3.7 or below
        if sys.version_info[1] <= 7:
            return super().get_num_tokens(text)
        try:
            import tiktoken
        except ImportError:
            raise ValueError(
                "Could not import tiktoken python package. "
                "This is needed in order to calculate get_num_tokens. "
                "Please install it with `pip install tiktoken`."
            )
        # create a GPT-3.5-Turbo encoder instance
        enc = tiktoken.encoding_for_model(self.model_name)
        # encode the text using the GPT-3.5-Turbo encoder
        tokenized_text = enc.encode(text)
        # calculate the number of tokens in the encoded text
        return len(tokenized_text)
[docs]    def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int:
        """Calculate num tokens for gpt-3.5-turbo and gpt-4 with tiktoken package.


Official documentation: https://github.com/openai/openai-cookbook/blob/
        main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb"""
        try:
            import tiktoken
        except ImportError:
            raise ValueError(
                "Could not import tiktoken python package. "
                "This is needed in order to calculate get_num_tokens. "
                "Please install it with `pip install tiktoken`."
            )
        model = self.model_name
        if model == "gpt-3.5-turbo":
            # gpt-3.5-turbo may change over time.
            # Returning num tokens assuming gpt-3.5-turbo-0301.
            model = "gpt-3.5-turbo-0301"
        elif model == "gpt-4":
            # gpt-4 may change over time.
            # Returning num tokens assuming gpt-4-0314.
            model = "gpt-4-0314"
        # Returns the number of tokens used by a list of messages.
        try:
            encoding = tiktoken.encoding_for_model(model)
        except KeyError:
            logger.warning("Warning: model not found. Using cl100k_base encoding.")
            encoding = tiktoken.get_encoding("cl100k_base")
        if model == "gpt-3.5-turbo-0301":
            # every message follows <im_start>{role/name}\n{content}<im_end>\n
            tokens_per_message = 4
            # if there's a name, the role is omitted
            tokens_per_name = -1
        elif model == "gpt-4-0314":
            tokens_per_message = 3
            tokens_per_name = 1
        else:


tokens_per_message = 3
            tokens_per_name = 1
        else:
            raise NotImplementedError(
                f"get_num_tokens_from_messages() is not presently implemented "
                f"for model {model}."
                "See https://github.com/openai/openai-python/blob/main/chatml.md for "
                "information on how messages are converted to tokens."
            )
        num_tokens = 0
        messages_dict = [_convert_message_to_dict(m) for m in messages]
        for message in messages_dict:
            num_tokens += tokens_per_message
            for key, value in message.items():
                num_tokens += len(encoding.encode(value))
                if key == "name":
                    num_tokens += tokens_per_name
        # every reply is primed with <im_start>assistant
        num_tokens += 3
        return num_tokens
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.notiondb
"""Notion DB loader for langchain"""
from typing import Any, Dict, List, Optional
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
NOTION_BASE_URL = "https://api.notion.com/v1"
DATABASE_URL = NOTION_BASE_URL + "/databases/{database_id}/query"
PAGE_URL = NOTION_BASE_URL + "/pages/{page_id}"
BLOCK_URL = NOTION_BASE_URL + "/blocks/{block_id}/children"
[docs]class NotionDBLoader(BaseLoader):
    """Notion DB Loader.
    Reads content from pages within a Noton Database.
    Args:
        integration_token (str): Notion integration token.
        database_id (str): Notion database id.
        request_timeout_sec (int): Timeout for Notion requests in seconds.
    """
    def __init__(
        self,
        integration_token: str,
        database_id: str,
        request_timeout_sec: Optional[int] = 10,
    ) -> None:
        """Initialize with parameters."""
        if not integration_token:
            raise ValueError("integration_token must be provided")
        if not database_id:
            raise ValueError("database_id must be provided")
        self.token = integration_token
        self.database_id = database_id
        self.headers = {
            "Authorization": "Bearer " + self.token,
            "Content-Type": "application/json",
            "Notion-Version": "2022-06-28",
        }
        self.request_timeout_sec = request_timeout_sec
[docs]    def load(self) -> List[Document]:
        """Load documents from the Notion database.
        Returns:
            List[Document]: List of documents.
        """


Returns:
            List[Document]: List of documents.
        """
        page_ids = self._retrieve_page_ids()
        return list(self.load_page(page_id) for page_id in page_ids)
    def _retrieve_page_ids(
        self, query_dict: Dict[str, Any] = {"page_size": 100}
    ) -> List[str]:
        """Get all the pages from a Notion database."""
        pages: List[Dict[str, Any]] = []
        while True:
            data = self._request(
                DATABASE_URL.format(database_id=self.database_id),
                method="POST",
                query_dict=query_dict,
            )
            pages.extend(data.get("results"))
            if not data.get("has_more"):
                break
            query_dict["start_cursor"] = data.get("next_cursor")
        page_ids = [page["id"] for page in pages]
        return page_ids
[docs]    def load_page(self, page_id: str) -> Document:
        """Read a page."""
        data = self._request(PAGE_URL.format(page_id=page_id))
        # load properties as metadata
        metadata: Dict[str, Any] = {}
        for prop_name, prop_data in data["properties"].items():
            prop_type = prop_data["type"]
            if prop_type == "rich_text":
                value = (
                    prop_data["rich_text"][0]["plain_text"]
                    if prop_data["rich_text"]
                    else None
                )
            elif prop_type == "title":
                value = (
                    prop_data["title"][0]["plain_text"] if prop_data["title"] else None
                )
            elif prop_type == "multi_select":
                value = (
                    [item["name"] for item in prop_data["multi_select"]]


value = (
                    [item["name"] for item in prop_data["multi_select"]]
                    if prop_data["multi_select"]
                    else []
                )
            elif prop_type == "url":
                value = prop_data["url"]
            else:
                value = None
            metadata[prop_name.lower()] = value
        metadata["id"] = page_id
        return Document(page_content=self._load_blocks(page_id), metadata=metadata)
    def _load_blocks(self, block_id: str, num_tabs: int = 0) -> str:
        """Read a block and its children."""
        result_lines_arr: List[str] = []
        cur_block_id: str = block_id
        while cur_block_id:
            data = self._request(BLOCK_URL.format(block_id=cur_block_id))
            for result in data["results"]:
                result_obj = result[result["type"]]
                if "rich_text" not in result_obj:
                    continue
                cur_result_text_arr: List[str] = []
                for rich_text in result_obj["rich_text"]:
                    if "text" in rich_text:
                        cur_result_text_arr.append(
                            "\t" * num_tabs + rich_text["text"]["content"]
                        )
                if result["has_children"]:
                    children_text = self._load_blocks(
                        result["id"], num_tabs=num_tabs + 1
                    )
                    cur_result_text_arr.append(children_text)
                result_lines_arr.append("\n".join(cur_result_text_arr))
            cur_block_id = data.get("next_cursor")
        return "\n".join(result_lines_arr)
    def _request(
        self, url: str, method: str = "GET", query_dict: Dict[str, Any] = {}
    ) -> Any:
        res = requests.request(


) -> Any:
        res = requests.request(
            method,
            url,
            headers=self.headers,
            json=query_dict,
            timeout=self.request_timeout_sec,
        )
        res.raise_for_status()
        return res.json()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.gcs_directory
"""Loading logic for loading documents from an GCS directory."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.gcs_file import GCSFileLoader
[docs]class GCSDirectoryLoader(BaseLoader):
    """Loading logic for loading documents from GCS."""
    def __init__(self, project_name: str, bucket: str, prefix: str = ""):
        """Initialize with bucket and key name."""
        self.project_name = project_name
        self.bucket = bucket
        self.prefix = prefix
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from google.cloud import storage
        except ImportError:
            raise ValueError(
                "Could not import google-cloud-storage python package. "
                "Please install it with `pip install google-cloud-storage`."
            )
        client = storage.Client(project=self.project_name)
        docs = []
        for blob in client.list_blobs(self.bucket, prefix=self.prefix):
            # we shall just skip directories since GCSFileLoader creates
            # intermediate directories on the fly
            if blob.name.endswith("/"):
                continue
            loader = GCSFileLoader(self.project_name, self.bucket, blob.name)
            docs.extend(loader.load())
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.modern_treasury
"""Loader that fetches data from Modern Treasury"""
import json
import urllib.request
from base64 import b64encode
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import get_from_env, stringify_value
MODERN_TREASURY_ENDPOINTS = {
    "payment_orders": "https://app.moderntreasury.com/api/payment_orders",
    "expected_payments": "https://app.moderntreasury.com/api/expected_payments",
    "returns": "https://app.moderntreasury.com/api/returns",
    "incoming_payment_details": "https://app.moderntreasury.com/api/\
incoming_payment_details",
    "counterparties": "https://app.moderntreasury.com/api/counterparties",
    "internal_accounts": "https://app.moderntreasury.com/api/internal_accounts",
    "external_accounts": "https://app.moderntreasury.com/api/external_accounts",
    "transactions": "https://app.moderntreasury.com/api/transactions",
    "ledgers": "https://app.moderntreasury.com/api/ledgers",
    "ledger_accounts": "https://app.moderntreasury.com/api/ledger_accounts",
    "ledger_transactions": "https://app.moderntreasury.com/api/ledger_transactions",
    "events": "https://app.moderntreasury.com/api/events",
    "invoices": "https://app.moderntreasury.com/api/invoices",
}
[docs]class ModernTreasuryLoader(BaseLoader):
    def __init__(
        self,
        resource: str,
        organization_id: Optional[str] = None,


self,
        resource: str,
        organization_id: Optional[str] = None,
        api_key: Optional[str] = None,
    ) -> None:
        self.resource = resource
        organization_id = organization_id or get_from_env(
            "organization_id", "MODERN_TREASURY_ORGANIZATION_ID"
        )
        api_key = api_key or get_from_env("api_key", "MODERN_TREASURY_API_KEY")
        credentials = f"{organization_id}:{api_key}".encode("utf-8")
        basic_auth_token = b64encode(credentials).decode("utf-8")
        self.headers = {"Authorization": f"Basic {basic_auth_token}"}
    def _make_request(self, url: str) -> List[Document]:
        request = urllib.request.Request(url, headers=self.headers)
        with urllib.request.urlopen(request) as response:
            json_data = json.loads(response.read().decode())
            text = stringify_value(json_data)
            metadata = {"source": url}
            return [Document(page_content=text, metadata=metadata)]
    def _get_resource(self) -> List[Document]:
        endpoint = MODERN_TREASURY_ENDPOINTS.get(self.resource)
        if endpoint is None:
            return []
        return self._make_request(endpoint)
[docs]    def load(self) -> List[Document]:
        return self._get_resource()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.azlyrics
"""Loader that loads AZLyrics."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class AZLyricsLoader(WebBaseLoader):
    """Loader that loads AZLyrics webpages."""
[docs]    def load(self) -> List[Document]:
        """Load webpage."""
        soup = self.scrape()
        title = soup.title.text
        lyrics = soup.find_all("div", {"class": ""})[2].text
        text = title + lyrics
        metadata = {"source": self.web_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.onedrive
"""Loader that loads data from OneDrive"""
from __future__ import annotations
import logging
import os
import tempfile
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING, Dict, List, Optional, Type, Union
from pydantic import BaseModel, BaseSettings, Field, FilePath, SecretStr
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.onedrive_file import OneDriveFileLoader
if TYPE_CHECKING:
    from O365 import Account
    from O365.drive import Drive, Folder
SCOPES = ["offline_access", "Files.Read.All"]
logger = logging.getLogger(__name__)
class _OneDriveSettings(BaseSettings):
    client_id: str = Field(..., env="O365_CLIENT_ID")
    client_secret: SecretStr = Field(..., env="O365_CLIENT_SECRET")
    class Config:
        env_prefix = ""
        case_sentive = False
        env_file = ".env"
class _OneDriveTokenStorage(BaseSettings):
    token_path: FilePath = Field(Path.home() / ".credentials" / "o365_token.txt")
class _FileType(str, Enum):
    DOC = "doc"
    DOCX = "docx"
    PDF = "pdf"
class _SupportedFileTypes(BaseModel):
    file_types: List[_FileType]
    def fetch_mime_types(self) -> Dict[str, str]:
        mime_types_mapping = {}
        for file_type in self.file_types:
            if file_type.value == "doc":
                mime_types_mapping[file_type.value] = "application/msword"
            elif file_type.value == "docx":
                mime_types_mapping[
                    file_type.value


mime_types_mapping[
                    file_type.value
                ] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"  # noqa: E501
            elif file_type.value == "pdf":
                mime_types_mapping[file_type.value] = "application/pdf"
        return mime_types_mapping
[docs]class OneDriveLoader(BaseLoader, BaseModel):
    settings: _OneDriveSettings = Field(default_factory=_OneDriveSettings)
    drive_id: str = Field(...)
    folder_path: Optional[str] = None
    object_ids: Optional[List[str]] = None
    auth_with_token: bool = False
    def _auth(self) -> Type[Account]:
        """
        Authenticates the OneDrive API client using the specified
        authentication method and returns the Account object.
        Returns:
            Type[Account]: The authenticated Account object.
        """
        try:
            from O365 import FileSystemTokenBackend
        except ImportError:
            raise ValueError(
                "O365 package not found, please install it with `pip install o365`"
            )
        if self.auth_with_token:
            token_storage = _OneDriveTokenStorage()
            token_path = token_storage.token_path
            token_backend = FileSystemTokenBackend(
                token_path=token_path.parent, token_filename=token_path.name
            )
            account = Account(
                credentials=(
                    self.settings.client_id,
                    self.settings.client_secret.get_secret_value(),
                ),
                scopes=SCOPES,
                token_backend=token_backend,
                **{"raise_http_errors": False},
            )
        else:
            token_backend = FileSystemTokenBackend(
                token_path=Path.home() / ".credentials"
            )
            account = Account(
                credentials=(
                    self.settings.client_id,


)
            account = Account(
                credentials=(
                    self.settings.client_id,
                    self.settings.client_secret.get_secret_value(),
                ),
                scopes=SCOPES,
                token_backend=token_backend,
                **{"raise_http_errors": False},
            )
            # make the auth
            account.authenticate()
        return account
    def _get_folder_from_path(self, drive: Type[Drive]) -> Union[Folder, Drive]:
        """
        Returns the folder or drive object located at the
        specified path relative to the given drive.
        Args:
            drive (Type[Drive]): The root drive from which the folder path is relative.
        Returns:
            Union[Folder, Drive]: The folder or drive object
            located at the specified path.
        Raises:
            FileNotFoundError: If the path does not exist.
        """
        subfolder_drive = drive
        if self.folder_path is None:
            return subfolder_drive
        subfolders = [f for f in self.folder_path.split("/") if f != ""]
        if len(subfolders) == 0:
            return subfolder_drive
        items = subfolder_drive.get_items()
        for subfolder in subfolders:
            try:
                subfolder_drive = list(filter(lambda x: subfolder in x.name, items))[0]
                items = subfolder_drive.get_items()
            except (IndexError, AttributeError):
                raise FileNotFoundError("Path {} not exist.".format(self.folder_path))
        return subfolder_drive
    def _load_from_folder(self, folder: Type[Folder]) -> List[Document]:
        """
        Loads all supported document files from the specified folder
        and returns a list of Document objects.
        Args:
            folder (Type[Folder]): The folder object to load the documents from.


Args:
            folder (Type[Folder]): The folder object to load the documents from.
        Returns:
            List[Document]: A list of Document objects representing
            the loaded documents.
        """
        docs = []
        file_types = _SupportedFileTypes(file_types=["doc", "docx", "pdf"])
        file_mime_types = file_types.fetch_mime_types()
        items = folder.get_items()
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            for file in items:
                if file.is_file:
                    if file.mime_type in list(file_mime_types.values()):
                        loader = OneDriveFileLoader(file=file)
                        docs.extend(loader.load())
        return docs
    def _load_from_object_ids(self, drive: Type[Drive]) -> List[Document]:
        """
        Loads all supported document files from the specified OneDrive
        drive based on their object IDs and returns a list
        of Document objects.
        Args:
            drive (Type[Drive]): The OneDrive drive object
            to load the documents from.
        Returns:
            List[Document]: A list of Document objects representing
            the loaded documents.
        """
        docs = []
        file_types = _SupportedFileTypes(file_types=["doc", "docx", "pdf"])
        file_mime_types = file_types.fetch_mime_types()
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            for object_id in self.object_ids if self.object_ids else [""]:
                file = drive.get_item(object_id)
                if not file:
                    logging.warning(


file = drive.get_item(object_id)
                if not file:
                    logging.warning(
                        "There isn't a file with "
                        f"object_id {object_id} in drive {drive}."
                    )
                    continue
                if file.is_file:
                    if file.mime_type in list(file_mime_types.values()):
                        loader = OneDriveFileLoader(file=file)
                        docs.extend(loader.load())
        return docs
[docs]    def load(self) -> List[Document]:
        """
        Loads all supported document files from the specified OneDrive drive a
        nd returns a list of Document objects.
        Returns:
            List[Document]: A list of Document objects
            representing the loaded documents.
        Raises:
            ValueError: If the specified drive ID
            does not correspond to a drive in the OneDrive storage.
        """
        account = self._auth()
        storage = account.storage()
        drive = storage.get_drive(self.drive_id)
        docs: List[Document] = []
        if not drive:
            raise ValueError(f"There isn't a drive with id {self.drive_id}.")
        if self.folder_path:
            folder = self._get_folder_from_path(drive=drive)
            docs.extend(self._load_from_folder(folder=folder))
        elif self.object_ids:
            docs.extend(self._load_from_object_ids(drive=drive))
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.blackboard
"""Loader that loads all documents from a blackboard course."""
import contextlib
import re
from pathlib import Path
from typing import Any, List, Optional, Tuple
from urllib.parse import unquote
from langchain.docstore.document import Document
from langchain.document_loaders.directory import DirectoryLoader
from langchain.document_loaders.pdf import PyPDFLoader
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class BlackboardLoader(WebBaseLoader):
    """Loader that loads all documents from a Blackboard course.
    This loader is not compatible with all Blackboard courses. It is only
    compatible with courses that use the new Blackboard interface.
    To use this loader, you must have the BbRouter cookie. You can get this
    cookie by logging into the course and then copying the value of the
    BbRouter cookie from the browser's developer tools.
    Example:
        .. code-block:: python
            from langchain.document_loaders import BlackboardLoader
            loader = BlackboardLoader(
                blackboard_course_url="https://blackboard.example.com/webapps/blackboard/execute/announcement?method=search&context=course_entry&course_id=_123456_1",
                bbrouter="expires:12345...",
            )
            documents = loader.load()
    """
    base_url: str
    folder_path: str
    load_all_recursively: bool
    def __init__(
        self,
        blackboard_course_url: str,
        bbrouter: str,
        load_all_recursively: bool = True,
        basic_auth: Optional[Tuple[str, str]] = None,
        cookies: Optional[dict] = None,
    ):
        """Initialize with blackboard course url.


):
        """Initialize with blackboard course url.
        The BbRouter cookie is required for most blackboard courses.
        Args:
            blackboard_course_url: Blackboard course url.
            bbrouter: BbRouter cookie.
            load_all_recursively: If True, load all documents recursively.
            basic_auth: Basic auth credentials.
            cookies: Cookies.
        Raises:
            ValueError: If blackboard course url is invalid.
        """
        super().__init__(blackboard_course_url)
        # Get base url
        try:
            self.base_url = blackboard_course_url.split("/webapps/blackboard")[0]
        except IndexError:
            raise ValueError(
                "Invalid blackboard course url. "
                "Please provide a url that starts with "
                "https://<blackboard_url>/webapps/blackboard"
            )
        if basic_auth is not None:
            self.session.auth = basic_auth
        # Combine cookies
        if cookies is None:
            cookies = {}
        cookies.update({"BbRouter": bbrouter})
        self.session.cookies.update(cookies)
        self.load_all_recursively = load_all_recursively
        self.check_bs4()
[docs]    def check_bs4(self) -> None:
        """Check if BeautifulSoup4 is installed.
        Raises:
            ImportError: If BeautifulSoup4 is not installed.
        """
        try:
            import bs4  # noqa: F401
        except ImportError:
            raise ImportError(
                "BeautifulSoup4 is required for BlackboardLoader. "
                "Please install it with `pip install beautifulsoup4`."
            )
[docs]    def load(self) -> List[Document]:
        """Load data into document objects.
        Returns:
            List of documents.


"""Load data into document objects.
        Returns:
            List of documents.
        """
        if self.load_all_recursively:
            soup_info = self.scrape()
            self.folder_path = self._get_folder_path(soup_info)
            relative_paths = self._get_paths(soup_info)
            documents = []
            for path in relative_paths:
                url = self.base_url + path
                print(f"Fetching documents from {url}")
                soup_info = self._scrape(url)
                with contextlib.suppress(ValueError):
                    documents.extend(self._get_documents(soup_info))
            return documents
        else:
            print(f"Fetching documents from {self.web_path}")
            soup_info = self.scrape()
            self.folder_path = self._get_folder_path(soup_info)
            return self._get_documents(soup_info)
    def _get_folder_path(self, soup: Any) -> str:
        """Get the folder path to save the documents in.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            Folder path.
        """
        # Get the course name
        course_name = soup.find("span", {"id": "crumb_1"})
        if course_name is None:
            raise ValueError("No course name found.")
        course_name = course_name.text.strip()
        # Prepare the folder path
        course_name_clean = (
            unquote(course_name)
            .replace(" ", "_")
            .replace("/", "_")
            .replace(":", "_")
            .replace(",", "_")
            .replace("?", "_")
            .replace("'", "_")
            .replace("!", "_")
            .replace('"', "_")
        )
        # Get the folder path
        folder_path = Path(".") / course_name_clean


)
        # Get the folder path
        folder_path = Path(".") / course_name_clean
        return str(folder_path)
    def _get_documents(self, soup: Any) -> List[Document]:
        """Fetch content from page and return Documents.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            List of documents.
        """
        attachments = self._get_attachments(soup)
        self._download_attachments(attachments)
        documents = self._load_documents()
        return documents
    def _get_attachments(self, soup: Any) -> List[str]:
        """Get all attachments from a page.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            List of attachments.
        """
        from bs4 import BeautifulSoup, Tag
        # Get content list
        content_list = soup.find("ul", {"class": "contentList"})
        if content_list is None:
            raise ValueError("No content list found.")
        content_list: BeautifulSoup  # type: ignore
        # Get all attachments
        attachments = []
        for attachment in content_list.find_all("ul", {"class": "attachments"}):
            attachment: Tag  # type: ignore
            for link in attachment.find_all("a"):
                link: Tag  # type: ignore
                href = link.get("href")
                # Only add if href is not None and does not start with #
                if href is not None and not href.startswith("#"):
                    attachments.append(href)
        return attachments
    def _download_attachments(self, attachments: List[str]) -> None:
        """Download all attachments.
        Args:
            attachments: List of attachments.
        """
        # Make sure the folder exists
        Path(self.folder_path).mkdir(parents=True, exist_ok=True)


Path(self.folder_path).mkdir(parents=True, exist_ok=True)
        # Download all attachments
        for attachment in attachments:
            self.download(attachment)
    def _load_documents(self) -> List[Document]:
        """Load all documents in the folder.
        Returns:
            List of documents.
        """
        # Create the document loader
        loader = DirectoryLoader(
            path=self.folder_path, glob="*.pdf", loader_cls=PyPDFLoader  # type: ignore
        )
        # Load the documents
        documents = loader.load()
        # Return all documents
        return documents
    def _get_paths(self, soup: Any) -> List[str]:
        """Get all relative paths in the navbar."""
        relative_paths = []
        course_menu = soup.find("ul", {"class": "courseMenu"})
        if course_menu is None:
            raise ValueError("No course menu found.")
        for link in course_menu.find_all("a"):
            href = link.get("href")
            if href is not None and href.startswith("/"):
                relative_paths.append(href)
        return relative_paths
[docs]    def download(self, path: str) -> None:
        """Download a file from a url.
        Args:
            path: Path to the file.
        """
        # Get the file content
        response = self.session.get(self.base_url + path, allow_redirects=True)
        # Get the filename
        filename = self.parse_filename(response.url)
        # Write the file to disk
        with open(Path(self.folder_path) / filename, "wb") as f:
            f.write(response.content)
[docs]    def parse_filename(self, url: str) -> str:
        """Parse the filename from a url.
        Args:


"""Parse the filename from a url.
        Args:
            url: Url to parse the filename from.
        Returns:
            The filename.
        """
        if (url_path := Path(url)) and url_path.suffix == ".pdf":
            return url_path.name
        else:
            return self._parse_filename_from_url(url)
    def _parse_filename_from_url(self, url: str) -> str:
        """Parse the filename from a url.
        Args:
            url: Url to parse the filename from.
        Returns:
            The filename.
        Raises:
            ValueError: If the filename could not be parsed.
        """
        filename_matches = re.search(r"filename%2A%3DUTF-8%27%27(.+)", url)
        if filename_matches:
            filename = filename_matches.group(1)
        else:
            raise ValueError(f"Could not parse filename from {url}")
        if ".pdf" not in filename:
            raise ValueError(f"Incorrect file type: {filename}")
        filename = filename.split(".pdf")[0] + ".pdf"
        filename = unquote(filename)
        filename = filename.replace("%20", " ")
        return filename
if __name__ == "__main__":
    loader = BlackboardLoader(
        "https://<YOUR BLACKBOARD URL"
        " HERE>/webapps/blackboard/content/listContent.jsp?course_id=_<YOUR COURSE ID"
        " HERE>_1&content_id=_<YOUR CONTENT ID HERE>_1&mode=reset",
        "<YOUR BBROUTER COOKIE HERE>",
        load_all_recursively=True,
    )
    documents = loader.load()
    print(f"Loaded {len(documents)} pages of PDFs from {loader.web_path}")
By Harrison Chase


By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.image_captions
"""
Loader that loads image captions
By default, the loader utilizes the pre-trained BLIP image captioning model.
https://huggingface.co/Salesforce/blip-image-captioning-base
"""
from typing import Any, List, Tuple, Union
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class ImageCaptionLoader(BaseLoader):
    """Loader that loads the captions of an image"""
    def __init__(
        self,
        path_images: Union[str, List[str]],
        blip_processor: str = "Salesforce/blip-image-captioning-base",
        blip_model: str = "Salesforce/blip-image-captioning-base",
    ):
        """
        Initialize with a list of image paths
        """
        if isinstance(path_images, str):
            self.image_paths = [path_images]
        else:
            self.image_paths = path_images
        self.blip_processor = blip_processor
        self.blip_model = blip_model
[docs]    def load(self) -> List[Document]:
        """
        Load from a list of image files
        """
        try:
            from transformers import BlipForConditionalGeneration, BlipProcessor
        except ImportError:
            raise ValueError(
                "`transformers` package not found, please install with "
                "`pip install transformers`."
            )
        processor = BlipProcessor.from_pretrained(self.blip_processor)
        model = BlipForConditionalGeneration.from_pretrained(self.blip_model)
        results = []
        for path_image in self.image_paths:
            caption, metadata = self._get_captions_and_metadata(
                model=model, processor=processor, path_image=path_image
            )


model=model, processor=processor, path_image=path_image
            )
            doc = Document(page_content=caption, metadata=metadata)
            results.append(doc)
        return results
    def _get_captions_and_metadata(
        self, model: Any, processor: Any, path_image: str
    ) -> Tuple[str, dict]:
        """
        Helper function for getting the captions and metadata of an image
        """
        try:
            from PIL import Image
        except ImportError:
            raise ValueError(
                "`PIL` package not found, please install with `pip install pillow`"
            )
        try:
            if path_image.startswith("http://") or path_image.startswith("https://"):
                image = Image.open(requests.get(path_image, stream=True).raw).convert(
                    "RGB"
                )
            else:
                image = Image.open(path_image).convert("RGB")
        except Exception:
            raise ValueError(f"Could not get image data for {path_image}")
        inputs = processor(image, "an image of", return_tensors="pt")
        output = model.generate(**inputs)
        caption: str = processor.decode(output[0])
        metadata: dict = {"image_path": path_image}
        return caption, metadata
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.rtf
"""Loader that loads rich text files."""
from typing import Any, List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    satisfies_min_unstructured_version,
)
[docs]class UnstructuredRTFLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load rtf files."""
    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        min_unstructured_version = "0.5.12"
        if not satisfies_min_unstructured_version(min_unstructured_version):
            raise ValueError(
                "Partitioning rtf files is only supported in "
                f"unstructured>={min_unstructured_version}."
            )
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.rtf import partition_rtf
        return partition_rtf(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.azure_blob_storage_container
"""Loading logic for loading documents from an Azure Blob Storage container."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.azure_blob_storage_file import (
    AzureBlobStorageFileLoader,
)
from langchain.document_loaders.base import BaseLoader
[docs]class AzureBlobStorageContainerLoader(BaseLoader):
    """Loading logic for loading documents from Azure Blob Storage."""
    def __init__(self, conn_str: str, container: str, prefix: str = ""):
        """Initialize with connection string, container and blob prefix."""
        self.conn_str = conn_str
        self.container = container
        self.prefix = prefix
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from azure.storage.blob import ContainerClient
        except ImportError as exc:
            raise ValueError(
                "Could not import azure storage blob python package. "
                "Please install it with `pip install azure-storage-blob`."
            ) from exc
        container = ContainerClient.from_connection_string(
            conn_str=self.conn_str, container_name=self.container
        )
        docs = []
        blob_list = container.list_blobs(name_starts_with=self.prefix)
        for blob in blob_list:
            loader = AzureBlobStorageFileLoader(
                self.conn_str, self.container, blob.name  # type: ignore
            )
            docs.extend(loader.load())
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.conllu
"""Load CoNLL-U files."""
import csv
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class CoNLLULoader(BaseLoader):
    """Load CoNLL-U files."""
    def __init__(self, file_path: str):
        """Initialize with file path."""
        self.file_path = file_path
[docs]    def load(self) -> List[Document]:
        """Load from file path."""
        with open(self.file_path, encoding="utf8") as f:
            tsv = list(csv.reader(f, delimiter="\t"))
            # If len(line) > 1, the line is not a comment
            lines = [line for line in tsv if len(line) > 1]
        text = ""
        for i, line in enumerate(lines):
            # Do not add a space after a punctuation mark or at the end of the sentence
            if line[9] == "SpaceAfter=No" or i == len(lines) - 1:
                text += line[1]
            else:
                text += line[1] + " "
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.epub
"""Loader that loads EPub files."""
from typing import List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    satisfies_min_unstructured_version,
)
[docs]class UnstructuredEPubLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load epub files."""
    def _get_elements(self) -> List:
        min_unstructured_version = "0.5.4"
        if not satisfies_min_unstructured_version(min_unstructured_version):
            raise ValueError(
                "Partitioning epub files is only supported in "
                f"unstructured>={min_unstructured_version}."
            )
        from unstructured.partition.epub import partition_epub
        return partition_epub(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.stripe
"""Loader that fetches data from Stripe"""
import json
import urllib.request
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import get_from_env, stringify_dict
STRIPE_ENDPOINTS = {
    "balance_transactions": "https://api.stripe.com/v1/balance_transactions",
    "charges": "https://api.stripe.com/v1/charges",
    "customers": "https://api.stripe.com/v1/customers",
    "events": "https://api.stripe.com/v1/events",
    "refunds": "https://api.stripe.com/v1/refunds",
    "disputes": "https://api.stripe.com/v1/disputes",
}
[docs]class StripeLoader(BaseLoader):
    def __init__(self, resource: str, access_token: Optional[str] = None) -> None:
        self.resource = resource
        access_token = access_token or get_from_env(
            "access_token", "STRIPE_ACCESS_TOKEN"
        )
        self.headers = {"Authorization": f"Bearer {access_token}"}
    def _make_request(self, url: str) -> List[Document]:
        request = urllib.request.Request(url, headers=self.headers)
        with urllib.request.urlopen(request) as response:
            json_data = json.loads(response.read().decode())
            text = stringify_dict(json_data)
            metadata = {"source": url}
            return [Document(page_content=text, metadata=metadata)]
    def _get_resource(self) -> List[Document]:
        endpoint = STRIPE_ENDPOINTS.get(self.resource)
        if endpoint is None:
            return []
        return self._make_request(endpoint)


if endpoint is None:
            return []
        return self._make_request(endpoint)
[docs]    def load(self) -> List[Document]:
        return self._get_resource()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.gcs_file
"""Loading logic for loading documents from a GCS file."""
import os
import tempfile
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class GCSFileLoader(BaseLoader):
    """Loading logic for loading documents from GCS."""
    def __init__(self, project_name: str, bucket: str, blob: str):
        """Initialize with bucket and key name."""
        self.bucket = bucket
        self.blob = blob
        self.project_name = project_name
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from google.cloud import storage
        except ImportError:
            raise ValueError(
                "Could not import google-cloud-storage python package. "
                "Please install it with `pip install google-cloud-storage`."
            )
        # Initialise a client
        storage_client = storage.Client(self.project_name)
        # Create a bucket object for our bucket
        bucket = storage_client.get_bucket(self.bucket)
        # Create a blob object from the filepath
        blob = bucket.blob(self.blob)
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}/{self.blob}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            # Download the file to a destination
            blob.download_to_filename(file_path)
            loader = UnstructuredFileLoader(file_path)
            return loader.load()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.pdf
"""Loader that loads PDF files."""
import json
import logging
import os
import tempfile
import time
from abc import ABC
from io import StringIO
from pathlib import Path
from typing import Any, Iterator, List, Mapping, Optional
from urllib.parse import urlparse
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.blob_loaders import Blob
from langchain.document_loaders.parsers.pdf import (
    PDFMinerParser,
    PDFPlumberParser,
    PyMuPDFParser,
    PyPDFium2Parser,
    PyPDFParser,
)
from langchain.document_loaders.unstructured import UnstructuredFileLoader
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__file__)
[docs]class UnstructuredPDFLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load PDF files."""
    def _get_elements(self) -> List:
        from unstructured.partition.pdf import partition_pdf
        return partition_pdf(filename=self.file_path, **self.unstructured_kwargs)
class BasePDFLoader(BaseLoader, ABC):
    """Base loader class for PDF files.
    Defaults to check for local file, but if the file is a web path, it will download it
    to a temporary file, and use that, then clean up the temporary file after completion
    """
    def __init__(self, file_path: str):
        """Initialize with file path."""
        self.file_path = file_path
        self.web_path = None
        if "~" in self.file_path:
            self.file_path = os.path.expanduser(self.file_path)
        # If the file is a web path, download it to a temporary file, and use that


# If the file is a web path, download it to a temporary file, and use that
        if not os.path.isfile(self.file_path) and self._is_valid_url(self.file_path):
            r = requests.get(self.file_path)
            if r.status_code != 200:
                raise ValueError(
                    "Check the url of your file; returned status code %s"
                    % r.status_code
                )
            self.web_path = self.file_path
            self.temp_file = tempfile.NamedTemporaryFile()
            self.temp_file.write(r.content)
            self.file_path = self.temp_file.name
        elif not os.path.isfile(self.file_path):
            raise ValueError("File path %s is not a valid file or url" % self.file_path)
    def __del__(self) -> None:
        if hasattr(self, "temp_file"):
            self.temp_file.close()
    @staticmethod
    def _is_valid_url(url: str) -> bool:
        """Check if the url is valid."""
        parsed = urlparse(url)
        return bool(parsed.netloc) and bool(parsed.scheme)
    @property
    def source(self) -> str:
        return self.web_path if self.web_path is not None else self.file_path
[docs]class OnlinePDFLoader(BasePDFLoader):
    """Loader that loads online PDFs."""
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        loader = UnstructuredPDFLoader(str(self.file_path))
        return loader.load()
[docs]class PyPDFLoader(BasePDFLoader):
    """Loads a PDF with pypdf and chunks at character level.
    Loader also stores page numbers in metadatas.
    """
    def __init__(self, file_path: str) -> None:
        """Initialize with file path."""


"""Initialize with file path."""
        try:
            import pypdf  # noqa:F401
        except ImportError:
            raise ValueError(
                "pypdf package not found, please install it with " "`pip install pypdf`"
            )
        self.parser = PyPDFParser()
        super().__init__(file_path)
[docs]    def load(self) -> List[Document]:
        """Load given path as pages."""
        return list(self.lazy_load())
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Lazy load given path as pages."""
        blob = Blob.from_path(self.file_path)
        yield from self.parser.parse(blob)
[docs]class PyPDFium2Loader(BasePDFLoader):
    """Loads a PDF with pypdfium2 and chunks at character level."""
    def __init__(self, file_path: str):
        """Initialize with file path."""
        super().__init__(file_path)
        self.parser = PyPDFium2Parser()
[docs]    def load(self) -> List[Document]:
        """Load given path as pages."""
        return list(self.lazy_load())
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Lazy load given path as pages."""
        blob = Blob.from_path(self.file_path)
        yield from self.parser.parse(blob)
[docs]class PyPDFDirectoryLoader(BaseLoader):
    """Loads a directory with PDF files with pypdf and chunks at character level.
    Loader also stores page numbers in metadatas.
    """
    def __init__(
        self,
        path: str,
        glob: str = "**/[!.]*.pdf",
        silent_errors: bool = False,


silent_errors: bool = False,
        load_hidden: bool = False,
        recursive: bool = False,
    ):
        self.path = path
        self.glob = glob
        self.load_hidden = load_hidden
        self.recursive = recursive
        self.silent_errors = silent_errors
    @staticmethod
    def _is_visible(path: Path) -> bool:
        return not any(part.startswith(".") for part in path.parts)
[docs]    def load(self) -> List[Document]:
        p = Path(self.path)
        docs = []
        items = p.rglob(self.glob) if self.recursive else p.glob(self.glob)
        for i in items:
            if i.is_file():
                if self._is_visible(i.relative_to(p)) or self.load_hidden:
                    try:
                        loader = PyPDFLoader(str(i))
                        sub_docs = loader.load()
                        for doc in sub_docs:
                            doc.metadata["source"] = str(i)
                        docs.extend(sub_docs)
                    except Exception as e:
                        if self.silent_errors:
                            logger.warning(e)
                        else:
                            raise e
        return docs
[docs]class PDFMinerLoader(BasePDFLoader):
    """Loader that uses PDFMiner to load PDF files."""
    def __init__(self, file_path: str) -> None:
        """Initialize with file path."""
        try:
            from pdfminer.high_level import extract_text  # noqa:F401
        except ImportError:
            raise ValueError(
                "`pdfminer` package not found, please install it with "
                "`pip install pdfminer.six`"
            )
        super().__init__(file_path)
        self.parser = PDFMinerParser()
[docs]    def load(self) -> List[Document]:


[docs]    def load(self) -> List[Document]:
        """Eagerly load the content."""
        return list(self.lazy_load())
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Lazily lod documents."""
        blob = Blob.from_path(self.file_path)
        yield from self.parser.parse(blob)
[docs]class PDFMinerPDFasHTMLLoader(BasePDFLoader):
    """Loader that uses PDFMiner to load PDF files as HTML content."""
    def __init__(self, file_path: str):
        """Initialize with file path."""
        try:
            from pdfminer.high_level import extract_text_to_fp  # noqa:F401
        except ImportError:
            raise ValueError(
                "`pdfminer` package not found, please install it with "
                "`pip install pdfminer.six`"
            )
        super().__init__(file_path)
[docs]    def load(self) -> List[Document]:
        """Load file."""
        from pdfminer.high_level import extract_text_to_fp
        from pdfminer.layout import LAParams
        from pdfminer.utils import open_filename
        output_string = StringIO()
        with open_filename(self.file_path, "rb") as fp:
            extract_text_to_fp(
                fp,  # type: ignore[arg-type]
                output_string,
                codec="",
                laparams=LAParams(),
                output_type="html",
            )
        metadata = {"source": self.file_path}
        return [Document(page_content=output_string.getvalue(), metadata=metadata)]
[docs]class PyMuPDFLoader(BasePDFLoader):
    """Loader that uses PyMuPDF to load PDF files."""
    def __init__(self, file_path: str) -> None:
        """Initialize with file path."""


"""Initialize with file path."""
        try:
            import fitz  # noqa:F401
        except ImportError:
            raise ValueError(
                "`PyMuPDF` package not found, please install it with "
                "`pip install pymupdf`"
            )
        super().__init__(file_path)
[docs]    def load(self, **kwargs: Optional[Any]) -> List[Document]:
        """Load file."""
        parser = PyMuPDFParser(text_kwargs=kwargs)
        blob = Blob.from_path(self.file_path)
        return parser.parse(blob)
# MathpixPDFLoader implementation taken largely from Daniel Gross's:
# https://gist.github.com/danielgross/3ab4104e14faccc12b49200843adab21
[docs]class MathpixPDFLoader(BasePDFLoader):
    def __init__(
        self,
        file_path: str,
        processed_file_format: str = "mmd",
        max_wait_time_seconds: int = 500,
        should_clean_pdf: bool = False,
        **kwargs: Any,
    ) -> None:
        super().__init__(file_path)
        self.mathpix_api_key = get_from_dict_or_env(
            kwargs, "mathpix_api_key", "MATHPIX_API_KEY"
        )
        self.mathpix_api_id = get_from_dict_or_env(
            kwargs, "mathpix_api_id", "MATHPIX_API_ID"
        )
        self.processed_file_format = processed_file_format
        self.max_wait_time_seconds = max_wait_time_seconds
        self.should_clean_pdf = should_clean_pdf
    @property
    def headers(self) -> dict:
        return {"app_id": self.mathpix_api_id, "app_key": self.mathpix_api_key}
    @property


@property
    def url(self) -> str:
        return "https://api.mathpix.com/v3/pdf"
    @property
    def data(self) -> dict:
        options = {"conversion_formats": {self.processed_file_format: True}}
        return {"options_json": json.dumps(options)}
[docs]    def send_pdf(self) -> str:
        with open(self.file_path, "rb") as f:
            files = {"file": f}
            response = requests.post(
                self.url, headers=self.headers, files=files, data=self.data
            )
        response_data = response.json()
        if "pdf_id" in response_data:
            pdf_id = response_data["pdf_id"]
            return pdf_id
        else:
            raise ValueError("Unable to send PDF to Mathpix.")
[docs]    def wait_for_processing(self, pdf_id: str) -> None:
        url = self.url + "/" + pdf_id
        for _ in range(0, self.max_wait_time_seconds, 5):
            response = requests.get(url, headers=self.headers)
            response_data = response.json()
            status = response_data.get("status", None)
            if status == "completed":
                return
            elif status == "error":
                raise ValueError("Unable to retrieve PDF from Mathpix")
            else:
                print(f"Status: {status}, waiting for processing to complete")
                time.sleep(5)
        raise TimeoutError
[docs]    def get_processed_pdf(self, pdf_id: str) -> str:
        self.wait_for_processing(pdf_id)
        url = f"{self.url}/{pdf_id}.{self.processed_file_format}"
        response = requests.get(url, headers=self.headers)
        return response.content.decode("utf-8")


return response.content.decode("utf-8")
[docs]    def clean_pdf(self, contents: str) -> str:
        contents = "\n".join(
            [line for line in contents.split("\n") if not line.startswith("![]")]
        )
        # replace \section{Title} with # Title
        contents = contents.replace("\\section{", "# ").replace("}", "")
        # replace the "\" slash that Mathpix adds to escape $, %, (, etc.
        contents = (
            contents.replace(r"\$", "$")
            .replace(r"\%", "%")
            .replace(r"\(", "(")
            .replace(r"\)", ")")
        )
        return contents
[docs]    def load(self) -> List[Document]:
        pdf_id = self.send_pdf()
        contents = self.get_processed_pdf(pdf_id)
        if self.should_clean_pdf:
            contents = self.clean_pdf(contents)
        metadata = {"source": self.source, "file_path": self.source}
        return [Document(page_content=contents, metadata=metadata)]
[docs]class PDFPlumberLoader(BasePDFLoader):
    """Loader that uses pdfplumber to load PDF files."""
    def __init__(
        self, file_path: str, text_kwargs: Optional[Mapping[str, Any]] = None
    ) -> None:
        """Initialize with file path."""
        try:
            import pdfplumber  # noqa:F401
        except ImportError:
            raise ValueError(
                "pdfplumber package not found, please install it with "
                "`pip install pdfplumber`"
            )
        super().__init__(file_path)
        self.text_kwargs = text_kwargs or {}
[docs]    def load(self) -> List[Document]:
        """Load file."""


[docs]    def load(self) -> List[Document]:
        """Load file."""
        parser = PDFPlumberParser(text_kwargs=self.text_kwargs)
        blob = Blob.from_path(self.file_path)
        return parser.parse(blob)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.college_confidential
"""Loader that loads College Confidential."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class CollegeConfidentialLoader(WebBaseLoader):
    """Loader that loads College Confidential webpages."""
[docs]    def load(self) -> List[Document]:
        """Load webpage."""
        soup = self.scrape()
        text = soup.select_one("main[class='skin-handler']").text
        metadata = {"source": self.web_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.powerpoint
"""Loader that loads powerpoint files."""
import os
from typing import List
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class UnstructuredPowerPointLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load powerpoint files."""
    def _get_elements(self) -> List:
        from unstructured.__version__ import __version__ as __unstructured_version__
        from unstructured.file_utils.filetype import FileType, detect_filetype
        unstructured_version = tuple(
            [int(x) for x in __unstructured_version__.split(".")]
        )
        # NOTE(MthwRobinson) - magic will raise an import error if the libmagic
        # system dependency isn't installed. If it's not installed, we'll just
        # check the file extension
        try:
            import magic  # noqa: F401
            is_ppt = detect_filetype(self.file_path) == FileType.PPT
        except ImportError:
            _, extension = os.path.splitext(self.file_path)
            is_ppt = extension == ".ppt"
        if is_ppt and unstructured_version < (0, 4, 11):
            raise ValueError(
                f"You are on unstructured version {__unstructured_version__}. "
                "Partitioning .ppt files is only supported in unstructured>=0.4.11. "
                "Please upgrade the unstructured package and try again."
            )
        if is_ppt:
            from unstructured.partition.ppt import partition_ppt
            return partition_ppt(filename=self.file_path, **self.unstructured_kwargs)
        else:
            from unstructured.partition.pptx import partition_pptx
            return partition_pptx(filename=self.file_path, **self.unstructured_kwargs)


return partition_pptx(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.csv_loader
import csv
from typing import Dict, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class CSVLoader(BaseLoader):
    """Loads a CSV file into a list of documents.
    Each document represents one row of the CSV file. Every row is converted into a
    key/value pair and outputted to a new line in the document's page_content.
    The source for each document loaded from csv is set to the value of the
    `file_path` argument for all doucments by default.
    You can override this by setting the `source_column` argument to the
    name of a column in the CSV file.
    The source of each document will then be set to the value of the column
    with the name specified in `source_column`.
    Output Example:
        .. code-block:: txt
            column1: value1
            column2: value2
            column3: value3
    """
    def __init__(
        self,
        file_path: str,
        source_column: Optional[str] = None,
        csv_args: Optional[Dict] = None,
        encoding: Optional[str] = None,
    ):
        self.file_path = file_path
        self.source_column = source_column
        self.encoding = encoding
        self.csv_args = csv_args or {}
[docs]    def load(self) -> List[Document]:
        """Load data into document objects."""
        docs = []
        with open(self.file_path, newline="", encoding=self.encoding) as csvfile:
            csv_reader = csv.DictReader(csvfile, **self.csv_args)  # type: ignore
            for i, row in enumerate(csv_reader):


for i, row in enumerate(csv_reader):
                content = "\n".join(f"{k.strip()}: {v.strip()}" for k, v in row.items())
                try:
                    source = (
                        row[self.source_column]
                        if self.source_column is not None
                        else self.file_path
                    )
                except KeyError:
                    raise ValueError(
                        f"Source column '{self.source_column}' not found in CSV file."
                    )
                metadata = {"source": source, "row": i}
                doc = Document(page_content=content, metadata=metadata)
                docs.append(doc)
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.directory
"""Loading logic for loading documents from a directory."""
import concurrent
import logging
from pathlib import Path
from typing import Any, List, Optional, Type, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.html_bs import BSHTMLLoader
from langchain.document_loaders.text import TextLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
FILE_LOADER_TYPE = Union[
    Type[UnstructuredFileLoader], Type[TextLoader], Type[BSHTMLLoader]
]
logger = logging.getLogger(__name__)
def _is_visible(p: Path) -> bool:
    parts = p.parts
    for _p in parts:
        if _p.startswith("."):
            return False
    return True
[docs]class DirectoryLoader(BaseLoader):
    """Loading logic for loading documents from a directory."""
    def __init__(
        self,
        path: str,
        glob: str = "**/[!.]*",
        silent_errors: bool = False,
        load_hidden: bool = False,
        loader_cls: FILE_LOADER_TYPE = UnstructuredFileLoader,
        loader_kwargs: Union[dict, None] = None,
        recursive: bool = False,
        show_progress: bool = False,
        use_multithreading: bool = False,
        max_concurrency: int = 4,
    ):
        """Initialize with path to directory and how to glob over it."""
        if loader_kwargs is None:
            loader_kwargs = {}
        self.path = path
        self.glob = glob
        self.load_hidden = load_hidden
        self.loader_cls = loader_cls
        self.loader_kwargs = loader_kwargs
        self.silent_errors = silent_errors


self.loader_kwargs = loader_kwargs
        self.silent_errors = silent_errors
        self.recursive = recursive
        self.show_progress = show_progress
        self.use_multithreading = use_multithreading
        self.max_concurrency = max_concurrency
[docs]    def load_file(
        self, item: Path, path: Path, docs: List[Document], pbar: Optional[Any]
    ) -> None:
        if item.is_file():
            if _is_visible(item.relative_to(path)) or self.load_hidden:
                try:
                    sub_docs = self.loader_cls(str(item), **self.loader_kwargs).load()
                    docs.extend(sub_docs)
                except Exception as e:
                    if self.silent_errors:
                        logger.warning(e)
                    else:
                        raise e
                finally:
                    if pbar:
                        pbar.update(1)
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        p = Path(self.path)
        docs: List[Document] = []
        items = list(p.rglob(self.glob) if self.recursive else p.glob(self.glob))
        pbar = None
        if self.show_progress:
            try:
                from tqdm import tqdm
                pbar = tqdm(total=len(items))
            except ImportError as e:
                logger.warning(
                    "To log the progress of DirectoryLoader you need to install tqdm, "
                    "`pip install tqdm`"
                )
                if self.silent_errors:
                    logger.warning(e)
                else:
                    raise e
        if self.use_multithreading:
            with concurrent.futures.ThreadPoolExecutor(
                max_workers=self.max_concurrency
            ) as executor:
                executor.map(lambda i: self.load_file(i, p, docs, pbar), items)


executor.map(lambda i: self.load_file(i, p, docs, pbar), items)
        else:
            for i in items:
                self.load_file(i, p, docs, pbar)
        if pbar:
            pbar.close()
        return docs
#
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.tomarkdown
"""Loader that loads HTML to markdown using 2markdown."""
from __future__ import annotations
from typing import Iterator, List
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class ToMarkdownLoader(BaseLoader):
    """Loader that loads HTML to markdown using 2markdown."""
    def __init__(self, url: str, api_key: str):
        """Initialize with url and api key."""
        self.url = url
        self.api_key = api_key
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Lazily load the file."""
        response = requests.post(
            "https://2markdown.com/api/2md",
            headers={"X-Api-Key": self.api_key},
            json={"url": self.url},
        )
        text = response.json()["article"]
        metadata = {"source": self.url}
        yield Document(page_content=text, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        """Load file."""
        return list(self.lazy_load())
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.markdown
"""Loader that loads Markdown files."""
from typing import List
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class UnstructuredMarkdownLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load markdown files."""
    def _get_elements(self) -> List:
        from unstructured.__version__ import __version__ as __unstructured_version__
        from unstructured.partition.md import partition_md
        # NOTE(MthwRobinson) - enables the loader to work when you're using pre-release
        # versions of unstructured like 0.4.17-dev1
        _unstructured_version = __unstructured_version__.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        if unstructured_version < (0, 4, 16):
            raise ValueError(
                f"You are on unstructured version {__unstructured_version__}. "
                "Partitioning markdown files is only supported in unstructured>=0.4.16."
            )
        return partition_md(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.


Source code for langchain.document_loaders.bigquery
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class BigQueryLoader(BaseLoader):
    """Loads a query result from BigQuery into a list of documents.
    Each document represents one row of the result. The `page_content_columns`
    are written into the `page_content` of the document. The `metadata_columns`
    are written into the `metadata` of the document. By default, all columns
    are written into the `page_content` and none into the `metadata`.
    """
    def __init__(
        self,
        query: str,
        project: Optional[str] = None,
        page_content_columns: Optional[List[str]] = None,
        metadata_columns: Optional[List[str]] = None,
    ):
        self.query = query
        self.project = project
        self.page_content_columns = page_content_columns
        self.metadata_columns = metadata_columns
[docs]    def load(self) -> List[Document]:
        try:
            from google.cloud import bigquery
        except ImportError as ex:
            raise ValueError(
                "Could not import google-cloud-bigquery python package. "
                "Please install it with `pip install google-cloud-bigquery`."
            ) from ex
        bq_client = bigquery.Client(self.project)
        query_result = bq_client.query(self.query).result()
        docs: List[Document] = []
        page_content_columns = self.page_content_columns
        metadata_columns = self.metadata_columns
        if page_content_columns is None:
            page_content_columns = [column.name for column in query_result.schema]
        if metadata_columns is None:
            metadata_columns = []
        for row in query_result:


metadata_columns = []
        for row in query_result:
            page_content = "\n".join(
                f"{k}: {v}" for k, v in row.items() if k in page_content_columns
            )
            metadata = {k: v for k, v in row.items() if k in metadata_columns}
            doc = Document(page_content=page_content, metadata=metadata)
            docs.append(doc)
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 18, 2023.