Source code for langchain.embeddings.self_hosted
"""Running custom embedding models on self-hosted remote hardware."""
from typing import Any, Callable, List
from pydantic import Extra
from langchain.embeddings.base import Embeddings
from langchain.llms import SelfHostedPipeline
def _embed_documents(pipeline: Any, *args: Any, **kwargs: Any) -> List[List[float]]:
    """Inference function to send to the remote hardware.
    Accepts a sentence_transformer model_id and
    returns a list of embeddings for each document in the batch.
    """
    return pipeline(*args, **kwargs)
[docs]class SelfHostedEmbeddings(SelfHostedPipeline, Embeddings):
    """Runs custom embedding models on self-hosted remote hardware.
    Supported hardware includes auto-launched instances on AWS, GCP, Azure,
    and Lambda, as well as servers specified
    by IP address and SSH credentials (such as on-prem, or another
    cloud like Paperspace, Coreweave, etc.).
    To use, you should have the ``runhouse`` python package installed.
    Example using a model load function:
        .. code-block:: python
            from langchain.embeddings import SelfHostedEmbeddings
            from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
            import runhouse as rh
            gpu = rh.cluster(name="rh-a10x", instance_type="A100:1")
            def get_pipeline():
                model_id = "facebook/bart-large"
                tokenizer = AutoTokenizer.from_pretrained(model_id)
                model = AutoModelForCausalLM.from_pretrained(model_id)
                return pipeline("feature-extraction", model=model, tokenizer=tokenizer)
            embeddings = SelfHostedEmbeddings(
                model_load_fn=get_pipeline,
                hardware=gpu


model_load_fn=get_pipeline,
                hardware=gpu
                model_reqs=["./", "torch", "transformers"],
            )
    Example passing in a pipeline path:
        .. code-block:: python
            from langchain.embeddings import SelfHostedHFEmbeddings
            import runhouse as rh
            from transformers import pipeline
            gpu = rh.cluster(name="rh-a10x", instance_type="A100:1")
            pipeline = pipeline(model="bert-base-uncased", task="feature-extraction")
            rh.blob(pickle.dumps(pipeline),
                path="models/pipeline.pkl").save().to(gpu, path="models")
            embeddings = SelfHostedHFEmbeddings.from_pipeline(
                pipeline="models/pipeline.pkl",
                hardware=gpu,
                model_reqs=["./", "torch", "transformers"],
            )
    """
    inference_fn: Callable = _embed_documents
    """Inference function to extract the embeddings on the remote hardware."""
    inference_kwargs: Any = None
    """Any kwargs to pass to the model's inference function."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Compute doc embeddings using a HuggingFace transformer model.
        Args:
            texts: The list of texts to embed.s
        Returns:
            List of embeddings, one for each text.
        """
        texts = list(map(lambda x: x.replace("\n", " "), texts))
        embeddings = self.client(self.pipeline_ref, texts)
        if not isinstance(embeddings, list):
            return embeddings.tolist()
        return embeddings
[docs]    def embed_query(self, text: str) -> List[float]:


[docs]    def embed_query(self, text: str) -> List[float]:
        """Compute query embeddings using a HuggingFace transformer model.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        text = text.replace("\n", " ")
        embeddings = self.client(self.pipeline_ref, text)
        if not isinstance(embeddings, list):
            return embeddings.tolist()
        return embeddings
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.embeddings.minimax
"""Wrapper around MiniMax APIs."""
from __future__ import annotations
import logging
from typing import Any, Callable, Dict, List, Optional
import requests
from pydantic import BaseModel, Extra, root_validator
from tenacity import (
    before_sleep_log,
    retry,
    stop_after_attempt,
    wait_exponential,
)
from langchain.embeddings.base import Embeddings
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
def _create_retry_decorator() -> Callable[[Any], Any]:
    """Returns a tenacity retry decorator."""
    multiplier = 1
    min_seconds = 1
    max_seconds = 4
    max_retries = 6
    return retry(
        reraise=True,
        stop=stop_after_attempt(max_retries),
        wait=wait_exponential(multiplier=multiplier, min=min_seconds, max=max_seconds),
        before_sleep=before_sleep_log(logger, logging.WARNING),
    )
def embed_with_retry(embeddings: MiniMaxEmbeddings, *args: Any, **kwargs: Any) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator()
    @retry_decorator
    def _embed_with_retry(*args: Any, **kwargs: Any) -> Any:
        return embeddings.embed(*args, **kwargs)
    return _embed_with_retry(*args, **kwargs)
[docs]class MiniMaxEmbeddings(BaseModel, Embeddings):
    """Wrapper around MiniMax's embedding inference service.
    To use, you should have the environment variable ``MINIMAX_GROUP_ID`` and
    ``MINIMAX_API_KEY`` set with your API token, or pass it as a named parameter to
    the constructor.


the constructor.
    Example:
        .. code-block:: python
            from langchain.embeddings import MiniMaxEmbeddings
            embeddings = MiniMaxEmbeddings()
            query_text = "This is a test query."
            query_result = embeddings.embed_query(query_text)
            document_text = "This is a test document."
            document_result = embeddings.embed_documents([document_text])
    """
    endpoint_url: str = "https://api.minimax.chat/v1/embeddings"
    """Endpoint URL to use."""
    model: str = "embo-01"
    """Embeddings model name to use."""
    embed_type_db: str = "db"
    """For embed_documents"""
    embed_type_query: str = "query"
    """For embed_query"""
    minimax_group_id: Optional[str] = None
    """Group ID for MiniMax API."""
    minimax_api_key: Optional[str] = None
    """API Key for MiniMax API."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that group id and api key exists in environment."""
        minimax_group_id = get_from_dict_or_env(
            values, "minimax_group_id", "MINIMAX_GROUP_ID"
        )
        minimax_api_key = get_from_dict_or_env(
            values, "minimax_api_key", "MINIMAX_API_KEY"
        )
        values["minimax_group_id"] = minimax_group_id
        values["minimax_api_key"] = minimax_api_key
        return values
    def embed(
        self,
        texts: List[str],
        embed_type: str,


self,
        texts: List[str],
        embed_type: str,
    ) -> List[List[float]]:
        payload = {
            "model": self.model,
            "type": embed_type,
            "texts": texts,
        }
        # HTTP headers for authorization
        headers = {
            "Authorization": f"Bearer {self.minimax_api_key}",
            "Content-Type": "application/json",
        }
        params = {
            "GroupId": self.minimax_group_id,
        }
        # send request
        response = requests.post(
            self.endpoint_url, params=params, headers=headers, json=payload
        )
        parsed_response = response.json()
        # check for errors
        if parsed_response["base_resp"]["status_code"] != 0:
            raise ValueError(
                f"MiniMax API returned an error: {parsed_response['base_resp']}"
            )
        embeddings = parsed_response["vectors"]
        return embeddings
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Embed documents using a MiniMax embedding endpoint.
        Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        embeddings = embed_with_retry(self, texts=texts, embed_type=self.embed_type_db)
        return embeddings
[docs]    def embed_query(self, text: str) -> List[float]:
        """Embed a query using a MiniMax embedding endpoint.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        embeddings = embed_with_retry(
            self, texts=[text], embed_type=self.embed_type_query
        )
        return embeddings[0]


)
        return embeddings[0]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.embeddings.openai
"""Wrapper around OpenAI embedding models."""
from __future__ import annotations
import logging
from typing import (
    Any,
    Callable,
    Dict,
    List,
    Literal,
    Optional,
    Sequence,
    Set,
    Tuple,
    Union,
)
import numpy as np
from pydantic import BaseModel, Extra, root_validator
from tenacity import (
    before_sleep_log,
    retry,
    retry_if_exception_type,
    stop_after_attempt,
    wait_exponential,
)
from langchain.embeddings.base import Embeddings
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
def _create_retry_decorator(embeddings: OpenAIEmbeddings) -> Callable[[Any], Any]:
    import openai
    min_seconds = 4
    max_seconds = 10
    # Wait 2^x * 1 second between each retry starting with
    # 4 seconds, then up to 10 seconds, then 10 seconds afterwards
    return retry(
        reraise=True,
        stop=stop_after_attempt(embeddings.max_retries),
        wait=wait_exponential(multiplier=1, min=min_seconds, max=max_seconds),
        retry=(
            retry_if_exception_type(openai.error.Timeout)
            | retry_if_exception_type(openai.error.APIError)
            | retry_if_exception_type(openai.error.APIConnectionError)
            | retry_if_exception_type(openai.error.RateLimitError)
            | retry_if_exception_type(openai.error.ServiceUnavailableError)
        ),
        before_sleep=before_sleep_log(logger, logging.WARNING),
    )
def embed_with_retry(embeddings: OpenAIEmbeddings, **kwargs: Any) -> Any:


"""Use tenacity to retry the embedding call."""
    retry_decorator = _create_retry_decorator(embeddings)
    @retry_decorator
    def _embed_with_retry(**kwargs: Any) -> Any:
        return embeddings.client.create(**kwargs)
    return _embed_with_retry(**kwargs)
[docs]class OpenAIEmbeddings(BaseModel, Embeddings):
    """Wrapper around OpenAI embedding models.
    To use, you should have the ``openai`` python package installed, and the
    environment variable ``OPENAI_API_KEY`` set with your API key or pass it
    as a named parameter to the constructor.
    Example:
        .. code-block:: python
            from langchain.embeddings import OpenAIEmbeddings
            openai = OpenAIEmbeddings(openai_api_key="my-api-key")
    In order to use the library with Microsoft Azure endpoints, you need to set
    the OPENAI_API_TYPE, OPENAI_API_BASE, OPENAI_API_KEY and OPENAI_API_VERSION.
    The OPENAI_API_TYPE must be set to 'azure' and the others correspond to
    the properties of your endpoint.
    In addition, the deployment name must be passed as the model parameter.
    Example:
        .. code-block:: python
            import os
            os.environ["OPENAI_API_TYPE"] = "azure"
            os.environ["OPENAI_API_BASE"] = "https://<your-endpoint.openai.azure.com/"
            os.environ["OPENAI_API_KEY"] = "your AzureOpenAI key"
            os.environ["OPENAI_API_VERSION"] = "2023-03-15-preview"
            os.environ["OPENAI_PROXY"] = "http://your-corporate-proxy:8080"
            from langchain.embeddings.openai import OpenAIEmbeddings
            embeddings = OpenAIEmbeddings(


embeddings = OpenAIEmbeddings(
                deployment="your-embeddings-deployment-name",
                model="your-embeddings-model-name",
                openai_api_base="https://your-endpoint.openai.azure.com/",
                openai_api_type="azure",
            )
            text = "This is a test query."
            query_result = embeddings.embed_query(text)
    """
    client: Any  #: :meta private:
    model: str = "text-embedding-ada-002"
    deployment: str = model  # to support Azure OpenAI Service custom deployment names
    openai_api_version: Optional[str] = None
    # to support Azure OpenAI Service custom endpoints
    openai_api_base: Optional[str] = None
    # to support Azure OpenAI Service custom endpoints
    openai_api_type: Optional[str] = None
    # to support explicit proxy for OpenAI
    openai_proxy: Optional[str] = None
    embedding_ctx_length: int = 8191
    openai_api_key: Optional[str] = None
    openai_organization: Optional[str] = None
    allowed_special: Union[Literal["all"], Set[str]] = set()
    disallowed_special: Union[Literal["all"], Set[str], Sequence[str]] = "all"
    chunk_size: int = 1000
    """Maximum number of texts to embed in each batch"""
    max_retries: int = 6
    """Maximum number of retries to make when generating."""
    request_timeout: Optional[Union[float, Tuple[float, float]]] = None
    """Timeout in seconds for the OpenAPI request."""
    headers: Any = None
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()


extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        values["openai_api_key"] = get_from_dict_or_env(
            values, "openai_api_key", "OPENAI_API_KEY"
        )
        values["openai_api_base"] = get_from_dict_or_env(
            values,
            "openai_api_base",
            "OPENAI_API_BASE",
            default="",
        )
        values["openai_api_type"] = get_from_dict_or_env(
            values,
            "openai_api_type",
            "OPENAI_API_TYPE",
            default="",
        )
        values["openai_proxy"] = get_from_dict_or_env(
            values,
            "openai_proxy",
            "OPENAI_PROXY",
            default="",
        )
        if values["openai_api_type"] in ("azure", "azure_ad", "azuread"):
            default_api_version = "2022-12-01"
        else:
            default_api_version = ""
        values["openai_api_version"] = get_from_dict_or_env(
            values,
            "openai_api_version",
            "OPENAI_API_VERSION",
            default=default_api_version,
        )
        values["openai_organization"] = get_from_dict_or_env(
            values,
            "openai_organization",
            "OPENAI_ORGANIZATION",
            default="",
        )
        try:
            import openai
            values["client"] = openai.Embedding
        except ImportError:
            raise ImportError(
                "Could not import openai python package. "
                "Please install it with `pip install openai`."
            )
        return values


)
        return values
    @property
    def _invocation_params(self) -> Dict:
        openai_args = {
            "engine": self.deployment,
            "request_timeout": self.request_timeout,
            "headers": self.headers,
            "api_key": self.openai_api_key,
            "organization": self.openai_organization,
            "api_base": self.openai_api_base,
            "api_type": self.openai_api_type,
            "api_version": self.openai_api_version,
        }
        if self.openai_proxy:
            import openai
            openai.proxy = {
                "http": self.openai_proxy,
                "https": self.openai_proxy,
            }  # type: ignore[assignment]  # noqa: E501
        return openai_args
    # please refer to
    # https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_long_inputs.ipynb
    def _get_len_safe_embeddings(
        self, texts: List[str], *, engine: str, chunk_size: Optional[int] = None
    ) -> List[List[float]]:
        embeddings: List[List[float]] = [[] for _ in range(len(texts))]
        try:
            import tiktoken
        except ImportError:
            raise ImportError(
                "Could not import tiktoken python package. "
                "This is needed in order to for OpenAIEmbeddings. "
                "Please install it with `pip install tiktoken`."
            )
        tokens = []
        indices = []
        encoding = tiktoken.model.encoding_for_model(self.model)
        for i, text in enumerate(texts):
            if self.model.endswith("001"):


for i, text in enumerate(texts):
            if self.model.endswith("001"):
                # See: https://github.com/openai/openai-python/issues/418#issuecomment-1525939500
                # replace newlines, which can negatively affect performance.
                text = text.replace("\n", " ")
            token = encoding.encode(
                text,
                allowed_special=self.allowed_special,
                disallowed_special=self.disallowed_special,
            )
            for j in range(0, len(token), self.embedding_ctx_length):
                tokens += [token[j : j + self.embedding_ctx_length]]
                indices += [i]
        batched_embeddings = []
        _chunk_size = chunk_size or self.chunk_size
        for i in range(0, len(tokens), _chunk_size):
            response = embed_with_retry(
                self,
                input=tokens[i : i + _chunk_size],
                **self._invocation_params,
            )
            batched_embeddings += [r["embedding"] for r in response["data"]]
        results: List[List[List[float]]] = [[] for _ in range(len(texts))]
        num_tokens_in_batch: List[List[int]] = [[] for _ in range(len(texts))]
        for i in range(len(indices)):
            results[indices[i]].append(batched_embeddings[i])
            num_tokens_in_batch[indices[i]].append(len(tokens[i]))
        for i in range(len(texts)):
            _result = results[i]
            if len(_result) == 0:
                average = embed_with_retry(
                    self,
                    input="",
                    **self._invocation_params,
                )[
                    "data"
                ][0]["embedding"]
            else:


)[
                    "data"
                ][0]["embedding"]
            else:
                average = np.average(_result, axis=0, weights=num_tokens_in_batch[i])
            embeddings[i] = (average / np.linalg.norm(average)).tolist()
        return embeddings
    def _embedding_func(self, text: str, *, engine: str) -> List[float]:
        """Call out to OpenAI's embedding endpoint."""
        # handle large input text
        if len(text) > self.embedding_ctx_length:
            return self._get_len_safe_embeddings([text], engine=engine)[0]
        else:
            if self.model.endswith("001"):
                # See: https://github.com/openai/openai-python/issues/418#issuecomment-1525939500
                # replace newlines, which can negatively affect performance.
                text = text.replace("\n", " ")
            return embed_with_retry(
                self,
                input=[text],
                **self._invocation_params,
            )[
                "data"
            ][0]["embedding"]
[docs]    def embed_documents(
        self, texts: List[str], chunk_size: Optional[int] = 0
    ) -> List[List[float]]:
        """Call out to OpenAI's embedding endpoint for embedding search docs.
        Args:
            texts: The list of texts to embed.
            chunk_size: The chunk size of embeddings. If None, will use the chunk size
                specified by the class.
        Returns:
            List of embeddings, one for each text.
        """
        # NOTE: to keep things simple, we assume the list may contain texts longer
        #       than the maximum context and use length-safe embedding function.
        return self._get_len_safe_embeddings(texts, engine=self.deployment)


return self._get_len_safe_embeddings(texts, engine=self.deployment)
[docs]    def embed_query(self, text: str) -> List[float]:
        """Call out to OpenAI's embedding endpoint for embedding query text.
        Args:
            text: The text to embed.
        Returns:
            Embedding for the text.
        """
        embedding = self._embedding_func(text, engine=self.deployment)
        return embedding
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.embeddings.mosaicml
"""Wrapper around MosaicML APIs."""
from typing import Any, Dict, List, Mapping, Optional, Tuple
import requests
from pydantic import BaseModel, Extra, root_validator
from langchain.embeddings.base import Embeddings
from langchain.utils import get_from_dict_or_env
[docs]class MosaicMLInstructorEmbeddings(BaseModel, Embeddings):
    """Wrapper around MosaicML's embedding inference service.
    To use, you should have the
    environment variable ``MOSAICML_API_TOKEN`` set with your API token, or pass
    it as a named parameter to the constructor.
    Example:
        .. code-block:: python
            from langchain.llms import MosaicMLInstructorEmbeddings
            endpoint_url = (
                "https://models.hosted-on.mosaicml.hosting/instructor-large/v1/predict"
            )
            mosaic_llm = MosaicMLInstructorEmbeddings(
                endpoint_url=endpoint_url,
                mosaicml_api_token="my-api-key"
            )
    """
    endpoint_url: str = (
        "https://models.hosted-on.mosaicml.hosting/instructor-large/v1/predict"
    )
    """Endpoint URL to use."""
    embed_instruction: str = "Represent the document for retrieval: "
    """Instruction used to embed documents."""
    query_instruction: str = (
        "Represent the question for retrieving supporting documents: "
    )
    """Instruction used to embed the query."""
    retry_sleep: float = 1.0
    """How long to try sleeping for if a rate limit is encountered"""
    mosaicml_api_token: Optional[str] = None
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid


"""Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        mosaicml_api_token = get_from_dict_or_env(
            values, "mosaicml_api_token", "MOSAICML_API_TOKEN"
        )
        values["mosaicml_api_token"] = mosaicml_api_token
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying parameters."""
        return {"endpoint_url": self.endpoint_url}
    def _embed(
        self, input: List[Tuple[str, str]], is_retry: bool = False
    ) -> List[List[float]]:
        payload = {"input_strings": input}
        # HTTP headers for authorization
        headers = {
            "Authorization": f"{self.mosaicml_api_token}",
            "Content-Type": "application/json",
        }
        # send request
        try:
            response = requests.post(self.endpoint_url, headers=headers, json=payload)
        except requests.exceptions.RequestException as e:
            raise ValueError(f"Error raised by inference endpoint: {e}")
        try:
            parsed_response = response.json()
            if "error" in parsed_response:
                # if we get rate limited, try sleeping for 1 second
                if (
                    not is_retry
                    and "rate limit exceeded" in parsed_response["error"].lower()
                ):
                    import time
                    time.sleep(self.retry_sleep)
                    return self._embed(input, is_retry=True)
                raise ValueError(
                    f"Error raised by inference API: {parsed_response['error']}"
                )


f"Error raised by inference API: {parsed_response['error']}"
                )
            if "data" not in parsed_response:
                raise ValueError(
                    f"Error raised by inference API, no key data: {parsed_response}"
                )
            embeddings = parsed_response["data"]
        except requests.exceptions.JSONDecodeError as e:
            raise ValueError(
                f"Error raised by inference API: {e}.\nResponse: {response.text}"
            )
        return embeddings
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Embed documents using a MosaicML deployed instructor embedding model.
        Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        instruction_pairs = [(self.embed_instruction, text) for text in texts]
        embeddings = self._embed(instruction_pairs)
        return embeddings
[docs]    def embed_query(self, text: str) -> List[float]:
        """Embed a query using a MosaicML deployed instructor embedding model.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        instruction_pair = (self.query_instruction, text)
        embedding = self._embed([instruction_pair])[0]
        return embedding
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.embeddings.sagemaker_endpoint
"""Wrapper around Sagemaker InvokeEndpoint API."""
from typing import Any, Dict, List, Optional
from pydantic import BaseModel, Extra, root_validator
from langchain.embeddings.base import Embeddings
from langchain.llms.sagemaker_endpoint import ContentHandlerBase
class EmbeddingsContentHandler(ContentHandlerBase[List[str], List[List[float]]]):
    """Content handler for LLM class."""
[docs]class SagemakerEndpointEmbeddings(BaseModel, Embeddings):
    """Wrapper around custom Sagemaker Inference Endpoints.
    To use, you must supply the endpoint name from your deployed
    Sagemaker model & the region where it is deployed.
    To authenticate, the AWS client uses the following methods to
    automatically load credentials:
    https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
    If a specific credential profile should be used, you must pass
    the name of the profile from the ~/.aws/credentials file that is to be used.
    Make sure the credentials / roles used have the required policies to
    access the Sagemaker endpoint.
    See: https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html
    """
    """
    Example:
        .. code-block:: python
            from langchain.embeddings import SagemakerEndpointEmbeddings
            endpoint_name = (
                "my-endpoint-name"
            )
            region_name = (
                "us-west-2"
            )
            credentials_profile_name = (
                "default"
            )
            se = SagemakerEndpointEmbeddings(
                endpoint_name=endpoint_name,
                region_name=region_name,
                credentials_profile_name=credentials_profile_name
            )
    """


credentials_profile_name=credentials_profile_name
            )
    """
    client: Any  #: :meta private:
    endpoint_name: str = ""
    """The name of the endpoint from the deployed Sagemaker model.
    Must be unique within an AWS Region."""
    region_name: str = ""
    """The aws region where the Sagemaker model is deployed, eg. `us-west-2`."""
    credentials_profile_name: Optional[str] = None
    """The name of the profile in the ~/.aws/credentials or ~/.aws/config files, which
    has either access keys or role information specified.
    If not specified, the default credential profile or, if on an EC2 instance,
    credentials from IMDS will be used.
    See: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
    """
    content_handler: EmbeddingsContentHandler
    """The content handler class that provides an input and
    output transform functions to handle formats between LLM
    and the endpoint.
    """
    """
     Example:
        .. code-block:: python
        from langchain.embeddings.sagemaker_endpoint import EmbeddingsContentHandler
        class ContentHandler(EmbeddingsContentHandler):
                content_type = "application/json"
                accepts = "application/json"
                def transform_input(self, prompts: List[str], model_kwargs: Dict) -> bytes:
                    input_str = json.dumps({prompts: prompts, **model_kwargs})
                    return input_str.encode('utf-8')
                def transform_output(self, output: bytes) -> List[List[float]]:
                    response_json = json.loads(output.read().decode("utf-8"))
                    return response_json["vectors"]
    """  # noqa: E501
    model_kwargs: Optional[Dict] = None


"""  # noqa: E501
    model_kwargs: Optional[Dict] = None
    """Key word arguments to pass to the model."""
    endpoint_kwargs: Optional[Dict] = None
    """Optional attributes passed to the invoke_endpoint
    function. See `boto3`_. docs for more info.
    .. _boto3: <https://boto3.amazonaws.com/v1/documentation/api/latest/index.html>
    """
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
        arbitrary_types_allowed = True
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that AWS credentials to and python package exists in environment."""
        try:
            import boto3
            try:
                if values["credentials_profile_name"] is not None:
                    session = boto3.Session(
                        profile_name=values["credentials_profile_name"]
                    )
                else:
                    # use default credentials
                    session = boto3.Session()
                values["client"] = session.client(
                    "sagemaker-runtime", region_name=values["region_name"]
                )
            except Exception as e:
                raise ValueError(
                    "Could not load credentials to authenticate with AWS client. "
                    "Please check that credentials in the specified "
                    "profile name are valid."
                ) from e
        except ImportError:
            raise ValueError(
                "Could not import boto3 python package. "
                "Please install it with `pip install boto3`."
            )
        return values
    def _embedding_func(self, texts: List[str]) -> List[List[float]]:
        """Call out to SageMaker Inference embedding endpoint."""
        # replace newlines, which can negatively affect performance.


# replace newlines, which can negatively affect performance.
        texts = list(map(lambda x: x.replace("\n", " "), texts))
        _model_kwargs = self.model_kwargs or {}
        _endpoint_kwargs = self.endpoint_kwargs or {}
        body = self.content_handler.transform_input(texts, _model_kwargs)
        content_type = self.content_handler.content_type
        accepts = self.content_handler.accepts
        # send request
        try:
            response = self.client.invoke_endpoint(
                EndpointName=self.endpoint_name,
                Body=body,
                ContentType=content_type,
                Accept=accepts,
                **_endpoint_kwargs,
            )
        except Exception as e:
            raise ValueError(f"Error raised by inference endpoint: {e}")
        return self.content_handler.transform_output(response["Body"])
[docs]    def embed_documents(
        self, texts: List[str], chunk_size: int = 64
    ) -> List[List[float]]:
        """Compute doc embeddings using a SageMaker Inference Endpoint.
        Args:
            texts: The list of texts to embed.
            chunk_size: The chunk size defines how many input texts will
                be grouped together as request. If None, will use the
                chunk size specified by the class.
        Returns:
            List of embeddings, one for each text.
        """
        results = []
        _chunk_size = len(texts) if chunk_size > len(texts) else chunk_size
        for i in range(0, len(texts), _chunk_size):
            response = self._embedding_func(texts[i : i + _chunk_size])
            results.extend(response)
        return results
[docs]    def embed_query(self, text: str) -> List[float]:
        """Compute query embeddings using a SageMaker inference endpoint.


"""Compute query embeddings using a SageMaker inference endpoint.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        return self._embedding_func([text])[0]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.sitemap
"""Loader that fetches a sitemap and loads those URLs."""
import itertools
import re
from typing import Any, Callable, Generator, Iterable, List, Optional
from langchain.document_loaders.web_base import WebBaseLoader
from langchain.schema import Document
def _default_parsing_function(content: Any) -> str:
    return str(content.get_text())
def _default_meta_function(meta: dict, _content: Any) -> dict:
    return {"source": meta["loc"], **meta}
def _batch_block(iterable: Iterable, size: int) -> Generator[List[dict], None, None]:
    it = iter(iterable)
    while item := list(itertools.islice(it, size)):
        yield item
[docs]class SitemapLoader(WebBaseLoader):
    """Loader that fetches a sitemap and loads those URLs."""
    def __init__(
        self,
        web_path: str,
        filter_urls: Optional[List[str]] = None,
        parsing_function: Optional[Callable] = None,
        blocksize: Optional[int] = None,
        blocknum: int = 0,
        meta_function: Optional[Callable] = None,
        is_local: bool = False,
    ):
        """Initialize with webpage path and optional filter URLs.
        Args:
            web_path: url of the sitemap. can also be a local path
            filter_urls: list of strings or regexes that will be applied to filter the
                urls that are parsed and loaded
            parsing_function: Function to parse bs4.Soup output
            blocksize: number of sitemap locations per block
            blocknum: the number of the block that should be loaded - zero indexed
            meta_function: Function to parse bs4.Soup output for metadata


meta_function: Function to parse bs4.Soup output for metadata
                remember when setting this method to also copy metadata["loc"]
                to metadata["source"] if you are using this field
            is_local: whether the sitemap is a local file
        """
        if blocksize is not None and blocksize < 1:
            raise ValueError("Sitemap blocksize should be at least 1")
        if blocknum < 0:
            raise ValueError("Sitemap blocknum can not be lower then 0")
        try:
            import lxml  # noqa:F401
        except ImportError:
            raise ImportError(
                "lxml package not found, please install it with " "`pip install lxml`"
            )
        super().__init__(web_path)
        self.filter_urls = filter_urls
        self.parsing_function = parsing_function or _default_parsing_function
        self.meta_function = meta_function or _default_meta_function
        self.blocksize = blocksize
        self.blocknum = blocknum
        self.is_local = is_local
[docs]    def parse_sitemap(self, soup: Any) -> List[dict]:
        """Parse sitemap xml and load into a list of dicts."""
        els = []
        for url in soup.find_all("url"):
            loc = url.find("loc")
            if not loc:
                continue
            # Strip leading and trailing whitespace and newlines
            loc_text = loc.text.strip()
            if self.filter_urls and not any(
                re.match(r, loc_text) for r in self.filter_urls
            ):
                continue
            els.append(
                {
                    tag: prop.text
                    for tag in ["loc", "lastmod", "changefreq", "priority"]
                    if (prop := url.find(tag))
                }
            )


if (prop := url.find(tag))
                }
            )
        for sitemap in soup.find_all("sitemap"):
            loc = sitemap.find("loc")
            if not loc:
                continue
            soup_child = self.scrape_all([loc.text], "xml")[0]
            els.extend(self.parse_sitemap(soup_child))
        return els
[docs]    def load(self) -> List[Document]:
        """Load sitemap."""
        if self.is_local:
            try:
                import bs4
            except ImportError:
                raise ImportError(
                    "beautifulsoup4 package not found, please install it"
                    " with `pip install beautifulsoup4`"
                )
            fp = open(self.web_path)
            soup = bs4.BeautifulSoup(fp, "xml")
        else:
            soup = self.scrape("xml")
        els = self.parse_sitemap(soup)
        if self.blocksize is not None:
            elblocks = list(_batch_block(els, self.blocksize))
            blockcount = len(elblocks)
            if blockcount - 1 < self.blocknum:
                raise ValueError(
                    "Selected sitemap does not contain enough blocks for given blocknum"
                )
            else:
                els = elblocks[self.blocknum]
        results = self.scrape_all([el["loc"].strip() for el in els if "loc" in el])
        return [
            Document(
                page_content=self.parsing_function(results[i]),
                metadata=self.meta_function(els[i], results[i]),
            )
            for i in range(len(results))
        ]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.image
"""Loader that loads image files."""
from typing import List
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class UnstructuredImageLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load image files, such as PNGs and JPGs."""
    def _get_elements(self) -> List:
        from unstructured.partition.image import partition_image
        return partition_image(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.modern_treasury
"""Loader that fetches data from Modern Treasury"""
import json
import urllib.request
from base64 import b64encode
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import get_from_env, stringify_value
MODERN_TREASURY_ENDPOINTS = {
    "payment_orders": "https://app.moderntreasury.com/api/payment_orders",
    "expected_payments": "https://app.moderntreasury.com/api/expected_payments",
    "returns": "https://app.moderntreasury.com/api/returns",
    "incoming_payment_details": "https://app.moderntreasury.com/api/\
incoming_payment_details",
    "counterparties": "https://app.moderntreasury.com/api/counterparties",
    "internal_accounts": "https://app.moderntreasury.com/api/internal_accounts",
    "external_accounts": "https://app.moderntreasury.com/api/external_accounts",
    "transactions": "https://app.moderntreasury.com/api/transactions",
    "ledgers": "https://app.moderntreasury.com/api/ledgers",
    "ledger_accounts": "https://app.moderntreasury.com/api/ledger_accounts",
    "ledger_transactions": "https://app.moderntreasury.com/api/ledger_transactions",
    "events": "https://app.moderntreasury.com/api/events",
    "invoices": "https://app.moderntreasury.com/api/invoices",
}
[docs]class ModernTreasuryLoader(BaseLoader):
    def __init__(
        self,
        resource: str,
        organization_id: Optional[str] = None,


resource: str,
        organization_id: Optional[str] = None,
        api_key: Optional[str] = None,
    ) -> None:
        self.resource = resource
        organization_id = organization_id or get_from_env(
            "organization_id", "MODERN_TREASURY_ORGANIZATION_ID"
        )
        api_key = api_key or get_from_env("api_key", "MODERN_TREASURY_API_KEY")
        credentials = f"{organization_id}:{api_key}".encode("utf-8")
        basic_auth_token = b64encode(credentials).decode("utf-8")
        self.headers = {"Authorization": f"Basic {basic_auth_token}"}
    def _make_request(self, url: str) -> List[Document]:
        request = urllib.request.Request(url, headers=self.headers)
        with urllib.request.urlopen(request) as response:
            json_data = json.loads(response.read().decode())
            text = stringify_value(json_data)
            metadata = {"source": url}
            return [Document(page_content=text, metadata=metadata)]
    def _get_resource(self) -> List[Document]:
        endpoint = MODERN_TREASURY_ENDPOINTS.get(self.resource)
        if endpoint is None:
            return []
        return self._make_request(endpoint)
[docs]    def load(self) -> List[Document]:
        return self._get_resource()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.odt
"""Loader that loads Open Office ODT files."""
from typing import Any, List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    validate_unstructured_version,
)
[docs]class UnstructuredODTLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load open office ODT files."""
    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        validate_unstructured_version(min_unstructured_version="0.6.3")
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.odt import partition_odt
        return partition_odt(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.slack_directory
"""Loader for documents from a Slack export."""
import json
import zipfile
from pathlib import Path
from typing import Dict, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class SlackDirectoryLoader(BaseLoader):
    """Loader for loading documents from a Slack directory dump."""
    def __init__(self, zip_path: str, workspace_url: Optional[str] = None):
        """Initialize the SlackDirectoryLoader.
        Args:
            zip_path (str): The path to the Slack directory dump zip file.
            workspace_url (Optional[str]): The Slack workspace URL.
              Including the URL will turn
              sources into links. Defaults to None.
        """
        self.zip_path = Path(zip_path)
        self.workspace_url = workspace_url
        self.channel_id_map = self._get_channel_id_map(self.zip_path)
    @staticmethod
    def _get_channel_id_map(zip_path: Path) -> Dict[str, str]:
        """Get a dictionary mapping channel names to their respective IDs."""
        with zipfile.ZipFile(zip_path, "r") as zip_file:
            try:
                with zip_file.open("channels.json", "r") as f:
                    channels = json.load(f)
                return {channel["name"]: channel["id"] for channel in channels}
            except KeyError:
                return {}
[docs]    def load(self) -> List[Document]:
        """Load and return documents from the Slack directory dump."""
        docs = []
        with zipfile.ZipFile(self.zip_path, "r") as zip_file:
            for channel_path in zip_file.namelist():
                channel_name = Path(channel_path).parent.name
                if not channel_name:
                    continue


if not channel_name:
                    continue
                if channel_path.endswith(".json"):
                    messages = self._read_json(zip_file, channel_path)
                    for message in messages:
                        document = self._convert_message_to_document(
                            message, channel_name
                        )
                        docs.append(document)
        return docs
    def _read_json(self, zip_file: zipfile.ZipFile, file_path: str) -> List[dict]:
        """Read JSON data from a zip subfile."""
        with zip_file.open(file_path, "r") as f:
            data = json.load(f)
        return data
    def _convert_message_to_document(
        self, message: dict, channel_name: str
    ) -> Document:
        """
        Convert a message to a Document object.
        Args:
            message (dict): A message in the form of a dictionary.
            channel_name (str): The name of the channel the message belongs to.
        Returns:
            Document: A Document object representing the message.
        """
        text = message.get("text", "")
        metadata = self._get_message_metadata(message, channel_name)
        return Document(
            page_content=text,
            metadata=metadata,
        )
    def _get_message_metadata(self, message: dict, channel_name: str) -> dict:
        """Create and return metadata for a given message and channel."""
        timestamp = message.get("ts", "")
        user = message.get("user", "")
        source = self._get_message_source(channel_name, user, timestamp)
        return {
            "source": source,
            "channel": channel_name,
            "timestamp": timestamp,
            "user": user,
        }


"timestamp": timestamp,
            "user": user,
        }
    def _get_message_source(self, channel_name: str, user: str, timestamp: str) -> str:
        """
        Get the message source as a string.
        Args:
            channel_name (str): The name of the channel the message belongs to.
            user (str): The user ID who sent the message.
            timestamp (str): The timestamp of the message.
        Returns:
            str: The message source.
        """
        if self.workspace_url:
            channel_id = self.channel_id_map.get(channel_name, "")
            return (
                f"{self.workspace_url}/archives/{channel_id}"
                + f"/p{timestamp.replace('.', '')}"
            )
        else:
            return f"{channel_name} - {user} - {timestamp}"
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.max_compute
from __future__ import annotations
from typing import Any, Iterator, List, Optional, Sequence
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utilities.max_compute import MaxComputeAPIWrapper
[docs]class MaxComputeLoader(BaseLoader):
    """Loads a query result from Alibaba Cloud MaxCompute table into documents."""
    def __init__(
        self,
        query: str,
        api_wrapper: MaxComputeAPIWrapper,
        *,
        page_content_columns: Optional[Sequence[str]] = None,
        metadata_columns: Optional[Sequence[str]] = None,
    ):
        """Initialize Alibaba Cloud MaxCompute document loader.
        Args:
            query: SQL query to execute.
            api_wrapper: MaxCompute API wrapper.
            page_content_columns: The columns to write into the `page_content` of the
                Document. If unspecified, all columns will be written to `page_content`.
            metadata_columns: The columns to write into the `metadata` of the Document.
                If unspecified, all columns not added to `page_content` will be written.
        """
        self.query = query
        self.api_wrapper = api_wrapper
        self.page_content_columns = page_content_columns
        self.metadata_columns = metadata_columns
[docs]    @classmethod
    def from_params(
        cls,
        query: str,
        endpoint: str,
        project: str,
        *,
        access_id: Optional[str] = None,
        secret_access_key: Optional[str] = None,
        **kwargs: Any,
    ) -> MaxComputeLoader:
        """Convenience constructor that builds the MaxCompute API wrapper from
            given parameters.
        Args:
            query: SQL query to execute.


given parameters.
        Args:
            query: SQL query to execute.
            endpoint: MaxCompute endpoint.
            project: A project is a basic organizational unit of MaxCompute, which is
                similar to a database.
            access_id: MaxCompute access ID. Should be passed in directly or set as the
                environment variable `MAX_COMPUTE_ACCESS_ID`.
            secret_access_key: MaxCompute secret access key. Should be passed in
                directly or set as the environment variable
                `MAX_COMPUTE_SECRET_ACCESS_KEY`.
        """
        api_wrapper = MaxComputeAPIWrapper.from_params(
            endpoint, project, access_id=access_id, secret_access_key=secret_access_key
        )
        return cls(query, api_wrapper, **kwargs)
[docs]    def lazy_load(self) -> Iterator[Document]:
        for row in self.api_wrapper.query(self.query):
            if self.page_content_columns:
                page_content_data = {
                    k: v for k, v in row.items() if k in self.page_content_columns
                }
            else:
                page_content_data = row
            page_content = "\n".join(f"{k}: {v}" for k, v in page_content_data.items())
            if self.metadata_columns:
                metadata = {k: v for k, v in row.items() if k in self.metadata_columns}
            else:
                metadata = {k: v for k, v in row.items() if k not in page_content_data}
            yield Document(page_content=page_content, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.toml
import json
from pathlib import Path
from typing import Iterator, List, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class TomlLoader(BaseLoader):
    """
    A TOML document loader that inherits from the BaseLoader class.
    This class can be initialized with either a single source file or a source
    directory containing TOML files.
    """
    def __init__(self, source: Union[str, Path]):
        """Initialize the TomlLoader with a source file or directory."""
        self.source = Path(source)
[docs]    def load(self) -> List[Document]:
        """Load and return all documents."""
        return list(self.lazy_load())
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Lazily load the TOML documents from the source file or directory."""
        import tomli
        if self.source.is_file() and self.source.suffix == ".toml":
            files = [self.source]
        elif self.source.is_dir():
            files = list(self.source.glob("**/*.toml"))
        else:
            raise ValueError("Invalid source path or file type")
        for file_path in files:
            with file_path.open("r", encoding="utf-8") as file:
                content = file.read()
                try:
                    data = tomli.loads(content)
                    doc = Document(
                        page_content=json.dumps(data),
                        metadata={"source": str(file_path)},
                    )
                    yield doc
                except tomli.TOMLDecodeError as e:
                    print(f"Error parsing TOML file {file_path}: {e}")
By Harrison Chase

      © Copyright 2023, Harrison Chase.


By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.epub
"""Loader that loads EPub files."""
from typing import List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    satisfies_min_unstructured_version,
)
[docs]class UnstructuredEPubLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load epub files."""
    def _get_elements(self) -> List:
        min_unstructured_version = "0.5.4"
        if not satisfies_min_unstructured_version(min_unstructured_version):
            raise ValueError(
                "Partitioning epub files is only supported in "
                f"unstructured>={min_unstructured_version}."
            )
        from unstructured.partition.epub import partition_epub
        return partition_epub(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.xml
"""Loader that loads Microsoft Excel files."""
from typing import Any, List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    validate_unstructured_version,
)
[docs]class UnstructuredXMLLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load XML files."""
    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        validate_unstructured_version(min_unstructured_version="0.6.7")
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.xml import partition_xml
        return partition_xml(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.notiondb
"""Notion DB loader for langchain"""
from typing import Any, Dict, List, Optional
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
NOTION_BASE_URL = "https://api.notion.com/v1"
DATABASE_URL = NOTION_BASE_URL + "/databases/{database_id}/query"
PAGE_URL = NOTION_BASE_URL + "/pages/{page_id}"
BLOCK_URL = NOTION_BASE_URL + "/blocks/{block_id}/children"
[docs]class NotionDBLoader(BaseLoader):
    """Notion DB Loader.
    Reads content from pages within a Noton Database.
    Args:
        integration_token (str): Notion integration token.
        database_id (str): Notion database id.
        request_timeout_sec (int): Timeout for Notion requests in seconds.
    """
    def __init__(
        self,
        integration_token: str,
        database_id: str,
        request_timeout_sec: Optional[int] = 10,
    ) -> None:
        """Initialize with parameters."""
        if not integration_token:
            raise ValueError("integration_token must be provided")
        if not database_id:
            raise ValueError("database_id must be provided")
        self.token = integration_token
        self.database_id = database_id
        self.headers = {
            "Authorization": "Bearer " + self.token,
            "Content-Type": "application/json",
            "Notion-Version": "2022-06-28",
        }
        self.request_timeout_sec = request_timeout_sec
[docs]    def load(self) -> List[Document]:
        """Load documents from the Notion database.
        Returns:
            List[Document]: List of documents.
        """


Returns:
            List[Document]: List of documents.
        """
        page_ids = self._retrieve_page_ids()
        return list(self.load_page(page_id) for page_id in page_ids)
    def _retrieve_page_ids(
        self, query_dict: Dict[str, Any] = {"page_size": 100}
    ) -> List[str]:
        """Get all the pages from a Notion database."""
        pages: List[Dict[str, Any]] = []
        while True:
            data = self._request(
                DATABASE_URL.format(database_id=self.database_id),
                method="POST",
                query_dict=query_dict,
            )
            pages.extend(data.get("results"))
            if not data.get("has_more"):
                break
            query_dict["start_cursor"] = data.get("next_cursor")
        page_ids = [page["id"] for page in pages]
        return page_ids
[docs]    def load_page(self, page_id: str) -> Document:
        """Read a page."""
        data = self._request(PAGE_URL.format(page_id=page_id))
        # load properties as metadata
        metadata: Dict[str, Any] = {}
        for prop_name, prop_data in data["properties"].items():
            prop_type = prop_data["type"]
            if prop_type == "rich_text":
                value = (
                    prop_data["rich_text"][0]["plain_text"]
                    if prop_data["rich_text"]
                    else None
                )
            elif prop_type == "title":
                value = (
                    prop_data["title"][0]["plain_text"] if prop_data["title"] else None
                )
            elif prop_type == "multi_select":
                value = (
                    [item["name"] for item in prop_data["multi_select"]]


[item["name"] for item in prop_data["multi_select"]]
                    if prop_data["multi_select"]
                    else []
                )
            elif prop_type == "url":
                value = prop_data["url"]
            else:
                value = None
            metadata[prop_name.lower()] = value
        metadata["id"] = page_id
        return Document(page_content=self._load_blocks(page_id), metadata=metadata)
    def _load_blocks(self, block_id: str, num_tabs: int = 0) -> str:
        """Read a block and its children."""
        result_lines_arr: List[str] = []
        cur_block_id: str = block_id
        while cur_block_id:
            data = self._request(BLOCK_URL.format(block_id=cur_block_id))
            for result in data["results"]:
                result_obj = result[result["type"]]
                if "rich_text" not in result_obj:
                    continue
                cur_result_text_arr: List[str] = []
                for rich_text in result_obj["rich_text"]:
                    if "text" in rich_text:
                        cur_result_text_arr.append(
                            "\t" * num_tabs + rich_text["text"]["content"]
                        )
                if result["has_children"]:
                    children_text = self._load_blocks(
                        result["id"], num_tabs=num_tabs + 1
                    )
                    cur_result_text_arr.append(children_text)
                result_lines_arr.append("\n".join(cur_result_text_arr))
            cur_block_id = data.get("next_cursor")
        return "\n".join(result_lines_arr)
    def _request(
        self, url: str, method: str = "GET", query_dict: Dict[str, Any] = {}
    ) -> Any:
        res = requests.request(
            method,


) -> Any:
        res = requests.request(
            method,
            url,
            headers=self.headers,
            json=query_dict,
            timeout=self.request_timeout_sec,
        )
        res.raise_for_status()
        return res.json()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.diffbot
"""Loader that uses Diffbot to load webpages in text format."""
import logging
from typing import Any, List
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class DiffbotLoader(BaseLoader):
    """Loader that loads Diffbot file json."""
    def __init__(
        self, api_token: str, urls: List[str], continue_on_failure: bool = True
    ):
        """Initialize with API token, ids, and key."""
        self.api_token = api_token
        self.urls = urls
        self.continue_on_failure = continue_on_failure
    def _diffbot_api_url(self, diffbot_api: str) -> str:
        return f"https://api.diffbot.com/v3/{diffbot_api}"
    def _get_diffbot_data(self, url: str) -> Any:
        """Get Diffbot file from Diffbot REST API."""
        # TODO: Add support for other Diffbot APIs
        diffbot_url = self._diffbot_api_url("article")
        params = {
            "token": self.api_token,
            "url": url,
        }
        response = requests.get(diffbot_url, params=params, timeout=10)
        # TODO: handle non-ok errors
        return response.json() if response.ok else {}
[docs]    def load(self) -> List[Document]:
        """Extract text from Diffbot on all the URLs and return Document instances"""
        docs: List[Document] = list()
        for url in self.urls:
            try:
                data = self._get_diffbot_data(url)
                text = data["objects"][0]["text"] if "objects" in data else ""


text = data["objects"][0]["text"] if "objects" in data else ""
                metadata = {"source": url}
                docs.append(Document(page_content=text, metadata=metadata))
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error fetching or processing {url}, exception: {e}")
                else:
                    raise e
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.git
import os
from typing import Callable, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class GitLoader(BaseLoader):
    """Loads files from a Git repository into a list of documents.
    Repository can be local on disk available at `repo_path`,
    or remote at `clone_url` that will be cloned to `repo_path`.
    Currently supports only text files.
    Each document represents one file in the repository. The `path` points to
    the local Git repository, and the `branch` specifies the branch to load
    files from. By default, it loads from the `main` branch.
    """
    def __init__(
        self,
        repo_path: str,
        clone_url: Optional[str] = None,
        branch: Optional[str] = "main",
        file_filter: Optional[Callable[[str], bool]] = None,
    ):
        self.repo_path = repo_path
        self.clone_url = clone_url
        self.branch = branch
        self.file_filter = file_filter
[docs]    def load(self) -> List[Document]:
        try:
            from git import Blob, Repo  # type: ignore
        except ImportError as ex:
            raise ImportError(
                "Could not import git python package. "
                "Please install it with `pip install GitPython`."
            ) from ex
        if not os.path.exists(self.repo_path) and self.clone_url is None:
            raise ValueError(f"Path {self.repo_path} does not exist")
        elif self.clone_url:
            repo = Repo.clone_from(self.clone_url, self.repo_path)
            repo.git.checkout(self.branch)
        else:
            repo = Repo(self.repo_path)


else:
            repo = Repo(self.repo_path)
            repo.git.checkout(self.branch)
        docs: List[Document] = []
        for item in repo.tree().traverse():
            if not isinstance(item, Blob):
                continue
            file_path = os.path.join(self.repo_path, item.path)
            ignored_files = repo.ignored([file_path])  # type: ignore
            if len(ignored_files):
                continue
            # uses filter to skip files
            if self.file_filter and not self.file_filter(file_path):
                continue
            rel_file_path = os.path.relpath(file_path, self.repo_path)
            try:
                with open(file_path, "rb") as f:
                    content = f.read()
                    file_type = os.path.splitext(item.name)[1]
                    # loads only text files
                    try:
                        text_content = content.decode("utf-8")
                    except UnicodeDecodeError:
                        continue
                    metadata = {
                        "source": rel_file_path,
                        "file_path": rel_file_path,
                        "file_name": item.name,
                        "file_type": file_type,
                    }
                    doc = Document(page_content=text_content, metadata=metadata)
                    docs.append(doc)
            except Exception as e:
                print(f"Error reading file {file_path}: {e}")
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.image_captions
"""
Loader that loads image captions
By default, the loader utilizes the pre-trained BLIP image captioning model.
https://huggingface.co/Salesforce/blip-image-captioning-base
"""
from typing import Any, List, Tuple, Union
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class ImageCaptionLoader(BaseLoader):
    """Loader that loads the captions of an image"""
    def __init__(
        self,
        path_images: Union[str, List[str]],
        blip_processor: str = "Salesforce/blip-image-captioning-base",
        blip_model: str = "Salesforce/blip-image-captioning-base",
    ):
        """
        Initialize with a list of image paths
        """
        if isinstance(path_images, str):
            self.image_paths = [path_images]
        else:
            self.image_paths = path_images
        self.blip_processor = blip_processor
        self.blip_model = blip_model
[docs]    def load(self) -> List[Document]:
        """
        Load from a list of image files
        """
        try:
            from transformers import BlipForConditionalGeneration, BlipProcessor
        except ImportError:
            raise ImportError(
                "`transformers` package not found, please install with "
                "`pip install transformers`."
            )
        processor = BlipProcessor.from_pretrained(self.blip_processor)
        model = BlipForConditionalGeneration.from_pretrained(self.blip_model)
        results = []
        for path_image in self.image_paths:
            caption, metadata = self._get_captions_and_metadata(
                model=model, processor=processor, path_image=path_image
            )


model=model, processor=processor, path_image=path_image
            )
            doc = Document(page_content=caption, metadata=metadata)
            results.append(doc)
        return results
    def _get_captions_and_metadata(
        self, model: Any, processor: Any, path_image: str
    ) -> Tuple[str, dict]:
        """
        Helper function for getting the captions and metadata of an image
        """
        try:
            from PIL import Image
        except ImportError:
            raise ImportError(
                "`PIL` package not found, please install with `pip install pillow`"
            )
        try:
            if path_image.startswith("http://") or path_image.startswith("https://"):
                image = Image.open(requests.get(path_image, stream=True).raw).convert(
                    "RGB"
                )
            else:
                image = Image.open(path_image).convert("RGB")
        except Exception:
            raise ValueError(f"Could not get image data for {path_image}")
        inputs = processor(image, "an image of", return_tensors="pt")
        output = model.generate(**inputs)
        caption: str = processor.decode(output[0])
        metadata: dict = {"image_path": path_image}
        return caption, metadata
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.weather
"""Simple reader that reads weather data from OpenWeatherMap API"""
from __future__ import annotations
from datetime import datetime
from typing import Iterator, List, Optional, Sequence
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utilities.openweathermap import OpenWeatherMapAPIWrapper
[docs]class WeatherDataLoader(BaseLoader):
    """Weather Reader.
    Reads the forecast & current weather of any location using OpenWeatherMap's free
    API. Checkout 'https://openweathermap.org/appid' for more on how to generate a free
    OpenWeatherMap API.
    """
    def __init__(
        self,
        client: OpenWeatherMapAPIWrapper,
        places: Sequence[str],
    ) -> None:
        """Initialize with parameters."""
        super().__init__()
        self.client = client
        self.places = places
[docs]    @classmethod
    def from_params(
        cls, places: Sequence[str], *, openweathermap_api_key: Optional[str] = None
    ) -> WeatherDataLoader:
        client = OpenWeatherMapAPIWrapper(openweathermap_api_key=openweathermap_api_key)
        return cls(client, places)
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Lazily load weather data for the given locations."""
        for place in self.places:
            metadata = {"queried_at": datetime.now()}
            content = self.client.run(place)
            yield Document(page_content=content, metadata=metadata)
[docs]    def load(
        self,
    ) -> List[Document]:
        """Load weather data for the given locations."""
        return list(self.lazy_load())
By Harrison Chase


return list(self.lazy_load())
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.gitbook
"""Loader that loads GitBook."""
from typing import Any, List, Optional
from urllib.parse import urljoin, urlparse
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class GitbookLoader(WebBaseLoader):
    """Load GitBook data.
    1. load from either a single page, or
    2. load all (relative) paths in the navbar.
    """
    def __init__(
        self,
        web_page: str,
        load_all_paths: bool = False,
        base_url: Optional[str] = None,
        content_selector: str = "main",
    ):
        """Initialize with web page and whether to load all paths.
        Args:
            web_page: The web page to load or the starting point from where
                relative paths are discovered.
            load_all_paths: If set to True, all relative paths in the navbar
                are loaded instead of only `web_page`.
            base_url: If `load_all_paths` is True, the relative paths are
                appended to this base url. Defaults to `web_page` if not set.
        """
        self.base_url = base_url or web_page
        if self.base_url.endswith("/"):
            self.base_url = self.base_url[:-1]
        if load_all_paths:
            # set web_path to the sitemap if we want to crawl all paths
            web_paths = f"{self.base_url}/sitemap.xml"
        else:
            web_paths = web_page
        super().__init__(web_paths)
        self.load_all_paths = load_all_paths
        self.content_selector = content_selector
[docs]    def load(self) -> List[Document]:


[docs]    def load(self) -> List[Document]:
        """Fetch text from one single GitBook page."""
        if self.load_all_paths:
            soup_info = self.scrape()
            relative_paths = self._get_paths(soup_info)
            documents = []
            for path in relative_paths:
                url = urljoin(self.base_url, path)
                print(f"Fetching text from {url}")
                soup_info = self._scrape(url)
                documents.append(self._get_document(soup_info, url))
            return [d for d in documents if d]
        else:
            soup_info = self.scrape()
            documents = [self._get_document(soup_info, self.web_path)]
            return [d for d in documents if d]
    def _get_document(
        self, soup: Any, custom_url: Optional[str] = None
    ) -> Optional[Document]:
        """Fetch content from page and return Document."""
        page_content_raw = soup.find(self.content_selector)
        if not page_content_raw:
            return None
        content = page_content_raw.get_text(separator="\n").strip()
        title_if_exists = page_content_raw.find("h1")
        title = title_if_exists.text if title_if_exists else ""
        metadata = {"source": custom_url or self.web_path, "title": title}
        return Document(page_content=content, metadata=metadata)
    def _get_paths(self, soup: Any) -> List[str]:
        """Fetch all relative paths in the navbar."""
        return [urlparse(loc.text).path for loc in soup.find_all("loc")]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.fauna
from typing import Iterator, List, Optional, Sequence
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class FaunaLoader(BaseLoader):
    """
    Attributes:
        query (str): The FQL query string to execute.
        page_content_field (str): The field that contains the content of each page.
        secret (str): The secret key for authenticating to FaunaDB.
        metadata_fields (Optional[Sequence[str]]):
            Optional list of field names to include in metadata.
    """
    def __init__(
        self,
        query: str,
        page_content_field: str,
        secret: str,
        metadata_fields: Optional[Sequence[str]] = None,
    ):
        self.query = query
        self.page_content_field = page_content_field
        self.secret = secret
        self.metadata_fields = metadata_fields
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
[docs]    def lazy_load(self) -> Iterator[Document]:
        try:
            from fauna import Page, fql
            from fauna.client import Client
            from fauna.encoding import QuerySuccess
        except ImportError:
            raise ImportError(
                "Could not import fauna python package. "
                "Please install it with `pip install fauna`."
            )
        # Create Fauna Client
        client = Client(secret=self.secret)
        # Run FQL Query
        response: QuerySuccess = client.query(fql(self.query))
        page: Page = response.data
        for result in page:
            if result is not None:
                document_dict = dict(result.items())
                page_content = ""
                for key, value in document_dict.items():


page_content = ""
                for key, value in document_dict.items():
                    if key == self.page_content_field:
                        page_content = value
                document: Document = Document(
                    page_content=page_content,
                    metadata={"id": result.id, "ts": result.ts},
                )
                yield document
        if page.after is not None:
            yield Document(
                page_content="Next Page Exists",
                metadata={"after": page.after},
            )
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.text
import logging
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.helpers import detect_file_encodings
logger = logging.getLogger(__name__)
[docs]class TextLoader(BaseLoader):
    """Load text files.
    Args:
        file_path: Path to the file to load.
        encoding: File encoding to use. If `None`, the file will be loaded
        with the default system encoding.
        autodetect_encoding: Whether to try to autodetect the file encoding
            if the specified encoding fails.
    """
    def __init__(
        self,
        file_path: str,
        encoding: Optional[str] = None,
        autodetect_encoding: bool = False,
    ):
        """Initialize with file path."""
        self.file_path = file_path
        self.encoding = encoding
        self.autodetect_encoding = autodetect_encoding
[docs]    def load(self) -> List[Document]:
        """Load from file path."""
        text = ""
        try:
            with open(self.file_path, encoding=self.encoding) as f:
                text = f.read()
        except UnicodeDecodeError as e:
            if self.autodetect_encoding:
                detected_encodings = detect_file_encodings(self.file_path)
                for encoding in detected_encodings:
                    logger.debug("Trying encoding: ", encoding.encoding)
                    try:
                        with open(self.file_path, encoding=encoding.encoding) as f:
                            text = f.read()
                        break
                    except UnicodeDecodeError:
                        continue
            else:
                raise RuntimeError(f"Error loading {self.file_path}") from e
        except Exception as e:


except Exception as e:
            raise RuntimeError(f"Error loading {self.file_path}") from e
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.apify_dataset
"""Logic for loading documents from Apify datasets."""
from typing import Any, Callable, Dict, List
from pydantic import BaseModel, root_validator
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class ApifyDatasetLoader(BaseLoader, BaseModel):
    """Logic for loading documents from Apify datasets."""
    apify_client: Any
    dataset_id: str
    """The ID of the dataset on the Apify platform."""
    dataset_mapping_function: Callable[[Dict], Document]
    """A custom function that takes a single dictionary (an Apify dataset item)
     and converts it to an instance of the Document class."""
    def __init__(
        self, dataset_id: str, dataset_mapping_function: Callable[[Dict], Document]
    ):
        """Initialize the loader with an Apify dataset ID and a mapping function.
        Args:
            dataset_id (str): The ID of the dataset on the Apify platform.
            dataset_mapping_function (Callable): A function that takes a single
                dictionary (an Apify dataset item) and converts it to an instance
                of the Document class.
        """
        super().__init__(
            dataset_id=dataset_id, dataset_mapping_function=dataset_mapping_function
        )
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate environment."""
        try:
            from apify_client import ApifyClient
            values["apify_client"] = ApifyClient()
        except ImportError:
            raise ImportError(
                "Could not import apify-client Python package. "
                "Please install it with `pip install apify-client`."
            )
        return values


)
        return values
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        dataset_items = self.apify_client.dataset(self.dataset_id).list_items().items
        return list(map(self.dataset_mapping_function, dataset_items))
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.stripe
"""Loader that fetches data from Stripe"""
import json
import urllib.request
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import get_from_env, stringify_dict
STRIPE_ENDPOINTS = {
    "balance_transactions": "https://api.stripe.com/v1/balance_transactions",
    "charges": "https://api.stripe.com/v1/charges",
    "customers": "https://api.stripe.com/v1/customers",
    "events": "https://api.stripe.com/v1/events",
    "refunds": "https://api.stripe.com/v1/refunds",
    "disputes": "https://api.stripe.com/v1/disputes",
}
[docs]class StripeLoader(BaseLoader):
    def __init__(self, resource: str, access_token: Optional[str] = None) -> None:
        self.resource = resource
        access_token = access_token or get_from_env(
            "access_token", "STRIPE_ACCESS_TOKEN"
        )
        self.headers = {"Authorization": f"Bearer {access_token}"}
    def _make_request(self, url: str) -> List[Document]:
        request = urllib.request.Request(url, headers=self.headers)
        with urllib.request.urlopen(request) as response:
            json_data = json.loads(response.read().decode())
            text = stringify_dict(json_data)
            metadata = {"source": url}
            return [Document(page_content=text, metadata=metadata)]
    def _get_resource(self) -> List[Document]:
        endpoint = STRIPE_ENDPOINTS.get(self.resource)
        if endpoint is None:
            return []
        return self._make_request(endpoint)


if endpoint is None:
            return []
        return self._make_request(endpoint)
[docs]    def load(self) -> List[Document]:
        return self._get_resource()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.airbyte_json
"""Loader that loads local airbyte json files."""
import json
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import stringify_dict
[docs]class AirbyteJSONLoader(BaseLoader):
    """Loader that loads local airbyte json files."""
    def __init__(self, file_path: str):
        """Initialize with file path. This should start with '/tmp/airbyte_local/'."""
        self.file_path = file_path
[docs]    def load(self) -> List[Document]:
        """Load file."""
        text = ""
        for line in open(self.file_path, "r"):
            data = json.loads(line)["_airbyte_data"]
            text += stringify_dict(data)
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.blackboard
"""Loader that loads all documents from a blackboard course."""
import contextlib
import re
from pathlib import Path
from typing import Any, List, Optional, Tuple
from urllib.parse import unquote
from langchain.docstore.document import Document
from langchain.document_loaders.directory import DirectoryLoader
from langchain.document_loaders.pdf import PyPDFLoader
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class BlackboardLoader(WebBaseLoader):
    """Loader that loads all documents from a Blackboard course.
    This loader is not compatible with all Blackboard courses. It is only
    compatible with courses that use the new Blackboard interface.
    To use this loader, you must have the BbRouter cookie. You can get this
    cookie by logging into the course and then copying the value of the
    BbRouter cookie from the browser's developer tools.
    Example:
        .. code-block:: python
            from langchain.document_loaders import BlackboardLoader
            loader = BlackboardLoader(
                blackboard_course_url="https://blackboard.example.com/webapps/blackboard/execute/announcement?method=search&context=course_entry&course_id=_123456_1",
                bbrouter="expires:12345...",
            )
            documents = loader.load()
    """
    base_url: str
    folder_path: str
    load_all_recursively: bool
    def __init__(
        self,
        blackboard_course_url: str,
        bbrouter: str,
        load_all_recursively: bool = True,
        basic_auth: Optional[Tuple[str, str]] = None,
        cookies: Optional[dict] = None,
    ):
        """Initialize with blackboard course url.


):
        """Initialize with blackboard course url.
        The BbRouter cookie is required for most blackboard courses.
        Args:
            blackboard_course_url: Blackboard course url.
            bbrouter: BbRouter cookie.
            load_all_recursively: If True, load all documents recursively.
            basic_auth: Basic auth credentials.
            cookies: Cookies.
        Raises:
            ValueError: If blackboard course url is invalid.
        """
        super().__init__(blackboard_course_url)
        # Get base url
        try:
            self.base_url = blackboard_course_url.split("/webapps/blackboard")[0]
        except IndexError:
            raise ValueError(
                "Invalid blackboard course url. "
                "Please provide a url that starts with "
                "https://<blackboard_url>/webapps/blackboard"
            )
        if basic_auth is not None:
            self.session.auth = basic_auth
        # Combine cookies
        if cookies is None:
            cookies = {}
        cookies.update({"BbRouter": bbrouter})
        self.session.cookies.update(cookies)
        self.load_all_recursively = load_all_recursively
        self.check_bs4()
[docs]    def check_bs4(self) -> None:
        """Check if BeautifulSoup4 is installed.
        Raises:
            ImportError: If BeautifulSoup4 is not installed.
        """
        try:
            import bs4  # noqa: F401
        except ImportError:
            raise ImportError(
                "BeautifulSoup4 is required for BlackboardLoader. "
                "Please install it with `pip install beautifulsoup4`."
            )
[docs]    def load(self) -> List[Document]:
        """Load data into document objects.
        Returns:
            List of documents.


"""Load data into document objects.
        Returns:
            List of documents.
        """
        if self.load_all_recursively:
            soup_info = self.scrape()
            self.folder_path = self._get_folder_path(soup_info)
            relative_paths = self._get_paths(soup_info)
            documents = []
            for path in relative_paths:
                url = self.base_url + path
                print(f"Fetching documents from {url}")
                soup_info = self._scrape(url)
                with contextlib.suppress(ValueError):
                    documents.extend(self._get_documents(soup_info))
            return documents
        else:
            print(f"Fetching documents from {self.web_path}")
            soup_info = self.scrape()
            self.folder_path = self._get_folder_path(soup_info)
            return self._get_documents(soup_info)
    def _get_folder_path(self, soup: Any) -> str:
        """Get the folder path to save the documents in.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            Folder path.
        """
        # Get the course name
        course_name = soup.find("span", {"id": "crumb_1"})
        if course_name is None:
            raise ValueError("No course name found.")
        course_name = course_name.text.strip()
        # Prepare the folder path
        course_name_clean = (
            unquote(course_name)
            .replace(" ", "_")
            .replace("/", "_")
            .replace(":", "_")
            .replace(",", "_")
            .replace("?", "_")
            .replace("'", "_")
            .replace("!", "_")
            .replace('"', "_")
        )
        # Get the folder path
        folder_path = Path(".") / course_name_clean


# Get the folder path
        folder_path = Path(".") / course_name_clean
        return str(folder_path)
    def _get_documents(self, soup: Any) -> List[Document]:
        """Fetch content from page and return Documents.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            List of documents.
        """
        attachments = self._get_attachments(soup)
        self._download_attachments(attachments)
        documents = self._load_documents()
        return documents
    def _get_attachments(self, soup: Any) -> List[str]:
        """Get all attachments from a page.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            List of attachments.
        """
        from bs4 import BeautifulSoup, Tag
        # Get content list
        content_list = soup.find("ul", {"class": "contentList"})
        if content_list is None:
            raise ValueError("No content list found.")
        content_list: BeautifulSoup  # type: ignore
        # Get all attachments
        attachments = []
        for attachment in content_list.find_all("ul", {"class": "attachments"}):
            attachment: Tag  # type: ignore
            for link in attachment.find_all("a"):
                link: Tag  # type: ignore
                href = link.get("href")
                # Only add if href is not None and does not start with #
                if href is not None and not href.startswith("#"):
                    attachments.append(href)
        return attachments
    def _download_attachments(self, attachments: List[str]) -> None:
        """Download all attachments.
        Args:
            attachments: List of attachments.
        """
        # Make sure the folder exists
        Path(self.folder_path).mkdir(parents=True, exist_ok=True)


Path(self.folder_path).mkdir(parents=True, exist_ok=True)
        # Download all attachments
        for attachment in attachments:
            self.download(attachment)
    def _load_documents(self) -> List[Document]:
        """Load all documents in the folder.
        Returns:
            List of documents.
        """
        # Create the document loader
        loader = DirectoryLoader(
            path=self.folder_path, glob="*.pdf", loader_cls=PyPDFLoader  # type: ignore
        )
        # Load the documents
        documents = loader.load()
        # Return all documents
        return documents
    def _get_paths(self, soup: Any) -> List[str]:
        """Get all relative paths in the navbar."""
        relative_paths = []
        course_menu = soup.find("ul", {"class": "courseMenu"})
        if course_menu is None:
            raise ValueError("No course menu found.")
        for link in course_menu.find_all("a"):
            href = link.get("href")
            if href is not None and href.startswith("/"):
                relative_paths.append(href)
        return relative_paths
[docs]    def download(self, path: str) -> None:
        """Download a file from a url.
        Args:
            path: Path to the file.
        """
        # Get the file content
        response = self.session.get(self.base_url + path, allow_redirects=True)
        # Get the filename
        filename = self.parse_filename(response.url)
        # Write the file to disk
        with open(Path(self.folder_path) / filename, "wb") as f:
            f.write(response.content)
[docs]    def parse_filename(self, url: str) -> str:
        """Parse the filename from a url.
        Args:


"""Parse the filename from a url.
        Args:
            url: Url to parse the filename from.
        Returns:
            The filename.
        """
        if (url_path := Path(url)) and url_path.suffix == ".pdf":
            return url_path.name
        else:
            return self._parse_filename_from_url(url)
    def _parse_filename_from_url(self, url: str) -> str:
        """Parse the filename from a url.
        Args:
            url: Url to parse the filename from.
        Returns:
            The filename.
        Raises:
            ValueError: If the filename could not be parsed.
        """
        filename_matches = re.search(r"filename%2A%3DUTF-8%27%27(.+)", url)
        if filename_matches:
            filename = filename_matches.group(1)
        else:
            raise ValueError(f"Could not parse filename from {url}")
        if ".pdf" not in filename:
            raise ValueError(f"Incorrect file type: {filename}")
        filename = filename.split(".pdf")[0] + ".pdf"
        filename = unquote(filename)
        filename = filename.replace("%20", " ")
        return filename
if __name__ == "__main__":
    loader = BlackboardLoader(
        "https://<YOUR BLACKBOARD URL"
        " HERE>/webapps/blackboard/content/listContent.jsp?course_id=_<YOUR COURSE ID"
        " HERE>_1&content_id=_<YOUR CONTENT ID HERE>_1&mode=reset",
        "<YOUR BBROUTER COOKIE HERE>",
        load_all_recursively=True,
    )
    documents = loader.load()
    print(f"Loaded {len(documents)} pages of PDFs from {loader.web_path}")
By Harrison Chase


By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.url
"""Loader that uses unstructured to load HTML files."""
import logging
from typing import Any, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class UnstructuredURLLoader(BaseLoader):
    """Loader that uses unstructured to load HTML files."""
    def __init__(
        self,
        urls: List[str],
        continue_on_failure: bool = True,
        mode: str = "single",
        **unstructured_kwargs: Any,
    ):
        """Initialize with file path."""
        try:
            import unstructured  # noqa:F401
            from unstructured.__version__ import __version__ as __unstructured_version__
            self.__version = __unstructured_version__
        except ImportError:
            raise ValueError(
                "unstructured package not found, please install it with "
                "`pip install unstructured`"
            )
        self._validate_mode(mode)
        self.mode = mode
        headers = unstructured_kwargs.pop("headers", {})
        if len(headers.keys()) != 0:
            warn_about_headers = False
            if self.__is_non_html_available():
                warn_about_headers = not self.__is_headers_available_for_non_html()
            else:
                warn_about_headers = not self.__is_headers_available_for_html()
            if warn_about_headers:
                logger.warning(
                    "You are using an old version of unstructured. "
                    "The headers parameter is ignored"
                )
        self.urls = urls
        self.continue_on_failure = continue_on_failure
        self.headers = headers
        self.unstructured_kwargs = unstructured_kwargs
    def _validate_mode(self, mode: str) -> None:


def _validate_mode(self, mode: str) -> None:
        _valid_modes = {"single", "elements"}
        if mode not in _valid_modes:
            raise ValueError(
                f"Got {mode} for `mode`, but should be one of `{_valid_modes}`"
            )
    def __is_headers_available_for_html(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 7)
    def __is_headers_available_for_non_html(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 13)
    def __is_non_html_available(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 12)
[docs]    def load(self) -> List[Document]:
        """Load file."""
        from unstructured.partition.auto import partition
        from unstructured.partition.html import partition_html
        docs: List[Document] = list()
        for url in self.urls:
            try:
                if self.__is_non_html_available():
                    if self.__is_headers_available_for_non_html():
                        elements = partition(
                            url=url, headers=self.headers, **self.unstructured_kwargs
                        )
                    else:
                        elements = partition(url=url, **self.unstructured_kwargs)
                else:


elements = partition(url=url, **self.unstructured_kwargs)
                else:
                    if self.__is_headers_available_for_html():
                        elements = partition_html(
                            url=url, headers=self.headers, **self.unstructured_kwargs
                        )
                    else:
                        elements = partition_html(url=url, **self.unstructured_kwargs)
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error fetching or processing {url}, exeption: {e}")
                    continue
                else:
                    raise e
            if self.mode == "single":
                text = "\n\n".join([str(el) for el in elements])
                metadata = {"source": url}
                docs.append(Document(page_content=text, metadata=metadata))
            elif self.mode == "elements":
                for element in elements:
                    metadata = element.metadata.to_dict()
                    metadata["category"] = element.category
                    docs.append(Document(page_content=str(element), metadata=metadata))
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.


Source code for langchain.document_loaders.gutenberg
"""Loader that loads .txt web files."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class GutenbergLoader(BaseLoader):
    """Loader that uses urllib to load .txt web files."""
    def __init__(self, file_path: str):
        """Initialize with file path."""
        if not file_path.startswith("https://www.gutenberg.org"):
            raise ValueError("file path must start with 'https://www.gutenberg.org'")
        if not file_path.endswith(".txt"):
            raise ValueError("file path must end with '.txt'")
        self.file_path = file_path
[docs]    def load(self) -> List[Document]:
        """Load file."""
        from urllib.request import urlopen
        elements = urlopen(self.file_path)
        text = "\n\n".join([str(el.decode("utf-8-sig")) for el in elements])
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 11, 2023.