Source code for langchain.document_loaders.notiondb
"""Notion DB loader for langchain"""
from typing import Any, Dict, List, Optional
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
NOTION_BASE_URL = "https://api.notion.com/v1"
DATABASE_URL = NOTION_BASE_URL + "/databases/{database_id}/query"
PAGE_URL = NOTION_BASE_URL + "/pages/{page_id}"
BLOCK_URL = NOTION_BASE_URL + "/blocks/{block_id}/children"
[docs]class NotionDBLoader(BaseLoader):
    """Notion DB Loader.
    Reads content from pages within a Noton Database.
    Args:
        integration_token (str): Notion integration token.
        database_id (str): Notion database id.
        request_timeout_sec (int): Timeout for Notion requests in seconds.
    """
    def __init__(
        self,
        integration_token: str,
        database_id: str,
        request_timeout_sec: Optional[int] = 10,
    ) -> None:
        """Initialize with parameters."""
        if not integration_token:
            raise ValueError("integration_token must be provided")
        if not database_id:
            raise ValueError("database_id must be provided")
        self.token = integration_token
        self.database_id = database_id
        self.headers = {
            "Authorization": "Bearer " + self.token,
            "Content-Type": "application/json",
            "Notion-Version": "2022-06-28",
        }
        self.request_timeout_sec = request_timeout_sec
[docs]    def load(self) -> List[Document]:
        """Load documents from the Notion database.
        Returns:
            List[Document]: List of documents.
        """


page_ids = self._retrieve_page_ids()
        return list(self.load_page(page_id) for page_id in page_ids)
    def _retrieve_page_ids(
        self, query_dict: Dict[str, Any] = {"page_size": 100}
    ) -> List[str]:
        """Get all the pages from a Notion database."""
        pages: List[Dict[str, Any]] = []
        while True:
            data = self._request(
                DATABASE_URL.format(database_id=self.database_id),
                method="POST",
                query_dict=query_dict,
            )
            pages.extend(data.get("results"))
            if not data.get("has_more"):
                break
            query_dict["start_cursor"] = data.get("next_cursor")
        page_ids = [page["id"] for page in pages]
        return page_ids
[docs]    def load_page(self, page_id: str) -> Document:
        """Read a page."""
        data = self._request(PAGE_URL.format(page_id=page_id))
        # load properties as metadata
        metadata: Dict[str, Any] = {}
        for prop_name, prop_data in data["properties"].items():
            prop_type = prop_data["type"]
            if prop_type == "rich_text":
                value = (
                    prop_data["rich_text"][0]["plain_text"]
                    if prop_data["rich_text"]
                    else None
                )
            elif prop_type == "title":
                value = (
                    prop_data["title"][0]["plain_text"] if prop_data["title"] else None
                )
            elif prop_type == "multi_select":
                value = (
                    [item["name"] for item in prop_data["multi_select"]]
                    if prop_data["multi_select"]
                    else []
                )


elif prop_type == "url":
                value = prop_data["url"]
            else:
                value = None
            metadata[prop_name.lower()] = value
        metadata["id"] = page_id
        return Document(page_content=self._load_blocks(page_id), metadata=metadata)
    def _load_blocks(self, block_id: str, num_tabs: int = 0) -> str:
        """Read a block and its children."""
        result_lines_arr: List[str] = []
        cur_block_id: str = block_id
        while cur_block_id:
            data = self._request(BLOCK_URL.format(block_id=cur_block_id))
            for result in data["results"]:
                result_obj = result[result["type"]]
                if "rich_text" not in result_obj:
                    continue
                cur_result_text_arr: List[str] = []
                for rich_text in result_obj["rich_text"]:
                    if "text" in rich_text:
                        cur_result_text_arr.append(
                            "\t" * num_tabs + rich_text["text"]["content"]
                        )
                if result["has_children"]:
                    children_text = self._load_blocks(
                        result["id"], num_tabs=num_tabs + 1
                    )
                    cur_result_text_arr.append(children_text)
                result_lines_arr.append("\n".join(cur_result_text_arr))
            cur_block_id = data.get("next_cursor")
        return "\n".join(result_lines_arr)
    def _request(
        self, url: str, method: str = "GET", query_dict: Dict[str, Any] = {}
    ) -> Any:
        res = requests.request(
            method,
            url,
            headers=self.headers,
            json=query_dict,
            timeout=self.request_timeout_sec,
        )


res.raise_for_status()
        return res.json()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.psychic
"""Loader that loads documents from Psychic.dev."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class PsychicLoader(BaseLoader):
    """Loader that loads documents from Psychic.dev."""
    def __init__(self, api_key: str, connector_id: str, connection_id: str):
        """Initialize with API key, connector id, and connection id."""
        try:
            from psychicapi import ConnectorId, Psychic  # noqa: F401
        except ImportError:
            raise ImportError(
                "`psychicapi` package not found, please run `pip install psychicapi`"
            )
        self.psychic = Psychic(secret_key=api_key)
        self.connector_id = ConnectorId(connector_id)
        self.connection_id = connection_id
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        psychic_docs = self.psychic.get_documents(self.connector_id, self.connection_id)
        return [
            Document(
                page_content=doc["content"],
                metadata={"title": doc["title"], "source": doc["uri"]},
            )
            for doc in psychic_docs
        ]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.hugging_face_dataset
"""Loader that loads HuggingFace datasets."""
from typing import Iterator, List, Mapping, Optional, Sequence, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class HuggingFaceDatasetLoader(BaseLoader):
    """Loading logic for loading documents from the Hugging Face Hub."""
    def __init__(
        self,
        path: str,
        page_content_column: str = "text",
        name: Optional[str] = None,
        data_dir: Optional[str] = None,
        data_files: Optional[
            Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]
        ] = None,
        cache_dir: Optional[str] = None,
        keep_in_memory: Optional[bool] = None,
        save_infos: bool = False,
        use_auth_token: Optional[Union[bool, str]] = None,
        num_proc: Optional[int] = None,
    ):
        """Initialize the HuggingFaceDatasetLoader.
        Args:
            path: Path or name of the dataset.
            page_content_column: Page content column name.
            name: Name of the dataset configuration.
            data_dir: Data directory of the dataset configuration.
            data_files: Path(s) to source data file(s).
            cache_dir: Directory to read/write data.
            keep_in_memory: Whether to copy the dataset in-memory.
            save_infos: Save the dataset information (checksums/size/splits/...).
            use_auth_token: Bearer token for remote files on the Datasets Hub.
            num_proc: Number of processes.
        """
        self.path = path
        self.page_content_column = page_content_column
        self.name = name


self.data_dir = data_dir
        self.data_files = data_files
        self.cache_dir = cache_dir
        self.keep_in_memory = keep_in_memory
        self.save_infos = save_infos
        self.use_auth_token = use_auth_token
        self.num_proc = num_proc
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Load documents lazily."""
        try:
            from datasets import load_dataset
        except ImportError:
            raise ImportError(
                "Could not import datasets python package. "
                "Please install it with `pip install datasets`."
            )
        dataset = load_dataset(
            path=self.path,
            name=self.name,
            data_dir=self.data_dir,
            data_files=self.data_files,
            cache_dir=self.cache_dir,
            keep_in_memory=self.keep_in_memory,
            save_infos=self.save_infos,
            use_auth_token=self.use_auth_token,
            num_proc=self.num_proc,
        )
        yield from (
            Document(
                page_content=row.pop(self.page_content_column),
                metadata=row,
            )
            for key in dataset.keys()
            for row in dataset[key]
        )
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        return list(self.lazy_load())
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.tomarkdown
"""Loader that loads HTML to markdown using 2markdown."""
from __future__ import annotations
from typing import Iterator, List
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class ToMarkdownLoader(BaseLoader):
    """Loader that loads HTML to markdown using 2markdown."""
    def __init__(self, url: str, api_key: str):
        """Initialize with url and api key."""
        self.url = url
        self.api_key = api_key
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Lazily load the file."""
        response = requests.post(
            "https://2markdown.com/api/2md",
            headers={"X-Api-Key": self.api_key},
            json={"url": self.url},
        )
        text = response.json()["article"]
        metadata = {"source": self.url}
        yield Document(page_content=text, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        """Load file."""
        return list(self.lazy_load())
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.joplin
import json
import urllib
from datetime import datetime
from typing import Iterator, List, Optional
from langchain.document_loaders.base import BaseLoader
from langchain.schema import Document
from langchain.utils import get_from_env
LINK_NOTE_TEMPLATE = "joplin://x-callback-url/openNote?id={id}"
[docs]class JoplinLoader(BaseLoader):
    """
    Loader that fetches notes from Joplin.
    In order to use this loader, you need to have Joplin running with the
    Web Clipper enabled (look for "Web Clipper" in the app settings).
    To get the access token, you need to go to the Web Clipper options and
    under "Advanced Options" you will find the access token.
    You can find more information about the Web Clipper service here:
    https://joplinapp.org/clipper/
    """
    def __init__(
        self,
        access_token: Optional[str] = None,
        port: int = 41184,
        host: str = "localhost",
    ) -> None:
        access_token = access_token or get_from_env(
            "access_token", "JOPLIN_ACCESS_TOKEN"
        )
        base_url = f"http://{host}:{port}"
        self._get_note_url = (
            f"{base_url}/notes?token={access_token}"
            f"&fields=id,parent_id,title,body,created_time,updated_time&page={{page}}"
        )
        self._get_folder_url = (
            f"{base_url}/folders/{{id}}?token={access_token}&fields=title"
        )
        self._get_tag_url = (


f"{base_url}/notes/{{id}}/tags?token={access_token}&fields=title"
        )
    def _get_notes(self) -> Iterator[Document]:
        has_more = True
        page = 1
        while has_more:
            req_note = urllib.request.Request(self._get_note_url.format(page=page))
            with urllib.request.urlopen(req_note) as response:
                json_data = json.loads(response.read().decode())
                for note in json_data["items"]:
                    metadata = {
                        "source": LINK_NOTE_TEMPLATE.format(id=note["id"]),
                        "folder": self._get_folder(note["parent_id"]),
                        "tags": self._get_tags(note["id"]),
                        "title": note["title"],
                        "created_time": self._convert_date(note["created_time"]),
                        "updated_time": self._convert_date(note["updated_time"]),
                    }
                    yield Document(page_content=note["body"], metadata=metadata)
                has_more = json_data["has_more"]
                page += 1
    def _get_folder(self, folder_id: str) -> str:
        req_folder = urllib.request.Request(self._get_folder_url.format(id=folder_id))
        with urllib.request.urlopen(req_folder) as response:
            json_data = json.loads(response.read().decode())
            return json_data["title"]
    def _get_tags(self, note_id: str) -> List[str]:
        req_tag = urllib.request.Request(self._get_tag_url.format(id=note_id))
        with urllib.request.urlopen(req_tag) as response:
            json_data = json.loads(response.read().decode())
            return [tag["title"] for tag in json_data["items"]]
    def _convert_date(self, date: int) -> str:


return datetime.fromtimestamp(date / 1000).strftime("%Y-%m-%d %H:%M:%S")
[docs]    def lazy_load(self) -> Iterator[Document]:
        yield from self._get_notes()
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.bibtex
import logging
import re
from pathlib import Path
from typing import Any, Iterator, List, Mapping, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utilities.bibtex import BibtexparserWrapper
logger = logging.getLogger(__name__)
[docs]class BibtexLoader(BaseLoader):
    """Loads a bibtex file into a list of Documents.
    Each document represents one entry from the bibtex file.
    If a PDF file is present in the `file` bibtex field, the original PDF
    is loaded into the document text. If no such file entry is present,
    the `abstract` field is used instead.
    """
    def __init__(
        self,
        file_path: str,
        *,
        parser: Optional[BibtexparserWrapper] = None,
        max_docs: Optional[int] = None,
        max_content_chars: Optional[int] = 4_000,
        load_extra_metadata: bool = False,
        file_pattern: str = r"[^:]+\.pdf",
    ):
        """Initialize the BibtexLoader.
        Args:
            file_path: Path to the bibtex file.
            max_docs: Max number of associated documents to load. Use -1 means
                           no limit.
        """
        self.file_path = file_path
        self.parser = parser or BibtexparserWrapper()
        self.max_docs = max_docs
        self.max_content_chars = max_content_chars
        self.load_extra_metadata = load_extra_metadata
        self.file_regex = re.compile(file_pattern)
    def _load_entry(self, entry: Mapping[str, Any]) -> Optional[Document]:
        import fitz


parent_dir = Path(self.file_path).parent
        # regex is useful for Zotero flavor bibtex files
        file_names = self.file_regex.findall(entry.get("file", ""))
        if not file_names:
            return None
        texts: List[str] = []
        for file_name in file_names:
            try:
                with fitz.open(parent_dir / file_name) as f:
                    texts.extend(page.get_text() for page in f)
            except FileNotFoundError as e:
                logger.debug(e)
        content = "\n".join(texts) or entry.get("abstract", "")
        if self.max_content_chars:
            content = content[: self.max_content_chars]
        metadata = self.parser.get_metadata(entry, load_extra=self.load_extra_metadata)
        return Document(
            page_content=content,
            metadata=metadata,
        )
[docs]    def lazy_load(self) -> Iterator[Document]:
        """Load bibtex file using bibtexparser and get the article texts plus the
        article metadata.
        See https://bibtexparser.readthedocs.io/en/master/
        Returns:
            a list of documents with the document.page_content in text format
        """
        try:
            import fitz  # noqa: F401
        except ImportError:
            raise ImportError(
                "PyMuPDF package not found, please install it with "
                "`pip install pymupdf`"
            )
        entries = self.parser.load_bibtex_entries(self.file_path)
        if self.max_docs:
            entries = entries[: self.max_docs]
        for entry in entries:
            doc = self._load_entry(entry)
            if doc:
                yield doc
[docs]    def load(self) -> List[Document]:


"""Load bibtex file documents from the given bibtex file path.
        See https://bibtexparser.readthedocs.io/en/master/
        Args:
            file_path: the path to the bibtex file
        Returns:
            a list of documents with the document.page_content in text format
        """
        return list(self.lazy_load())
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.image_captions
"""
Loader that loads image captions
By default, the loader utilizes the pre-trained BLIP image captioning model.
https://huggingface.co/Salesforce/blip-image-captioning-base
"""
from typing import Any, List, Tuple, Union
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class ImageCaptionLoader(BaseLoader):
    """Loader that loads the captions of an image"""
    def __init__(
        self,
        path_images: Union[str, List[str]],
        blip_processor: str = "Salesforce/blip-image-captioning-base",
        blip_model: str = "Salesforce/blip-image-captioning-base",
    ):
        """
        Initialize with a list of image paths
        """
        if isinstance(path_images, str):
            self.image_paths = [path_images]
        else:
            self.image_paths = path_images
        self.blip_processor = blip_processor
        self.blip_model = blip_model
[docs]    def load(self) -> List[Document]:
        """
        Load from a list of image files
        """
        try:
            from transformers import BlipForConditionalGeneration, BlipProcessor
        except ImportError:
            raise ImportError(
                "`transformers` package not found, please install with "
                "`pip install transformers`."
            )
        processor = BlipProcessor.from_pretrained(self.blip_processor)
        model = BlipForConditionalGeneration.from_pretrained(self.blip_model)
        results = []
        for path_image in self.image_paths:
            caption, metadata = self._get_captions_and_metadata(
                model=model, processor=processor, path_image=path_image
            )


doc = Document(page_content=caption, metadata=metadata)
            results.append(doc)
        return results
    def _get_captions_and_metadata(
        self, model: Any, processor: Any, path_image: str
    ) -> Tuple[str, dict]:
        """
        Helper function for getting the captions and metadata of an image
        """
        try:
            from PIL import Image
        except ImportError:
            raise ImportError(
                "`PIL` package not found, please install with `pip install pillow`"
            )
        try:
            if path_image.startswith("http://") or path_image.startswith("https://"):
                image = Image.open(requests.get(path_image, stream=True).raw).convert(
                    "RGB"
                )
            else:
                image = Image.open(path_image).convert("RGB")
        except Exception:
            raise ValueError(f"Could not get image data for {path_image}")
        inputs = processor(image, "an image of", return_tensors="pt")
        output = model.generate(**inputs)
        caption: str = processor.decode(output[0])
        metadata: dict = {"image_path": path_image}
        return caption, metadata
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.csv_loader
import csv
from typing import Dict, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class CSVLoader(BaseLoader):
    """Loads a CSV file into a list of documents.
    Each document represents one row of the CSV file. Every row is converted into a
    key/value pair and outputted to a new line in the document's page_content.
    The source for each document loaded from csv is set to the value of the
    `file_path` argument for all doucments by default.
    You can override this by setting the `source_column` argument to the
    name of a column in the CSV file.
    The source of each document will then be set to the value of the column
    with the name specified in `source_column`.
    Output Example:
        .. code-block:: txt
            column1: value1
            column2: value2
            column3: value3
    """
    def __init__(
        self,
        file_path: str,
        source_column: Optional[str] = None,
        csv_args: Optional[Dict] = None,
        encoding: Optional[str] = None,
    ):
        self.file_path = file_path
        self.source_column = source_column
        self.encoding = encoding
        self.csv_args = csv_args or {}
[docs]    def load(self) -> List[Document]:
        """Load data into document objects."""
        docs = []
        with open(self.file_path, newline="", encoding=self.encoding) as csvfile:
            csv_reader = csv.DictReader(csvfile, **self.csv_args)  # type: ignore
            for i, row in enumerate(csv_reader):


content = "\n".join(f"{k.strip()}: {v.strip()}" for k, v in row.items())
                try:
                    source = (
                        row[self.source_column]
                        if self.source_column is not None
                        else self.file_path
                    )
                except KeyError:
                    raise ValueError(
                        f"Source column '{self.source_column}' not found in CSV file."
                    )
                metadata = {"source": source, "row": i}
                doc = Document(page_content=content, metadata=metadata)
                docs.append(doc)
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.s3_file
"""Loading logic for loading documents from an s3 file."""
import os
import tempfile
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class S3FileLoader(BaseLoader):
    """Loading logic for loading documents from s3."""
    def __init__(self, bucket: str, key: str):
        """Initialize with bucket and key name."""
        self.bucket = bucket
        self.key = key
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            import boto3
        except ImportError:
            raise ImportError(
                "Could not import `boto3` python package. "
                "Please install it with `pip install boto3`."
            )
        s3 = boto3.client("s3")
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}/{self.key}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            s3.download_file(self.bucket, self.key, file_path)
            loader = UnstructuredFileLoader(file_path)
            return loader.load()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.googledrive
"""Loader that loads data from Google Drive."""
# Prerequisites:
# 1. Create a Google Cloud project
# 2. Enable the Google Drive API:
#   https://console.cloud.google.com/flows/enableapi?apiid=drive.googleapis.com
# 3. Authorize credentials for desktop app:
#   https://developers.google.com/drive/api/quickstart/python#authorize_credentials_for_a_desktop_application # noqa: E501
# 4. For service accounts visit
#   https://cloud.google.com/iam/docs/service-accounts-create
from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence, Union
from pydantic import BaseModel, root_validator, validator
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
SCOPES = ["https://www.googleapis.com/auth/drive.readonly"]
[docs]class GoogleDriveLoader(BaseLoader, BaseModel):
    """Loader that loads Google Docs from Google Drive."""
    service_account_key: Path = Path.home() / ".credentials" / "keys.json"
    credentials_path: Path = Path.home() / ".credentials" / "credentials.json"
    token_path: Path = Path.home() / ".credentials" / "token.json"
    folder_id: Optional[str] = None
    document_ids: Optional[List[str]] = None
    file_ids: Optional[List[str]] = None
    recursive: bool = False
    file_types: Optional[Sequence[str]] = None
    load_trashed_files: bool = False
    @root_validator
    def validate_inputs(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Validate that either folder_id or document_ids is set, but not both."""
        if values.get("folder_id") and (


values.get("document_ids") or values.get("file_ids")
        ):
            raise ValueError(
                "Cannot specify both folder_id and document_ids nor "
                "folder_id and file_ids"
            )
        if (
            not values.get("folder_id")
            and not values.get("document_ids")
            and not values.get("file_ids")
        ):
            raise ValueError("Must specify either folder_id, document_ids, or file_ids")
        file_types = values.get("file_types")
        if file_types:
            if values.get("document_ids") or values.get("file_ids"):
                raise ValueError(
                    "file_types can only be given when folder_id is given,"
                    " (not when document_ids or file_ids are given)."
                )
            type_mapping = {
                "document": "application/vnd.google-apps.document",
                "sheet": "application/vnd.google-apps.spreadsheet",
                "pdf": "application/pdf",
            }
            allowed_types = list(type_mapping.keys()) + list(type_mapping.values())
            short_names = ", ".join([f"'{x}'" for x in type_mapping.keys()])
            full_names = ", ".join([f"'{x}'" for x in type_mapping.values()])
            for file_type in file_types:
                if file_type not in allowed_types:
                    raise ValueError(
                        f"Given file type {file_type} is not supported. "
                        f"Supported values are: {short_names}; and "
                        f"their full-form names: {full_names}"
                    )
            # replace short-form file types by full-form file types
            def full_form(x: str) -> str:
                return type_mapping[x] if x in type_mapping else x


values["file_types"] = [full_form(file_type) for file_type in file_types]
        return values
    @validator("credentials_path")
    def validate_credentials_path(cls, v: Any, **kwargs: Any) -> Any:
        """Validate that credentials_path exists."""
        if not v.exists():
            raise ValueError(f"credentials_path {v} does not exist")
        return v
    def _load_credentials(self) -> Any:
        """Load credentials."""
        # Adapted from https://developers.google.com/drive/api/v3/quickstart/python
        try:
            from google.auth.transport.requests import Request
            from google.oauth2 import service_account
            from google.oauth2.credentials import Credentials
            from google_auth_oauthlib.flow import InstalledAppFlow
        except ImportError:
            raise ImportError(
                "You must run "
                "`pip install --upgrade "
                "google-api-python-client google-auth-httplib2 "
                "google-auth-oauthlib` "
                "to use the Google Drive loader."
            )
        creds = None
        if self.service_account_key.exists():
            return service_account.Credentials.from_service_account_file(
                str(self.service_account_key), scopes=SCOPES
            )
        if self.token_path.exists():
            creds = Credentials.from_authorized_user_file(str(self.token_path), SCOPES)
        if not creds or not creds.valid:
            if creds and creds.expired and creds.refresh_token:
                creds.refresh(Request())
            else:
                flow = InstalledAppFlow.from_client_secrets_file(
                    str(self.credentials_path), SCOPES
                )
                creds = flow.run_local_server(port=0)
            with open(self.token_path, "w") as token:
                token.write(creds.to_json())
        return creds


def _load_sheet_from_id(self, id: str) -> List[Document]:
        """Load a sheet and all tabs from an ID."""
        from googleapiclient.discovery import build
        creds = self._load_credentials()
        sheets_service = build("sheets", "v4", credentials=creds)
        spreadsheet = sheets_service.spreadsheets().get(spreadsheetId=id).execute()
        sheets = spreadsheet.get("sheets", [])
        documents = []
        for sheet in sheets:
            sheet_name = sheet["properties"]["title"]
            result = (
                sheets_service.spreadsheets()
                .values()
                .get(spreadsheetId=id, range=sheet_name)
                .execute()
            )
            values = result.get("values", [])
            header = values[0]
            for i, row in enumerate(values[1:], start=1):
                metadata = {
                    "source": (
                        f"https://docs.google.com/spreadsheets/d/{id}/"
                        f"edit?gid={sheet['properties']['sheetId']}"
                    ),
                    "title": f"{spreadsheet['properties']['title']} - {sheet_name}",
                    "row": i,
                }
                content = []
                for j, v in enumerate(row):
                    title = header[j].strip() if len(header) > j else ""
                    content.append(f"{title}: {v.strip()}")
                page_content = "\n".join(content)
                documents.append(Document(page_content=page_content, metadata=metadata))
        return documents
    def _load_document_from_id(self, id: str) -> Document:
        """Load a document from an ID."""
        from io import BytesIO
        from googleapiclient.discovery import build


from googleapiclient.errors import HttpError
        from googleapiclient.http import MediaIoBaseDownload
        creds = self._load_credentials()
        service = build("drive", "v3", credentials=creds)
        file = service.files().get(fileId=id, supportsAllDrives=True).execute()
        request = service.files().export_media(fileId=id, mimeType="text/plain")
        fh = BytesIO()
        downloader = MediaIoBaseDownload(fh, request)
        done = False
        try:
            while done is False:
                status, done = downloader.next_chunk()
        except HttpError as e:
            if e.resp.status == 404:
                print("File not found: {}".format(id))
            else:
                print("An error occurred: {}".format(e))
        text = fh.getvalue().decode("utf-8")
        metadata = {
            "source": f"https://docs.google.com/document/d/{id}/edit",
            "title": f"{file.get('name')}",
        }
        return Document(page_content=text, metadata=metadata)
    def _load_documents_from_folder(
        self, folder_id: str, *, file_types: Optional[Sequence[str]] = None
    ) -> List[Document]:
        """Load documents from a folder."""
        from googleapiclient.discovery import build
        creds = self._load_credentials()
        service = build("drive", "v3", credentials=creds)
        files = self._fetch_files_recursive(service, folder_id)
        # If file types filter is provided, we'll filter by the file type.
        if file_types:
            _files = [f for f in files if f["mimeType"] in file_types]  # type: ignore
        else:
            _files = files


returns = []
        for file in files:
            if file["trashed"] and not self.load_trashed_files:
                continue
            elif file["mimeType"] == "application/vnd.google-apps.document":
                returns.append(self._load_document_from_id(file["id"]))  # type: ignore
            elif file["mimeType"] == "application/vnd.google-apps.spreadsheet":
                returns.extend(self._load_sheet_from_id(file["id"]))  # type: ignore
            elif file["mimeType"] == "application/pdf":
                returns.extend(self._load_file_from_id(file["id"]))  # type: ignore
            else:
                pass
        return returns
    def _fetch_files_recursive(
        self, service: Any, folder_id: str
    ) -> List[Dict[str, Union[str, List[str]]]]:
        """Fetch all files and subfolders recursively."""
        results = (
            service.files()
            .list(
                q=f"'{folder_id}' in parents",
                pageSize=1000,
                includeItemsFromAllDrives=True,
                supportsAllDrives=True,
                fields="nextPageToken, files(id, name, mimeType, parents, trashed)",
            )
            .execute()
        )
        files = results.get("files", [])
        returns = []
        for file in files:
            if file["mimeType"] == "application/vnd.google-apps.folder":
                if self.recursive:
                    returns.extend(self._fetch_files_recursive(service, file["id"]))
            else:
                returns.append(file)
        return returns
    def _load_documents_from_ids(self) -> List[Document]:
        """Load documents from a list of IDs."""
        if not self.document_ids:
            raise ValueError("document_ids must be set")


return [self._load_document_from_id(doc_id) for doc_id in self.document_ids]
    def _load_file_from_id(self, id: str) -> List[Document]:
        """Load a file from an ID."""
        from io import BytesIO
        from googleapiclient.discovery import build
        from googleapiclient.http import MediaIoBaseDownload
        creds = self._load_credentials()
        service = build("drive", "v3", credentials=creds)
        file = service.files().get(fileId=id, supportsAllDrives=True).execute()
        request = service.files().get_media(fileId=id)
        fh = BytesIO()
        downloader = MediaIoBaseDownload(fh, request)
        done = False
        while done is False:
            status, done = downloader.next_chunk()
        content = fh.getvalue()
        from PyPDF2 import PdfReader
        pdf_reader = PdfReader(BytesIO(content))
        return [
            Document(
                page_content=page.extract_text(),
                metadata={
                    "source": f"https://drive.google.com/file/d/{id}/view",
                    "title": f"{file.get('name')}",
                    "page": i,
                },
            )
            for i, page in enumerate(pdf_reader.pages)
        ]
    def _load_file_from_ids(self) -> List[Document]:
        """Load files from a list of IDs."""
        if not self.file_ids:
            raise ValueError("file_ids must be set")
        docs = []
        for file_id in self.file_ids:
            docs.extend(self._load_file_from_id(file_id))
        return docs
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        if self.folder_id:


return self._load_documents_from_folder(
                self.folder_id, file_types=self.file_types
            )
        elif self.document_ids:
            return self._load_documents_from_ids()
        else:
            return self._load_file_from_ids()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.markdown
"""Loader that loads Markdown files."""
from typing import List
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class UnstructuredMarkdownLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load markdown files."""
    def _get_elements(self) -> List:
        from unstructured.__version__ import __version__ as __unstructured_version__
        from unstructured.partition.md import partition_md
        # NOTE(MthwRobinson) - enables the loader to work when you're using pre-release
        # versions of unstructured like 0.4.17-dev1
        _unstructured_version = __unstructured_version__.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        if unstructured_version < (0, 4, 16):
            raise ValueError(
                f"You are on unstructured version {__unstructured_version__}. "
                "Partitioning markdown files is only supported in unstructured>=0.4.16."
            )
        return partition_md(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.apify_dataset
"""Logic for loading documents from Apify datasets."""
from typing import Any, Callable, Dict, List
from pydantic import BaseModel, root_validator
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class ApifyDatasetLoader(BaseLoader, BaseModel):
    """Logic for loading documents from Apify datasets."""
    apify_client: Any
    dataset_id: str
    """The ID of the dataset on the Apify platform."""
    dataset_mapping_function: Callable[[Dict], Document]
    """A custom function that takes a single dictionary (an Apify dataset item)
     and converts it to an instance of the Document class."""
    def __init__(
        self, dataset_id: str, dataset_mapping_function: Callable[[Dict], Document]
    ):
        """Initialize the loader with an Apify dataset ID and a mapping function.
        Args:
            dataset_id (str): The ID of the dataset on the Apify platform.
            dataset_mapping_function (Callable): A function that takes a single
                dictionary (an Apify dataset item) and converts it to an instance
                of the Document class.
        """
        super().__init__(
            dataset_id=dataset_id, dataset_mapping_function=dataset_mapping_function
        )
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate environment."""
        try:
            from apify_client import ApifyClient
            values["apify_client"] = ApifyClient()
        except ImportError:
            raise ImportError(
                "Could not import apify-client Python package. "
                "Please install it with `pip install apify-client`."
            )
        return values


[docs]    def load(self) -> List[Document]:
        """Load documents."""
        dataset_items = self.apify_client.dataset(self.dataset_id).list_items().items
        return list(map(self.dataset_mapping_function, dataset_items))
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.mediawikidump
"""Load Data from a MediaWiki dump xml."""
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class MWDumpLoader(BaseLoader):
    """
    Load MediaWiki dump from XML file
    Example:
        .. code-block:: python
            from langchain.document_loaders import MWDumpLoader
            loader = MWDumpLoader(
                file_path="myWiki.xml",
                encoding="utf8"
            )
            docs = loader.load()
            from langchain.text_splitter import RecursiveCharacterTextSplitter
            text_splitter = RecursiveCharacterTextSplitter(
                chunk_size=1000, chunk_overlap=0
            )
            texts = text_splitter.split_documents(docs)
    :param file_path: XML local file path
    :type file_path: str
    :param encoding: Charset encoding, defaults to "utf8"
    :type encoding: str, optional
    """
    def __init__(self, file_path: str, encoding: Optional[str] = "utf8"):
        """Initialize with file path."""
        self.file_path = file_path
        self.encoding = encoding
[docs]    def load(self) -> List[Document]:
        """Load from file path."""
        import mwparserfromhell
        import mwxml
        dump = mwxml.Dump.from_file(open(self.file_path, encoding=self.encoding))
        docs = []
        for page in dump.pages:
            for revision in page:
                code = mwparserfromhell.parse(revision.text)
                text = code.strip_code(
                    normalize=True, collapse=True, keep_template_params=False
                )
                metadata = {"source": page.title}


docs.append(Document(page_content=text, metadata=metadata))
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.facebook_chat
"""Loader that loads Facebook chat json dump."""
import datetime
import json
from pathlib import Path
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
def concatenate_rows(row: dict) -> str:
    """Combine message information in a readable format ready to be used."""
    sender = row["sender_name"]
    text = row["content"]
    date = datetime.datetime.fromtimestamp(row["timestamp_ms"] / 1000).strftime(
        "%Y-%m-%d %H:%M:%S"
    )
    return f"{sender} on {date}: {text}\n\n"
[docs]class FacebookChatLoader(BaseLoader):
    """Loader that loads Facebook messages json directory dump."""
    def __init__(self, path: str):
        """Initialize with path."""
        self.file_path = path
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        p = Path(self.file_path)
        with open(p, encoding="utf8") as f:
            d = json.load(f)
        text = "".join(
            concatenate_rows(message)
            for message in d["messages"]
            if message.get("content") and isinstance(message["content"], str)
        )
        metadata = {"source": str(p)}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.gitbook
"""Loader that loads GitBook."""
from typing import Any, List, Optional
from urllib.parse import urljoin, urlparse
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class GitbookLoader(WebBaseLoader):
    """Load GitBook data.
    1. load from either a single page, or
    2. load all (relative) paths in the navbar.
    """
    def __init__(
        self,
        web_page: str,
        load_all_paths: bool = False,
        base_url: Optional[str] = None,
        content_selector: str = "main",
    ):
        """Initialize with web page and whether to load all paths.
        Args:
            web_page: The web page to load or the starting point from where
                relative paths are discovered.
            load_all_paths: If set to True, all relative paths in the navbar
                are loaded instead of only `web_page`.
            base_url: If `load_all_paths` is True, the relative paths are
                appended to this base url. Defaults to `web_page` if not set.
        """
        self.base_url = base_url or web_page
        if self.base_url.endswith("/"):
            self.base_url = self.base_url[:-1]
        if load_all_paths:
            # set web_path to the sitemap if we want to crawl all paths
            web_paths = f"{self.base_url}/sitemap.xml"
        else:
            web_paths = web_page
        super().__init__(web_paths)
        self.load_all_paths = load_all_paths
        self.content_selector = content_selector
[docs]    def load(self) -> List[Document]:


"""Fetch text from one single GitBook page."""
        if self.load_all_paths:
            soup_info = self.scrape()
            relative_paths = self._get_paths(soup_info)
            documents = []
            for path in relative_paths:
                url = urljoin(self.base_url, path)
                print(f"Fetching text from {url}")
                soup_info = self._scrape(url)
                documents.append(self._get_document(soup_info, url))
            return [d for d in documents if d]
        else:
            soup_info = self.scrape()
            documents = [self._get_document(soup_info, self.web_path)]
            return [d for d in documents if d]
    def _get_document(
        self, soup: Any, custom_url: Optional[str] = None
    ) -> Optional[Document]:
        """Fetch content from page and return Document."""
        page_content_raw = soup.find(self.content_selector)
        if not page_content_raw:
            return None
        content = page_content_raw.get_text(separator="\n").strip()
        title_if_exists = page_content_raw.find("h1")
        title = title_if_exists.text if title_if_exists else ""
        metadata = {"source": custom_url or self.web_path, "title": title}
        return Document(page_content=content, metadata=metadata)
    def _get_paths(self, soup: Any) -> List[str]:
        """Fetch all relative paths in the navbar."""
        return [urlparse(loc.text).path for loc in soup.find_all("loc")]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.blockchain
import os
import re
import time
from enum import Enum
from typing import List, Optional
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
class BlockchainType(Enum):
    ETH_MAINNET = "eth-mainnet"
    ETH_GOERLI = "eth-goerli"
    POLYGON_MAINNET = "polygon-mainnet"
    POLYGON_MUMBAI = "polygon-mumbai"
[docs]class BlockchainDocumentLoader(BaseLoader):
    """Loads elements from a blockchain smart contract into Langchain documents.
    The supported blockchains are: Ethereum mainnet, Ethereum Goerli testnet,
    Polygon mainnet, and Polygon Mumbai testnet.
    If no BlockchainType is specified, the default is Ethereum mainnet.
    The Loader uses the Alchemy API to interact with the blockchain.
    ALCHEMY_API_KEY environment variable must be set to use this loader.
    The API returns 100 NFTs per request and can be paginated using the
    startToken parameter.
    If get_all_tokens is set to True, the loader will get all tokens
    on the contract.  Note that for contracts with a large number of tokens,
    this may take a long time (e.g. 10k tokens is 100 requests).
    Default value is false for this reason.
    The max_execution_time (sec) can be set to limit the execution time
    of the loader.
    Future versions of this loader can:
        - Support additional Alchemy APIs (e.g. getTransactions, etc.)
        - Support additional blockain APIs (e.g. Infura, Opensea, etc.)
    """
    def __init__(
        self,
        contract_address: str,


blockchainType: BlockchainType = BlockchainType.ETH_MAINNET,
        api_key: str = "docs-demo",
        startToken: str = "",
        get_all_tokens: bool = False,
        max_execution_time: Optional[int] = None,
    ):
        self.contract_address = contract_address
        self.blockchainType = blockchainType.value
        self.api_key = os.environ.get("ALCHEMY_API_KEY") or api_key
        self.startToken = startToken
        self.get_all_tokens = get_all_tokens
        self.max_execution_time = max_execution_time
        if not self.api_key:
            raise ValueError("Alchemy API key not provided.")
        if not re.match(r"^0x[a-fA-F0-9]{40}$", self.contract_address):
            raise ValueError(f"Invalid contract address {self.contract_address}")
[docs]    def load(self) -> List[Document]:
        result = []
        current_start_token = self.startToken
        start_time = time.time()
        while True:
            url = (
                f"https://{self.blockchainType}.g.alchemy.com/nft/v2/"
                f"{self.api_key}/getNFTsForCollection?withMetadata="
                f"True&contractAddress={self.contract_address}"
                f"&startToken={current_start_token}"
            )
            response = requests.get(url)
            if response.status_code != 200:
                raise ValueError(
                    f"Request failed with status code {response.status_code}"
                )
            items = response.json()["nfts"]
            if not items:
                break
            for item in items:
                content = str(item)
                tokenId = item["id"]["tokenId"]
                metadata = {
                    "source": self.contract_address,
                    "blockchain": self.blockchainType,


"tokenId": tokenId,
                }
                result.append(Document(page_content=content, metadata=metadata))
            # exit after the first API call if get_all_tokens is False
            if not self.get_all_tokens:
                break
            # get the start token for the next API call from the last item in array
            current_start_token = self._get_next_tokenId(result[-1].metadata["tokenId"])
            if (
                self.max_execution_time is not None
                and (time.time() - start_time) > self.max_execution_time
            ):
                raise RuntimeError("Execution time exceeded the allowed time limit.")
        if not result:
            raise ValueError(
                f"No NFTs found for contract address {self.contract_address}"
            )
        return result
    # add one to the tokenId, ensuring the correct tokenId format is used
    def _get_next_tokenId(self, tokenId: str) -> str:
        value_type = self._detect_value_type(tokenId)
        if value_type == "hex_0x":
            value_int = int(tokenId, 16)
        elif value_type == "hex_0xbf":
            value_int = int(tokenId[2:], 16)
        else:
            value_int = int(tokenId)
        result = value_int + 1
        if value_type == "hex_0x":
            return "0x" + format(result, "0" + str(len(tokenId) - 2) + "x")
        elif value_type == "hex_0xbf":
            return "0xbf" + format(result, "0" + str(len(tokenId) - 4) + "x")
        else:
            return str(result)
    # A smart contract can use different formats for the tokenId
    @staticmethod


def _detect_value_type(tokenId: str) -> str:
        if isinstance(tokenId, int):
            return "int"
        elif tokenId.startswith("0x"):
            return "hex_0x"
        elif tokenId.startswith("0xbf"):
            return "hex_0xbf"
        else:
            return "hex_0xbf"
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.spreedly
"""Loader that fetches data from Spreedly API."""
import json
import urllib.request
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import stringify_dict
SPREEDLY_ENDPOINTS = {
    "gateways_options": "https://core.spreedly.com/v1/gateways_options.json",
    "gateways": "https://core.spreedly.com/v1/gateways.json",
    "receivers_options": "https://core.spreedly.com/v1/receivers_options.json",
    "receivers": "https://core.spreedly.com/v1/receivers.json",
    "payment_methods": "https://core.spreedly.com/v1/payment_methods.json",
    "certificates": "https://core.spreedly.com/v1/certificates.json",
    "transactions": "https://core.spreedly.com/v1/transactions.json",
    "environments": "https://core.spreedly.com/v1/environments.json",
}
[docs]class SpreedlyLoader(BaseLoader):
    def __init__(self, access_token: str, resource: str) -> None:
        self.access_token = access_token
        self.resource = resource
        self.headers = {
            "Authorization": f"Bearer {self.access_token}",
            "Accept": "application/json",
        }
    def _make_request(self, url: str) -> List[Document]:
        request = urllib.request.Request(url, headers=self.headers)
        with urllib.request.urlopen(request) as response:
            json_data = json.loads(response.read().decode())
            text = stringify_dict(json_data)
            metadata = {"source": url}


return [Document(page_content=text, metadata=metadata)]
    def _get_resource(self) -> List[Document]:
        endpoint = SPREEDLY_ENDPOINTS.get(self.resource)
        if endpoint is None:
            return []
        return self._make_request(endpoint)
[docs]    def load(self) -> List[Document]:
        return self._get_resource()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.discord
"""Load from Discord chat dump"""
from __future__ import annotations
from typing import TYPE_CHECKING, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    import pandas as pd
[docs]class DiscordChatLoader(BaseLoader):
    """Load Discord chat logs."""
    def __init__(self, chat_log: pd.DataFrame, user_id_col: str = "ID"):
        """Initialize with a Pandas DataFrame containing chat logs."""
        if not isinstance(chat_log, pd.DataFrame):
            raise ValueError(
                f"Expected chat_log to be a pd.DataFrame, got {type(chat_log)}"
            )
        self.chat_log = chat_log
        self.user_id_col = user_id_col
[docs]    def load(self) -> List[Document]:
        """Load all chat messages."""
        result = []
        for _, row in self.chat_log.iterrows():
            user_id = row[self.user_id_col]
            metadata = row.to_dict()
            metadata.pop(self.user_id_col)
            result.append(Document(page_content=user_id, metadata=metadata))
        return result
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.rtf
"""Loader that loads rich text files."""
from typing import Any, List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    satisfies_min_unstructured_version,
)
[docs]class UnstructuredRTFLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load rtf files."""
    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        min_unstructured_version = "0.5.12"
        if not satisfies_min_unstructured_version(min_unstructured_version):
            raise ValueError(
                "Partitioning rtf files is only supported in "
                f"unstructured>={min_unstructured_version}."
            )
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.rtf import partition_rtf
        return partition_rtf(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.s3_directory
"""Loading logic for loading documents from an s3 directory."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.s3_file import S3FileLoader
[docs]class S3DirectoryLoader(BaseLoader):
    """Loading logic for loading documents from s3."""
    def __init__(self, bucket: str, prefix: str = ""):
        """Initialize with bucket and key name."""
        self.bucket = bucket
        self.prefix = prefix
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            import boto3
        except ImportError:
            raise ImportError(
                "Could not import boto3 python package. "
                "Please install it with `pip install boto3`."
            )
        s3 = boto3.resource("s3")
        bucket = s3.Bucket(self.bucket)
        docs = []
        for obj in bucket.objects.filter(Prefix=self.prefix):
            loader = S3FileLoader(self.bucket, obj.key)
            docs.extend(loader.load())
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.bilibili
import json
import re
import warnings
from typing import List, Tuple
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class BiliBiliLoader(BaseLoader):
    """Loader that loads bilibili transcripts."""
    def __init__(self, video_urls: List[str]):
        """Initialize with bilibili url."""
        self.video_urls = video_urls
[docs]    def load(self) -> List[Document]:
        """Load from bilibili url."""
        results = []
        for url in self.video_urls:
            transcript, video_info = self._get_bilibili_subs_and_info(url)
            doc = Document(page_content=transcript, metadata=video_info)
            results.append(doc)
        return results
    def _get_bilibili_subs_and_info(self, url: str) -> Tuple[str, dict]:
        try:
            from bilibili_api import sync, video
        except ImportError:
            raise ValueError(
                "requests package not found, please install it with "
                "`pip install bilibili-api-python`"
            )
        bvid = re.search(r"BV\w+", url)
        if bvid is not None:
            v = video.Video(bvid=bvid.group())
        else:
            aid = re.search(r"av[0-9]+", url)
            if aid is not None:
                try:
                    v = video.Video(aid=int(aid.group()[2:]))
                except AttributeError:
                    raise ValueError(f"{url} is not bilibili url.")
            else:
                raise ValueError(f"{url} is not bilibili url.")
        video_info = sync(v.get_info())


video_info.update({"url": url})
        # Get subtitle url
        subtitle = video_info.pop("subtitle")
        sub_list = subtitle["list"]
        if sub_list:
            sub_url = sub_list[0]["subtitle_url"]
            result = requests.get(sub_url)
            raw_sub_titles = json.loads(result.content)["body"]
            raw_transcript = " ".join([c["content"] for c in raw_sub_titles])
            raw_transcript_with_meta_info = (
                f"Video Title: {video_info['title']},"
                f"description: {video_info['desc']}\n\n"
                f"Transcript: {raw_transcript}"
            )
            return raw_transcript_with_meta_info, video_info
        else:
            raw_transcript = ""
            warnings.warn(
                f"""
                No subtitles found for video: {url}.
                Return Empty transcript.
                """
            )
            return raw_transcript, video_info
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.arxiv
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utilities.arxiv import ArxivAPIWrapper
[docs]class ArxivLoader(BaseLoader):
    """Loads a query result from arxiv.org into a list of Documents.
    Each document represents one Document.
    The loader converts the original PDF format into the text.
    """
    def __init__(
        self,
        query: str,
        load_max_docs: Optional[int] = 100,
        load_all_available_meta: Optional[bool] = False,
    ):
        self.query = query
        self.load_max_docs = load_max_docs
        self.load_all_available_meta = load_all_available_meta
[docs]    def load(self) -> List[Document]:
        arxiv_client = ArxivAPIWrapper(
            load_max_docs=self.load_max_docs,
            load_all_available_meta=self.load_all_available_meta,
        )
        docs = arxiv_client.load(self.query)
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.onedrive_file
from __future__ import annotations
import tempfile
from typing import TYPE_CHECKING, List
from pydantic import BaseModel, Field
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
if TYPE_CHECKING:
    from O365.drive import File
CHUNK_SIZE = 1024 * 1024 * 5
[docs]class OneDriveFileLoader(BaseLoader, BaseModel):
    file: File = Field(...)
    class Config:
        arbitrary_types_allowed = True
[docs]    def load(self) -> List[Document]:
        """Load Documents"""
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}/{self.file.name}"
            self.file.download(to_path=temp_dir, chunk_size=CHUNK_SIZE)
            loader = UnstructuredFileLoader(file_path)
            return loader.load()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.odt
"""Loader that loads Open Office ODT files."""
from typing import Any, List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    validate_unstructured_version,
)
[docs]class UnstructuredODTLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load open office ODT files."""
    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        validate_unstructured_version(min_unstructured_version="0.6.3")
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.odt import partition_odt
        return partition_odt(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.college_confidential
"""Loader that loads College Confidential."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class CollegeConfidentialLoader(WebBaseLoader):
    """Loader that loads College Confidential webpages."""
[docs]    def load(self) -> List[Document]:
        """Load webpage."""
        soup = self.scrape()
        text = soup.select_one("main[class='skin-handler']").text
        metadata = {"source": self.web_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.whatsapp_chat
import re
from pathlib import Path
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
def concatenate_rows(date: str, sender: str, text: str) -> str:
    """Combine message information in a readable format ready to be used."""
    return f"{sender} on {date}: {text}\n\n"
[docs]class WhatsAppChatLoader(BaseLoader):
    """Loader that loads WhatsApp messages text file."""
    def __init__(self, path: str):
        """Initialize with path."""
        self.file_path = path
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        p = Path(self.file_path)
        text_content = ""
        with open(p, encoding="utf8") as f:
            lines = f.readlines()
        message_line_regex = r"""
            \[?
            (
                \d{1,2}
                [\/.]
                \d{1,2}
                [\/.]
                \d{2,4}
                ,\s
                \d{1,2}
                :\d{2}
                (?:
                    :\d{2}
                )?
                (?:[ _](?:AM|PM))?
            )
            \]?
            [\s-]*
            ([~\w\s]+)
            [:]+
            \s
            (.+)
        """
        for line in lines:
            result = re.match(message_line_regex, line.strip(), flags=re.VERBOSE)
            if result:
                date, sender, text = result.groups()
                text_content += concatenate_rows(date, sender, text)


metadata = {"source": str(p)}
        return [Document(page_content=text_content, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.stripe
"""Loader that fetches data from Stripe"""
import json
import urllib.request
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import get_from_env, stringify_dict
STRIPE_ENDPOINTS = {
    "balance_transactions": "https://api.stripe.com/v1/balance_transactions",
    "charges": "https://api.stripe.com/v1/charges",
    "customers": "https://api.stripe.com/v1/customers",
    "events": "https://api.stripe.com/v1/events",
    "refunds": "https://api.stripe.com/v1/refunds",
    "disputes": "https://api.stripe.com/v1/disputes",
}
[docs]class StripeLoader(BaseLoader):
    def __init__(self, resource: str, access_token: Optional[str] = None) -> None:
        self.resource = resource
        access_token = access_token or get_from_env(
            "access_token", "STRIPE_ACCESS_TOKEN"
        )
        self.headers = {"Authorization": f"Bearer {access_token}"}
    def _make_request(self, url: str) -> List[Document]:
        request = urllib.request.Request(url, headers=self.headers)
        with urllib.request.urlopen(request) as response:
            json_data = json.loads(response.read().decode())
            text = stringify_dict(json_data)
            metadata = {"source": url}
            return [Document(page_content=text, metadata=metadata)]
    def _get_resource(self) -> List[Document]:
        endpoint = STRIPE_ENDPOINTS.get(self.resource)
        if endpoint is None:
            return []
        return self._make_request(endpoint)


[docs]    def load(self) -> List[Document]:
        return self._get_resource()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.conllu
"""Load CoNLL-U files."""
import csv
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class CoNLLULoader(BaseLoader):
    """Load CoNLL-U files."""
    def __init__(self, file_path: str):
        """Initialize with file path."""
        self.file_path = file_path
[docs]    def load(self) -> List[Document]:
        """Load from file path."""
        with open(self.file_path, encoding="utf8") as f:
            tsv = list(csv.reader(f, delimiter="\t"))
            # If len(line) > 1, the line is not a comment
            lines = [line for line in tsv if len(line) > 1]
        text = ""
        for i, line in enumerate(lines):
            # Do not add a space after a punctuation mark or at the end of the sentence
            if line[9] == "SpaceAfter=No" or i == len(lines) - 1:
                text += line[1]
            else:
                text += line[1] + " "
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.max_compute
from __future__ import annotations
from typing import Any, Iterator, List, Optional, Sequence
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utilities.max_compute import MaxComputeAPIWrapper
[docs]class MaxComputeLoader(BaseLoader):
    """Loads a query result from Alibaba Cloud MaxCompute table into documents."""
    def __init__(
        self,
        query: str,
        api_wrapper: MaxComputeAPIWrapper,
        *,
        page_content_columns: Optional[Sequence[str]] = None,
        metadata_columns: Optional[Sequence[str]] = None,
    ):
        """Initialize Alibaba Cloud MaxCompute document loader.
        Args:
            query: SQL query to execute.
            api_wrapper: MaxCompute API wrapper.
            page_content_columns: The columns to write into the `page_content` of the
                Document. If unspecified, all columns will be written to `page_content`.
            metadata_columns: The columns to write into the `metadata` of the Document.
                If unspecified, all columns not added to `page_content` will be written.
        """
        self.query = query
        self.api_wrapper = api_wrapper
        self.page_content_columns = page_content_columns
        self.metadata_columns = metadata_columns
[docs]    @classmethod
    def from_params(
        cls,
        query: str,
        endpoint: str,
        project: str,
        *,
        access_id: Optional[str] = None,
        secret_access_key: Optional[str] = None,
        **kwargs: Any,
    ) -> MaxComputeLoader:
        """Convenience constructor that builds the MaxCompute API wrapper from
            given parameters.
        Args:
            query: SQL query to execute.


endpoint: MaxCompute endpoint.
            project: A project is a basic organizational unit of MaxCompute, which is
                similar to a database.
            access_id: MaxCompute access ID. Should be passed in directly or set as the
                environment variable `MAX_COMPUTE_ACCESS_ID`.
            secret_access_key: MaxCompute secret access key. Should be passed in
                directly or set as the environment variable
                `MAX_COMPUTE_SECRET_ACCESS_KEY`.
        """
        api_wrapper = MaxComputeAPIWrapper.from_params(
            endpoint, project, access_id=access_id, secret_access_key=secret_access_key
        )
        return cls(query, api_wrapper, **kwargs)
[docs]    def lazy_load(self) -> Iterator[Document]:
        for row in self.api_wrapper.query(self.query):
            if self.page_content_columns:
                page_content_data = {
                    k: v for k, v in row.items() if k in self.page_content_columns
                }
            else:
                page_content_data = row
            page_content = "\n".join(f"{k}: {v}" for k, v in page_content_data.items())
            if self.metadata_columns:
                metadata = {k: v for k, v in row.items() if k in self.metadata_columns}
            else:
                metadata = {k: v for k, v in row.items() if k not in page_content_data}
            yield Document(page_content=page_content, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        return list(self.lazy_load())
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.roam
"""Loader that loads Roam directory dump."""
from pathlib import Path
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class RoamLoader(BaseLoader):
    """Loader that loads Roam files from disk."""
    def __init__(self, path: str):
        """Initialize with path."""
        self.file_path = path
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        ps = list(Path(self.file_path).glob("**/*.md"))
        docs = []
        for p in ps:
            with open(p) as f:
                text = f.read()
            metadata = {"source": str(p)}
            docs.append(Document(page_content=text, metadata=metadata))
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.gcs_file
"""Loading logic for loading documents from a GCS file."""
import os
import tempfile
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class GCSFileLoader(BaseLoader):
    """Loading logic for loading documents from GCS."""
    def __init__(self, project_name: str, bucket: str, blob: str):
        """Initialize with bucket and key name."""
        self.bucket = bucket
        self.blob = blob
        self.project_name = project_name
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from google.cloud import storage
        except ImportError:
            raise ValueError(
                "Could not import google-cloud-storage python package. "
                "Please install it with `pip install google-cloud-storage`."
            )
        # Initialise a client
        storage_client = storage.Client(self.project_name)
        # Create a bucket object for our bucket
        bucket = storage_client.get_bucket(self.bucket)
        # Create a blob object from the filepath
        blob = bucket.blob(self.blob)
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}/{self.blob}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            # Download the file to a destination
            blob.download_to_filename(file_path)
            loader = UnstructuredFileLoader(file_path)
            return loader.load()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.excel
"""Loader that loads Microsoft Excel files."""
from typing import Any, List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    validate_unstructured_version,
)
[docs]class UnstructuredExcelLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load Microsoft Excel files."""
    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        validate_unstructured_version(min_unstructured_version="0.6.7")
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.xlsx import partition_xlsx
        return partition_xlsx(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.azlyrics
"""Loader that loads AZLyrics."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class AZLyricsLoader(WebBaseLoader):
    """Loader that loads AZLyrics webpages."""
[docs]    def load(self) -> List[Document]:
        """Load webpage."""
        soup = self.scrape()
        title = soup.title.text
        lyrics = soup.find_all("div", {"class": ""})[2].text
        text = title + lyrics
        metadata = {"source": self.web_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.ifixit
"""Loader that loads iFixit data."""
from typing import List, Optional
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.web_base import WebBaseLoader
IFIXIT_BASE_URL = "https://www.ifixit.com/api/2.0"
[docs]class IFixitLoader(BaseLoader):
    """Load iFixit repair guides, device wikis and answers.
    iFixit is the largest, open repair community on the web. The site contains nearly
    100k repair manuals, 200k Questions & Answers on 42k devices, and all the data is
    licensed under CC-BY.
    This loader will allow you to download the text of a repair guide, text of Q&A's
    and wikis from devices on iFixit using their open APIs and web scraping.
    """
    def __init__(self, web_path: str):
        """Initialize with web path."""
        if not web_path.startswith("https://www.ifixit.com"):
            raise ValueError("web path must start with 'https://www.ifixit.com'")
        path = web_path.replace("https://www.ifixit.com", "")
        allowed_paths = ["/Device", "/Guide", "/Answers", "/Teardown"]
        """ TODO: Add /Wiki """
        if not any(path.startswith(allowed_path) for allowed_path in allowed_paths):
            raise ValueError(
                "web path must start with /Device, /Guide, /Teardown or /Answers"
            )
        pieces = [x for x in path.split("/") if x]
        """Teardowns are just guides by a different name"""


self.page_type = pieces[0] if pieces[0] != "Teardown" else "Guide"
        if self.page_type == "Guide" or self.page_type == "Answers":
            self.id = pieces[2]
        else:
            self.id = pieces[1]
        self.web_path = web_path
[docs]    def load(self) -> List[Document]:
        if self.page_type == "Device":
            return self.load_device()
        elif self.page_type == "Guide" or self.page_type == "Teardown":
            return self.load_guide()
        elif self.page_type == "Answers":
            return self.load_questions_and_answers()
        else:
            raise ValueError("Unknown page type: " + self.page_type)
[docs]    @staticmethod
    def load_suggestions(query: str = "", doc_type: str = "all") -> List[Document]:
        res = requests.get(
            IFIXIT_BASE_URL + "/suggest/" + query + "?doctypes=" + doc_type
        )
        if res.status_code != 200:
            raise ValueError(
                'Could not load suggestions for "' + query + '"\n' + res.json()
            )
        data = res.json()
        results = data["results"]
        output = []
        for result in results:
            try:
                loader = IFixitLoader(result["url"])
                if loader.page_type == "Device":
                    output += loader.load_device(include_guides=False)
                else:
                    output += loader.load()
            except ValueError:
                continue
        return output
[docs]    def load_questions_and_answers(
        self, url_override: Optional[str] = None
    ) -> List[Document]:


loader = WebBaseLoader(self.web_path if url_override is None else url_override)
        soup = loader.scrape()
        output = []
        title = soup.find("h1", "post-title").text
        output.append("# " + title)
        output.append(soup.select_one(".post-content .post-text").text.strip())
        answersHeader = soup.find("div", "post-answers-header")
        if answersHeader:
            output.append("\n## " + answersHeader.text.strip())
        for answer in soup.select(".js-answers-list .post.post-answer"):
            if answer.has_attr("itemprop") and "acceptedAnswer" in answer["itemprop"]:
                output.append("\n### Accepted Answer")
            elif "post-helpful" in answer["class"]:
                output.append("\n### Most Helpful Answer")
            else:
                output.append("\n### Other Answer")
            output += [
                a.text.strip() for a in answer.select(".post-content .post-text")
            ]
            output.append("\n")
        text = "\n".join(output).strip()
        metadata = {"source": self.web_path, "title": title}
        return [Document(page_content=text, metadata=metadata)]
[docs]    def load_device(
        self, url_override: Optional[str] = None, include_guides: bool = True
    ) -> List[Document]:
        documents = []
        if url_override is None:
            url = IFIXIT_BASE_URL + "/wikis/CATEGORY/" + self.id
        else:
            url = url_override
        res = requests.get(url)
        data = res.json()
        text = "\n".join(
            [
                data[key]
                for key in ["title", "description", "contents_raw"]


if key in data
            ]
        ).strip()
        metadata = {"source": self.web_path, "title": data["title"]}
        documents.append(Document(page_content=text, metadata=metadata))
        if include_guides:
            """Load and return documents for each guide linked to from the device"""
            guide_urls = [guide["url"] for guide in data["guides"]]
            for guide_url in guide_urls:
                documents.append(IFixitLoader(guide_url).load()[0])
        return documents
[docs]    def load_guide(self, url_override: Optional[str] = None) -> List[Document]:
        if url_override is None:
            url = IFIXIT_BASE_URL + "/guides/" + self.id
        else:
            url = url_override
        res = requests.get(url)
        if res.status_code != 200:
            raise ValueError(
                "Could not load guide: " + self.web_path + "\n" + res.json()
            )
        data = res.json()
        doc_parts = ["# " + data["title"], data["introduction_raw"]]
        doc_parts.append("\n\n###Tools Required:")
        if len(data["tools"]) == 0:
            doc_parts.append("\n - None")
        else:
            for tool in data["tools"]:
                doc_parts.append("\n - " + tool["text"])
        doc_parts.append("\n\n###Parts Required:")
        if len(data["parts"]) == 0:
            doc_parts.append("\n - None")
        else:
            for part in data["parts"]:
                doc_parts.append("\n - " + part["text"])
        for row in data["steps"]:
            doc_parts.append(
                "\n\n## "
                + (


row["title"]
                    if row["title"] != ""
                    else "Step {}".format(row["orderby"])
                )
            )
            for line in row["lines"]:
                doc_parts.append(line["text_raw"])
        doc_parts.append(data["conclusion_raw"])
        text = "\n".join(doc_parts)
        metadata = {"source": self.web_path, "title": data["title"]}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.azure_blob_storage_container
"""Loading logic for loading documents from an Azure Blob Storage container."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.azure_blob_storage_file import (
    AzureBlobStorageFileLoader,
)
from langchain.document_loaders.base import BaseLoader
[docs]class AzureBlobStorageContainerLoader(BaseLoader):
    """Loading logic for loading documents from Azure Blob Storage."""
    def __init__(self, conn_str: str, container: str, prefix: str = ""):
        """Initialize with connection string, container and blob prefix."""
        self.conn_str = conn_str
        self.container = container
        self.prefix = prefix
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from azure.storage.blob import ContainerClient
        except ImportError as exc:
            raise ValueError(
                "Could not import azure storage blob python package. "
                "Please install it with `pip install azure-storage-blob`."
            ) from exc
        container = ContainerClient.from_connection_string(
            conn_str=self.conn_str, container_name=self.container
        )
        docs = []
        blob_list = container.list_blobs(name_starts_with=self.prefix)
        for blob in blob_list:
            loader = AzureBlobStorageFileLoader(
                self.conn_str, self.container, blob.name  # type: ignore
            )
            docs.extend(loader.load())
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.document_loaders.duckdb_loader
from typing import Dict, List, Optional, cast
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class DuckDBLoader(BaseLoader):
    """Loads a query result from DuckDB into a list of documents.
    Each document represents one row of the result. The `page_content_columns`
    are written into the `page_content` of the document. The `metadata_columns`
    are written into the `metadata` of the document. By default, all columns
    are written into the `page_content` and none into the `metadata`.
    """
    def __init__(
        self,
        query: str,
        database: str = ":memory:",
        read_only: bool = False,
        config: Optional[Dict[str, str]] = None,
        page_content_columns: Optional[List[str]] = None,
        metadata_columns: Optional[List[str]] = None,
    ):
        self.query = query
        self.database = database
        self.read_only = read_only
        self.config = config or {}
        self.page_content_columns = page_content_columns
        self.metadata_columns = metadata_columns
[docs]    def load(self) -> List[Document]:
        try:
            import duckdb
        except ImportError:
            raise ImportError(
                "Could not import duckdb python package. "
                "Please install it with `pip install duckdb`."
            )
        docs = []
        with duckdb.connect(
            database=self.database, read_only=self.read_only, config=self.config
        ) as con:
            query_result = con.execute(self.query)
            results = query_result.fetchall()
            description = cast(list, query_result.description)


field_names = [c[0] for c in description]
            if self.page_content_columns is None:
                page_content_columns = field_names
            else:
                page_content_columns = self.page_content_columns
            if self.metadata_columns is None:
                metadata_columns = []
            else:
                metadata_columns = self.metadata_columns
            for result in results:
                page_content = "\n".join(
                    f"{column}: {result[field_names.index(column)]}"
                    for column in page_content_columns
                )
                metadata = {
                    column: result[field_names.index(column)]
                    for column in metadata_columns
                }
                doc = Document(page_content=page_content, metadata=metadata)
                docs.append(doc)
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 04, 2023.


Source code for langchain.llms.databricks
import os
from abc import ABC, abstractmethod
from typing import Any, Callable, Dict, List, Optional
import requests
from pydantic import BaseModel, Extra, Field, PrivateAttr, root_validator, validator
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
__all__ = ["Databricks"]
class _DatabricksClientBase(BaseModel, ABC):
    """A base JSON API client that talks to Databricks."""
    api_url: str
    api_token: str
    def post_raw(self, request: Any) -> Any:
        headers = {"Authorization": f"Bearer {self.api_token}"}
        response = requests.post(self.api_url, headers=headers, json=request)
        # TODO: error handling and automatic retries
        if not response.ok:
            raise ValueError(f"HTTP {response.status_code} error: {response.text}")
        return response.json()
    @abstractmethod
    def post(self, request: Any) -> Any:
        ...
class _DatabricksServingEndpointClient(_DatabricksClientBase):
    """An API client that talks to a Databricks serving endpoint."""
    host: str
    endpoint_name: str
    @root_validator(pre=True)
    def set_api_url(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        if "api_url" not in values:
            host = values["host"]
            endpoint_name = values["endpoint_name"]
            api_url = f"https://{host}/serving-endpoints/{endpoint_name}/invocations"
            values["api_url"] = api_url
        return values
    def post(self, request: Any) -> Any:


# See https://docs.databricks.com/machine-learning/model-serving/score-model-serving-endpoints.html
        wrapped_request = {"dataframe_records": [request]}
        response = self.post_raw(wrapped_request)["predictions"]
        # For a single-record query, the result is not a list.
        if isinstance(response, list):
            response = response[0]
        return response
class _DatabricksClusterDriverProxyClient(_DatabricksClientBase):
    """An API client that talks to a Databricks cluster driver proxy app."""
    host: str
    cluster_id: str
    cluster_driver_port: str
    @root_validator(pre=True)
    def set_api_url(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        if "api_url" not in values:
            host = values["host"]
            cluster_id = values["cluster_id"]
            port = values["cluster_driver_port"]
            api_url = f"https://{host}/driver-proxy-api/o/0/{cluster_id}/{port}"
            values["api_url"] = api_url
        return values
    def post(self, request: Any) -> Any:
        return self.post_raw(request)
def get_repl_context() -> Any:
    """Gets the notebook REPL context if running inside a Databricks notebook.
    Returns None otherwise.
    """
    try:
        from dbruntime.databricks_repl_context import get_context
        return get_context()
    except ImportError:
        raise ValueError(
            "Cannot access dbruntime, not running inside a Databricks notebook."
        )
def get_default_host() -> str:
    """Gets the default Databricks workspace hostname.
    Raises an error if the hostname cannot be automatically determined.
    """


host = os.getenv("DATABRICKS_HOST")
    if not host:
        try:
            host = get_repl_context().browserHostName
            if not host:
                raise ValueError("context doesn't contain browserHostName.")
        except Exception as e:
            raise ValueError(
                "host was not set and cannot be automatically inferred. Set "
                f"environment variable 'DATABRICKS_HOST'. Received error: {e}"
            )
    # TODO: support Databricks CLI profile
    host = host.lstrip("https://").lstrip("http://").rstrip("/")
    return host
def get_default_api_token() -> str:
    """Gets the default Databricks personal access token.
    Raises an error if the token cannot be automatically determined.
    """
    if api_token := os.getenv("DATABRICKS_API_TOKEN"):
        return api_token
    try:
        api_token = get_repl_context().apiToken
        if not api_token:
            raise ValueError("context doesn't contain apiToken.")
    except Exception as e:
        raise ValueError(
            "api_token was not set and cannot be automatically inferred. Set "
            f"environment variable 'DATABRICKS_API_TOKEN'. Received error: {e}"
        )
    # TODO: support Databricks CLI profile
    return api_token
[docs]class Databricks(LLM):
    """LLM wrapper around a Databricks serving endpoint or a cluster driver proxy app.
    It supports two endpoint types:
    * **Serving endpoint** (recommended for both production and development).
      We assume that an LLM was registered and deployed to a serving endpoint.
      To wrap it as an LLM you must have "Can Query" permission to the endpoint.


Set ``endpoint_name`` accordingly and do not set ``cluster_id`` and
      ``cluster_driver_port``.
      The expected model signature is:
      * inputs::
          [{"name": "prompt", "type": "string"},
           {"name": "stop", "type": "list[string]"}]
      * outputs: ``[{"type": "string"}]``
    * **Cluster driver proxy app** (recommended for interactive development).
      One can load an LLM on a Databricks interactive cluster and start a local HTTP
      server on the driver node to serve the model at ``/`` using HTTP POST method
      with JSON input/output.
      Please use a port number between ``[3000, 8000]`` and let the server listen to
      the driver IP address or simply ``0.0.0.0`` instead of localhost only.
      To wrap it as an LLM you must have "Can Attach To" permission to the cluster.
      Set ``cluster_id`` and ``cluster_driver_port`` and do not set ``endpoint_name``.
      The expected server schema (using JSON schema) is:
      * inputs::
          {"type": "object",
           "properties": {
              "prompt": {"type": "string"},
              "stop": {"type": "array", "items": {"type": "string"}}},
           "required": ["prompt"]}`
      * outputs: ``{"type": "string"}``
    If the endpoint model signature is different or you want to set extra params,
    you can use `transform_input_fn` and `transform_output_fn` to apply necessary
    transformations before and after the query.
    """
    host: str = Field(default_factory=get_default_host)
    """Databricks workspace hostname.


If not provided, the default value is determined by
    * the ``DATABRICKS_HOST`` environment variable if present, or
    * the hostname of the current Databricks workspace if running inside
      a Databricks notebook attached to an interactive cluster in "single user"
      or "no isolation shared" mode.
    """
    api_token: str = Field(default_factory=get_default_api_token)
    """Databricks personal access token.
    If not provided, the default value is determined by
    * the ``DATABRICKS_API_TOKEN`` environment variable if present, or
    * an automatically generated temporary token if running inside a Databricks
      notebook attached to an interactive cluster in "single user" or
      "no isolation shared" mode.
    """
    endpoint_name: Optional[str] = None
    """Name of the model serving endpont.
    You must specify the endpoint name to connect to a model serving endpoint.
    You must not set both ``endpoint_name`` and ``cluster_id``.
    """
    cluster_id: Optional[str] = None
    """ID of the cluster if connecting to a cluster driver proxy app.
    If neither ``endpoint_name`` nor ``cluster_id`` is not provided and the code runs
    inside a Databricks notebook attached to an interactive cluster in "single user"
    or "no isolation shared" mode, the current cluster ID is used as default.
    You must not set both ``endpoint_name`` and ``cluster_id``.
    """
    cluster_driver_port: Optional[str] = None
    """The port number used by the HTTP server running on the cluster driver node.
    The server should listen on the driver IP address or simply ``0.0.0.0`` to connect.


We recommend the server using a port number between ``[3000, 8000]``.
    """
    model_kwargs: Optional[Dict[str, Any]] = None
    """Extra parameters to pass to the endpoint."""
    transform_input_fn: Optional[Callable] = None
    """A function that transforms ``{prompt, stop, **kwargs}`` into a JSON-compatible
    request object that the endpoint accepts.
    For example, you can apply a prompt template to the input prompt.
    """
    transform_output_fn: Optional[Callable[..., str]] = None
    """A function that transforms the output from the endpoint to the generated text.
    """
    _client: _DatabricksClientBase = PrivateAttr()
    class Config:
        extra = Extra.forbid
        underscore_attrs_are_private = True
    @validator("cluster_id", always=True)
    def set_cluster_id(cls, v: Any, values: Dict[str, Any]) -> Optional[str]:
        if v and values["endpoint_name"]:
            raise ValueError("Cannot set both endpoint_name and cluster_id.")
        elif values["endpoint_name"]:
            return None
        elif v:
            return v
        else:
            try:
                if v := get_repl_context().clusterId:
                    return v
                raise ValueError("Context doesn't contain clusterId.")
            except Exception as e:
                raise ValueError(
                    "Neither endpoint_name nor cluster_id was set. "
                    "And the cluster_id cannot be automatically determined. Received"
                    f" error: {e}"
                )
    @validator("cluster_driver_port", always=True)
    def set_cluster_driver_port(cls, v: Any, values: Dict[str, Any]) -> Optional[str]:
        if v and values["endpoint_name"]: