Source code for langchain.document_loaders.url
"""Loader that uses unstructured to load HTML files."""
import logging
from typing import Any, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class UnstructuredURLLoader(BaseLoader):
    """Loader that uses unstructured to load HTML files."""
    def __init__(
        self,
        urls: List[str],
        continue_on_failure: bool = True,
        mode: str = "single",
        **unstructured_kwargs: Any,
    ):
        """Initialize with file path."""
        try:
            import unstructured  # noqa:F401
            from unstructured.__version__ import __version__ as __unstructured_version__
            self.__version = __unstructured_version__
        except ImportError:
            raise ValueError(
                "unstructured package not found, please install it with "
                "`pip install unstructured`"
            )
        self._validate_mode(mode)
        self.mode = mode
        headers = unstructured_kwargs.pop("headers", {})
        if len(headers.keys()) != 0:
            warn_about_headers = False
            if self.__is_non_html_available():
                warn_about_headers = not self.__is_headers_available_for_non_html()
            else:
                warn_about_headers = not self.__is_headers_available_for_html()
            if warn_about_headers:
                logger.warning(
                    "You are using an old version of unstructured. "
                    "The headers parameter is ignored"
                )
        self.urls = urls
        self.continue_on_failure = continue_on_failure
        self.headers = headers
        self.unstructured_kwargs = unstructured_kwargs
    def _validate_mode(self, mode: str) -> None:


def _validate_mode(self, mode: str) -> None:
        _valid_modes = {"single", "elements"}
        if mode not in _valid_modes:
            raise ValueError(
                f"Got {mode} for `mode`, but should be one of `{_valid_modes}`"
            )
    def __is_headers_available_for_html(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 7)
    def __is_headers_available_for_non_html(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 13)
    def __is_non_html_available(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 12)
[docs]    def load(self) -> List[Document]:
        """Load file."""
        from unstructured.partition.auto import partition
        from unstructured.partition.html import partition_html
        docs: List[Document] = list()
        for url in self.urls:
            try:
                if self.__is_non_html_available():
                    if self.__is_headers_available_for_non_html():
                        elements = partition(
                            url=url, headers=self.headers, **self.unstructured_kwargs
                        )
                    else:
                        elements = partition(url=url, **self.unstructured_kwargs)
                else:


elements = partition(url=url, **self.unstructured_kwargs)
                else:
                    if self.__is_headers_available_for_html():
                        elements = partition_html(
                            url=url, headers=self.headers, **self.unstructured_kwargs
                        )
                    else:
                        elements = partition_html(url=url, **self.unstructured_kwargs)
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error fetching or processing {url}, exeption: {e}")
                    continue
                else:
                    raise e
            if self.mode == "single":
                text = "\n\n".join([str(el) for el in elements])
                metadata = {"source": url}
                docs.append(Document(page_content=text, metadata=metadata))
            elif self.mode == "elements":
                for element in elements:
                    metadata = element.metadata.to_dict()
                    metadata["category"] = element.category
                    docs.append(Document(page_content=str(element), metadata=metadata))
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.blockchain
import os
import re
from enum import Enum
from typing import List
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
class BlockchainType(Enum):
    ETH_MAINNET = "eth-mainnet"
    ETH_GOERLI = "eth-goerli"
    POLYGON_MAINNET = "polygon-mainnet"
    POLYGON_MUMBAI = "polygon-mumbai"
[docs]class BlockchainDocumentLoader(BaseLoader):
    """Loads elements from a blockchain smart contract into Langchain documents.
    The supported blockchains are: Ethereum mainnet, Ethereum Goerli testnet,
    Polygon mainnet, and Polygon Mumbai testnet.
    If no BlockchainType is specified, the default is Ethereum mainnet.
    The Loader uses the Alchemy API to interact with the blockchain.
    The API returns 100 NFTs per request and can be paginated using the
    startToken parameter.
    ALCHEMY_API_KEY environment variable must be set to use this loader.
    Future versions of this loader can:
        - Support additional Alchemy APIs (e.g. getTransactions, etc.)
        - Support additional blockain APIs (e.g. Infura, Opensea, etc.)
    """
    def __init__(
        self,
        contract_address: str,
        blockchainType: BlockchainType = BlockchainType.ETH_MAINNET,
        api_key: str = "docs-demo",
        startToken: str = "",
    ):
        self.contract_address = contract_address
        self.blockchainType = blockchainType.value
        self.api_key = os.environ.get("ALCHEMY_API_KEY") or api_key
        self.startToken = startToken
        if not self.api_key:


self.startToken = startToken
        if not self.api_key:
            raise ValueError("Alchemy API key not provided.")
        if not re.match(r"^0x[a-fA-F0-9]{40}$", self.contract_address):
            raise ValueError(f"Invalid contract address {self.contract_address}")
[docs]    def load(self) -> List[Document]:
        url = (
            f"https://{self.blockchainType}.g.alchemy.com/nft/v2/"
            f"{self.api_key}/getNFTsForCollection?withMetadata="
            f"True&contractAddress={self.contract_address}"
            f"&startToken={self.startToken}"
        )
        response = requests.get(url)
        if response.status_code != 200:
            raise ValueError(f"Request failed with status code {response.status_code}")
        items = response.json()["nfts"]
        if not (items):
            raise ValueError(
                f"No NFTs found for contract address {self.contract_address}"
            )
        result = []
        for item in items:
            content = str(item)
            tokenId = item["id"]["tokenId"]
            metadata = {
                "source": self.contract_address,
                "blockchain": self.blockchainType,
                "tokenId": tokenId,
            }
            result.append(Document(page_content=content, metadata=metadata))
        return result
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.reddit
"""Reddit document loader."""
from __future__ import annotations
from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    import praw
def _dependable_praw_import() -> praw:
    try:
        import praw
    except ImportError:
        raise ValueError(
            "praw package not found, please install it with `pip install praw`"
        )
    return praw
[docs]class RedditPostsLoader(BaseLoader):
    """Reddit posts loader.
    Read posts on a subreddit.
    First you need to go to
    https://www.reddit.com/prefs/apps/
    and create your application
    """
    def __init__(
        self,
        client_id: str,
        client_secret: str,
        user_agent: str,
        search_queries: Sequence[str],
        mode: str,
        categories: Sequence[str] = ["new"],
        number_posts: Optional[int] = 10,
    ):
        self.client_id = client_id
        self.client_secret = client_secret
        self.user_agent = user_agent
        self.search_queries = search_queries
        self.mode = mode
        self.categories = categories
        self.number_posts = number_posts
[docs]    def load(self) -> List[Document]:
        """Load reddits."""
        praw = _dependable_praw_import()
        reddit = praw.Reddit(
            client_id=self.client_id,
            client_secret=self.client_secret,
            user_agent=self.user_agent,
        )
        results: List[Document] = []
        if self.mode == "subreddit":
            for search_query in self.search_queries:


if self.mode == "subreddit":
            for search_query in self.search_queries:
                for category in self.categories:
                    docs = self._subreddit_posts_loader(
                        search_query=search_query, category=category, reddit=reddit
                    )
                    results.extend(docs)
        elif self.mode == "username":
            for search_query in self.search_queries:
                for category in self.categories:
                    docs = self._user_posts_loader(
                        search_query=search_query, category=category, reddit=reddit
                    )
                    results.extend(docs)
        else:
            raise ValueError(
                "mode not correct, please enter 'username' or 'subreddit' as mode"
            )
        return results
    def _subreddit_posts_loader(
        self, search_query: str, category: str, reddit: praw.reddit.Reddit
    ) -> Iterable[Document]:
        subreddit = reddit.subreddit(search_query)
        method = getattr(subreddit, category)
        cat_posts = method(limit=self.number_posts)
        """Format reddit posts into a string."""
        for post in cat_posts:
            metadata = {
                "post_subreddit": post.subreddit_name_prefixed,
                "post_category": category,
                "post_title": post.title,
                "post_score": post.score,
                "post_id": post.id,
                "post_url": post.url,
                "post_author": post.author,
            }
            yield Document(
                page_content=post.selftext,
                metadata=metadata,
            )
    def _user_posts_loader(
        self, search_query: str, category: str, reddit: praw.reddit.Reddit
    ) -> Iterable[Document]:
        user = reddit.redditor(search_query)
        method = getattr(user.submissions, category)


method = getattr(user.submissions, category)
        cat_posts = method(limit=self.number_posts)
        """Format reddit posts into a string."""
        for post in cat_posts:
            metadata = {
                "post_subreddit": post.subreddit_name_prefixed,
                "post_category": category,
                "post_title": post.title,
                "post_score": post.score,
                "post_id": post.id,
                "post_url": post.url,
                "post_author": post.author,
            }
            yield Document(
                page_content=post.selftext,
                metadata=metadata,
            )
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.discord
"""Load from Discord chat dump"""
from __future__ import annotations
from typing import TYPE_CHECKING, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    import pandas as pd
[docs]class DiscordChatLoader(BaseLoader):
    """Load Discord chat logs."""
    def __init__(self, chat_log: pd.DataFrame, user_id_col: str = "ID"):
        """Initialize with a Pandas DataFrame containing chat logs."""
        if not isinstance(chat_log, pd.DataFrame):
            raise ValueError(
                f"Expected chat_log to be a pd.DataFrame, got {type(chat_log)}"
            )
        self.chat_log = chat_log
        self.user_id_col = user_id_col
[docs]    def load(self) -> List[Document]:
        """Load all chat messages."""
        result = []
        for _, row in self.chat_log.iterrows():
            user_id = row[self.user_id_col]
            metadata = row.to_dict()
            metadata.pop(self.user_id_col)
            result.append(Document(page_content=user_id, metadata=metadata))
        return result
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.html_bs
"""Loader that uses bs4 to load HTML files, enriching metadata with page title."""
import logging
from typing import Dict, List, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class BSHTMLLoader(BaseLoader):
    """Loader that uses beautiful soup to parse HTML files."""
    def __init__(
        self,
        file_path: str,
        open_encoding: Union[str, None] = None,
        bs_kwargs: Union[dict, None] = None,
        get_text_separator: str = "",
    ) -> None:
        """Initialise with path, and optionally, file encoding to use, and any kwargs
        to pass to the BeautifulSoup object."""
        try:
            import bs4  # noqa:F401
        except ImportError:
            raise ValueError(
                "beautifulsoup4 package not found, please install it with "
                "`pip install beautifulsoup4`"
            )
        self.file_path = file_path
        self.open_encoding = open_encoding
        if bs_kwargs is None:
            bs_kwargs = {"features": "lxml"}
        self.bs_kwargs = bs_kwargs
        self.get_text_separator = get_text_separator
[docs]    def load(self) -> List[Document]:
        from bs4 import BeautifulSoup
        """Load HTML document into document objects."""
        with open(self.file_path, "r", encoding=self.open_encoding) as f:
            soup = BeautifulSoup(f, **self.bs_kwargs)
        text = soup.get_text(self.get_text_separator)
        if soup.title:
            title = str(soup.title.string)
        else:
            title = ""
        metadata: Dict[str, Union[str, None]] = {


title = ""
        metadata: Dict[str, Union[str, None]] = {
            "source": self.file_path,
            "title": title,
        }
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.googledrive
"""Loader that loads data from Google Drive."""
# Prerequisites:
# 1. Create a Google Cloud project
# 2. Enable the Google Drive API:
#   https://console.cloud.google.com/flows/enableapi?apiid=drive.googleapis.com
# 3. Authorize credentials for desktop app:
#   https://developers.google.com/drive/api/quickstart/python#authorize_credentials_for_a_desktop_application # noqa: E501
# 4. For service accounts visit
#   https://cloud.google.com/iam/docs/service-accounts-create
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from pydantic import BaseModel, root_validator, validator
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
SCOPES = ["https://www.googleapis.com/auth/drive.readonly"]
[docs]class GoogleDriveLoader(BaseLoader, BaseModel):
    """Loader that loads Google Docs from Google Drive."""
    service_account_key: Path = Path.home() / ".credentials" / "keys.json"
    credentials_path: Path = Path.home() / ".credentials" / "credentials.json"
    token_path: Path = Path.home() / ".credentials" / "token.json"
    folder_id: Optional[str] = None
    document_ids: Optional[List[str]] = None
    file_ids: Optional[List[str]] = None
    recursive: bool = False
    @root_validator
    def validate_folder_id_or_document_ids(
        cls, values: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Validate that either folder_id or document_ids is set, but not both."""
        if values.get("folder_id") and (


if values.get("folder_id") and (
            values.get("document_ids") or values.get("file_ids")
        ):
            raise ValueError(
                "Cannot specify both folder_id and document_ids nor "
                "folder_id and file_ids"
            )
        if (
            not values.get("folder_id")
            and not values.get("document_ids")
            and not values.get("file_ids")
        ):
            raise ValueError("Must specify either folder_id, document_ids, or file_ids")
        return values
    @validator("credentials_path")
    def validate_credentials_path(cls, v: Any, **kwargs: Any) -> Any:
        """Validate that credentials_path exists."""
        if not v.exists():
            raise ValueError(f"credentials_path {v} does not exist")
        return v
    def _load_credentials(self) -> Any:
        """Load credentials."""
        # Adapted from https://developers.google.com/drive/api/v3/quickstart/python
        try:
            from google.auth.transport.requests import Request
            from google.oauth2 import service_account
            from google.oauth2.credentials import Credentials
            from google_auth_oauthlib.flow import InstalledAppFlow
        except ImportError:
            raise ImportError(
                "You must run"
                "`pip install --upgrade "
                "google-api-python-client google-auth-httplib2 "
                "google-auth-oauthlib`"
                "to use the Google Drive loader."
            )
        creds = None
        if self.service_account_key.exists():
            return service_account.Credentials.from_service_account_file(
                str(self.service_account_key), scopes=SCOPES
            )
        if self.token_path.exists():
            creds = Credentials.from_authorized_user_file(str(self.token_path), SCOPES)
        if not creds or not creds.valid:


if not creds or not creds.valid:
            if creds and creds.expired and creds.refresh_token:
                creds.refresh(Request())
            else:
                flow = InstalledAppFlow.from_client_secrets_file(
                    str(self.credentials_path), SCOPES
                )
                creds = flow.run_local_server(port=0)
            with open(self.token_path, "w") as token:
                token.write(creds.to_json())
        return creds
    def _load_sheet_from_id(self, id: str) -> List[Document]:
        """Load a sheet and all tabs from an ID."""
        from googleapiclient.discovery import build
        creds = self._load_credentials()
        sheets_service = build("sheets", "v4", credentials=creds)
        spreadsheet = sheets_service.spreadsheets().get(spreadsheetId=id).execute()
        sheets = spreadsheet.get("sheets", [])
        documents = []
        for sheet in sheets:
            sheet_name = sheet["properties"]["title"]
            result = (
                sheets_service.spreadsheets()
                .values()
                .get(spreadsheetId=id, range=sheet_name)
                .execute()
            )
            values = result.get("values", [])
            header = values[0]
            for i, row in enumerate(values[1:], start=1):
                metadata = {
                    "source": (
                        f"https://docs.google.com/spreadsheets/d/{id}/"
                        f"edit?gid={sheet['properties']['sheetId']}"
                    ),
                    "title": f"{spreadsheet['properties']['title']} - {sheet_name}",
                    "row": i,
                }
                content = []
                for j, v in enumerate(row):
                    title = header[j].strip() if len(header) > j else ""


title = header[j].strip() if len(header) > j else ""
                    content.append(f"{title}: {v.strip()}")
                page_content = "\n".join(content)
                documents.append(Document(page_content=page_content, metadata=metadata))
        return documents
    def _load_document_from_id(self, id: str) -> Document:
        """Load a document from an ID."""
        from io import BytesIO
        from googleapiclient.discovery import build
        from googleapiclient.errors import HttpError
        from googleapiclient.http import MediaIoBaseDownload
        creds = self._load_credentials()
        service = build("drive", "v3", credentials=creds)
        file = service.files().get(fileId=id, supportsAllDrives=True).execute()
        request = service.files().export_media(fileId=id, mimeType="text/plain")
        fh = BytesIO()
        downloader = MediaIoBaseDownload(fh, request)
        done = False
        try:
            while done is False:
                status, done = downloader.next_chunk()
        except HttpError as e:
            if e.resp.status == 404:
                print("File not found: {}".format(id))
            else:
                print("An error occurred: {}".format(e))
        text = fh.getvalue().decode("utf-8")
        metadata = {
            "source": f"https://docs.google.com/document/d/{id}/edit",
            "title": f"{file.get('name')}",
        }
        return Document(page_content=text, metadata=metadata)
    def _load_documents_from_folder(self, folder_id: str) -> List[Document]:
        """Load documents from a folder."""
        from googleapiclient.discovery import build
        creds = self._load_credentials()


from googleapiclient.discovery import build
        creds = self._load_credentials()
        service = build("drive", "v3", credentials=creds)
        files = self._fetch_files_recursive(service, folder_id)
        returns = []
        for file in files:
            if file["mimeType"] == "application/vnd.google-apps.document":
                returns.append(self._load_document_from_id(file["id"]))  # type: ignore
            elif file["mimeType"] == "application/vnd.google-apps.spreadsheet":
                returns.extend(self._load_sheet_from_id(file["id"]))  # type: ignore
            elif file["mimeType"] == "application/pdf":
                returns.extend(self._load_file_from_id(file["id"]))  # type: ignore
            else:
                pass
        return returns
    def _fetch_files_recursive(
        self, service: Any, folder_id: str
    ) -> List[Dict[str, Union[str, List[str]]]]:
        """Fetch all files and subfolders recursively."""
        results = (
            service.files()
            .list(
                q=f"'{folder_id}' in parents",
                pageSize=1000,
                includeItemsFromAllDrives=True,
                supportsAllDrives=True,
                fields="nextPageToken, files(id, name, mimeType, parents)",
            )
            .execute()
        )
        files = results.get("files", [])
        returns = []
        for file in files:
            if file["mimeType"] == "application/vnd.google-apps.folder":
                if self.recursive:
                    returns.extend(self._fetch_files_recursive(service, file["id"]))
            else:
                returns.append(file)
        return returns
    def _load_documents_from_ids(self) -> List[Document]:


return returns
    def _load_documents_from_ids(self) -> List[Document]:
        """Load documents from a list of IDs."""
        if not self.document_ids:
            raise ValueError("document_ids must be set")
        return [self._load_document_from_id(doc_id) for doc_id in self.document_ids]
    def _load_file_from_id(self, id: str) -> List[Document]:
        """Load a file from an ID."""
        from io import BytesIO
        from googleapiclient.discovery import build
        from googleapiclient.http import MediaIoBaseDownload
        creds = self._load_credentials()
        service = build("drive", "v3", credentials=creds)
        file = service.files().get(fileId=id, supportsAllDrives=True).execute()
        request = service.files().get_media(fileId=id)
        fh = BytesIO()
        downloader = MediaIoBaseDownload(fh, request)
        done = False
        while done is False:
            status, done = downloader.next_chunk()
        content = fh.getvalue()
        from PyPDF2 import PdfReader
        pdf_reader = PdfReader(BytesIO(content))
        return [
            Document(
                page_content=page.extract_text(),
                metadata={
                    "source": f"https://drive.google.com/file/d/{id}/view",
                    "title": f"{file.get('name')}",
                    "page": i,
                },
            )
            for i, page in enumerate(pdf_reader.pages)
        ]
    def _load_file_from_ids(self) -> List[Document]:
        """Load files from a list of IDs."""
        if not self.file_ids:
            raise ValueError("file_ids must be set")
        docs = []
        for file_id in self.file_ids:


docs = []
        for file_id in self.file_ids:
            docs.extend(self._load_file_from_id(file_id))
        return docs
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        if self.folder_id:
            return self._load_documents_from_folder(self.folder_id)
        elif self.document_ids:
            return self._load_documents_from_ids()
        else:
            return self._load_file_from_ids()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.csv_loader
import csv
from typing import Dict, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class CSVLoader(BaseLoader):
    """Loads a CSV file into a list of documents.
    Each document represents one row of the CSV file. Every row is converted into a
    key/value pair and outputted to a new line in the document's page_content.
    The source for each document loaded from csv is set to the value of the
    `file_path` argument for all doucments by default.
    You can override this by setting the `source_column` argument to the
    name of a column in the CSV file.
    The source of each document will then be set to the value of the column
    with the name specified in `source_column`.
    Output Example:
        .. code-block:: txt
            column1: value1
            column2: value2
            column3: value3
    """
    def __init__(
        self,
        file_path: str,
        source_column: Optional[str] = None,
        csv_args: Optional[Dict] = None,
        encoding: Optional[str] = None,
    ):
        self.file_path = file_path
        self.source_column = source_column
        self.encoding = encoding
        if csv_args is None:
            self.csv_args = {
                "delimiter": csv.Dialect.delimiter,
                "quotechar": csv.Dialect.quotechar,
            }
        else:
            self.csv_args = csv_args
[docs]    def load(self) -> List[Document]:
        """Load data into document objects."""
        docs = []
        with open(self.file_path, newline="", encoding=self.encoding) as csvfile:


with open(self.file_path, newline="", encoding=self.encoding) as csvfile:
            csv_reader = csv.DictReader(csvfile, **self.csv_args)  # type: ignore
            for i, row in enumerate(csv_reader):
                content = "\n".join(f"{k.strip()}: {v.strip()}" for k, v in row.items())
                try:
                    source = (
                        row[self.source_column]
                        if self.source_column is not None
                        else self.file_path
                    )
                except KeyError:
                    raise ValueError(
                        f"Source column '{self.source_column}' not found in CSV file."
                    )
                metadata = {"source": source, "row": i}
                doc = Document(page_content=content, metadata=metadata)
                docs.append(doc)
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.twitter
"""Twitter document loader."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Sequence, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
if TYPE_CHECKING:
    import tweepy
    from tweepy import OAuth2BearerHandler, OAuthHandler
def _dependable_tweepy_import() -> tweepy:
    try:
        import tweepy
    except ImportError:
        raise ValueError(
            "tweepy package not found, please install it with `pip install tweepy`"
        )
    return tweepy
[docs]class TwitterTweetLoader(BaseLoader):
    """Twitter tweets loader.
    Read tweets of user twitter handle.
    First you need to go to
    `https://developer.twitter.com/en/docs/twitter-api
    /getting-started/getting-access-to-the-twitter-api`
    to get your token. And create a v2 version of the app.
    """
    def __init__(
        self,
        auth_handler: Union[OAuthHandler, OAuth2BearerHandler],
        twitter_users: Sequence[str],
        number_tweets: Optional[int] = 100,
    ):
        self.auth = auth_handler
        self.twitter_users = twitter_users
        self.number_tweets = number_tweets
[docs]    def load(self) -> List[Document]:
        """Load tweets."""
        tweepy = _dependable_tweepy_import()
        api = tweepy.API(self.auth, parser=tweepy.parsers.JSONParser())
        results: List[Document] = []
        for username in self.twitter_users:
            tweets = api.user_timeline(screen_name=username, count=self.number_tweets)
            user = api.get_user(screen_name=username)


user = api.get_user(screen_name=username)
            docs = self._format_tweets(tweets, user)
            results.extend(docs)
        return results
    def _format_tweets(
        self, tweets: List[Dict[str, Any]], user_info: dict
    ) -> Iterable[Document]:
        """Format tweets into a string."""
        for tweet in tweets:
            metadata = {
                "created_at": tweet["created_at"],
                "user_info": user_info,
            }
            yield Document(
                page_content=tweet["text"],
                metadata=metadata,
            )
[docs]    @classmethod
    def from_bearer_token(
        cls,
        oauth2_bearer_token: str,
        twitter_users: Sequence[str],
        number_tweets: Optional[int] = 100,
    ) -> TwitterTweetLoader:
        """Create a TwitterTweetLoader from OAuth2 bearer token."""
        tweepy = _dependable_tweepy_import()
        auth = tweepy.OAuth2BearerHandler(oauth2_bearer_token)
        return cls(
            auth_handler=auth,
            twitter_users=twitter_users,
            number_tweets=number_tweets,
        )
[docs]    @classmethod
    def from_secrets(
        cls,
        access_token: str,
        access_token_secret: str,
        consumer_key: str,
        consumer_secret: str,
        twitter_users: Sequence[str],
        number_tweets: Optional[int] = 100,
    ) -> TwitterTweetLoader:
        """Create a TwitterTweetLoader from access tokens and secrets."""
        tweepy = _dependable_tweepy_import()
        auth = tweepy.OAuthHandler(
            access_token=access_token,
            access_token_secret=access_token_secret,


access_token=access_token,
            access_token_secret=access_token_secret,
            consumer_key=consumer_key,
            consumer_secret=consumer_secret,
        )
        return cls(
            auth_handler=auth,
            twitter_users=twitter_users,
            number_tweets=number_tweets,
        )
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.duckdb_loader
from typing import Dict, List, Optional, cast
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class DuckDBLoader(BaseLoader):
    """Loads a query result from DuckDB into a list of documents.
    Each document represents one row of the result. The `page_content_columns`
    are written into the `page_content` of the document. The `metadata_columns`
    are written into the `metadata` of the document. By default, all columns
    are written into the `page_content` and none into the `metadata`.
    """
    def __init__(
        self,
        query: str,
        database: str = ":memory:",
        read_only: bool = False,
        config: Optional[Dict[str, str]] = None,
        page_content_columns: Optional[List[str]] = None,
        metadata_columns: Optional[List[str]] = None,
    ):
        self.query = query
        self.database = database
        self.read_only = read_only
        self.config = config or {}
        self.page_content_columns = page_content_columns
        self.metadata_columns = metadata_columns
[docs]    def load(self) -> List[Document]:
        try:
            import duckdb
        except ImportError:
            raise ValueError(
                "Could not import duckdb python package. "
                "Please install it with `pip install duckdb`."
            )
        docs = []
        with duckdb.connect(
            database=self.database, read_only=self.read_only, config=self.config
        ) as con:
            query_result = con.execute(self.query)
            results = query_result.fetchall()
            description = cast(list, query_result.description)


results = query_result.fetchall()
            description = cast(list, query_result.description)
            field_names = [c[0] for c in description]
            if self.page_content_columns is None:
                page_content_columns = field_names
            else:
                page_content_columns = self.page_content_columns
            if self.metadata_columns is None:
                metadata_columns = []
            else:
                metadata_columns = self.metadata_columns
            for result in results:
                page_content = "\n".join(
                    f"{column}: {result[field_names.index(column)]}"
                    for column in page_content_columns
                )
                metadata = {
                    column: result[field_names.index(column)]
                    for column in metadata_columns
                }
                doc = Document(page_content=page_content, metadata=metadata)
                docs.append(doc)
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.notebook
"""Loader that loads .ipynb notebook files."""
import json
from pathlib import Path
from typing import Any, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
def concatenate_cells(
    cell: dict, include_outputs: bool, max_output_length: int, traceback: bool
) -> str:
    """Combine cells information in a readable format ready to be used."""
    cell_type = cell["cell_type"]
    source = cell["source"]
    output = cell["outputs"]
    if include_outputs and cell_type == "code" and output:
        if "ename" in output[0].keys():
            error_name = output[0]["ename"]
            error_value = output[0]["evalue"]
            if traceback:
                traceback = output[0]["traceback"]
                return (
                    f"'{cell_type}' cell: '{source}'\n, gives error '{error_name}',"
                    f" with description '{error_value}'\n"
                    f"and traceback '{traceback}'\n\n"
                )
            else:
                return (
                    f"'{cell_type}' cell: '{source}'\n, gives error '{error_name}',"
                    f"with description '{error_value}'\n\n"
                )
        elif output[0]["output_type"] == "stream":
            output = output[0]["text"]
            min_output = min(max_output_length, len(output))
            return (
                f"'{cell_type}' cell: '{source}'\n with "
                f"output: '{output[:min_output]}'\n\n"
            )
    else:
        return f"'{cell_type}' cell: '{source}'\n\n"


return f"'{cell_type}' cell: '{source}'\n\n"
    return ""
def remove_newlines(x: Any) -> Any:
    """Remove recursively newlines, no matter the data structure they are stored in."""
    import pandas as pd
    if isinstance(x, str):
        return x.replace("\n", "")
    elif isinstance(x, list):
        return [remove_newlines(elem) for elem in x]
    elif isinstance(x, pd.DataFrame):
        return x.applymap(remove_newlines)
    else:
        return x
[docs]class NotebookLoader(BaseLoader):
    """Loader that loads .ipynb notebook files."""
    def __init__(
        self,
        path: str,
        include_outputs: bool = False,
        max_output_length: int = 10,
        remove_newline: bool = False,
        traceback: bool = False,
    ):
        """Initialize with path."""
        self.file_path = path
        self.include_outputs = include_outputs
        self.max_output_length = max_output_length
        self.remove_newline = remove_newline
        self.traceback = traceback
[docs]    def load(
        self,
    ) -> List[Document]:
        """Load documents."""
        try:
            import pandas as pd
        except ImportError:
            raise ValueError(
                "pandas is needed for Notebook Loader, "
                "please install with `pip install pandas`"
            )
        p = Path(self.file_path)
        with open(p, encoding="utf8") as f:
            d = json.load(f)
        data = pd.json_normalize(d["cells"])
        filtered_data = data[["cell_type", "source", "outputs"]]
        if self.remove_newline:


if self.remove_newline:
            filtered_data = filtered_data.applymap(remove_newlines)
        text = filtered_data.apply(
            lambda x: concatenate_cells(
                x, self.include_outputs, self.max_output_length, self.traceback
            ),
            axis=1,
        ).str.cat(sep=" ")
        metadata = {"source": str(p)}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.modern_treasury
"""Loader that fetches data from Modern Treasury"""
import json
import urllib.request
from base64 import b64encode
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import get_from_env, stringify_value
MODERN_TREASURY_ENDPOINTS = {
    "payment_orders": "https://app.moderntreasury.com/api/payment_orders",
    "expected_payments": "https://app.moderntreasury.com/api/expected_payments",
    "returns": "https://app.moderntreasury.com/api/returns",
    "incoming_payment_details": "https://app.moderntreasury.com/api/\
incoming_payment_details",
    "counterparties": "https://app.moderntreasury.com/api/counterparties",
    "internal_accounts": "https://app.moderntreasury.com/api/internal_accounts",
    "external_accounts": "https://app.moderntreasury.com/api/external_accounts",
    "transactions": "https://app.moderntreasury.com/api/transactions",
    "ledgers": "https://app.moderntreasury.com/api/ledgers",
    "ledger_accounts": "https://app.moderntreasury.com/api/ledger_accounts",
    "ledger_transactions": "https://app.moderntreasury.com/api/ledger_transactions",
    "events": "https://app.moderntreasury.com/api/events",
    "invoices": "https://app.moderntreasury.com/api/invoices",
}
[docs]class ModernTreasuryLoader(BaseLoader):
    def __init__(
        self,
        resource: str,
        organization_id: Optional[str] = None,


self,
        resource: str,
        organization_id: Optional[str] = None,
        api_key: Optional[str] = None,
    ) -> None:
        self.resource = resource
        organization_id = organization_id or get_from_env(
            "organization_id", "MODERN_TREASURY_ORGANIZATION_ID"
        )
        api_key = api_key or get_from_env("api_key", "MODERN_TREASURY_API_KEY")
        credentials = f"{organization_id}:{api_key}".encode("utf-8")
        basic_auth_token = b64encode(credentials).decode("utf-8")
        self.headers = {"Authorization": f"Basic {basic_auth_token}"}
    def _make_request(self, url: str) -> List[Document]:
        request = urllib.request.Request(url, headers=self.headers)
        with urllib.request.urlopen(request) as response:
            json_data = json.loads(response.read().decode())
            text = stringify_value(json_data)
            metadata = {"source": url}
            return [Document(page_content=text, metadata=metadata)]
    def _get_resource(self) -> List[Document]:
        endpoint = MODERN_TREASURY_ENDPOINTS.get(self.resource)
        if endpoint is None:
            return []
        return self._make_request(endpoint)
[docs]    def load(self) -> List[Document]:
        return self._get_resource()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.roam
"""Loader that loads Roam directory dump."""
from pathlib import Path
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class RoamLoader(BaseLoader):
    """Loader that loads Roam files from disk."""
    def __init__(self, path: str):
        """Initialize with path."""
        self.file_path = path
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        ps = list(Path(self.file_path).glob("**/*.md"))
        docs = []
        for p in ps:
            with open(p) as f:
                text = f.read()
            metadata = {"source": str(p)}
            docs.append(Document(page_content=text, metadata=metadata))
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.rtf
"""Loader that loads rich text files."""
from typing import Any, List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    satisfies_min_unstructured_version,
)
[docs]class UnstructuredRTFLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load rtf files."""
    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        min_unstructured_version = "0.5.12"
        if not satisfies_min_unstructured_version(min_unstructured_version):
            raise ValueError(
                "Partitioning rtf files is only supported in "
                f"unstructured>={min_unstructured_version}."
            )
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.rtf import partition_rtf
        return partition_rtf(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.facebook_chat
"""Loader that loads Facebook chat json dump."""
import datetime
import json
from pathlib import Path
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
def concatenate_rows(row: dict) -> str:
    """Combine message information in a readable format ready to be used."""
    sender = row["sender_name"]
    text = row["content"]
    date = datetime.datetime.fromtimestamp(row["timestamp_ms"] / 1000).strftime(
        "%Y-%m-%d %H:%M:%S"
    )
    return f"{sender} on {date}: {text}\n\n"
[docs]class FacebookChatLoader(BaseLoader):
    """Loader that loads Facebook messages json directory dump."""
    def __init__(self, path: str):
        """Initialize with path."""
        self.file_path = path
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            import pandas as pd
        except ImportError:
            raise ValueError(
                "pandas is needed for Facebook chat loader, "
                "please install with `pip install pandas`"
            )
        p = Path(self.file_path)
        with open(p, encoding="utf8") as f:
            d = json.load(f)
        normalized_messages = pd.json_normalize(d["messages"])
        df_normalized_messages = pd.DataFrame(normalized_messages)
        # Only keep plain text messages
        # (no services, nor links, hashtags, code, bold ...)
        df_filtered = df_normalized_messages[
            (df_normalized_messages.content.apply(lambda x: type(x) == str))
        ]
        df_filtered = df_filtered[["timestamp_ms", "content", "sender_name"]]


df_filtered = df_filtered[["timestamp_ms", "content", "sender_name"]]
        text = df_filtered.apply(concatenate_rows, axis=1).str.cat(sep="")
        metadata = {"source": str(p)}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.bilibili
import json
import re
import warnings
from typing import List, Tuple
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class BiliBiliLoader(BaseLoader):
    """Loader that loads bilibili transcripts."""
    def __init__(self, video_urls: List[str]):
        """Initialize with bilibili url."""
        self.video_urls = video_urls
[docs]    def load(self) -> List[Document]:
        """Load from bilibili url."""
        results = []
        for url in self.video_urls:
            transcript, video_info = self._get_bilibili_subs_and_info(url)
            doc = Document(page_content=transcript, metadata=video_info)
            results.append(doc)
        return results
    def _get_bilibili_subs_and_info(self, url: str) -> Tuple[str, dict]:
        try:
            from bilibili_api import sync, video
        except ImportError:
            raise ValueError(
                "requests package not found, please install it with "
                "`pip install bilibili-api`"
            )
        bvid = re.search(r"BV\w+", url)
        if bvid is not None:
            v = video.Video(bvid=bvid.group())
        else:
            aid = re.search(r"av[0-9]+", url)
            if aid is not None:
                try:
                    v = video.Video(aid=int(aid.group()[2:]))
                except AttributeError:
                    raise ValueError(f"{url} is not bilibili url.")
            else:
                raise ValueError(f"{url} is not bilibili url.")
        video_info = sync(v.get_info())
        video_info.update({"url": url})


video_info = sync(v.get_info())
        video_info.update({"url": url})
        # Get subtitle url
        subtitle = video_info.pop("subtitle")
        sub_list = subtitle["list"]
        if sub_list:
            sub_url = sub_list[0]["subtitle_url"]
            result = requests.get(sub_url)
            raw_sub_titles = json.loads(result.content)["body"]
            raw_transcript = " ".join([c["content"] for c in raw_sub_titles])
            raw_transcript_with_meta_info = f"""
                Video Title: {video_info['title']},
                description: {video_info['desc']}\n
                Transcript: {raw_transcript}
                """
            return raw_transcript_with_meta_info, video_info
        else:
            raw_transcript = ""
            warnings.warn(
                f"""
                No subtitles found for video: {url}.
                Return Empty transcript.
                """
            )
            return raw_transcript, video_info
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.notion
"""Loader that loads Notion directory dump."""
from pathlib import Path
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class NotionDirectoryLoader(BaseLoader):
    """Loader that loads Notion directory dump."""
    def __init__(self, path: str):
        """Initialize with path."""
        self.file_path = path
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        ps = list(Path(self.file_path).glob("**/*.md"))
        docs = []
        for p in ps:
            with open(p) as f:
                text = f.read()
            metadata = {"source": str(p)}
            docs.append(Document(page_content=text, metadata=metadata))
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.diffbot
"""Loader that uses Diffbot to load webpages in text format."""
import logging
from typing import Any, List
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class DiffbotLoader(BaseLoader):
    """Loader that loads Diffbot file json."""
    def __init__(
        self, api_token: str, urls: List[str], continue_on_failure: bool = True
    ):
        """Initialize with API token, ids, and key."""
        self.api_token = api_token
        self.urls = urls
        self.continue_on_failure = continue_on_failure
    def _diffbot_api_url(self, diffbot_api: str) -> str:
        return f"https://api.diffbot.com/v3/{diffbot_api}"
    def _get_diffbot_data(self, url: str) -> Any:
        """Get Diffbot file from Diffbot REST API."""
        # TODO: Add support for other Diffbot APIs
        diffbot_url = self._diffbot_api_url("article")
        params = {
            "token": self.api_token,
            "url": url,
        }
        response = requests.get(diffbot_url, params=params, timeout=10)
        # TODO: handle non-ok errors
        return response.json() if response.ok else {}
[docs]    def load(self) -> List[Document]:
        """Extract text from Diffbot on all the URLs and return Document instances"""
        docs: List[Document] = list()
        for url in self.urls:
            try:
                data = self._get_diffbot_data(url)
                text = data["objects"][0]["text"] if "objects" in data else ""


text = data["objects"][0]["text"] if "objects" in data else ""
                metadata = {"source": url}
                docs.append(Document(page_content=text, metadata=metadata))
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error fetching or processing {url}, exception: {e}")
                else:
                    raise e
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.email
"""Loader that loads email files."""
import os
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    satisfies_min_unstructured_version,
)
[docs]class UnstructuredEmailLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load email files."""
    def _get_elements(self) -> List:
        from unstructured.file_utils.filetype import FileType, detect_filetype
        filetype = detect_filetype(self.file_path)
        if filetype == FileType.EML:
            from unstructured.partition.email import partition_email
            return partition_email(filename=self.file_path, **self.unstructured_kwargs)
        elif satisfies_min_unstructured_version("0.5.8") and filetype == FileType.MSG:
            from unstructured.partition.msg import partition_msg
            return partition_msg(filename=self.file_path, **self.unstructured_kwargs)
        else:
            raise ValueError(
                f"Filetype {filetype} is not supported in UnstructuredEmailLoader."
            )
[docs]class OutlookMessageLoader(BaseLoader):
    """
    Loader that loads Outlook Message files using extract_msg.
    https://github.com/TeamMsgExtractor/msg-extractor
    """
    def __init__(self, file_path: str):
        """Initialize with file path."""
        self.file_path = file_path
        if not os.path.isfile(self.file_path):
            raise ValueError("File path %s is not a valid file" % self.file_path)
        try:
            import extract_msg  # noqa:F401
        except ImportError:
            raise ImportError(
                "extract_msg is not installed. Please install it with "
                "`pip install extract_msg`"


"`pip install extract_msg`"
            )
[docs]    def load(self) -> List[Document]:
        """Load data into document objects."""
        import extract_msg
        msg = extract_msg.Message(self.file_path)
        return [
            Document(
                page_content=msg.body,
                metadata={
                    "subject": msg.subject,
                    "sender": msg.sender,
                    "date": msg.date,
                },
            )
        ]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.whatsapp_chat
import re
from pathlib import Path
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
def concatenate_rows(date: str, sender: str, text: str) -> str:
    """Combine message information in a readable format ready to be used."""
    return f"{sender} on {date}: {text}\n\n"
[docs]class WhatsAppChatLoader(BaseLoader):
    """Loader that loads WhatsApp messages text file."""
    def __init__(self, path: str):
        """Initialize with path."""
        self.file_path = path
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        p = Path(self.file_path)
        text_content = ""
        with open(p, encoding="utf8") as f:
            lines = f.readlines()
        message_line_regex = (
            r"(\d{1,2}/\d{1,2}/\d{2,4}, "
            r"\d{1,2}:\d{1,2}[ _]?(?:AM|PM)?) - "
            r"(.*?): (.*)"
        )
        for line in lines:
            result = re.match(
                message_line_regex,
                line.strip(),
            )
            if result:
                date, sender, text = result.groups()
                text_content += concatenate_rows(date, sender, text)
        metadata = {"source": str(p)}
        return [Document(page_content=text_content, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.image
"""Loader that loads image files."""
from typing import List
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class UnstructuredImageLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load image files, such as PNGs and JPGs."""
    def _get_elements(self) -> List:
        from unstructured.partition.image import partition_image
        return partition_image(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.blackboard
"""Loader that loads all documents from a blackboard course."""
import contextlib
import re
from pathlib import Path
from typing import Any, List, Optional, Tuple
from urllib.parse import unquote
from langchain.docstore.document import Document
from langchain.document_loaders.directory import DirectoryLoader
from langchain.document_loaders.pdf import PyPDFLoader
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class BlackboardLoader(WebBaseLoader):
    """Loader that loads all documents from a Blackboard course.
    This loader is not compatible with all Blackboard courses. It is only
    compatible with courses that use the new Blackboard interface.
    To use this loader, you must have the BbRouter cookie. You can get this
    cookie by logging into the course and then copying the value of the
    BbRouter cookie from the browser's developer tools.
    Example:
        .. code-block:: python
            from langchain.document_loaders import BlackboardLoader
            loader = BlackboardLoader(
                blackboard_course_url="https://blackboard.example.com/webapps/blackboard/execute/announcement?method=search&context=course_entry&course_id=_123456_1",
                bbrouter="expires:12345...",
            )
            documents = loader.load()
    """
    base_url: str
    folder_path: str
    load_all_recursively: bool
    def __init__(
        self,
        blackboard_course_url: str,
        bbrouter: str,
        load_all_recursively: bool = True,
        basic_auth: Optional[Tuple[str, str]] = None,
        cookies: Optional[dict] = None,
    ):
        """Initialize with blackboard course url.


):
        """Initialize with blackboard course url.
        The BbRouter cookie is required for most blackboard courses.
        Args:
            blackboard_course_url: Blackboard course url.
            bbrouter: BbRouter cookie.
            load_all_recursively: If True, load all documents recursively.
            basic_auth: Basic auth credentials.
            cookies: Cookies.
        Raises:
            ValueError: If blackboard course url is invalid.
        """
        super().__init__(blackboard_course_url)
        # Get base url
        try:
            self.base_url = blackboard_course_url.split("/webapps/blackboard")[0]
        except IndexError:
            raise ValueError(
                "Invalid blackboard course url. "
                "Please provide a url that starts with "
                "https://<blackboard_url>/webapps/blackboard"
            )
        if basic_auth is not None:
            self.session.auth = basic_auth
        # Combine cookies
        if cookies is None:
            cookies = {}
        cookies.update({"BbRouter": bbrouter})
        self.session.cookies.update(cookies)
        self.load_all_recursively = load_all_recursively
        self.check_bs4()
[docs]    def check_bs4(self) -> None:
        """Check if BeautifulSoup4 is installed.
        Raises:
            ImportError: If BeautifulSoup4 is not installed.
        """
        try:
            import bs4  # noqa: F401
        except ImportError:
            raise ImportError(
                "BeautifulSoup4 is required for BlackboardLoader. "
                "Please install it with `pip install beautifulsoup4`."
            )
[docs]    def load(self) -> List[Document]:
        """Load data into document objects.
        Returns:
            List of documents.


"""Load data into document objects.
        Returns:
            List of documents.
        """
        if self.load_all_recursively:
            soup_info = self.scrape()
            self.folder_path = self._get_folder_path(soup_info)
            relative_paths = self._get_paths(soup_info)
            documents = []
            for path in relative_paths:
                url = self.base_url + path
                print(f"Fetching documents from {url}")
                soup_info = self._scrape(url)
                with contextlib.suppress(ValueError):
                    documents.extend(self._get_documents(soup_info))
            return documents
        else:
            print(f"Fetching documents from {self.web_path}")
            soup_info = self.scrape()
            self.folder_path = self._get_folder_path(soup_info)
            return self._get_documents(soup_info)
    def _get_folder_path(self, soup: Any) -> str:
        """Get the folder path to save the documents in.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            Folder path.
        """
        # Get the course name
        course_name = soup.find("span", {"id": "crumb_1"})
        if course_name is None:
            raise ValueError("No course name found.")
        course_name = course_name.text.strip()
        # Prepare the folder path
        course_name_clean = (
            unquote(course_name)
            .replace(" ", "_")
            .replace("/", "_")
            .replace(":", "_")
            .replace(",", "_")
            .replace("?", "_")
            .replace("'", "_")
            .replace("!", "_")
            .replace('"', "_")
        )
        # Get the folder path
        folder_path = Path(".") / course_name_clean


)
        # Get the folder path
        folder_path = Path(".") / course_name_clean
        return str(folder_path)
    def _get_documents(self, soup: Any) -> List[Document]:
        """Fetch content from page and return Documents.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            List of documents.
        """
        attachments = self._get_attachments(soup)
        self._download_attachments(attachments)
        documents = self._load_documents()
        return documents
    def _get_attachments(self, soup: Any) -> List[str]:
        """Get all attachments from a page.
        Args:
            soup: BeautifulSoup4 soup object.
        Returns:
            List of attachments.
        """
        from bs4 import BeautifulSoup, Tag
        # Get content list
        content_list = soup.find("ul", {"class": "contentList"})
        if content_list is None:
            raise ValueError("No content list found.")
        content_list: BeautifulSoup  # type: ignore
        # Get all attachments
        attachments = []
        for attachment in content_list.find_all("ul", {"class": "attachments"}):
            attachment: Tag  # type: ignore
            for link in attachment.find_all("a"):
                link: Tag  # type: ignore
                href = link.get("href")
                # Only add if href is not None and does not start with #
                if href is not None and not href.startswith("#"):
                    attachments.append(href)
        return attachments
    def _download_attachments(self, attachments: List[str]) -> None:
        """Download all attachments.
        Args:
            attachments: List of attachments.
        """
        # Make sure the folder exists
        Path(self.folder_path).mkdir(parents=True, exist_ok=True)


Path(self.folder_path).mkdir(parents=True, exist_ok=True)
        # Download all attachments
        for attachment in attachments:
            self.download(attachment)
    def _load_documents(self) -> List[Document]:
        """Load all documents in the folder.
        Returns:
            List of documents.
        """
        # Create the document loader
        loader = DirectoryLoader(
            path=self.folder_path, glob="*.pdf", loader_cls=PyPDFLoader  # type: ignore
        )
        # Load the documents
        documents = loader.load()
        # Return all documents
        return documents
    def _get_paths(self, soup: Any) -> List[str]:
        """Get all relative paths in the navbar."""
        relative_paths = []
        course_menu = soup.find("ul", {"class": "courseMenu"})
        if course_menu is None:
            raise ValueError("No course menu found.")
        for link in course_menu.find_all("a"):
            href = link.get("href")
            if href is not None and href.startswith("/"):
                relative_paths.append(href)
        return relative_paths
[docs]    def download(self, path: str) -> None:
        """Download a file from a url.
        Args:
            path: Path to the file.
        """
        # Get the file content
        response = self.session.get(self.base_url + path, allow_redirects=True)
        # Get the filename
        filename = self.parse_filename(response.url)
        # Write the file to disk
        with open(Path(self.folder_path) / filename, "wb") as f:
            f.write(response.content)
[docs]    def parse_filename(self, url: str) -> str:
        """Parse the filename from a url.
        Args:


"""Parse the filename from a url.
        Args:
            url: Url to parse the filename from.
        Returns:
            The filename.
        """
        if (url_path := Path(url)) and url_path.suffix == ".pdf":
            return url_path.name
        else:
            return self._parse_filename_from_url(url)
    def _parse_filename_from_url(self, url: str) -> str:
        """Parse the filename from a url.
        Args:
            url: Url to parse the filename from.
        Returns:
            The filename.
        Raises:
            ValueError: If the filename could not be parsed.
        """
        filename_matches = re.search(r"filename%2A%3DUTF-8%27%27(.+)", url)
        if filename_matches:
            filename = filename_matches.group(1)
        else:
            raise ValueError(f"Could not parse filename from {url}")
        if ".pdf" not in filename:
            raise ValueError(f"Incorrect file type: {filename}")
        filename = filename.split(".pdf")[0] + ".pdf"
        filename = unquote(filename)
        filename = filename.replace("%20", " ")
        return filename
if __name__ == "__main__":
    loader = BlackboardLoader(
        "https://<YOUR BLACKBOARD URL"
        " HERE>/webapps/blackboard/content/listContent.jsp?course_id=_<YOUR COURSE ID"
        " HERE>_1&content_id=_<YOUR CONTENT ID HERE>_1&mode=reset",
        "<YOUR BBROUTER COOKIE HERE>",
        load_all_recursively=True,
    )
    documents = loader.load()
    print(f"Loaded {len(documents)} pages of PDFs from {loader.web_path}")
By Harrison Chase


By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.slack_directory
"""Loader for documents from a Slack export."""
import json
import zipfile
from pathlib import Path
from typing import Dict, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class SlackDirectoryLoader(BaseLoader):
    """Loader for loading documents from a Slack directory dump."""
    def __init__(self, zip_path: str, workspace_url: Optional[str] = None):
        """Initialize the SlackDirectoryLoader.
        Args:
            zip_path (str): The path to the Slack directory dump zip file.
            workspace_url (Optional[str]): The Slack workspace URL.
              Including the URL will turn
              sources into links. Defaults to None.
        """
        self.zip_path = Path(zip_path)
        self.workspace_url = workspace_url
        self.channel_id_map = self._get_channel_id_map(self.zip_path)
    @staticmethod
    def _get_channel_id_map(zip_path: Path) -> Dict[str, str]:
        """Get a dictionary mapping channel names to their respective IDs."""
        with zipfile.ZipFile(zip_path, "r") as zip_file:
            try:
                with zip_file.open("channels.json", "r") as f:
                    channels = json.load(f)
                return {channel["name"]: channel["id"] for channel in channels}
            except KeyError:
                return {}
[docs]    def load(self) -> List[Document]:
        """Load and return documents from the Slack directory dump."""
        docs = []
        with zipfile.ZipFile(self.zip_path, "r") as zip_file:
            for channel_path in zip_file.namelist():
                channel_name = Path(channel_path).parent.name
                if not channel_name:
                    continue


channel_name = Path(channel_path).parent.name
                if not channel_name:
                    continue
                if channel_path.endswith(".json"):
                    messages = self._read_json(zip_file, channel_path)
                    for message in messages:
                        document = self._convert_message_to_document(
                            message, channel_name
                        )
                        docs.append(document)
        return docs
    def _read_json(self, zip_file: zipfile.ZipFile, file_path: str) -> List[dict]:
        """Read JSON data from a zip subfile."""
        with zip_file.open(file_path, "r") as f:
            data = json.load(f)
        return data
    def _convert_message_to_document(
        self, message: dict, channel_name: str
    ) -> Document:
        """
        Convert a message to a Document object.
        Args:
            message (dict): A message in the form of a dictionary.
            channel_name (str): The name of the channel the message belongs to.
        Returns:
            Document: A Document object representing the message.
        """
        text = message.get("text", "")
        metadata = self._get_message_metadata(message, channel_name)
        return Document(
            page_content=text,
            metadata=metadata,
        )
    def _get_message_metadata(self, message: dict, channel_name: str) -> dict:
        """Create and return metadata for a given message and channel."""
        timestamp = message.get("ts", "")
        user = message.get("user", "")
        source = self._get_message_source(channel_name, user, timestamp)
        return {
            "source": source,
            "channel": channel_name,
            "timestamp": timestamp,
            "user": user,
        }


"timestamp": timestamp,
            "user": user,
        }
    def _get_message_source(self, channel_name: str, user: str, timestamp: str) -> str:
        """
        Get the message source as a string.
        Args:
            channel_name (str): The name of the channel the message belongs to.
            user (str): The user ID who sent the message.
            timestamp (str): The timestamp of the message.
        Returns:
            str: The message source.
        """
        if self.workspace_url:
            channel_id = self.channel_id_map.get(channel_name, "")
            return (
                f"{self.workspace_url}/archives/{channel_id}"
                + f"/p{timestamp.replace('.', '')}"
            )
        else:
            return f"{channel_name} - {user} - {timestamp}"
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.s3_file
"""Loading logic for loading documents from an s3 file."""
import os
import tempfile
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class S3FileLoader(BaseLoader):
    """Loading logic for loading documents from s3."""
    def __init__(self, bucket: str, key: str):
        """Initialize with bucket and key name."""
        self.bucket = bucket
        self.key = key
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            import boto3
        except ImportError:
            raise ValueError(
                "Could not import boto3 python package. "
                "Please install it with `pip install boto3`."
            )
        s3 = boto3.client("s3")
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}/{self.key}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            s3.download_file(self.bucket, self.key, file_path)
            loader = UnstructuredFileLoader(file_path)
            return loader.load()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.epub
"""Loader that loads EPub files."""
from typing import List
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    satisfies_min_unstructured_version,
)
[docs]class UnstructuredEPubLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load epub files."""
    def _get_elements(self) -> List:
        min_unstructured_version = "0.5.4"
        if not satisfies_min_unstructured_version(min_unstructured_version):
            raise ValueError(
                "Partitioning epub files is only supported in "
                f"unstructured>={min_unstructured_version}."
            )
        from unstructured.partition.epub import partition_epub
        return partition_epub(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.conllu
"""Load CoNLL-U files."""
import csv
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class CoNLLULoader(BaseLoader):
    """Load CoNLL-U files."""
    def __init__(self, file_path: str):
        """Initialize with file path."""
        self.file_path = file_path
[docs]    def load(self) -> List[Document]:
        """Load from file path."""
        with open(self.file_path, encoding="utf8") as f:
            tsv = list(csv.reader(f, delimiter="\t"))
            # If len(line) > 1, the line is not a comment
            lines = [line for line in tsv if len(line) > 1]
        text = ""
        for i, line in enumerate(lines):
            # Do not add a space after a punctuation mark or at the end of the sentence
            if line[9] == "SpaceAfter=No" or i == len(lines) - 1:
                text += line[1]
            else:
                text += line[1] + " "
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.unstructured
"""Loader that uses unstructured to load files."""
from abc import ABC, abstractmethod
from typing import IO, Any, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
def satisfies_min_unstructured_version(min_version: str) -> bool:
    """Checks to see if the installed unstructured version exceeds the minimum version
    for the feature in question."""
    from unstructured.__version__ import __version__ as __unstructured_version__
    min_version_tuple = tuple([int(x) for x in min_version.split(".")])
    # NOTE(MthwRobinson) - enables the loader to work when you're using pre-release
    # versions of unstructured like 0.4.17-dev1
    _unstructured_version = __unstructured_version__.split("-")[0]
    unstructured_version_tuple = tuple(
        [int(x) for x in _unstructured_version.split(".")]
    )
    return unstructured_version_tuple >= min_version_tuple
class UnstructuredBaseLoader(BaseLoader, ABC):
    """Loader that uses unstructured to load files."""
    def __init__(self, mode: str = "single", **unstructured_kwargs: Any):
        """Initialize with file path."""
        try:
            import unstructured  # noqa:F401
        except ImportError:
            raise ValueError(
                "unstructured package not found, please install it with "
                "`pip install unstructured`"
            )
        _valid_modes = {"single", "elements"}
        if mode not in _valid_modes:
            raise ValueError(
                f"Got {mode} for `mode`, but should be one of `{_valid_modes}`"
            )
        self.mode = mode


)
        self.mode = mode
        if not satisfies_min_unstructured_version("0.5.4"):
            if "strategy" in unstructured_kwargs:
                unstructured_kwargs.pop("strategy")
        self.unstructured_kwargs = unstructured_kwargs
    @abstractmethod
    def _get_elements(self) -> List:
        """Get elements."""
    @abstractmethod
    def _get_metadata(self) -> dict:
        """Get metadata."""
    def load(self) -> List[Document]:
        """Load file."""
        elements = self._get_elements()
        if self.mode == "elements":
            docs: List[Document] = list()
            for element in elements:
                metadata = self._get_metadata()
                # NOTE(MthwRobinson) - the attribute check is for backward compatibility
                # with unstructured<0.4.9. The metadata attributed was added in 0.4.9.
                if hasattr(element, "metadata"):
                    metadata.update(element.metadata.to_dict())
                if hasattr(element, "category"):
                    metadata["category"] = element.category
                docs.append(Document(page_content=str(element), metadata=metadata))
        elif self.mode == "single":
            metadata = self._get_metadata()
            text = "\n\n".join([str(el) for el in elements])
            docs = [Document(page_content=text, metadata=metadata)]
        else:
            raise ValueError(f"mode of {self.mode} not supported.")
        return docs
[docs]class UnstructuredFileLoader(UnstructuredBaseLoader):
    """Loader that uses unstructured to load files."""
    def __init__(
        self, file_path: str, mode: str = "single", **unstructured_kwargs: Any
    ):
        """Initialize with file path."""
        self.file_path = file_path


):
        """Initialize with file path."""
        self.file_path = file_path
        super().__init__(mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.auto import partition
        return partition(filename=self.file_path, **self.unstructured_kwargs)
    def _get_metadata(self) -> dict:
        return {"source": self.file_path}
[docs]class UnstructuredAPIFileLoader(UnstructuredFileLoader):
    """Loader that uses the unstructured web API to load files."""
    def __init__(
        self,
        file_path: str,
        mode: str = "single",
        url: str = "https://api.unstructured.io/general/v0/general",
        api_key: str = "",
        **unstructured_kwargs: Any,
    ):
        """Initialize with file path."""
        min_unstructured_version = "0.6.2"
        if not satisfies_min_unstructured_version(min_unstructured_version):
            raise ValueError(
                "Partitioning via API is only supported in "
                f"unstructured>={min_unstructured_version}."
            )
        self.url = url
        self.api_key = api_key
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.api import partition_via_api
        return partition_via_api(
            filename=self.file_path,
            api_key=self.api_key,
            api_url=self.url,
            **self.unstructured_kwargs,
        )
[docs]class UnstructuredFileIOLoader(UnstructuredBaseLoader):
    """Loader that uses unstructured to load file IO objects."""


"""Loader that uses unstructured to load file IO objects."""
    def __init__(self, file: IO, mode: str = "single", **unstructured_kwargs: Any):
        """Initialize with file path."""
        self.file = file
        super().__init__(mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.auto import partition
        return partition(file=self.file, **self.unstructured_kwargs)
    def _get_metadata(self) -> dict:
        return {}
[docs]class UnstructuredAPIFileIOLoader(UnstructuredFileIOLoader):
    """Loader that uses the unstructured web API to load file IO objects."""
    def __init__(
        self,
        file: IO,
        mode: str = "single",
        url: str = "https://api.unstructured.io/general/v0/general",
        api_key: str = "",
        **unstructured_kwargs: Any,
    ):
        """Initialize with file path."""
        min_unstructured_version = "0.6.2"
        if not satisfies_min_unstructured_version(min_unstructured_version):
            raise ValueError(
                "Partitioning via API is only supported in "
                f"unstructured>={min_unstructured_version}."
            )
        self.url = url
        self.api_key = api_key
        super().__init__(file=file, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.api import partition_via_api
        return partition_via_api(
            file=self.file,
            api_key=self.api_key,
            api_url=self.url,
            **self.unstructured_kwargs,
        )
By Harrison Chase

      © Copyright 2023, Harrison Chase.


Source code for langchain.document_loaders.bigquery
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class BigQueryLoader(BaseLoader):
    """Loads a query result from BigQuery into a list of documents.
    Each document represents one row of the result. The `page_content_columns`
    are written into the `page_content` of the document. The `metadata_columns`
    are written into the `metadata` of the document. By default, all columns
    are written into the `page_content` and none into the `metadata`.
    """
    def __init__(
        self,
        query: str,
        project: Optional[str] = None,
        page_content_columns: Optional[List[str]] = None,
        metadata_columns: Optional[List[str]] = None,
    ):
        self.query = query
        self.project = project
        self.page_content_columns = page_content_columns
        self.metadata_columns = metadata_columns
[docs]    def load(self) -> List[Document]:
        try:
            from google.cloud import bigquery
        except ImportError as ex:
            raise ValueError(
                "Could not import google-cloud-bigquery python package. "
                "Please install it with `pip install google-cloud-bigquery`."
            ) from ex
        bq_client = bigquery.Client(self.project)
        query_result = bq_client.query(self.query).result()
        docs: List[Document] = []
        page_content_columns = self.page_content_columns
        metadata_columns = self.metadata_columns
        if page_content_columns is None:
            page_content_columns = [column.name for column in query_result.schema]
        if metadata_columns is None:
            metadata_columns = []
        for row in query_result:


metadata_columns = []
        for row in query_result:
            page_content = "\n".join(
                f"{k}: {v}" for k, v in row.items() if k in page_content_columns
            )
            metadata = {k: v for k, v in row.items() if k in metadata_columns}
            doc = Document(page_content=page_content, metadata=metadata)
            docs.append(doc)
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.gutenberg
"""Loader that loads .txt web files."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class GutenbergLoader(BaseLoader):
    """Loader that uses urllib to load .txt web files."""
    def __init__(self, file_path: str):
        """Initialize with file path."""
        if not file_path.startswith("https://www.gutenberg.org"):
            raise ValueError("file path must start with 'https://www.gutenberg.org'")
        if not file_path.endswith(".txt"):
            raise ValueError("file path must end with '.txt'")
        self.file_path = file_path
[docs]    def load(self) -> List[Document]:
        """Load file."""
        from urllib.request import urlopen
        elements = urlopen(self.file_path)
        text = "\n\n".join([str(el.decode("utf-8-sig")) for el in elements])
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.evernote
"""Load documents from Evernote.
https://gist.github.com/foxmask/7b29c43a161e001ff04afdb2f181e31c
"""
import hashlib
from base64 import b64decode
from time import strptime
from typing import Any, Dict, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
def _parse_content(content: str) -> str:
    from pypandoc import convert_text
    text = convert_text(content, "org", format="html")
    return text
def _parse_resource(resource: list) -> dict:
    rsc_dict: Dict[str, Any] = {}
    for elem in resource:
        if elem.tag == "data":
            # Some times elem.text is None
            rsc_dict[elem.tag] = b64decode(elem.text) if elem.text else b""
            rsc_dict["hash"] = hashlib.md5(rsc_dict[elem.tag]).hexdigest()
        else:
            rsc_dict[elem.tag] = elem.text
    return rsc_dict
def _parse_note(note: List) -> dict:
    note_dict: Dict[str, Any] = {}
    resources = []
    for elem in note:
        if elem.tag == "content":
            note_dict[elem.tag] = _parse_content(elem.text)
            # A copy of original content
            note_dict["content-raw"] = elem.text
        elif elem.tag == "resource":
            resources.append(_parse_resource(elem))
        elif elem.tag == "created" or elem.tag == "updated":
            note_dict[elem.tag] = strptime(elem.text, "%Y%m%dT%H%M%SZ")
        else:


else:
            note_dict[elem.tag] = elem.text
    note_dict["resource"] = resources
    return note_dict
def _parse_note_xml(xml_file: str) -> str:
    """Parse Evernote xml."""
    # Without huge_tree set to True, parser may complain about huge text node
    # Try to recover, because there may be " ", which will cause
    # "XMLSyntaxError: Entity 'nbsp' not defined"
    from lxml import etree
    context = etree.iterparse(
        xml_file, encoding="utf-8", strip_cdata=False, huge_tree=True, recover=True
    )
    result_string = ""
    for action, elem in context:
        if elem.tag == "note":
            result_string += _parse_note(elem)["content"]
    return result_string
[docs]class EverNoteLoader(BaseLoader):
    """Loader to load in EverNote files.."""
    def __init__(self, file_path: str):
        """Initialize with file path."""
        self.file_path = file_path
[docs]    def load(self) -> List[Document]:
        """Load document from EverNote file."""
        text = _parse_note_xml(self.file_path)
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.s3_directory
"""Loading logic for loading documents from an s3 directory."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.s3_file import S3FileLoader
[docs]class S3DirectoryLoader(BaseLoader):
    """Loading logic for loading documents from s3."""
    def __init__(self, bucket: str, prefix: str = ""):
        """Initialize with bucket and key name."""
        self.bucket = bucket
        self.prefix = prefix
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            import boto3
        except ImportError:
            raise ValueError(
                "Could not import boto3 python package. "
                "Please install it with `pip install boto3`."
            )
        s3 = boto3.resource("s3")
        bucket = s3.Bucket(self.bucket)
        docs = []
        for obj in bucket.objects.filter(Prefix=self.prefix):
            loader = S3FileLoader(self.bucket, obj.key)
            docs.extend(loader.load())
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.gcs_file
"""Loading logic for loading documents from a GCS file."""
import os
import tempfile
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class GCSFileLoader(BaseLoader):
    """Loading logic for loading documents from GCS."""
    def __init__(self, project_name: str, bucket: str, blob: str):
        """Initialize with bucket and key name."""
        self.bucket = bucket
        self.blob = blob
        self.project_name = project_name
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from google.cloud import storage
        except ImportError:
            raise ValueError(
                "Could not import google-cloud-storage python package. "
                "Please install it with `pip install google-cloud-storage`."
            )
        # Initialise a client
        storage_client = storage.Client(self.project_name)
        # Create a bucket object for our bucket
        bucket = storage_client.get_bucket(self.bucket)
        # Create a blob object from the filepath
        blob = bucket.blob(self.blob)
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}/{self.blob}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            # Download the file to a destination
            blob.download_to_filename(file_path)
            loader = UnstructuredFileLoader(file_path)
            return loader.load()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.notiondb
"""Notion DB loader for langchain"""
from typing import Any, Dict, List
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
NOTION_BASE_URL = "https://api.notion.com/v1"
DATABASE_URL = NOTION_BASE_URL + "/databases/{database_id}/query"
PAGE_URL = NOTION_BASE_URL + "/pages/{page_id}"
BLOCK_URL = NOTION_BASE_URL + "/blocks/{block_id}/children"
[docs]class NotionDBLoader(BaseLoader):
    """Notion DB Loader.
    Reads content from pages within a Noton Database.
    Args:
        integration_token (str): Notion integration token.
        database_id (str): Notion database id.
    """
    def __init__(self, integration_token: str, database_id: str) -> None:
        """Initialize with parameters."""
        if not integration_token:
            raise ValueError("integration_token must be provided")
        if not database_id:
            raise ValueError("database_id must be provided")
        self.token = integration_token
        self.database_id = database_id
        self.headers = {
            "Authorization": "Bearer " + self.token,
            "Content-Type": "application/json",
            "Notion-Version": "2022-06-28",
        }
[docs]    def load(self) -> List[Document]:
        """Load documents from the Notion database.
        Returns:
            List[Document]: List of documents.
        """
        page_ids = self._retrieve_page_ids()
        return list(self.load_page(page_id) for page_id in page_ids)
    def _retrieve_page_ids(


def _retrieve_page_ids(
        self, query_dict: Dict[str, Any] = {"page_size": 100}
    ) -> List[str]:
        """Get all the pages from a Notion database."""
        pages: List[Dict[str, Any]] = []
        while True:
            data = self._request(
                DATABASE_URL.format(database_id=self.database_id),
                method="POST",
                query_dict=query_dict,
            )
            pages.extend(data.get("results"))
            if not data.get("has_more"):
                break
            query_dict["start_cursor"] = data.get("next_cursor")
        page_ids = [page["id"] for page in pages]
        return page_ids
[docs]    def load_page(self, page_id: str) -> Document:
        """Read a page."""
        data = self._request(PAGE_URL.format(page_id=page_id))
        # load properties as metadata
        metadata: Dict[str, Any] = {}
        for prop_name, prop_data in data["properties"].items():
            prop_type = prop_data["type"]
            if prop_type == "rich_text":
                value = (
                    prop_data["rich_text"][0]["plain_text"]
                    if prop_data["rich_text"]
                    else None
                )
            elif prop_type == "title":
                value = (
                    prop_data["title"][0]["plain_text"] if prop_data["title"] else None
                )
            elif prop_type == "multi_select":
                value = (
                    [item["name"] for item in prop_data["multi_select"]]
                    if prop_data["multi_select"]
                    else []
                )
            else:
                value = None
            metadata[prop_name.lower()] = value
        metadata["id"] = page_id


metadata[prop_name.lower()] = value
        metadata["id"] = page_id
        return Document(page_content=self._load_blocks(page_id), metadata=metadata)
    def _load_blocks(self, block_id: str, num_tabs: int = 0) -> str:
        """Read a block and its children."""
        result_lines_arr: List[str] = []
        cur_block_id: str = block_id
        while cur_block_id:
            data = self._request(BLOCK_URL.format(block_id=cur_block_id))
            for result in data["results"]:
                result_obj = result[result["type"]]
                if "rich_text" not in result_obj:
                    continue
                cur_result_text_arr: List[str] = []
                for rich_text in result_obj["rich_text"]:
                    if "text" in rich_text:
                        cur_result_text_arr.append(
                            "\t" * num_tabs + rich_text["text"]["content"]
                        )
                if result["has_children"]:
                    children_text = self._load_blocks(
                        result["id"], num_tabs=num_tabs + 1
                    )
                    cur_result_text_arr.append(children_text)
                result_lines_arr.append("\n".join(cur_result_text_arr))
            cur_block_id = data.get("next_cursor")
        return "\n".join(result_lines_arr)
    def _request(
        self, url: str, method: str = "GET", query_dict: Dict[str, Any] = {}
    ) -> Any:
        res = requests.request(
            method,
            url,
            headers=self.headers,
            json=query_dict,
            timeout=10,
        )
        res.raise_for_status()
        return res.json()
By Harrison Chase

      © Copyright 2023, Harrison Chase.


By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on May 02, 2023.


Source code for langchain.document_loaders.youtube
"""Loader that loads YouTube transcript."""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any, Dict, List, Optional
from pydantic import root_validator
from pydantic.dataclasses import dataclass
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
SCOPES = ["https://www.googleapis.com/auth/youtube.readonly"]
[docs]@dataclass
class GoogleApiClient:
    """A Generic Google Api Client.
    To use, you should have the ``google_auth_oauthlib,youtube_transcript_api,google``
    python package installed.
    As the google api expects credentials you need to set up a google account and
    register your Service. "https://developers.google.com/docs/api/quickstart/python"
    Example:
        .. code-block:: python
            from langchain.document_loaders import GoogleApiClient
            google_api_client = GoogleApiClient(
                service_account_path=Path("path_to_your_sec_file.json")
            )
    """
    credentials_path: Path = Path.home() / ".credentials" / "credentials.json"
    service_account_path: Path = Path.home() / ".credentials" / "credentials.json"
    token_path: Path = Path.home() / ".credentials" / "token.json"
    def __post_init__(self) -> None:
        self.creds = self._load_credentials()
[docs]    @root_validator
    def validate_channel_or_videoIds_is_set(
        cls, values: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Validate that either folder_id or document_ids is set, but not both."""
        if not values.get("credentials_path") and not values.get(


if not values.get("credentials_path") and not values.get(
            "service_account_path"
        ):
            raise ValueError("Must specify either channel_name or video_ids")
        return values
    def _load_credentials(self) -> Any:
        """Load credentials."""
        # Adapted from https://developers.google.com/drive/api/v3/quickstart/python
        try:
            from google.auth.transport.requests import Request
            from google.oauth2 import service_account
            from google.oauth2.credentials import Credentials
            from google_auth_oauthlib.flow import InstalledAppFlow
            from youtube_transcript_api import YouTubeTranscriptApi  # noqa: F401
        except ImportError:
            raise ImportError(
                "You must run"
                "`pip install --upgrade "
                "google-api-python-client google-auth-httplib2 "
                "google-auth-oauthlib"
                "youtube-transcript-api`"
                "to use the Google Drive loader"
            )
        creds = None
        if self.service_account_path.exists():
            return service_account.Credentials.from_service_account_file(
                str(self.service_account_path)
            )
        if self.token_path.exists():
            creds = Credentials.from_authorized_user_file(str(self.token_path), SCOPES)
        if not creds or not creds.valid:
            if creds and creds.expired and creds.refresh_token:
                creds.refresh(Request())
            else:
                flow = InstalledAppFlow.from_client_secrets_file(
                    str(self.credentials_path), SCOPES
                )
                creds = flow.run_local_server(port=0)
            with open(self.token_path, "w") as token:
                token.write(creds.to_json())
        return creds
[docs]class YoutubeLoader(BaseLoader):
    """Loader that loads Youtube transcripts."""
    def __init__(