text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        try:
            images = convert_from_bytes(response.content)
        except ValueError:
            return text
        for i, image in enumerate(images):
            image_text = pytesseract.image_to_string(image)
            text += f"Page {i + 1}:\n{image_text}\n\n"
        return text
[docs]    def process_image(self, link: str) -> str:
        try:
            import pytesseract  # noqa: F401
            from PIL import Image  # noqa: F401
        except ImportError:
            raise ImportError(
                "`pytesseract` or `Pillow` package not found, "
                "please run `pip install pytesseract Pillow`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        try:
            image = Image.open(BytesIO(response.content))
        except OSError:
            return text
        return pytesseract.image_to_string(image)
[docs]    def process_doc(self, link: str) -> str:
        try:
            import docx2txt  # noqa: F401
        except ImportError:
            raise ImportError(
                "`docx2txt` package not found, please run `pip install docx2txt`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""


if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        file_data = BytesIO(response.content)
        return docx2txt.process(file_data)
[docs]    def process_xls(self, link: str) -> str:
        try:
            import xlrd  # noqa: F401
        except ImportError:
            raise ImportError("`xlrd` package not found, please run `pip install xlrd`")
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        workbook = xlrd.open_workbook(file_contents=response.content)
        for sheet in workbook.sheets():
            text += f"{sheet.name}:\n"
            for row in range(sheet.nrows):
                for col in range(sheet.ncols):
                    text += f"{sheet.cell_value(row, col)}\t"
                text += "\n"
            text += "\n"
        return text
[docs]    def process_svg(self, link: str) -> str:
        try:
            import pytesseract  # noqa: F401
            from PIL import Image  # noqa: F401
            from reportlab.graphics import renderPM  # noqa: F401
            from svglib.svglib import svg2rlg  # noqa: F401
        except ImportError:
            raise ImportError(
                "`pytesseract`, `Pillow`, `reportlab` or `svglib` package not found, "
                "please run `pip install pytesseract Pillow reportlab svglib`"
            )


"please run `pip install pytesseract Pillow reportlab svglib`"
            )
        response = self.confluence.request(path=link, absolute=True)
        text = ""
        if (
            response.status_code != 200
            or response.content == b""
            or response.content is None
        ):
            return text
        drawing = svg2rlg(BytesIO(response.content))
        img_data = BytesIO()
        renderPM.drawToFile(drawing, img_data, fmt="PNG")
        img_data.seek(0)
        image = Image.open(img_data)
        return pytesseract.image_to_string(image)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.slack_directory
"""Loader for documents from a Slack export."""
import json
import zipfile
from pathlib import Path
from typing import Dict, List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class SlackDirectoryLoader(BaseLoader):
    """Loader for loading documents from a Slack directory dump."""
    def __init__(self, zip_path: str, workspace_url: Optional[str] = None):
        """Initialize the SlackDirectoryLoader.
        Args:
            zip_path (str): The path to the Slack directory dump zip file.
            workspace_url (Optional[str]): The Slack workspace URL.
              Including the URL will turn
              sources into links. Defaults to None.
        """
        self.zip_path = Path(zip_path)
        self.workspace_url = workspace_url
        self.channel_id_map = self._get_channel_id_map(self.zip_path)
    @staticmethod
    def _get_channel_id_map(zip_path: Path) -> Dict[str, str]:
        """Get a dictionary mapping channel names to their respective IDs."""
        with zipfile.ZipFile(zip_path, "r") as zip_file:
            try:
                with zip_file.open("channels.json", "r") as f:
                    channels = json.load(f)
                return {channel["name"]: channel["id"] for channel in channels}
            except KeyError:
                return {}
[docs]    def load(self) -> List[Document]:
        """Load and return documents from the Slack directory dump."""
        docs = []
        with zipfile.ZipFile(self.zip_path, "r") as zip_file:
            for channel_path in zip_file.namelist():
                channel_name = Path(channel_path).parent.name
                if not channel_name:
                    continue


if not channel_name:
                    continue
                if channel_path.endswith(".json"):
                    messages = self._read_json(zip_file, channel_path)
                    for message in messages:
                        document = self._convert_message_to_document(
                            message, channel_name
                        )
                        docs.append(document)
        return docs
    def _read_json(self, zip_file: zipfile.ZipFile, file_path: str) -> List[dict]:
        """Read JSON data from a zip subfile."""
        with zip_file.open(file_path, "r") as f:
            data = json.load(f)
        return data
    def _convert_message_to_document(
        self, message: dict, channel_name: str
    ) -> Document:
        """
        Convert a message to a Document object.
        Args:
            message (dict): A message in the form of a dictionary.
            channel_name (str): The name of the channel the message belongs to.
        Returns:
            Document: A Document object representing the message.
        """
        text = message.get("text", "")
        metadata = self._get_message_metadata(message, channel_name)
        return Document(
            page_content=text,
            metadata=metadata,
        )
    def _get_message_metadata(self, message: dict, channel_name: str) -> dict:
        """Create and return metadata for a given message and channel."""
        timestamp = message.get("ts", "")
        user = message.get("user", "")
        source = self._get_message_source(channel_name, user, timestamp)
        return {
            "source": source,
            "channel": channel_name,
            "timestamp": timestamp,
            "user": user,
        }


"timestamp": timestamp,
            "user": user,
        }
    def _get_message_source(self, channel_name: str, user: str, timestamp: str) -> str:
        """
        Get the message source as a string.
        Args:
            channel_name (str): The name of the channel the message belongs to.
            user (str): The user ID who sent the message.
            timestamp (str): The timestamp of the message.
        Returns:
            str: The message source.
        """
        if self.workspace_url:
            channel_id = self.channel_id_map.get(channel_name, "")
            return (
                f"{self.workspace_url}/archives/{channel_id}"
                + f"/p{timestamp.replace('.', '')}"
            )
        else:
            return f"{channel_name} - {user} - {timestamp}"
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.hn
"""Loader that loads HN."""
from typing import Any, List
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class HNLoader(WebBaseLoader):
    """Load Hacker News data from either main page results or the comments page."""
[docs]    def load(self) -> List[Document]:
        """Get important HN webpage information.
        Components are:
            - title
            - content
            - source url,
            - time of post
            - author of the post
            - number of comments
            - rank of the post
        """
        soup_info = self.scrape()
        if "item" in self.web_path:
            return self.load_comments(soup_info)
        else:
            return self.load_results(soup_info)
[docs]    def load_comments(self, soup_info: Any) -> List[Document]:
        """Load comments from a HN post."""
        comments = soup_info.select("tr[class='athing comtr']")
        title = soup_info.select_one("tr[id='pagespace']").get("title")
        return [
            Document(
                page_content=comment.text.strip(),
                metadata={"source": self.web_path, "title": title},
            )
            for comment in comments
        ]
[docs]    def load_results(self, soup: Any) -> List[Document]:
        """Load items from an HN page."""
        items = soup.select("tr[class='athing']")
        documents = []
        for lineItem in items:
            ranking = lineItem.select_one("span[class='rank']").text
            link = lineItem.find("span", {"class": "titleline"}).find("a").get("href")


title = lineItem.find("span", {"class": "titleline"}).text.strip()
            metadata = {
                "source": self.web_path,
                "title": title,
                "link": link,
                "ranking": ranking,
            }
            documents.append(
                Document(
                    page_content=title, link=link, ranking=ranking, metadata=metadata
                )
            )
        return documents
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.gutenberg
"""Loader that loads .txt web files."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class GutenbergLoader(BaseLoader):
    """Loader that uses urllib to load .txt web files."""
    def __init__(self, file_path: str):
        """Initialize with file path."""
        if not file_path.startswith("https://www.gutenberg.org"):
            raise ValueError("file path must start with 'https://www.gutenberg.org'")
        if not file_path.endswith(".txt"):
            raise ValueError("file path must end with '.txt'")
        self.file_path = file_path
[docs]    def load(self) -> List[Document]:
        """Load file."""
        from urllib.request import urlopen
        elements = urlopen(self.file_path)
        text = "\n\n".join([str(el.decode("utf-8-sig")) for el in elements])
        metadata = {"source": self.file_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.email
"""Loader that loads email files."""
import os
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import (
    UnstructuredFileLoader,
    satisfies_min_unstructured_version,
)
[docs]class UnstructuredEmailLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load email files."""
    def _get_elements(self) -> List:
        from unstructured.file_utils.filetype import FileType, detect_filetype
        filetype = detect_filetype(self.file_path)
        if filetype == FileType.EML:
            from unstructured.partition.email import partition_email
            return partition_email(filename=self.file_path, **self.unstructured_kwargs)
        elif satisfies_min_unstructured_version("0.5.8") and filetype == FileType.MSG:
            from unstructured.partition.msg import partition_msg
            return partition_msg(filename=self.file_path, **self.unstructured_kwargs)
        else:
            raise ValueError(
                f"Filetype {filetype} is not supported in UnstructuredEmailLoader."
            )
[docs]class OutlookMessageLoader(BaseLoader):
    """
    Loader that loads Outlook Message files using extract_msg.
    https://github.com/TeamMsgExtractor/msg-extractor
    """
    def __init__(self, file_path: str):
        """Initialize with file path."""
        self.file_path = file_path
        if not os.path.isfile(self.file_path):
            raise ValueError("File path %s is not a valid file" % self.file_path)
        try:
            import extract_msg  # noqa:F401
        except ImportError:
            raise ImportError(
                "extract_msg is not installed. Please install it with "
                "`pip install extract_msg`"


"`pip install extract_msg`"
            )
[docs]    def load(self) -> List[Document]:
        """Load data into document objects."""
        import extract_msg
        msg = extract_msg.Message(self.file_path)
        return [
            Document(
                page_content=msg.body,
                metadata={
                    "subject": msg.subject,
                    "sender": msg.sender,
                    "date": msg.date,
                },
            )
        ]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.sitemap
"""Loader that fetches a sitemap and loads those URLs."""
import itertools
import re
from typing import Any, Callable, Generator, Iterable, List, Optional
from langchain.document_loaders.web_base import WebBaseLoader
from langchain.schema import Document
def _default_parsing_function(content: Any) -> str:
    return str(content.get_text())
def _default_meta_function(meta: dict, _content: Any) -> dict:
    return {"source": meta["loc"], **meta}
def _batch_block(iterable: Iterable, size: int) -> Generator[List[dict], None, None]:
    it = iter(iterable)
    while item := list(itertools.islice(it, size)):
        yield item
[docs]class SitemapLoader(WebBaseLoader):
    """Loader that fetches a sitemap and loads those URLs."""
    def __init__(
        self,
        web_path: str,
        filter_urls: Optional[List[str]] = None,
        parsing_function: Optional[Callable] = None,
        blocksize: Optional[int] = None,
        blocknum: int = 0,
        meta_function: Optional[Callable] = None,
        is_local: bool = False,
    ):
        """Initialize with webpage path and optional filter URLs.
        Args:
            web_path: url of the sitemap. can also be a local path
            filter_urls: list of strings or regexes that will be applied to filter the
                urls that are parsed and loaded
            parsing_function: Function to parse bs4.Soup output
            blocksize: number of sitemap locations per block
            blocknum: the number of the block that should be loaded - zero indexed
            meta_function: Function to parse bs4.Soup output for metadata


meta_function: Function to parse bs4.Soup output for metadata
                remember when setting this method to also copy metadata["loc"]
                to metadata["source"] if you are using this field
            is_local: whether the sitemap is a local file
        """
        if blocksize is not None and blocksize < 1:
            raise ValueError("Sitemap blocksize should be at least 1")
        if blocknum < 0:
            raise ValueError("Sitemap blocknum can not be lower then 0")
        try:
            import lxml  # noqa:F401
        except ImportError:
            raise ImportError(
                "lxml package not found, please install it with " "`pip install lxml`"
            )
        super().__init__(web_path)
        self.filter_urls = filter_urls
        self.parsing_function = parsing_function or _default_parsing_function
        self.meta_function = meta_function or _default_meta_function
        self.blocksize = blocksize
        self.blocknum = blocknum
        self.is_local = is_local
[docs]    def parse_sitemap(self, soup: Any) -> List[dict]:
        """Parse sitemap xml and load into a list of dicts."""
        els = []
        for url in soup.find_all("url"):
            loc = url.find("loc")
            if not loc:
                continue
            if self.filter_urls and not any(
                re.match(r, loc.text) for r in self.filter_urls
            ):
                continue
            els.append(
                {
                    tag: prop.text
                    for tag in ["loc", "lastmod", "changefreq", "priority"]
                    if (prop := url.find(tag))
                }
            )
        for sitemap in soup.find_all("sitemap"):


}
            )
        for sitemap in soup.find_all("sitemap"):
            loc = sitemap.find("loc")
            if not loc:
                continue
            soup_child = self.scrape_all([loc.text], "xml")[0]
            els.extend(self.parse_sitemap(soup_child))
        return els
[docs]    def load(self) -> List[Document]:
        """Load sitemap."""
        if self.is_local:
            try:
                import bs4
            except ImportError:
                raise ImportError(
                    "beautifulsoup4 package not found, please install it"
                    " with `pip install beautifulsoup4`"
                )
            fp = open(self.web_path)
            soup = bs4.BeautifulSoup(fp, "xml")
        else:
            soup = self.scrape("xml")
        els = self.parse_sitemap(soup)
        if self.blocksize is not None:
            elblocks = list(_batch_block(els, self.blocksize))
            blockcount = len(elblocks)
            if blockcount - 1 < self.blocknum:
                raise ValueError(
                    "Selected sitemap does not contain enough blocks for given blocknum"
                )
            else:
                els = elblocks[self.blocknum]
        results = self.scrape_all([el["loc"].strip() for el in els if "loc" in el])
        return [
            Document(
                page_content=self.parsing_function(results[i]),
                metadata=self.meta_function(els[i], results[i]),
            )
            for i in range(len(results))
        ]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.word_document
"""Loader that loads word documents."""
import os
import tempfile
from abc import ABC
from typing import List
from urllib.parse import urlparse
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
[docs]class Docx2txtLoader(BaseLoader, ABC):
    """Loads a DOCX with docx2txt and chunks at character level.
    Defaults to check for local file, but if the file is a web path, it will download it
    to a temporary file, and use that, then clean up the temporary file after completion
    """
    def __init__(self, file_path: str):
        """Initialize with file path."""
        self.file_path = file_path
        if "~" in self.file_path:
            self.file_path = os.path.expanduser(self.file_path)
        # If the file is a web path, download it to a temporary file, and use that
        if not os.path.isfile(self.file_path) and self._is_valid_url(self.file_path):
            r = requests.get(self.file_path)
            if r.status_code != 200:
                raise ValueError(
                    "Check the url of your file; returned status code %s"
                    % r.status_code
                )
            self.web_path = self.file_path
            self.temp_file = tempfile.NamedTemporaryFile()
            self.temp_file.write(r.content)
            self.file_path = self.temp_file.name
        elif not os.path.isfile(self.file_path):
            raise ValueError("File path %s is not a valid file or url" % self.file_path)
    def __del__(self) -> None:
        if hasattr(self, "temp_file"):
            self.temp_file.close()


if hasattr(self, "temp_file"):
            self.temp_file.close()
[docs]    def load(self) -> List[Document]:
        """Load given path as single page."""
        import docx2txt
        return [
            Document(
                page_content=docx2txt.process(self.file_path),
                metadata={"source": self.file_path},
            )
        ]
    @staticmethod
    def _is_valid_url(url: str) -> bool:
        """Check if the url is valid."""
        parsed = urlparse(url)
        return bool(parsed.netloc) and bool(parsed.scheme)
[docs]class UnstructuredWordDocumentLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load word documents."""
    def _get_elements(self) -> List:
        from unstructured.__version__ import __version__ as __unstructured_version__
        from unstructured.file_utils.filetype import FileType, detect_filetype
        unstructured_version = tuple(
            [int(x) for x in __unstructured_version__.split(".")]
        )
        # NOTE(MthwRobinson) - magic will raise an import error if the libmagic
        # system dependency isn't installed. If it's not installed, we'll just
        # check the file extension
        try:
            import magic  # noqa: F401
            is_doc = detect_filetype(self.file_path) == FileType.DOC
        except ImportError:
            _, extension = os.path.splitext(str(self.file_path))
            is_doc = extension == ".doc"
        if is_doc and unstructured_version < (0, 4, 11):
            raise ValueError(
                f"You are on unstructured version {__unstructured_version__}. "


f"You are on unstructured version {__unstructured_version__}. "
                "Partitioning .doc files is only supported in unstructured>=0.4.11. "
                "Please upgrade the unstructured package and try again."
            )
        if is_doc:
            from unstructured.partition.doc import partition_doc
            return partition_doc(filename=self.file_path, **self.unstructured_kwargs)
        else:
            from unstructured.partition.docx import partition_docx
            return partition_docx(filename=self.file_path, **self.unstructured_kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.pyspark_dataframe
"""Load from a Spark Dataframe object"""
import itertools
import logging
import sys
from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Tuple
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__file__)
if TYPE_CHECKING:
    from pyspark.sql import SparkSession
[docs]class PySparkDataFrameLoader(BaseLoader):
    """Load PySpark DataFrames"""
    def __init__(
        self,
        spark_session: Optional["SparkSession"] = None,
        df: Optional[Any] = None,
        page_content_column: str = "text",
        fraction_of_memory: float = 0.1,
    ):
        """Initialize with a Spark DataFrame object."""
        try:
            from pyspark.sql import DataFrame, SparkSession
        except ImportError:
            raise ImportError(
                "pyspark is not installed. "
                "Please install it with `pip install pyspark`"
            )
        self.spark = (
            spark_session if spark_session else SparkSession.builder.getOrCreate()
        )
        if not isinstance(df, DataFrame):
            raise ValueError(
                f"Expected data_frame to be a PySpark DataFrame, got {type(df)}"
            )
        self.df = df
        self.page_content_column = page_content_column
        self.fraction_of_memory = fraction_of_memory
        self.num_rows, self.max_num_rows = self.get_num_rows()
        self.rdd_df = self.df.rdd.map(list)
        self.column_names = self.df.columns
[docs]    def get_num_rows(self) -> Tuple[int, int]:
        """Gets the amount of "feasible" rows for the DataFrame"""
        try:


"""Gets the amount of "feasible" rows for the DataFrame"""
        try:
            import psutil
        except ImportError as e:
            raise ImportError(
                "psutil not installed. Please install it with `pip install psutil`."
            ) from e
        row = self.df.limit(1).collect()[0]
        estimated_row_size = sys.getsizeof(row)
        mem_info = psutil.virtual_memory()
        available_memory = mem_info.available
        max_num_rows = int(
            (available_memory / estimated_row_size) * self.fraction_of_memory
        )
        return min(max_num_rows, self.df.count()), max_num_rows
[docs]    def lazy_load(self) -> Iterator[Document]:
        """A lazy loader for document content."""
        for row in self.rdd_df.toLocalIterator():
            metadata = {self.column_names[i]: row[i] for i in range(len(row))}
            text = metadata[self.page_content_column]
            metadata.pop(self.page_content_column)
            yield Document(page_content=text, metadata=metadata)
[docs]    def load(self) -> List[Document]:
        """Load from the dataframe."""
        if self.df.count() > self.max_num_rows:
            logger.warning(
                f"The number of DataFrame rows is {self.df.count()}, "
                f"but we will only include the amount "
                f"of rows that can reasonably fit in memory: {self.num_rows}."
            )
        lazy_load_iterator = self.lazy_load()
        return list(itertools.islice(lazy_load_iterator, self.num_rows))
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.dataframe
"""Load from Dataframe object"""
from typing import Any, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class DataFrameLoader(BaseLoader):
    """Load Pandas DataFrames."""
    def __init__(self, data_frame: Any, page_content_column: str = "text"):
        """Initialize with dataframe object."""
        import pandas as pd
        if not isinstance(data_frame, pd.DataFrame):
            raise ValueError(
                f"Expected data_frame to be a pd.DataFrame, got {type(data_frame)}"
            )
        self.data_frame = data_frame
        self.page_content_column = page_content_column
[docs]    def load(self) -> List[Document]:
        """Load from the dataframe."""
        result = []
        # For very large dataframes, this needs to yield instead of building a list
        # but that would require chaging return type to a generator for BaseLoader
        # and all its subclasses, which is a bigger refactor. Marking as future TODO.
        # This change will allow us to extend this to Spark and Dask dataframes.
        for _, row in self.data_frame.iterrows():
            text = row[self.page_content_column]
            metadata = row.to_dict()
            metadata.pop(self.page_content_column)
            result.append(Document(page_content=text, metadata=metadata))
        return result
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.web_base
"""Web base loader class."""
import asyncio
import logging
import warnings
from typing import Any, Dict, List, Optional, Union
import aiohttp
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
default_header_template = {
    "User-Agent": "",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*"
    ";q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
    "Referer": "https://www.google.com/",
    "DNT": "1",
    "Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
}
def _build_metadata(soup: Any, url: str) -> dict:
    """Build metadata from BeautifulSoup output."""
    metadata = {"source": url}
    if title := soup.find("title"):
        metadata["title"] = title.get_text()
    if description := soup.find("meta", attrs={"name": "description"}):
        metadata["description"] = description.get("content", None)
    if html := soup.find("html"):
        metadata["language"] = html.get("lang", None)
    return metadata
[docs]class WebBaseLoader(BaseLoader):
    """Loader that uses urllib and beautiful soup to load webpages."""
    web_paths: List[str]
    requests_per_second: int = 2
    """Max number of concurrent requests to make."""
    default_parser: str = "html.parser"
    """Default parser to use for BeautifulSoup."""
    requests_kwargs: Dict[str, Any] = {}
    """kwargs for requests"""
    def __init__(


"""kwargs for requests"""
    def __init__(
        self, web_path: Union[str, List[str]], header_template: Optional[dict] = None
    ):
        """Initialize with webpage path."""
        # TODO: Deprecate web_path in favor of web_paths, and remove this
        # left like this because there are a number of loaders that expect single
        # urls
        if isinstance(web_path, str):
            self.web_paths = [web_path]
        elif isinstance(web_path, List):
            self.web_paths = web_path
        self.session = requests.Session()
        try:
            import bs4  # noqa:F401
        except ImportError:
            raise ValueError(
                "bs4 package not found, please install it with " "`pip install bs4`"
            )
        headers = header_template or default_header_template
        if not headers.get("User-Agent"):
            try:
                from fake_useragent import UserAgent
                headers["User-Agent"] = UserAgent().random
            except ImportError:
                logger.info(
                    "fake_useragent not found, using default user agent."
                    "To get a realistic header for requests, "
                    "`pip install fake_useragent`."
                )
        self.session.headers = dict(headers)
    @property
    def web_path(self) -> str:
        if len(self.web_paths) > 1:
            raise ValueError("Multiple webpaths found.")
        return self.web_paths[0]
    async def _fetch(
        self, url: str, retries: int = 3, cooldown: int = 2, backoff: float = 1.5
    ) -> str:
        async with aiohttp.ClientSession() as session:
            for i in range(retries):
                try:


for i in range(retries):
                try:
                    async with session.get(
                        url, headers=self.session.headers
                    ) as response:
                        return await response.text()
                except aiohttp.ClientConnectionError as e:
                    if i == retries - 1:
                        raise
                    else:
                        logger.warning(
                            f"Error fetching {url} with attempt "
                            f"{i + 1}/{retries}: {e}. Retrying..."
                        )
                        await asyncio.sleep(cooldown * backoff**i)
        raise ValueError("retry count exceeded")
    async def _fetch_with_rate_limit(
        self, url: str, semaphore: asyncio.Semaphore
    ) -> str:
        async with semaphore:
            return await self._fetch(url)
[docs]    async def fetch_all(self, urls: List[str]) -> Any:
        """Fetch all urls concurrently with rate limiting."""
        semaphore = asyncio.Semaphore(self.requests_per_second)
        tasks = []
        for url in urls:
            task = asyncio.ensure_future(self._fetch_with_rate_limit(url, semaphore))
            tasks.append(task)
        try:
            from tqdm.asyncio import tqdm_asyncio
            return await tqdm_asyncio.gather(
                *tasks, desc="Fetching pages", ascii=True, mininterval=1
            )
        except ImportError:
            warnings.warn("For better logging of progress, `pip install tqdm`")
            return await asyncio.gather(*tasks)
    @staticmethod
    def _check_parser(parser: str) -> None:
        """Check that parser is valid for bs4."""
        valid_parsers = ["html.parser", "lxml", "xml", "lxml-xml", "html5lib"]
        if parser not in valid_parsers:
            raise ValueError(


if parser not in valid_parsers:
            raise ValueError(
                "`parser` must be one of " + ", ".join(valid_parsers) + "."
            )
[docs]    def scrape_all(self, urls: List[str], parser: Union[str, None] = None) -> List[Any]:
        """Fetch all urls, then return soups for all results."""
        from bs4 import BeautifulSoup
        results = asyncio.run(self.fetch_all(urls))
        final_results = []
        for i, result in enumerate(results):
            url = urls[i]
            if parser is None:
                if url.endswith(".xml"):
                    parser = "xml"
                else:
                    parser = self.default_parser
                self._check_parser(parser)
            final_results.append(BeautifulSoup(result, parser))
        return final_results
    def _scrape(self, url: str, parser: Union[str, None] = None) -> Any:
        from bs4 import BeautifulSoup
        if parser is None:
            if url.endswith(".xml"):
                parser = "xml"
            else:
                parser = self.default_parser
        self._check_parser(parser)
        html_doc = self.session.get(url, **self.requests_kwargs)
        html_doc.encoding = html_doc.apparent_encoding
        return BeautifulSoup(html_doc.text, parser)
[docs]    def scrape(self, parser: Union[str, None] = None) -> Any:
        """Scrape data from webpage and return it in BeautifulSoup format."""
        if parser is None:
            parser = self.default_parser
        return self._scrape(self.web_path, parser)
[docs]    def load(self) -> List[Document]:
        """Load text from the url(s) in web_path."""
        docs = []
        for path in self.web_paths:


docs = []
        for path in self.web_paths:
            soup = self._scrape(path)
            text = soup.get_text()
            metadata = _build_metadata(soup, path)
            docs.append(Document(page_content=text, metadata=metadata))
        return docs
[docs]    def aload(self) -> List[Document]:
        """Load text from the urls in web_path async into Documents."""
        results = self.scrape_all(self.web_paths)
        docs = []
        for i in range(len(results)):
            soup = results[i]
            text = soup.get_text()
            metadata = _build_metadata(soup, self.web_paths[i])
            docs.append(Document(page_content=text, metadata=metadata))
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.wikipedia
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utilities.wikipedia import WikipediaAPIWrapper
[docs]class WikipediaLoader(BaseLoader):
    """Loads a query result from www.wikipedia.org into a list of Documents.
    The hard limit on the number of downloaded Documents is 300 for now.
    Each wiki page represents one Document.
    """
    def __init__(
        self,
        query: str,
        lang: str = "en",
        load_max_docs: Optional[int] = 100,
        load_all_available_meta: Optional[bool] = False,
    ):
        self.query = query
        self.lang = lang
        self.load_max_docs = load_max_docs
        self.load_all_available_meta = load_all_available_meta
[docs]    def load(self) -> List[Document]:
        client = WikipediaAPIWrapper(
            lang=self.lang,
            top_k_results=self.load_max_docs,
            load_all_available_meta=self.load_all_available_meta,
        )
        docs = client.load(self.query)
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.onedrive
"""Loader that loads data from OneDrive"""
from __future__ import annotations
import logging
import os
import tempfile
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING, Dict, List, Optional, Type, Union
from pydantic import BaseModel, BaseSettings, Field, FilePath, SecretStr
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.onedrive_file import OneDriveFileLoader
if TYPE_CHECKING:
    from O365 import Account
    from O365.drive import Drive, Folder
SCOPES = ["offline_access", "Files.Read.All"]
logger = logging.getLogger(__name__)
class _OneDriveSettings(BaseSettings):
    client_id: str = Field(..., env="O365_CLIENT_ID")
    client_secret: SecretStr = Field(..., env="O365_CLIENT_SECRET")
    class Config:
        env_prefix = ""
        case_sentive = False
        env_file = ".env"
class _OneDriveTokenStorage(BaseSettings):
    token_path: FilePath = Field(Path.home() / ".credentials" / "o365_token.txt")
class _FileType(str, Enum):
    DOC = "doc"
    DOCX = "docx"
    PDF = "pdf"
class _SupportedFileTypes(BaseModel):
    file_types: List[_FileType]
    def fetch_mime_types(self) -> Dict[str, str]:
        mime_types_mapping = {}
        for file_type in self.file_types:
            if file_type.value == "doc":
                mime_types_mapping[file_type.value] = "application/msword"
            elif file_type.value == "docx":
                mime_types_mapping[
                    file_type.value


mime_types_mapping[
                    file_type.value
                ] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"  # noqa: E501
            elif file_type.value == "pdf":
                mime_types_mapping[file_type.value] = "application/pdf"
        return mime_types_mapping
[docs]class OneDriveLoader(BaseLoader, BaseModel):
    settings: _OneDriveSettings = Field(default_factory=_OneDriveSettings)
    drive_id: str = Field(...)
    folder_path: Optional[str] = None
    object_ids: Optional[List[str]] = None
    auth_with_token: bool = False
    def _auth(self) -> Type[Account]:
        """
        Authenticates the OneDrive API client using the specified
        authentication method and returns the Account object.
        Returns:
            Type[Account]: The authenticated Account object.
        """
        try:
            from O365 import FileSystemTokenBackend
        except ImportError:
            raise ImportError(
                "O365 package not found, please install it with `pip install o365`"
            )
        if self.auth_with_token:
            token_storage = _OneDriveTokenStorage()
            token_path = token_storage.token_path
            token_backend = FileSystemTokenBackend(
                token_path=token_path.parent, token_filename=token_path.name
            )
            account = Account(
                credentials=(
                    self.settings.client_id,
                    self.settings.client_secret.get_secret_value(),
                ),
                scopes=SCOPES,
                token_backend=token_backend,
                **{"raise_http_errors": False},
            )
        else:
            token_backend = FileSystemTokenBackend(
                token_path=Path.home() / ".credentials"
            )
            account = Account(
                credentials=(
                    self.settings.client_id,


)
            account = Account(
                credentials=(
                    self.settings.client_id,
                    self.settings.client_secret.get_secret_value(),
                ),
                scopes=SCOPES,
                token_backend=token_backend,
                **{"raise_http_errors": False},
            )
            # make the auth
            account.authenticate()
        return account
    def _get_folder_from_path(self, drive: Type[Drive]) -> Union[Folder, Drive]:
        """
        Returns the folder or drive object located at the
        specified path relative to the given drive.
        Args:
            drive (Type[Drive]): The root drive from which the folder path is relative.
        Returns:
            Union[Folder, Drive]: The folder or drive object
            located at the specified path.
        Raises:
            FileNotFoundError: If the path does not exist.
        """
        subfolder_drive = drive
        if self.folder_path is None:
            return subfolder_drive
        subfolders = [f for f in self.folder_path.split("/") if f != ""]
        if len(subfolders) == 0:
            return subfolder_drive
        items = subfolder_drive.get_items()
        for subfolder in subfolders:
            try:
                subfolder_drive = list(filter(lambda x: subfolder in x.name, items))[0]
                items = subfolder_drive.get_items()
            except (IndexError, AttributeError):
                raise FileNotFoundError("Path {} not exist.".format(self.folder_path))
        return subfolder_drive
    def _load_from_folder(self, folder: Type[Folder]) -> List[Document]:
        """
        Loads all supported document files from the specified folder
        and returns a list of Document objects.
        Args:
            folder (Type[Folder]): The folder object to load the documents from.


folder (Type[Folder]): The folder object to load the documents from.
        Returns:
            List[Document]: A list of Document objects representing
            the loaded documents.
        """
        docs = []
        file_types = _SupportedFileTypes(file_types=["doc", "docx", "pdf"])
        file_mime_types = file_types.fetch_mime_types()
        items = folder.get_items()
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            for file in items:
                if file.is_file:
                    if file.mime_type in list(file_mime_types.values()):
                        loader = OneDriveFileLoader(file=file)
                        docs.extend(loader.load())
        return docs
    def _load_from_object_ids(self, drive: Type[Drive]) -> List[Document]:
        """
        Loads all supported document files from the specified OneDrive
        drive based on their object IDs and returns a list
        of Document objects.
        Args:
            drive (Type[Drive]): The OneDrive drive object
            to load the documents from.
        Returns:
            List[Document]: A list of Document objects representing
            the loaded documents.
        """
        docs = []
        file_types = _SupportedFileTypes(file_types=["doc", "docx", "pdf"])
        file_mime_types = file_types.fetch_mime_types()
        with tempfile.TemporaryDirectory() as temp_dir:
            file_path = f"{temp_dir}"
            os.makedirs(os.path.dirname(file_path), exist_ok=True)
            for object_id in self.object_ids if self.object_ids else [""]:
                file = drive.get_item(object_id)
                if not file:
                    logging.warning(
                        "There isn't a file with "


logging.warning(
                        "There isn't a file with "
                        f"object_id {object_id} in drive {drive}."
                    )
                    continue
                if file.is_file:
                    if file.mime_type in list(file_mime_types.values()):
                        loader = OneDriveFileLoader(file=file)
                        docs.extend(loader.load())
        return docs
[docs]    def load(self) -> List[Document]:
        """
        Loads all supported document files from the specified OneDrive drive a
        nd returns a list of Document objects.
        Returns:
            List[Document]: A list of Document objects
            representing the loaded documents.
        Raises:
            ValueError: If the specified drive ID
            does not correspond to a drive in the OneDrive storage.
        """
        account = self._auth()
        storage = account.storage()
        drive = storage.get_drive(self.drive_id)
        docs: List[Document] = []
        if not drive:
            raise ValueError(f"There isn't a drive with id {self.drive_id}.")
        if self.folder_path:
            folder = self._get_folder_from_path(drive=drive)
            docs.extend(self._load_from_folder(folder=folder))
        elif self.object_ids:
            docs.extend(self._load_from_object_ids(drive=drive))
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.directory
"""Loading logic for loading documents from a directory."""
import concurrent
import logging
from pathlib import Path
from typing import Any, List, Optional, Type, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.html_bs import BSHTMLLoader
from langchain.document_loaders.text import TextLoader
from langchain.document_loaders.unstructured import UnstructuredFileLoader
FILE_LOADER_TYPE = Union[
    Type[UnstructuredFileLoader], Type[TextLoader], Type[BSHTMLLoader]
]
logger = logging.getLogger(__name__)
def _is_visible(p: Path) -> bool:
    parts = p.parts
    for _p in parts:
        if _p.startswith("."):
            return False
    return True
[docs]class DirectoryLoader(BaseLoader):
    """Loading logic for loading documents from a directory."""
    def __init__(
        self,
        path: str,
        glob: str = "**/[!.]*",
        silent_errors: bool = False,
        load_hidden: bool = False,
        loader_cls: FILE_LOADER_TYPE = UnstructuredFileLoader,
        loader_kwargs: Union[dict, None] = None,
        recursive: bool = False,
        show_progress: bool = False,
        use_multithreading: bool = False,
        max_concurrency: int = 4,
    ):
        """Initialize with path to directory and how to glob over it."""
        if loader_kwargs is None:
            loader_kwargs = {}
        self.path = path
        self.glob = glob
        self.load_hidden = load_hidden
        self.loader_cls = loader_cls
        self.loader_kwargs = loader_kwargs
        self.silent_errors = silent_errors


self.loader_kwargs = loader_kwargs
        self.silent_errors = silent_errors
        self.recursive = recursive
        self.show_progress = show_progress
        self.use_multithreading = use_multithreading
        self.max_concurrency = max_concurrency
[docs]    def load_file(
        self, item: Path, path: Path, docs: List[Document], pbar: Optional[Any]
    ) -> None:
        if item.is_file():
            if _is_visible(item.relative_to(path)) or self.load_hidden:
                try:
                    sub_docs = self.loader_cls(str(item), **self.loader_kwargs).load()
                    docs.extend(sub_docs)
                except Exception as e:
                    if self.silent_errors:
                        logger.warning(e)
                    else:
                        raise e
                finally:
                    if pbar:
                        pbar.update(1)
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        p = Path(self.path)
        if not p.exists():
            raise FileNotFoundError(f"Directory not found: '{self.path}'")
        if not p.is_dir():
            raise ValueError(f"Expected directory, got file: '{self.path}'")
        docs: List[Document] = []
        items = list(p.rglob(self.glob) if self.recursive else p.glob(self.glob))
        pbar = None
        if self.show_progress:
            try:
                from tqdm import tqdm
                pbar = tqdm(total=len(items))
            except ImportError as e:
                logger.warning(
                    "To log the progress of DirectoryLoader you need to install tqdm, "
                    "`pip install tqdm`"
                )
                if self.silent_errors:
                    logger.warning(e)
                else:
                    raise e


logger.warning(e)
                else:
                    raise e
        if self.use_multithreading:
            with concurrent.futures.ThreadPoolExecutor(
                max_workers=self.max_concurrency
            ) as executor:
                executor.map(lambda i: self.load_file(i, p, docs, pbar), items)
        else:
            for i in items:
                self.load_file(i, p, docs, pbar)
        if pbar:
            pbar.close()
        return docs
#
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.iugu
"""Loader that fetches data from IUGU"""
import json
import urllib.request
from typing import List, Optional
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import get_from_env, stringify_dict
IUGU_ENDPOINTS = {
    "invoices": "https://api.iugu.com/v1/invoices",
    "customers": "https://api.iugu.com/v1/customers",
    "charges": "https://api.iugu.com/v1/charges",
    "subscriptions": "https://api.iugu.com/v1/subscriptions",
    "plans": "https://api.iugu.com/v1/plans",
}
[docs]class IuguLoader(BaseLoader):
    def __init__(self, resource: str, api_token: Optional[str] = None) -> None:
        self.resource = resource
        api_token = api_token or get_from_env("api_token", "IUGU_API_TOKEN")
        self.headers = {"Authorization": f"Bearer {api_token}"}
    def _make_request(self, url: str) -> List[Document]:
        request = urllib.request.Request(url, headers=self.headers)
        with urllib.request.urlopen(request) as response:
            json_data = json.loads(response.read().decode())
            text = stringify_dict(json_data)
            metadata = {"source": url}
            return [Document(page_content=text, metadata=metadata)]
    def _get_resource(self) -> List[Document]:
        endpoint = IUGU_ENDPOINTS.get(self.resource)
        if endpoint is None:
            return []
        return self._make_request(endpoint)
[docs]    def load(self) -> List[Document]:
        return self._get_resource()
By Harrison Chase


return self._get_resource()
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.html_bs
"""Loader that uses bs4 to load HTML files, enriching metadata with page title."""
import logging
from typing import Dict, List, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class BSHTMLLoader(BaseLoader):
    """Loader that uses beautiful soup to parse HTML files."""
    def __init__(
        self,
        file_path: str,
        open_encoding: Union[str, None] = None,
        bs_kwargs: Union[dict, None] = None,
        get_text_separator: str = "",
    ) -> None:
        """Initialise with path, and optionally, file encoding to use, and any kwargs
        to pass to the BeautifulSoup object."""
        try:
            import bs4  # noqa:F401
        except ImportError:
            raise ValueError(
                "beautifulsoup4 package not found, please install it with "
                "`pip install beautifulsoup4`"
            )
        self.file_path = file_path
        self.open_encoding = open_encoding
        if bs_kwargs is None:
            bs_kwargs = {"features": "lxml"}
        self.bs_kwargs = bs_kwargs
        self.get_text_separator = get_text_separator
[docs]    def load(self) -> List[Document]:
        from bs4 import BeautifulSoup
        """Load HTML document into document objects."""
        with open(self.file_path, "r", encoding=self.open_encoding) as f:
            soup = BeautifulSoup(f, **self.bs_kwargs)
        text = soup.get_text(self.get_text_separator)
        if soup.title:
            title = str(soup.title.string)
        else:
            title = ""
        metadata: Dict[str, Union[str, None]] = {


title = ""
        metadata: Dict[str, Union[str, None]] = {
            "source": self.file_path,
            "title": title,
        }
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.json_loader
"""Loader that loads data from JSON."""
import json
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
[docs]class JSONLoader(BaseLoader):
    """Loads a JSON file and references a jq schema provided to load the text into
    documents.
    Example:
        [{"text": ...}, {"text": ...}, {"text": ...}] -> schema = .[].text
        {"key": [{"text": ...}, {"text": ...}, {"text": ...}]} -> schema = .key[].text
        ["", "", ""] -> schema = .[]
    """
    def __init__(
        self,
        file_path: Union[str, Path],
        jq_schema: str,
        content_key: Optional[str] = None,
        metadata_func: Optional[Callable[[Dict, Dict], Dict]] = None,
        text_content: bool = True,
    ):
        """Initialize the JSONLoader.
        Args:
            file_path (Union[str, Path]): The path to the JSON file.
            jq_schema (str): The jq schema to use to extract the data or text from
                the JSON.
            content_key (str): The key to use to extract the content from the JSON if
                the jq_schema results to a list of objects (dict).
            metadata_func (Callable[Dict, Dict]): A function that takes in the JSON
                object extracted by the jq_schema and the default metadata and returns
                a dict of the updated metadata.
            text_content (bool): Boolean flag to indicates whether the content is in
                string format, default to True
        """
        try:
            import jq  # noqa:F401


"""
        try:
            import jq  # noqa:F401
        except ImportError:
            raise ImportError(
                "jq package not found, please install it with `pip install jq`"
            )
        self.file_path = Path(file_path).resolve()
        self._jq_schema = jq.compile(jq_schema)
        self._content_key = content_key
        self._metadata_func = metadata_func
        self._text_content = text_content
[docs]    def load(self) -> List[Document]:
        """Load and return documents from the JSON file."""
        data = self._jq_schema.input(json.loads(self.file_path.read_text()))
        # Perform some validation
        # This is not a perfect validation, but it should catch most cases
        # and prevent the user from getting a cryptic error later on.
        if self._content_key is not None:
            self._validate_content_key(data)
        docs = []
        for i, sample in enumerate(data, 1):
            metadata = dict(
                source=str(self.file_path),
                seq_num=i,
            )
            text = self._get_text(sample=sample, metadata=metadata)
            docs.append(Document(page_content=text, metadata=metadata))
        return docs
    def _get_text(self, sample: Any, metadata: dict) -> str:
        """Convert sample to string format"""
        if self._content_key is not None:
            content = sample.get(self._content_key)
            if self._metadata_func is not None:
                # We pass in the metadata dict to the metadata_func
                # so that the user can customize the default metadata
                # based on the content of the JSON object.
                metadata = self._metadata_func(sample, metadata)
        else:
            content = sample


else:
            content = sample
        if self._text_content and not isinstance(content, str):
            raise ValueError(
                f"Expected page_content is string, got {type(content)} instead. \
                    Set `text_content=False` if the desired input for \
                    `page_content` is not a string"
            )
        # In case the text is None, set it to an empty string
        elif isinstance(content, str):
            return content
        elif isinstance(content, dict):
            return json.dumps(content) if content else ""
        else:
            return str(content) if content is not None else ""
    def _validate_content_key(self, data: Any) -> None:
        """Check if content key is valid"""
        sample = data.first()
        if not isinstance(sample, dict):
            raise ValueError(
                f"Expected the jq schema to result in a list of objects (dict), \
                    so sample must be a dict but got `{type(sample)}`"
            )
        if sample.get(self._content_key) is None:
            raise ValueError(
                f"Expected the jq schema to result in a list of objects (dict) \
                    with the key `{self._content_key}`"
            )
        if self._metadata_func is not None:
            sample_metadata = self._metadata_func(sample, {})
            if not isinstance(sample_metadata, dict):
                raise ValueError(
                    f"Expected the metadata_func to return a dict but got \
                        `{type(sample_metadata)}`"
                )
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.weather
"""Simple reader that reads weather data from OpenWeatherMap API"""
from __future__ import annotations
from datetime import datetime
from typing import Iterator, List, Optional, Sequence
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utilities.openweathermap import OpenWeatherMapAPIWrapper
[docs]class WeatherDataLoader(BaseLoader):
    """Weather Reader.
    Reads the forecast & current weather of any location using OpenWeatherMap's free
    API. Checkout 'https://openweathermap.org/appid' for more on how to generate a free
    OpenWeatherMap API.
    """
    def __init__(
        self,
        client: OpenWeatherMapAPIWrapper,
        places: Sequence[str],
    ) -> None:
        """Initialize with parameters."""
        super().__init__()
        self.client = client
        self.places = places
[docs]    @classmethod
    def from_params(
        cls, places: Sequence[str], *, openweathermap_api_key: Optional[str] = None
    ) -> WeatherDataLoader:
        client = OpenWeatherMapAPIWrapper(openweathermap_api_key=openweathermap_api_key)
        return cls(client, places)
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Lazily load weather data for the given locations."""
        for place in self.places:
            metadata = {"queried_at": datetime.now()}
            content = self.client.run(place)
            yield Document(page_content=content, metadata=metadata)
[docs]    def load(
        self,
    ) -> List[Document]:
        """Load weather data for the given locations."""
        return list(self.lazy_load())
By Harrison Chase


return list(self.lazy_load())
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.url
"""Loader that uses unstructured to load HTML files."""
import logging
from typing import Any, List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
logger = logging.getLogger(__name__)
[docs]class UnstructuredURLLoader(BaseLoader):
    """Loader that uses unstructured to load HTML files."""
    def __init__(
        self,
        urls: List[str],
        continue_on_failure: bool = True,
        mode: str = "single",
        **unstructured_kwargs: Any,
    ):
        """Initialize with file path."""
        try:
            import unstructured  # noqa:F401
            from unstructured.__version__ import __version__ as __unstructured_version__
            self.__version = __unstructured_version__
        except ImportError:
            raise ValueError(
                "unstructured package not found, please install it with "
                "`pip install unstructured`"
            )
        self._validate_mode(mode)
        self.mode = mode
        headers = unstructured_kwargs.pop("headers", {})
        if len(headers.keys()) != 0:
            warn_about_headers = False
            if self.__is_non_html_available():
                warn_about_headers = not self.__is_headers_available_for_non_html()
            else:
                warn_about_headers = not self.__is_headers_available_for_html()
            if warn_about_headers:
                logger.warning(
                    "You are using an old version of unstructured. "
                    "The headers parameter is ignored"
                )
        self.urls = urls
        self.continue_on_failure = continue_on_failure
        self.headers = headers
        self.unstructured_kwargs = unstructured_kwargs
    def _validate_mode(self, mode: str) -> None:


def _validate_mode(self, mode: str) -> None:
        _valid_modes = {"single", "elements"}
        if mode not in _valid_modes:
            raise ValueError(
                f"Got {mode} for `mode`, but should be one of `{_valid_modes}`"
            )
    def __is_headers_available_for_html(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 7)
    def __is_headers_available_for_non_html(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 13)
    def __is_non_html_available(self) -> bool:
        _unstructured_version = self.__version.split("-")[0]
        unstructured_version = tuple([int(x) for x in _unstructured_version.split(".")])
        return unstructured_version >= (0, 5, 12)
[docs]    def load(self) -> List[Document]:
        """Load file."""
        from unstructured.partition.auto import partition
        from unstructured.partition.html import partition_html
        docs: List[Document] = list()
        for url in self.urls:
            try:
                if self.__is_non_html_available():
                    if self.__is_headers_available_for_non_html():
                        elements = partition(
                            url=url, headers=self.headers, **self.unstructured_kwargs
                        )
                    else:
                        elements = partition(url=url, **self.unstructured_kwargs)
                else:


elements = partition(url=url, **self.unstructured_kwargs)
                else:
                    if self.__is_headers_available_for_html():
                        elements = partition_html(
                            url=url, headers=self.headers, **self.unstructured_kwargs
                        )
                    else:
                        elements = partition_html(url=url, **self.unstructured_kwargs)
            except Exception as e:
                if self.continue_on_failure:
                    logger.error(f"Error fetching or processing {url}, exeption: {e}")
                    continue
                else:
                    raise e
            if self.mode == "single":
                text = "\n\n".join([str(el) for el in elements])
                metadata = {"source": url}
                docs.append(Document(page_content=text, metadata=metadata))
            elif self.mode == "elements":
                for element in elements:
                    metadata = element.metadata.to_dict()
                    metadata["category"] = element.category
                    docs.append(Document(page_content=str(element), metadata=metadata))
        return docs
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.imsdb
"""Loader that loads IMSDb."""
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.web_base import WebBaseLoader
[docs]class IMSDbLoader(WebBaseLoader):
    """Loader that loads IMSDb webpages."""
[docs]    def load(self) -> List[Document]:
        """Load webpage."""
        soup = self.scrape()
        text = soup.select_one("td[class='scrtext']").text
        metadata = {"source": self.web_path}
        return [Document(page_content=text, metadata=metadata)]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.unstructured
"""Loader that uses unstructured to load files."""
import collections
from abc import ABC, abstractmethod
from typing import IO, Any, List, Sequence, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
def satisfies_min_unstructured_version(min_version: str) -> bool:
    """Checks to see if the installed unstructured version exceeds the minimum version
    for the feature in question."""
    from unstructured.__version__ import __version__ as __unstructured_version__
    min_version_tuple = tuple([int(x) for x in min_version.split(".")])
    # NOTE(MthwRobinson) - enables the loader to work when you're using pre-release
    # versions of unstructured like 0.4.17-dev1
    _unstructured_version = __unstructured_version__.split("-")[0]
    unstructured_version_tuple = tuple(
        [int(x) for x in _unstructured_version.split(".")]
    )
    return unstructured_version_tuple >= min_version_tuple
def validate_unstructured_version(min_unstructured_version: str) -> None:
    """Raises an error if the unstructured version does not exceed the
    specified minimum."""
    if not satisfies_min_unstructured_version(min_unstructured_version):
        raise ValueError(
            f"unstructured>={min_unstructured_version} is required in this loader."
        )
class UnstructuredBaseLoader(BaseLoader, ABC):
    """Loader that uses unstructured to load files."""
    def __init__(self, mode: str = "single", **unstructured_kwargs: Any):
        """Initialize with file path."""
        try:
            import unstructured  # noqa:F401
        except ImportError:
            raise ValueError(


import unstructured  # noqa:F401
        except ImportError:
            raise ValueError(
                "unstructured package not found, please install it with "
                "`pip install unstructured`"
            )
        _valid_modes = {"single", "elements"}
        if mode not in _valid_modes:
            raise ValueError(
                f"Got {mode} for `mode`, but should be one of `{_valid_modes}`"
            )
        self.mode = mode
        if not satisfies_min_unstructured_version("0.5.4"):
            if "strategy" in unstructured_kwargs:
                unstructured_kwargs.pop("strategy")
        self.unstructured_kwargs = unstructured_kwargs
    @abstractmethod
    def _get_elements(self) -> List:
        """Get elements."""
    @abstractmethod
    def _get_metadata(self) -> dict:
        """Get metadata."""
    def load(self) -> List[Document]:
        """Load file."""
        elements = self._get_elements()
        if self.mode == "elements":
            docs: List[Document] = list()
            for element in elements:
                metadata = self._get_metadata()
                # NOTE(MthwRobinson) - the attribute check is for backward compatibility
                # with unstructured<0.4.9. The metadata attributed was added in 0.4.9.
                if hasattr(element, "metadata"):
                    metadata.update(element.metadata.to_dict())
                if hasattr(element, "category"):
                    metadata["category"] = element.category
                docs.append(Document(page_content=str(element), metadata=metadata))
        elif self.mode == "single":
            metadata = self._get_metadata()
            text = "\n\n".join([str(el) for el in elements])
            docs = [Document(page_content=text, metadata=metadata)]


docs = [Document(page_content=text, metadata=metadata)]
        else:
            raise ValueError(f"mode of {self.mode} not supported.")
        return docs
[docs]class UnstructuredFileLoader(UnstructuredBaseLoader):
    """Loader that uses unstructured to load files."""
    def __init__(
        self,
        file_path: Union[str, List[str]],
        mode: str = "single",
        **unstructured_kwargs: Any,
    ):
        """Initialize with file path."""
        self.file_path = file_path
        super().__init__(mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.auto import partition
        return partition(filename=self.file_path, **self.unstructured_kwargs)
    def _get_metadata(self) -> dict:
        return {"source": self.file_path}
def get_elements_from_api(
    file_path: Union[str, List[str], None] = None,
    file: Union[IO, Sequence[IO], None] = None,
    api_url: str = "https://api.unstructured.io/general/v0/general",
    api_key: str = "",
    **unstructured_kwargs: Any,
) -> List:
    """Retrieves a list of elements from the Unstructured API."""
    if isinstance(file, collections.abc.Sequence) or isinstance(file_path, list):
        from unstructured.partition.api import partition_multiple_via_api
        _doc_elements = partition_multiple_via_api(
            filenames=file_path,
            files=file,
            api_key=api_key,
            api_url=api_url,
            **unstructured_kwargs,
        )
        elements = []
        for _elements in _doc_elements:
            elements.extend(_elements)
        return elements
    else:


elements.extend(_elements)
        return elements
    else:
        from unstructured.partition.api import partition_via_api
        return partition_via_api(
            filename=file_path,
            file=file,
            api_key=api_key,
            api_url=api_url,
            **unstructured_kwargs,
        )
[docs]class UnstructuredAPIFileLoader(UnstructuredFileLoader):
    """Loader that uses the unstructured web API to load files."""
    def __init__(
        self,
        file_path: Union[str, List[str]] = "",
        mode: str = "single",
        url: str = "https://api.unstructured.io/general/v0/general",
        api_key: str = "",
        **unstructured_kwargs: Any,
    ):
        """Initialize with file path."""
        if isinstance(file_path, str):
            validate_unstructured_version(min_unstructured_version="0.6.2")
        else:
            validate_unstructured_version(min_unstructured_version="0.6.3")
        self.url = url
        self.api_key = api_key
        super().__init__(file_path=file_path, mode=mode, **unstructured_kwargs)
    def _get_metadata(self) -> dict:
        return {"source": self.file_path}
    def _get_elements(self) -> List:
        return get_elements_from_api(
            file_path=self.file_path,
            api_key=self.api_key,
            api_url=self.url,
            **self.unstructured_kwargs,
        )
[docs]class UnstructuredFileIOLoader(UnstructuredBaseLoader):
    """Loader that uses unstructured to load file IO objects."""
    def __init__(
        self,
        file: Union[IO, Sequence[IO]],
        mode: str = "single",


mode: str = "single",
        **unstructured_kwargs: Any,
    ):
        """Initialize with file path."""
        self.file = file
        super().__init__(mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        from unstructured.partition.auto import partition
        return partition(file=self.file, **self.unstructured_kwargs)
    def _get_metadata(self) -> dict:
        return {}
[docs]class UnstructuredAPIFileIOLoader(UnstructuredFileIOLoader):
    """Loader that uses the unstructured web API to load file IO objects."""
    def __init__(
        self,
        file: Union[IO, Sequence[IO]],
        mode: str = "single",
        url: str = "https://api.unstructured.io/general/v0/general",
        api_key: str = "",
        **unstructured_kwargs: Any,
    ):
        """Initialize with file path."""
        if isinstance(file, collections.abc.Sequence):
            validate_unstructured_version(min_unstructured_version="0.6.3")
        if file:
            validate_unstructured_version(min_unstructured_version="0.6.2")
        self.url = url
        self.api_key = api_key
        super().__init__(file=file, mode=mode, **unstructured_kwargs)
    def _get_elements(self) -> List:
        return get_elements_from_api(
            file=self.file,
            api_key=self.api_key,
            api_url=self.url,
            **self.unstructured_kwargs,
        )
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.trello
"""Loader that loads cards from Trello"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, List, Literal, Optional, Tuple
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import get_from_env
if TYPE_CHECKING:
    from trello import Board, Card, TrelloClient
[docs]class TrelloLoader(BaseLoader):
    """Trello loader. Reads all cards from a Trello board."""
    def __init__(
        self,
        client: TrelloClient,
        board_name: str,
        *,
        include_card_name: bool = True,
        include_comments: bool = True,
        include_checklist: bool = True,
        card_filter: Literal["closed", "open", "all"] = "all",
        extra_metadata: Tuple[str, ...] = ("due_date", "labels", "list", "closed"),
    ):
        """Initialize Trello loader.
        Args:
            client: Trello API client.
            board_name: The name of the Trello board.
            include_card_name: Whether to include the name of the card in the document.
            include_comments: Whether to include the comments on the card in the
                document.
            include_checklist: Whether to include the checklist on the card in the
                document.
            card_filter: Filter on card status. Valid values are "closed", "open",
                "all".
            extra_metadata: List of additional metadata fields to include as document
                metadata.Valid values are "due_date", "labels", "list", "closed".
        """
        self.client = client
        self.board_name = board_name
        self.include_card_name = include_card_name


self.board_name = board_name
        self.include_card_name = include_card_name
        self.include_comments = include_comments
        self.include_checklist = include_checklist
        self.extra_metadata = extra_metadata
        self.card_filter = card_filter
[docs]    @classmethod
    def from_credentials(
        cls,
        board_name: str,
        *,
        api_key: Optional[str] = None,
        token: Optional[str] = None,
        **kwargs: Any,
    ) -> TrelloLoader:
        """Convenience constructor that builds TrelloClient init param for you.
        Args:
            board_name: The name of the Trello board.
            api_key: Trello API key. Can also be specified as environment variable
                TRELLO_API_KEY.
            token: Trello token. Can also be specified as environment variable
                TRELLO_TOKEN.
            include_card_name: Whether to include the name of the card in the document.
            include_comments: Whether to include the comments on the card in the
                document.
            include_checklist: Whether to include the checklist on the card in the
                document.
            card_filter: Filter on card status. Valid values are "closed", "open",
                "all".
            extra_metadata: List of additional metadata fields to include as document
                metadata.Valid values are "due_date", "labels", "list", "closed".
        """
        try:
            from trello import TrelloClient  # type: ignore
        except ImportError as ex:
            raise ImportError(
                "Could not import trello python package. "
                "Please install it with `pip install py-trello`."
            ) from ex
        api_key = api_key or get_from_env("api_key", "TRELLO_API_KEY")


token = token or get_from_env("token", "TRELLO_TOKEN")
        client = TrelloClient(api_key=api_key, token=token)
        return cls(client, board_name, **kwargs)
[docs]    def load(self) -> List[Document]:
        """Loads all cards from the specified Trello board.
        You can filter the cards, metadata and text included by using the optional
            parameters.
         Returns:
            A list of documents, one for each card in the board.
        """
        try:
            from bs4 import BeautifulSoup  # noqa: F401
        except ImportError as ex:
            raise ImportError(
                "`beautifulsoup4` package not found, please run"
                " `pip install beautifulsoup4`"
            ) from ex
        board = self._get_board()
        # Create a dictionary with the list IDs as keys and the list names as values
        list_dict = {list_item.id: list_item.name for list_item in board.list_lists()}
        # Get Cards on the board
        cards = board.get_cards(card_filter=self.card_filter)
        return [self._card_to_doc(card, list_dict) for card in cards]
    def _get_board(self) -> Board:
        # Find the first board with a matching name
        board = next(
            (b for b in self.client.list_boards() if b.name == self.board_name), None
        )
        if not board:
            raise ValueError(f"Board `{self.board_name}` not found.")
        return board
    def _card_to_doc(self, card: Card, list_dict: dict) -> Document:
        from bs4 import BeautifulSoup  # type: ignore
        text_content = ""
        if self.include_card_name:
            text_content = card.name + "\n"


if self.include_card_name:
            text_content = card.name + "\n"
        if card.description.strip():
            text_content += BeautifulSoup(card.description, "lxml").get_text()
        if self.include_checklist:
            # Get all the checklist items on the card
            for checklist in card.checklists:
                if checklist.items:
                    items = [
                        f"{item['name']}:{item['state']}" for item in checklist.items
                    ]
                    text_content += f"\n{checklist.name}\n" + "\n".join(items)
        if self.include_comments:
            # Get all the comments on the card
            comments = [
                BeautifulSoup(comment["data"]["text"], "lxml").get_text()
                for comment in card.comments
            ]
            text_content += "Comments:" + "\n".join(comments)
        # Default metadata fields
        metadata = {
            "title": card.name,
            "id": card.id,
            "url": card.url,
        }
        # Extra metadata fields. Card object is not subscriptable.
        if "labels" in self.extra_metadata:
            metadata["labels"] = [label.name for label in card.labels]
        if "list" in self.extra_metadata:
            if card.list_id in list_dict:
                metadata["list"] = list_dict[card.list_id]
        if "closed" in self.extra_metadata:
            metadata["closed"] = card.closed
        if "due_date" in self.extra_metadata:
            metadata["due_date"] = card.due_date
        return Document(page_content=text_content, metadata=metadata)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.pdf
"""Loader that loads PDF files."""
import json
import logging
import os
import tempfile
import time
from abc import ABC
from io import StringIO
from pathlib import Path
from typing import Any, Iterator, List, Mapping, Optional
from urllib.parse import urlparse
import requests
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.document_loaders.blob_loaders import Blob
from langchain.document_loaders.parsers.pdf import (
    PDFMinerParser,
    PDFPlumberParser,
    PyMuPDFParser,
    PyPDFium2Parser,
    PyPDFParser,
)
from langchain.document_loaders.unstructured import UnstructuredFileLoader
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__file__)
[docs]class UnstructuredPDFLoader(UnstructuredFileLoader):
    """Loader that uses unstructured to load PDF files."""
    def _get_elements(self) -> List:
        from unstructured.partition.pdf import partition_pdf
        return partition_pdf(filename=self.file_path, **self.unstructured_kwargs)
class BasePDFLoader(BaseLoader, ABC):
    """Base loader class for PDF files.
    Defaults to check for local file, but if the file is a web path, it will download it
    to a temporary file, and use that, then clean up the temporary file after completion
    """
    def __init__(self, file_path: str):
        """Initialize with file path."""
        self.file_path = file_path
        self.web_path = None
        if "~" in self.file_path:
            self.file_path = os.path.expanduser(self.file_path)
        # If the file is a web path, download it to a temporary file, and use that


if not os.path.isfile(self.file_path) and self._is_valid_url(self.file_path):
            r = requests.get(self.file_path)
            if r.status_code != 200:
                raise ValueError(
                    "Check the url of your file; returned status code %s"
                    % r.status_code
                )
            self.web_path = self.file_path
            self.temp_file = tempfile.NamedTemporaryFile()
            self.temp_file.write(r.content)
            self.file_path = self.temp_file.name
        elif not os.path.isfile(self.file_path):
            raise ValueError("File path %s is not a valid file or url" % self.file_path)
    def __del__(self) -> None:
        if hasattr(self, "temp_file"):
            self.temp_file.close()
    @staticmethod
    def _is_valid_url(url: str) -> bool:
        """Check if the url is valid."""
        parsed = urlparse(url)
        return bool(parsed.netloc) and bool(parsed.scheme)
    @property
    def source(self) -> str:
        return self.web_path if self.web_path is not None else self.file_path
[docs]class OnlinePDFLoader(BasePDFLoader):
    """Loader that loads online PDFs."""
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        loader = UnstructuredPDFLoader(str(self.file_path))
        return loader.load()
[docs]class PyPDFLoader(BasePDFLoader):
    """Loads a PDF with pypdf and chunks at character level.
    Loader also stores page numbers in metadatas.
    """
    def __init__(self, file_path: str) -> None:
        """Initialize with file path."""
        try:
            import pypdf  # noqa:F401
        except ImportError:


try:
            import pypdf  # noqa:F401
        except ImportError:
            raise ImportError(
                "pypdf package not found, please install it with " "`pip install pypdf`"
            )
        self.parser = PyPDFParser()
        super().__init__(file_path)
[docs]    def load(self) -> List[Document]:
        """Load given path as pages."""
        return list(self.lazy_load())
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Lazy load given path as pages."""
        blob = Blob.from_path(self.file_path)
        yield from self.parser.parse(blob)
[docs]class PyPDFium2Loader(BasePDFLoader):
    """Loads a PDF with pypdfium2 and chunks at character level."""
    def __init__(self, file_path: str):
        """Initialize with file path."""
        super().__init__(file_path)
        self.parser = PyPDFium2Parser()
[docs]    def load(self) -> List[Document]:
        """Load given path as pages."""
        return list(self.lazy_load())
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Lazy load given path as pages."""
        blob = Blob.from_path(self.file_path)
        yield from self.parser.parse(blob)
[docs]class PyPDFDirectoryLoader(BaseLoader):
    """Loads a directory with PDF files with pypdf and chunks at character level.
    Loader also stores page numbers in metadatas.
    """
    def __init__(
        self,
        path: str,
        glob: str = "**/[!.]*.pdf",
        silent_errors: bool = False,
        load_hidden: bool = False,


silent_errors: bool = False,
        load_hidden: bool = False,
        recursive: bool = False,
    ):
        self.path = path
        self.glob = glob
        self.load_hidden = load_hidden
        self.recursive = recursive
        self.silent_errors = silent_errors
    @staticmethod
    def _is_visible(path: Path) -> bool:
        return not any(part.startswith(".") for part in path.parts)
[docs]    def load(self) -> List[Document]:
        p = Path(self.path)
        docs = []
        items = p.rglob(self.glob) if self.recursive else p.glob(self.glob)
        for i in items:
            if i.is_file():
                if self._is_visible(i.relative_to(p)) or self.load_hidden:
                    try:
                        loader = PyPDFLoader(str(i))
                        sub_docs = loader.load()
                        for doc in sub_docs:
                            doc.metadata["source"] = str(i)
                        docs.extend(sub_docs)
                    except Exception as e:
                        if self.silent_errors:
                            logger.warning(e)
                        else:
                            raise e
        return docs
[docs]class PDFMinerLoader(BasePDFLoader):
    """Loader that uses PDFMiner to load PDF files."""
    def __init__(self, file_path: str) -> None:
        """Initialize with file path."""
        try:
            from pdfminer.high_level import extract_text  # noqa:F401
        except ImportError:
            raise ImportError(
                "`pdfminer` package not found, please install it with "
                "`pip install pdfminer.six`"
            )
        super().__init__(file_path)
        self.parser = PDFMinerParser()
[docs]    def load(self) -> List[Document]:


[docs]    def load(self) -> List[Document]:
        """Eagerly load the content."""
        return list(self.lazy_load())
[docs]    def lazy_load(
        self,
    ) -> Iterator[Document]:
        """Lazily lod documents."""
        blob = Blob.from_path(self.file_path)
        yield from self.parser.parse(blob)
[docs]class PDFMinerPDFasHTMLLoader(BasePDFLoader):
    """Loader that uses PDFMiner to load PDF files as HTML content."""
    def __init__(self, file_path: str):
        """Initialize with file path."""
        try:
            from pdfminer.high_level import extract_text_to_fp  # noqa:F401
        except ImportError:
            raise ImportError(
                "`pdfminer` package not found, please install it with "
                "`pip install pdfminer.six`"
            )
        super().__init__(file_path)
[docs]    def load(self) -> List[Document]:
        """Load file."""
        from pdfminer.high_level import extract_text_to_fp
        from pdfminer.layout import LAParams
        from pdfminer.utils import open_filename
        output_string = StringIO()
        with open_filename(self.file_path, "rb") as fp:
            extract_text_to_fp(
                fp,  # type: ignore[arg-type]
                output_string,
                codec="",
                laparams=LAParams(),
                output_type="html",
            )
        metadata = {"source": self.file_path}
        return [Document(page_content=output_string.getvalue(), metadata=metadata)]
[docs]class PyMuPDFLoader(BasePDFLoader):
    """Loader that uses PyMuPDF to load PDF files."""
    def __init__(self, file_path: str) -> None:


def __init__(self, file_path: str) -> None:
        """Initialize with file path."""
        try:
            import fitz  # noqa:F401
        except ImportError:
            raise ImportError(
                "`PyMuPDF` package not found, please install it with "
                "`pip install pymupdf`"
            )
        super().__init__(file_path)
[docs]    def load(self, **kwargs: Optional[Any]) -> List[Document]:
        """Load file."""
        parser = PyMuPDFParser(text_kwargs=kwargs)
        blob = Blob.from_path(self.file_path)
        return parser.parse(blob)
# MathpixPDFLoader implementation taken largely from Daniel Gross's:
# https://gist.github.com/danielgross/3ab4104e14faccc12b49200843adab21
[docs]class MathpixPDFLoader(BasePDFLoader):
    def __init__(
        self,
        file_path: str,
        processed_file_format: str = "mmd",
        max_wait_time_seconds: int = 500,
        should_clean_pdf: bool = False,
        **kwargs: Any,
    ) -> None:
        super().__init__(file_path)
        self.mathpix_api_key = get_from_dict_or_env(
            kwargs, "mathpix_api_key", "MATHPIX_API_KEY"
        )
        self.mathpix_api_id = get_from_dict_or_env(
            kwargs, "mathpix_api_id", "MATHPIX_API_ID"
        )
        self.processed_file_format = processed_file_format
        self.max_wait_time_seconds = max_wait_time_seconds
        self.should_clean_pdf = should_clean_pdf
    @property
    def headers(self) -> dict:


@property
    def headers(self) -> dict:
        return {"app_id": self.mathpix_api_id, "app_key": self.mathpix_api_key}
    @property
    def url(self) -> str:
        return "https://api.mathpix.com/v3/pdf"
    @property
    def data(self) -> dict:
        options = {"conversion_formats": {self.processed_file_format: True}}
        return {"options_json": json.dumps(options)}
[docs]    def send_pdf(self) -> str:
        with open(self.file_path, "rb") as f:
            files = {"file": f}
            response = requests.post(
                self.url, headers=self.headers, files=files, data=self.data
            )
        response_data = response.json()
        if "pdf_id" in response_data:
            pdf_id = response_data["pdf_id"]
            return pdf_id
        else:
            raise ValueError("Unable to send PDF to Mathpix.")
[docs]    def wait_for_processing(self, pdf_id: str) -> None:
        url = self.url + "/" + pdf_id
        for _ in range(0, self.max_wait_time_seconds, 5):
            response = requests.get(url, headers=self.headers)
            response_data = response.json()
            status = response_data.get("status", None)
            if status == "completed":
                return
            elif status == "error":
                raise ValueError("Unable to retrieve PDF from Mathpix")
            else:
                print(f"Status: {status}, waiting for processing to complete")
                time.sleep(5)
        raise TimeoutError
[docs]    def get_processed_pdf(self, pdf_id: str) -> str:
        self.wait_for_processing(pdf_id)


self.wait_for_processing(pdf_id)
        url = f"{self.url}/{pdf_id}.{self.processed_file_format}"
        response = requests.get(url, headers=self.headers)
        return response.content.decode("utf-8")
[docs]    def clean_pdf(self, contents: str) -> str:
        contents = "\n".join(
            [line for line in contents.split("\n") if not line.startswith("![]")]
        )
        # replace \section{Title} with # Title
        contents = contents.replace("\\section{", "# ").replace("}", "")
        # replace the "\" slash that Mathpix adds to escape $, %, (, etc.
        contents = (
            contents.replace(r"\$", "$")
            .replace(r"\%", "%")
            .replace(r"\(", "(")
            .replace(r"\)", ")")
        )
        return contents
[docs]    def load(self) -> List[Document]:
        pdf_id = self.send_pdf()
        contents = self.get_processed_pdf(pdf_id)
        if self.should_clean_pdf:
            contents = self.clean_pdf(contents)
        metadata = {"source": self.source, "file_path": self.source}
        return [Document(page_content=contents, metadata=metadata)]
[docs]class PDFPlumberLoader(BasePDFLoader):
    """Loader that uses pdfplumber to load PDF files."""
    def __init__(
        self, file_path: str, text_kwargs: Optional[Mapping[str, Any]] = None
    ) -> None:
        """Initialize with file path."""
        try:
            import pdfplumber  # noqa:F401
        except ImportError:
            raise ImportError(
                "pdfplumber package not found, please install it with "
                "`pip install pdfplumber`"
            )


"`pip install pdfplumber`"
            )
        super().__init__(file_path)
        self.text_kwargs = text_kwargs or {}
[docs]    def load(self) -> List[Document]:
        """Load file."""
        parser = PDFPlumberParser(text_kwargs=self.text_kwargs)
        blob = Blob.from_path(self.file_path)
        return parser.parse(blob)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 05, 2023.


Source code for langchain.document_loaders.telegram
"""Loader that loads Telegram chat json dump."""
from __future__ import annotations
import asyncio
import json
from pathlib import Path
from typing import TYPE_CHECKING, Dict, List, Optional, Union
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
if TYPE_CHECKING:
    import pandas as pd
    from telethon.hints import EntityLike
def concatenate_rows(row: dict) -> str:
    """Combine message information in a readable format ready to be used."""
    date = row["date"]
    sender = row["from"]
    text = row["text"]
    return f"{sender} on {date}: {text}\n\n"
[docs]class TelegramChatFileLoader(BaseLoader):
    """Loader that loads Telegram chat json directory dump."""
    def __init__(self, path: str):
        """Initialize with path."""
        self.file_path = path
[docs]    def load(self) -> List[Document]:
        """Load documents."""
        p = Path(self.file_path)
        with open(p, encoding="utf8") as f:
            d = json.load(f)
        text = "".join(
            concatenate_rows(message)
            for message in d["messages"]
            if message["type"] == "message" and isinstance(message["text"], str)
        )
        metadata = {"source": str(p)}
        return [Document(page_content=text, metadata=metadata)]
def text_to_docs(text: Union[str, List[str]]) -> List[Document]:
    """Converts a string or list of strings to a list of Documents with metadata."""
    if isinstance(text, str):
        # Take a single string as one page


if isinstance(text, str):
        # Take a single string as one page
        text = [text]
    page_docs = [Document(page_content=page) for page in text]
    # Add page numbers as metadata
    for i, doc in enumerate(page_docs):
        doc.metadata["page"] = i + 1
    # Split pages into chunks
    doc_chunks = []
    for doc in page_docs:
        text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=800,
            separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""],
            chunk_overlap=20,
        )
        chunks = text_splitter.split_text(doc.page_content)
        for i, chunk in enumerate(chunks):
            doc = Document(
                page_content=chunk, metadata={"page": doc.metadata["page"], "chunk": i}
            )
            # Add sources a metadata
            doc.metadata["source"] = f"{doc.metadata['page']}-{doc.metadata['chunk']}"
            doc_chunks.append(doc)
    return doc_chunks
[docs]class TelegramChatApiLoader(BaseLoader):
    """Loader that loads Telegram chat json directory dump."""
    def __init__(
        self,
        chat_entity: Optional[EntityLike] = None,
        api_id: Optional[int] = None,
        api_hash: Optional[str] = None,
        username: Optional[str] = None,
        file_path: str = "telegram_data.json",
    ):
        """Initialize with API parameters."""
        self.chat_entity = chat_entity
        self.api_id = api_id
        self.api_hash = api_hash
        self.username = username
        self.file_path = file_path
[docs]    async def fetch_data_from_telegram(self) -> None:


[docs]    async def fetch_data_from_telegram(self) -> None:
        """Fetch data from Telegram API and save it as a JSON file."""
        from telethon.sync import TelegramClient
        data = []
        async with TelegramClient(self.username, self.api_id, self.api_hash) as client:
            async for message in client.iter_messages(self.chat_entity):
                is_reply = message.reply_to is not None
                reply_to_id = message.reply_to.reply_to_msg_id if is_reply else None
                data.append(
                    {
                        "sender_id": message.sender_id,
                        "text": message.text,
                        "date": message.date.isoformat(),
                        "message.id": message.id,
                        "is_reply": is_reply,
                        "reply_to_id": reply_to_id,
                    }
                )
        with open(self.file_path, "w", encoding="utf-8") as f:
            json.dump(data, f, ensure_ascii=False, indent=4)
    def _get_message_threads(self, data: pd.DataFrame) -> dict:
        """Create a dictionary of message threads from the given data.
        Args:
            data (pd.DataFrame): A DataFrame containing the conversation \
                data with columns:
                - message.sender_id
                - text
                - date
                - message.id
                - is_reply
                - reply_to_id
        Returns:
            dict: A dictionary where the key is the parent message ID and \
                the value is a list of message IDs in ascending order.
        """
        def find_replies(parent_id: int, reply_data: pd.DataFrame) -> List[int]:
            """
            Recursively find all replies to a given parent message ID.
            Args:
                parent_id (int): The parent message ID.