File size: 4,509 Bytes
a80f6e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | import glob
import sys
from pathlib import Path
import requests
import yaml
#################
# CONFIGURATION #
#################
# packages to ignore / exclude from the table
IGNORE_PACKGAGES = {
# top-level packages
"langchain-core",
"langchain-text-splitters",
"langchain",
"langchain-community",
"langchain-experimental",
"langchain-cli",
"langchain-tests",
# integration packages that don't have a provider index
# do NOT add to these. These were merged before having a
# provider index was required
# can remove these once they have a provider index
"langchain-yt-dlp",
}
#####################
# END CONFIGURATION #
#####################
DOCS_DIR = Path(__file__).parents[1]
PACKAGE_YML = Path(__file__).parents[2] / "libs" / "packages.yml"
# for now, only include packages that are in the langchain-ai org
# because we don't have a policy for inclusion in this table yet,
# and including all packages will make the list too long
def _get_type(package: dict) -> str:
if package["name"] in IGNORE_PACKGAGES:
return "ignore"
if package["repo"] == "langchain-ai/langchain":
return "B"
if package["repo"].startswith("langchain-ai/"):
return "C"
return "D"
def _enrich_package(p: dict) -> dict | None:
p["name_short"] = (
p["name"][10:] if p["name"].startswith("langchain-") else p["name"]
)
p["name_title"] = p.get("name_title") or p["name_short"].title().replace(
"-", " "
).replace("db", "DB").replace("Db", "DB").replace("ai", "AI").replace("Ai", "AI")
p["type"] = _get_type(p)
if p["type"] == "ignore":
return None
p["js_exists"] = bool(p.get("js"))
custom_provider_page = p.get("provider_page")
default_provider_page = f"/docs/integrations/providers/{p['name_short']}/"
default_provider_page_exists = bool(
glob.glob(str(DOCS_DIR / f"docs/integrations/providers/{p['name_short']}.*"))
)
p["provider_page"] = custom_provider_page or (
default_provider_page if default_provider_page_exists else None
)
if p["provider_page"] is None:
msg = (
f"Provider page not found for {p['name_short']}. "
f"Please add one at docs/integrations/providers/{p['name_short']}.{{mdx,ipynb}}"
)
raise ValueError(msg)
if p["type"] in ("B", "C"):
p["package_url"] = (
f"https://python.langchain.com/api_reference/{p['name_short'].replace('-', '_')}/"
)
else:
p["package_url"] = f"https://pypi.org/project/{p['name']}/"
return p
with open(PACKAGE_YML) as f:
data = yaml.safe_load(f)
packages_n = [_enrich_package(p) for p in data["packages"]]
packages = [p for p in packages_n if p is not None]
# sort by downloads
packages_sorted = sorted(packages, key=lambda p: p.get("downloads", 0), reverse=True)
def package_row(p: dict) -> str:
js = "✅" if p["js_exists"] else "❌"
link = p["provider_page"]
title = p["name_title"]
provider = f"[{title}]({link})" if link else title
return f"| {provider} | [{p['name']}]({p['package_url']}) |  |  | {js} |"
def table() -> str:
header = """| Provider | Package | Downloads | Latest | [JS](https://js.langchain.com/docs/integrations/platforms/) |
| :--- | :---: | :---: | :---: | :---: |
"""
return header + "\n".join(package_row(p) for p in packages_sorted)
def doc() -> str:
return f"""\
---
sidebar_position: 0
sidebar_class_name: hidden
---
# Providers
:::info
If you'd like to write your own integration, see [Extending LangChain](/docs/how_to/#custom).
If you'd like to contribute an integration, see [Contributing integrations](/docs/contributing/how_to/integrations/).
:::
LangChain integrates with many providers.
## Integration Packages
These providers have standalone `langchain-{{provider}}` packages for improved versioning, dependency management and testing.
{table()}
## All Providers
Click [here](/docs/integrations/providers/all) to see all providers. Or search for a
provider using the Search field in the top-right corner of the screen.
"""
if __name__ == "__main__":
output_dir = Path(sys.argv[1]) / "integrations" / "providers"
with open(output_dir / "index.mdx", "w") as f:
f.write(doc())
|