third_party/transformers/setup.py

# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/main/setup.py

To create the package for pypi.

1. Create the release branch named: v<RELEASE>-release, for example v4.19-release. For a patch release checkout the
   current release branch.

   If releasing on a special branch, copy the updated README.md on the main branch for the commit you will make
   for the post-release and run `make fix-repo` on the main branch as well.

2. Run `make pre-release` (or `make pre-patch` for a patch release) and commit these changes with the message:
   "Release: <VERSION>" and push.

3. Go back to the main branch and run `make post-release` then `make fix-repo`. Commit these changes with the
   message "v<NEXT_VERSION>.dev.0" and push to main.

# If you were just cutting the branch in preparation for a release, you can stop here for now.

4. Wait for the tests on the release branch to be completed and be green (otherwise revert and fix bugs)

5. On the release branch, add a tag in git to mark the release: "git tag v<VERSION> -m 'Adds tag v<VERSION> for pypi' "
   Push the tag to git: git push --tags origin v<RELEASE>-release

6. Have a core maintainer review and approve the deployment to pypi.
"""

import re
import shutil
import sys
from pathlib import Path

from setuptools import Command, find_packages, setup


# Supported Python version range (min, max)
SUPPORTED_PYTHON_VERSIONS = (10, 14)  # 3.10 to 3.14

PYTHON_MINOR_VERSION = sys.version_info.minor

# Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466
stale_egg_info = Path(__file__).parent / "transformers.egg-info"
if stale_egg_info.exists():
    print(
        f"Warning: {stale_egg_info} exists.\n\n"
        "If you recently updated transformers to 3.0 or later, this is expected,\n"
        "but it may prevent transformers from installing in editable mode.\n\n"
        "This directory is automatically generated by Python's packaging tools.\n"
        "I will remove it now.\n\n"
        "See https://github.com/pypa/pip/issues/5466 for details.\n"
    )
    shutil.rmtree(stale_egg_info)


# IMPORTANT:
# 1. all dependencies should be listed here with their version requirements if any
# 2. once modified, run: `make fix-repo` to update src/transformers/dependency_versions_table.py
_deps = [
    "Pillow>=10.0.1,<=15.0",
    "accelerate>=1.1.0",
    "av",
    "beautifulsoup4",
    "blobfile",
    "codecarbon>=2.8.1",
    "datasets>=2.15.0",  # We need either this pin or pyarrow<21.0.0
    "deepspeed>=0.9.3",
    "dill<0.3.5",
    "evaluate>=0.4.6",
    "faiss-cpu",
    "fastapi",
    "filelock",
    "fugashi>=1.0",
    "GitPython<3.1.19",
    "hf-doc-builder",
    "huggingface-hub>=1.5.0,<2.0",
    "ipadic>=1.0.0,<2.0",
    "jinja2>=3.1.0",
    "jmespath>=1.0.1",
    "kenlm",
    "kernels>=0.12.0,<0.13",
    "librosa",
    "mistral-common[image]>=1.10.0",
    "nltk<=3.8.1",
    "num2words",
    "numpy>=1.17",
    "openai>=1.98.0",
    "opencv-python",
    "optimum-benchmark>=0.3.0",
    "optuna",
    "pandas<2.3.0",  # `datasets` requires `pandas` while `pandas==2.3.0` has issues with CircleCI on 2025/06/05
    "packaging>=20.0",
    "parameterized>=0.9",  # older version of parameterized cause pytest collection to fail on .expand
    "peft>=0.18.0",
    "phonemizer",
    "protobuf",
    "psutil",
    "pyyaml>=5.1",
    "pydantic>=2",
    "pytest>=7.2.0,<9.0.0",
    "pytest-asyncio>=1.2.0",
    "pytest-random-order",
    "pytest-rerunfailures<16.0",
    "pytest-timeout",
    "pytest-env",
    "pytest-xdist",
    "pytest-order",
    "python>=3.10.0",
    "regex>=2025.10.22",
    "rhoknp>=1.1.0,<1.3.1",
    "rjieba",
    "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
    "ruff==0.14.10",
    "ty==0.0.20",
    # `sacrebleu` not used in `transformers`. However, it is needed in several tests, when a test calls
    # `evaluate.load("sacrebleu")`. This metric is used in the examples that we use to test the `Trainer` with, in the
    # `Trainer` tests (see references to `run_translation.py`).
    "sacrebleu>=1.4.12,<2.0.0",
    "sacremoses",
    "safetensors>=0.4.3",
    "sagemaker>=2.31.0",
    "schedulefree>=1.2.6",
    "scikit-learn",
    "scipy",
    "sentencepiece>=0.1.91,!=0.1.92",
    "starlette",
    "sudachipy>=0.6.6",
    "sudachidict_core>=20220729",
    "tensorboard",
    "timeout-decorator",
    "tomli",
    "tiktoken",
    "timm>=1.0.23",
    "tokenizers>=0.22.0,<=0.23.0",
    "torch>=2.4",
    "torchaudio",
    "torchvision",
    "pyctcdecode>=0.4.0",
    "tqdm>=4.27",
    "typer",
    "unidic>=1.0.2",
    "unidic_lite>=1.0.7",
    "urllib3<2.0.0",
    "uvicorn",
    "pytest-rich",
    "libcst",
    "rich",
    "ray[tune]>=2.7.0",
    "opentelemetry-api",
    "opentelemetry-exporter-otlp",
    "opentelemetry-sdk",
]

# This is a lookup table with items like: {"tokenizers": "tokenizers==0.9.4", "packaging": "packaging"}, i.e.
# some of the values are versioned whereas others aren't.
deps = {b: a for a, b in (re.findall(r"^(([^!=<>~ ]+)(?:[!=<>~ ].*)?$)", x)[0] for x in _deps)}


def deps_list(*pkgs):
    return [deps[pkg] for pkg in pkgs]


extras = {}

extras["torch"] = deps_list("torch", "accelerate")
extras["vision"] = deps_list("torchvision", "Pillow")
extras["audio"] = deps_list("torchaudio", "librosa", "pyctcdecode", "phonemizer")
if PYTHON_MINOR_VERSION < 13:
    extras["audio"] += deps_list("kenlm")
extras["video"] = deps_list("av")
extras["timm"] = deps_list("timm")
extras["quality"] = deps_list("datasets", "ruff", "GitPython", "urllib3", "libcst", "rich", "ty", "tomli")
extras["docs"] = deps_list("hf-doc-builder")
extras["kernels"] = deps_list("kernels")
extras["sentencepiece"] = deps_list("sentencepiece", "protobuf")
extras["tiktoken"] = deps_list("tiktoken", "blobfile")
extras["mistral-common"] = deps_list("mistral-common[image]")
extras["chat_template"] = deps_list("jinja2", "jmespath")
extras["sklearn"] = deps_list("scikit-learn")
extras["accelerate"] = deps_list("accelerate")
extras["retrieval"] = deps_list("faiss-cpu", "datasets")
extras["sagemaker"] = deps_list("sagemaker")
extras["deepspeed"] = deps_list("deepspeed", "accelerate")
extras["optuna"] = deps_list("optuna")
extras["integrations"] = deps_list("kernels", "optuna", "codecarbon")
if PYTHON_MINOR_VERSION < 14:
    extras["ray"] = deps_list("ray[tune]")
    extras["integrations"] += extras["ray"]
extras["codecarbon"] = deps_list("codecarbon")
extras["serving"] = deps_list("openai", "pydantic", "uvicorn", "fastapi", "starlette", "rich") + extras["torch"]
extras["num2words"] = deps_list("num2words")
extras["benchmark"] = deps_list("optimum-benchmark")
extras["ja"] = deps_list("fugashi", "ipadic", "unidic_lite", "unidic", "rhoknp")
if PYTHON_MINOR_VERSION < 14:
    extras["ja"] += deps_list("sudachipy", "sudachidict_core")
# OpenTelemetry dependencies for metrics collection in continuous batching
extras["open-telemetry"] = deps_list("opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk")

extras["testing"] = (
    deps_list(
        "pytest",
        "pytest-asyncio",
        "pytest-random-order",
        "pytest-rich",
        "pytest-xdist",
        "pytest-order",
        "pytest-rerunfailures",
        "pytest-timeout",
        "pytest-env",
        "timeout-decorator",
        "parameterized",
        "psutil",
        "dill",
        "evaluate",
        "rouge-score",
        "nltk",
        "sacremoses",
        "rjieba",
        "beautifulsoup4",
        "tensorboard",
        "sacrebleu",  # needed in trainer tests, see references to `run_translation.py`
        "filelock",  # filesystem locks, e.g., to prevent parallel downloads
    )
    + extras["docs"]
    + extras["quality"]
    + extras["retrieval"]
    + extras["sentencepiece"]
    + extras["serving"]
)
extras["testing"] += extras["mistral-common"]

extras["deepspeed-testing"] = extras["deepspeed"] + extras["testing"] + extras["optuna"] + extras["sentencepiece"]
extras["all"] = (
    extras["torch"]
    + extras["vision"]
    + extras["audio"]
    + extras["video"]
    + extras["kernels"]
    + extras["timm"]
    + extras["sentencepiece"]
    + extras["tiktoken"]
    + extras["chat_template"]
    + extras["num2words"]
)
extras["all"] += extras["mistral-common"]

extras["dev"] = extras["all"] + extras["testing"] + extras["ja"] + extras["sklearn"]

# Those define the hard dependencies of `transformers`
install_requires = [
    deps["huggingface-hub"],
    deps["numpy"],
    deps["packaging"],  # utilities from PyPA to e.g., compare versions
    deps["pyyaml"],  # used for the model cards metadata
    deps["regex"],  # for OpenAI GPT
    deps["tokenizers"],
    deps["typer"],  # CLI utilities. In practice, already a dependency of huggingface_hub but we use it as well
    deps["safetensors"],
    deps["tqdm"],  # progress bars in model download and training scripts
]


class DepsTableUpdateCommand(Command):
    """
    A custom distutils command that updates the dependency table.
    usage: python setup.py deps_table_update
    """

    description = "build runtime dependency table"
    user_options = [
        # format: (long option, short option, description).
        ("dep-table-update", None, "updates src/transformers/dependency_versions_table.py"),
    ]

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        if SUPPORTED_PYTHON_VERSIONS[0] >= PYTHON_MINOR_VERSION:
            print(
                f"Table updated only when running 3.{SUPPORTED_PYTHON_VERSIONS[0]}.x, detected version is {sys.version}."
            )
            return

        entries = "\n".join([f'    "{k}": "{v}",' for k, v in deps.items()])
        content = [
            "# THIS FILE HAS BEEN AUTOGENERATED. To update:",
            "# 1. modify the `_deps` dict in setup.py",
            "# 2. run `make fix-repo``",
            "deps = {",
            entries,
            "}",
            "",
        ]
        target = "src/transformers/dependency_versions_table.py"
        with open(target, "w", encoding="utf-8", newline="\n") as f:
            f.write("\n".join(content))


if __name__ == "__main__":
    # Generate python_requires from supported version range
    min_version, max_version = SUPPORTED_PYTHON_VERSIONS
    python_requires = f">=3.{min_version}.0"

    # Generate Python version classifiers dynamically
    python_classifiers = ["Programming Language :: Python :: 3"]
    for minor in range(min_version, max_version + 1):
        python_classifiers.append(f"Programming Language :: Python :: 3.{minor}")

    setup(
        name="transformers",
        version="5.6.0.dev0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
        author="The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)",
        author_email="transformers@huggingface.co",
        description="Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training.",
        long_description=open("README.md", "r", encoding="utf-8").read(),
        long_description_content_type="text/markdown",
        keywords="machine-learning nlp python pytorch transformer llm vlm deep-learning inference training model-hub pretrained-models llama gemma qwen",
        license="Apache 2.0 License",
        url="https://github.com/huggingface/transformers",
        package_dir={"": "src"},
        packages=find_packages("src"),
        include_package_data=True,
        package_data={"": ["**/*.cu", "**/*.cpp", "**/*.cuh", "**/*.h", "**/*.pyx", "py.typed"]},
        zip_safe=False,
        extras_require=extras,
        entry_points={"console_scripts": ["transformers=transformers.cli.transformers:main"]},
        python_requires=python_requires,
        install_requires=list(install_requires),
        classifiers=[
            "Development Status :: 5 - Production/Stable",
            "Intended Audience :: Developers",
            "Intended Audience :: Education",
            "Intended Audience :: Science/Research",
            "Operating System :: OS Independent",
        ]
        + python_classifiers
        + [
            "Topic :: Scientific/Engineering :: Artificial Intelligence",
        ],
        cmdclass={"deps_table_update": DepsTableUpdateCommand},
    )