python/helpers/lib/parser.py

import glob
import io
import json
import os.path
import re

import configparser
import setuptools
import pip._internal.req.req_file
from pip._internal.network.session import PipSession
from pip._internal.req.constructors import (
    install_req_from_line,
    install_req_from_parsed_requirement,
)

from packaging.requirements import InvalidRequirement, Requirement
# TODO: Replace 3p package `tomli` with 3.11's new stdlib `tomllib` once we
#       drop support for Python 3.10.
import tomli

# Inspired by pips internal check:
# https://github.com/pypa/pip/blob/0bb3ac87f5bb149bd75cceac000844128b574385/src/pip/_internal/req/req_file.py#L35
COMMENT_RE = re.compile(r'(^|\s+)#.*$')


def parse_pep621_pep735_dependencies(pyproject_path):
    with open(pyproject_path, "rb") as file:
        project_toml = tomli.load(file)

    def version_from_req(specifier_set):
        if (len(specifier_set) == 1 and
                next(iter(specifier_set)).operator in {"==", "==="}):
            return next(iter(specifier_set)).version

    def parse_requirement(entry, pyproject_path, requirement_type=None):
        try:
            req = Requirement(entry)
        except InvalidRequirement as e:
            print(json.dumps({"error": repr(e)}))
            exit(1)
        else:
            data = {
                "name": req.name,
                "version": version_from_req(req.specifier),
                "markers": str(req.marker) or None,
                "file": pyproject_path,
                "requirement": str(req.specifier),
                "extras": sorted(list(req.extras)),
                "requirement_type": requirement_type,
            }
            return data

    def parse_toml_section_pep621_dependencies(
        pyproject_path, dependencies, requirement_type=None
    ):
        requirement_packages = []

        for dependency in dependencies:
            parsed_dependency = parse_requirement(
                dependency, pyproject_path, requirement_type
            )
            requirement_packages.append(parsed_dependency)

        return requirement_packages

    def parse_toml_section_pep735_dependencies(
        pyproject_path,
        dependency_groups,
        group_name,
        visited=None,
    ):
        requirement_packages = []
        visited = visited or set()

        if group_name in visited:
            return requirement_packages

        visited.add(group_name)
        dependencies = dependency_groups.get(group_name, [])
        for entry in dependencies:
            # Handle direct requirement
            if isinstance(entry, str):
                parsed_dependency = parse_requirement(
                    entry, pyproject_path, group_name
                )
                requirement_packages.append(parsed_dependency)
            # Handle include-group directive
            elif isinstance(entry, dict) and "include-group" in entry:
                included_group = entry["include-group"]
                requirement_packages.extend(
                    parse_toml_section_pep735_dependencies(
                        pyproject_path,
                        dependency_groups,
                        included_group,
                        visited
                    )
                )

        return requirement_packages

    dependencies = []

    if 'project' in project_toml:
        project_section = project_toml['project']

        if 'dependencies' in project_section:
            dependencies_toml = project_section['dependencies']
            runtime_dependencies = parse_toml_section_pep621_dependencies(
                pyproject_path,
                dependencies_toml,
                "dependencies"
            )
            dependencies.extend(runtime_dependencies)

        if 'optional-dependencies' in project_section:
            optional_dependencies_toml = project_section[
                'optional-dependencies'
            ]
            for group in optional_dependencies_toml:
                group_dependencies = parse_toml_section_pep621_dependencies(
                    pyproject_path,
                    optional_dependencies_toml[group],
                    group
                )
                dependencies.extend(group_dependencies)

    if 'dependency-groups' in project_toml:
        dependency_groups = project_toml['dependency-groups']
        for group_name in dependency_groups:
            group_dependencies = parse_toml_section_pep735_dependencies(
                pyproject_path, dependency_groups, group_name
            )
            dependencies.extend(group_dependencies)

    if 'build-system' in project_toml:
        build_system_section = project_toml['build-system']
        if 'requires' in build_system_section:
            build_system_dependencies = parse_toml_section_pep621_dependencies(
                pyproject_path,
                build_system_section['requires'],
                "build-system.requires"
            )
            dependencies.extend(build_system_dependencies)

    return json.dumps({"result": dependencies})


def parse_requirements(directory):
    # Parse the requirements.txt
    requirement_packages = []
    requirement_files = glob.glob(os.path.join(directory, '*.txt')) \
        + glob.glob(os.path.join(directory, '**', '*.txt'))

    pip_compile_files = glob.glob(os.path.join(directory, '*.in')) \
        + glob.glob(os.path.join(directory, '**', '*.in'))

    def version_from_install_req(install_req):
        if install_req.is_pinned:
            return next(iter(install_req.specifier)).version

    for reqs_file in requirement_files + pip_compile_files:
        try:
            requirements = pip._internal.req.req_file.parse_requirements(
                reqs_file,
                session=PipSession()
            )
            for parsed_req in requirements:
                install_req = install_req_from_parsed_requirement(parsed_req)
                if install_req.req is None:
                    continue

                # Ignore file: requirements
                if install_req.link is not None and install_req.link.is_file:
                    continue

                pattern = r"-[cr] (.*) \(line \d+\)"
                abs_path = re.search(pattern, install_req.comes_from).group(1)

                # Ignore dependencies from remote constraint files
                if not os.path.isfile(abs_path):
                    continue

                rel_path = os.path.relpath(abs_path, directory)

                requirement_packages.append({
                    "name": install_req.req.name,
                    "version": version_from_install_req(install_req),
                    "markers": str(install_req.markers) or None,
                    "file": rel_path,
                    "requirement": str(install_req.specifier) or None,
                    "extras": sorted(list(install_req.extras))
                })
        except Exception as e:
            print(json.dumps({"error": repr(e)}))
            exit(1)

    return json.dumps({"result": requirement_packages})


def parse_setup(directory):
    def version_from_install_req(install_req):
        if install_req.is_pinned:
            return next(iter(install_req.specifier)).version

    def parse_requirement(req, req_type, filename):
        install_req = install_req_from_line(req)
        if install_req.original_link:
            return

        setup_packages.append(
            {
                "name": install_req.req.name,
                "version": version_from_install_req(install_req),
                "markers": str(install_req.markers) or None,
                "file": filename,
                "requirement": str(install_req.specifier) or None,
                "requirement_type": req_type,
                "extras": sorted(list(install_req.extras)),
            }
        )

    def parse_requirements(requires, req_type, filename):
        for req in requires:
            req = COMMENT_RE.sub('', req)
            req = req.strip()
            parse_requirement(req, req_type, filename)

    # Parse the setup.py and setup.cfg
    setup_py = "setup.py"
    setup_py_path = os.path.join(directory, setup_py)
    setup_cfg = "setup.cfg"
    setup_cfg_path = os.path.join(directory, setup_cfg)
    setup_packages = []

    if os.path.isfile(setup_py_path):

        def setup(*args, **kwargs):
            for arg in ["setup_requires", "install_requires", "tests_require"]:
                requires = kwargs.get(arg, [])
                parse_requirements(requires, arg, setup_py)
            extras_require_dict = kwargs.get("extras_require", {})
            for key, value in extras_require_dict.items():
                parse_requirements(
                    value, "extras_require:{}".format(key), setup_py
                )

        setuptools.setup = setup

        def noop(*args, **kwargs):
            pass

        def fake_parse(*args, **kwargs):
            return []

        global fake_open

        def fake_open(*args, **kwargs):
            content = (
                "VERSION = ('0', '0', '1+dependabot')\n"
                "__version__ = '0.0.1+dependabot'\n"
                "__author__ = 'someone'\n"
                "__title__ = 'something'\n"
                "__description__ = 'something'\n"
                "__author_email__ = 'something'\n"
                "__license__ = 'something'\n"
                "__url__ = 'something'\n"
            )
            return io.StringIO(content)

        content = open(setup_py_path, "r").read()

        # Remove `print`, `open`, `log` and import statements
        content = re.sub(r"print\s*\(", "noop(", content)
        content = re.sub(r"log\s*(\.\w+)*\(", "noop(", content)
        content = re.sub(r"\b(\w+\.)*(open|file)\s*\(", "fake_open(", content)
        content = content.replace("parse_requirements(", "fake_parse(")
        version_re = re.compile(r"^.*import.*__version__.*$", re.MULTILINE)
        content = re.sub(version_re, "", content)

        # Set variables likely to be imported
        __version__ = "0.0.1+dependabot"
        __author__ = "someone"
        __title__ = "something"
        __description__ = "something"
        __author_email__ = "something"
        __license__ = "something"
        __url__ = "something"

        # Run as main (since setup.py is a script)
        __name__ = "__main__"

        # Exec the setup.py
        exec(content) in globals(), locals()

    if os.path.isfile(setup_cfg_path):
        try:
            config = configparser.ConfigParser()
            config.read(setup_cfg_path)

            for req_type in [
                "setup_requires",
                "install_requires",
                "tests_require",
            ]:
                requires = config.get(
                    'options',
                    req_type, fallback='').splitlines()
                requires = [req for req in requires if req.strip()]
                parse_requirements(requires, req_type, setup_cfg)

            if config.has_section('options.extras_require'):
                extras_require = config._sections['options.extras_require']
                for key, value in extras_require.items():
                    requires = value.splitlines()
                    requires = [req for req in requires if req.strip()]
                    parse_requirements(
                        requires,
                        f"extras_require:{key}",
                        setup_cfg
                    )

        except Exception as e:
            print(json.dumps({"error": repr(e)}))
            exit(1)

    return json.dumps({"result": setup_packages})