Initialize the project

.bumpversion.cfg

@@ -0,0 +1,16 @@
+[bumpversion]
+current_version = 0.1.0
+commit = False
+tag = False
+
+[bumpversion:file:pyproject.toml]
+search = version = "{current_version}"
+replace = version = "{new_version}"
+
+[bumpversion:file:src/deep_package_detection/__init__.py]
+search = __version__ = "{current_version}"
+replace = __version__ = "{new_version}"
+
+[bumpversion:file:tests/test_deep_package_detection.py]
+search = __version__ == "{current_version}"
+replace = __version__ == "{new_version}"

.gitignore

@@ -0,0 +1,24 @@
+# Pycharm files
+.idea/
+
+# poetry
+poetry.lock
+
+# Unit test files
+.coverage
+.coverage.*
+.cache
+.pytest_cache/
+
+# mypy files
+.mypy_cache/
+
+# Probable environments files
+.env
+.venv
+env/
+venv/
+
+
+
+

.pre-commit-config.yaml

@@ -0,0 +1,74 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+default_language_version:
+    python: python3
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0  # the release, git tag, or commit you want to use
+    hooks:
+      - id: check-toml
+      - id: check-yaml
+      - id: check-json
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: no-commit-to-branch
+      - id: check-executables-have-shebangs
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: pretty-format-json
+        args:
+        - --autofix
+      - id: check-symlinks
+      - id: check-ast
+      - id: detect-private-key
+  - repo: https://github.com/psf/black
+    rev: 24.1.1
+    hooks:
+      - id: black
+        language: python
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.8.0
+    hooks:
+    - id: mypy
+      language: system
+      args: [--strict, --ignore-missing-imports]
+  - repo: https://github.com/pycqa/pylint
+    rev: v3.0.3
+    hooks:
+    - id: pylint
+      language: system
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: v1.5.4
+    hooks:
+    - id: forbid-crlf
+    - id: remove-crlf
+    - id: forbid-tabs
+    - id: remove-tabs
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.7
+    hooks:
+    - id: bandit
+      args: ["--skip=B101"]
+  - repo: https://github.com/Lucas-C/pre-commit-hooks-markup
+    rev: v1.0.1
+    hooks:
+    - id: rst-linter
+  - repo: https://github.com/Yelp/detect-secrets
+    rev: v1.4.0
+    hooks:
+    - id: detect-secrets
+      language: python
+      exclude: "poetry.lock"
+      # args: ['--baseline', '.secrets.baseline']
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.9.0.6
+    hooks:
+    - id: shellcheck
+      args: ["--external-sources"]
+  - repo: https://github.com/python-poetry/poetry
+    rev: '1.7.1'
+    hooks:
+    - id: poetry-check
+    - id: poetry-lock
+      args: ["--no-update"]

Dockerfile

@@ -0,0 +1,143 @@
+## syntax=docker/dockerfile:1.1.7-experimental
+
+################
+# Base builder #
+################
+FROM python:3.10-bookworm as base_build
+
+ENV \
+  # locale environment variables
+  LC_ALL=C.UTF-8 \
+  # python environemnt variables
+  PYTHONFAULTHANDLER=1 \
+  PYTHONUNBUFFERED=1 \
+  PYTHONHASHSEED=random \
+  # pip environmental variables
+  PIP_NO_CACHE_DIR=off \
+  PIP_DISABLE_PIP_VERSION_CHECK=on \
+  PIP_DEFAULT_TIMEOUT=100 \
+  # poetry version
+  POETRY_VERSION=1.5.0
+
+# Install requirements
+RUN apt-get update && apt-get install -y \
+        curl \
+        git \
+        bash \
+        build-essential \
+        libffi-dev \
+        libssl-dev \
+        tini \
+        openssh-client \
+        cargo \
+        musl-dev \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/* \
+    # github ssh key setting
+    && mkdir -p -m 0700 ~/.ssh && ssh-keyscan github.com | sort > ~/.ssh/known_hosts \
+    # Installing poetry and set the PATH
+    && curl -sSL https://install.python-poetry.org | python3 - \
+    && echo 'export PATH="/root/.local/bin:$PATH"' >>/root/.profile \
+    && export PATH="/root/.local/bin:$PATH" \
+    && true
+SHELL ["/bin/bash", "-lc"]
+
+# Copy poetry lock and pyproject config files to the container
+WORKDIR /pysetup
+COPY ./poetry.lock ./pyproject.toml /pysetup/
+# Install pip/wheel/virtualenv and build the wheels based on the poetry lock
+RUN --mount=type=ssh pip3 install wheel virtualenv poetry-plugin-export \
+    && poetry export -f requirements.txt --without-hashes -o /tmp/requirements.txt \
+    && pip3 wheel --wheel-dir=/tmp/wheelhouse --trusted-host 172.17.0.1 --find-links=http://172.17.0.1:3141/debian/ -r /tmp/requirements.txt \
+    && virtualenv /.venv && source /.venv/bin/activate && echo 'source /.venv/bin/activate' >>/root/.profile \
+    && pip3 install --no-deps --trusted-host 172.17.0.1 --find-links=http://172.17.0.1:3141/debian/ --find-links=/tmp/wheelhouse/ /tmp/wheelhouse/*.whl \
+    && true
+
+
+###########################
+# Production base builder #
+###########################
+FROM base_build as production_build
+# Copy entrypoint script to the container and src files to the app directory
+COPY ./docker/entrypoint.sh /docker-entrypoint.sh
+COPY . /app/
+WORKDIR /app
+# Build the wheel packages with poetry and add them to the wheelhouse
+RUN --mount=type=ssh source /.venv/bin/activate \
+    && poetry build -f wheel --no-interaction --no-ansi \
+    && cp dist/*.whl /tmp/wheelhouse \
+    && chmod a+x /docker-entrypoint.sh \
+    && true
+
+
+
+########################
+# Production Container #
+########################
+FROM python:3.10-bookworm as production
+COPY --from=production_build /tmp/wheelhouse /tmp/wheelhouse
+COPY --from=production_build /docker-entrypoint.sh /docker-entrypoint.sh
+WORKDIR /app
+# Install system level deps for running the package and install the wheels we built in the previous step.
+RUN --mount=type=ssh apt-get update && apt-get install -y \
+        bash \
+        libffi8 \
+        libgl1 \
+        tini \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && chmod a+x /docker-entrypoint.sh \
+    && WHEELFILE=`echo /tmp/wheelhouse/deep_package_detection-*.whl` \
+    && pip3 install --trusted-host 172.17.0.1 --find-links=http://172.17.0.1:3141/debian/ --find-links=/tmp/wheelhouse/ "$WHEELFILE"[all] \
+    && rm -rf /tmp/wheelhouse/ \
+    && true
+ENTRYPOINT ["/usr/bin/tini", "--", "/docker-entrypoint.sh"]
+
+
+
+############################
+# Development base builder #
+############################
+FROM base_build as development_build
+# Copy src to app directory
+COPY . /app
+WORKDIR /app
+# Install dependencies from poetry lock
+RUN --mount=type=ssh source /.venv/bin/activate \
+    && apt-get update && apt-get install -y libgl1 \
+    && export PIP_FIND_LINKS=http://172.17.0.1:3141/debian/ \
+    && export PIP_TRUSTED_HOST=172.17.0.1 \
+    && pip3 install nvidia-cublas-cu12 nvidia-cusparse-cu12 triton nvidia-nccl-cu12 nvidia-cudnn-cu12 nvidia-cufft-cu12 nvidia-cusolver-cu12 \
+    && poetry install --no-interaction --no-ansi \
+    && true
+
+
+
+###################
+# Tests Container #
+###################
+FROM development_build as test
+RUN --mount=type=ssh source /.venv/bin/activate \
+    && chmod a+x docker/*.sh \
+    && docker/pre_commit_init.sh \
+    && true
+ENTRYPOINT ["/usr/bin/tini", "--", "docker/entrypoint-test.sh"]
+
+
+#########################
+# Development Container #
+#########################
+FROM development_build as development
+RUN apt-get update && apt-get install -y zsh \
+    && sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" \
+    && echo "if [ \"\$NO_WHEELHOUSE\" = \"1\" ]" >>/root/.profile \
+    && echo "then" >>/root/.profile \
+    && echo "  echo \"Wheelhouse disabled\"" >>/root/.profile \
+    && echo "else">>/root/.profile \
+    && echo "  export PIP_TRUSTED_HOST=172.17.0.1" >>/root/.profile \
+    && echo "  export PIP_FIND_LINKS=http://172.17.0.1:3141/debian/" >>/root/.profile \
+    && echo "fi" >>/root/.profile \
+    && echo "source /root/.profile" >>/root/.zshrc \
+    && pip3 install git-up \
+    && true
+ENTRYPOINT ["/bin/zsh", "-l"]

README.md

@@ -0,0 +1,20 @@
+# Deep-Package-Detection
+This is a deep model for detecting different types of packages.
+
+## How to Develop
+Do the following only once after creating your project:
+- Init the git repo with `git init`.
+- Add files with `git add .`.
+- Then `git commit -m 'initialize the project'`.
+- Add remote url with `git remote add origin REPO_URL`.
+- Then `git branch -M master`.
+- `git push origin main`.
+Then create a branch with `git checkout -b BRANCH_NAME` for further developments.
+- Install poetry if you do not have it in your system from [here](https://python-poetry.org/docs/#installing-with-pipx).
+- Create a virtual env preferably with virtualenv wrapper and `mkvirtualenv -p $(which python3.10) ENVNAME`.
+- Then `git add poetry.lock`.
+- Then `pre-commit install`.
+- For applying changes use `pre-commit run --all-files`.
+
+## Docker Container
+Under development.

docker/entrypoint-test.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -l
+set -e
+if [ "$#" -eq 0 ]; then
+  # Kill cache, pytest complains about it if running local and docker tests in mapped volume
+  find tests  -type d -name '__pycache__' -print0 | xargs -0 rm -rf {}
+  # Make sure the service itself is installed
+  poetry install
+  # Make sure pre-commit checks were not missed and run tests
+  git config --global --add safe.directory /app
+  poetry run pre-commit install
+  pre-commit run --all-files
+  pytest -v --junitxml=pytest.xml tests/
+else
+  exec "$@"
+fi

docker/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -l
+set -e
+if [ "$#" -eq 0 ]; then
+  exec deep_package_detection --help
+else
+  exec "$@"
+fi

docker/pre_commit_init.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -l
+if [ ! -d .git ]
+then
+  git init
+  git checkout -b precommit_init
+  git add .
+fi
+set -e
+poetry run pre-commit install
+SKIP="poetry-lock" poetry run pre-commit run --all-files

pyproject.toml

@@ -0,0 +1,57 @@
+[tool.poetry]
+name = "deep_package_detection"
+version = "0.1.0"
+description = "This is a deep model for detecting different types of packages."
+authors = ["Afshin Dini <Afshin Dini>"]
+readme = "README.md"
+packages = [{include = "deep_package_detection", from = "src"}]
+
+[tool.poetry.scripts]
+deep_package_detection = "deep_package_detection.main:deep_package_detection_cli"
+
+[tool.pylint.format]
+max-line-length=150     # This defines the maximum number of characters on a single line in pylint
+
+[tool.pylint.design]
+max-attributes=10
+max-positional-arguments=6
+max-args=6
+
+[tool.pylint.messages_control]
+disable=["fixme"]
+
+[tool.pylint.similarities]
+min-similarity-lines = 8    # Minimum lines number of a similarity.
+ignore-imports = true       # Ignore imports when computing similarities.
+
+[tool.pytest.ini_options]
+junit_family="xunit2"
+addopts="--cov=deep_package_detection --cov-fail-under=65 --cov-branch"
+asyncio_mode="strict"
+
+
+[tool.coverage.run]
+omit = ["tests/*"]
+branch = true
+
+
+[tool.poetry.dependencies]
+python = "^3.10"
+click = "^8.1.7"
+
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.3"
+coverage = "^7.6"
+pytest-cov = "^6.0"
+pylint = "^3.3.1"
+black = "^24.10.0"
+mypy = "^1.13.0"
+bump2version = "^1.0.1"
+bandit = "^1.7.10"
+pre-commit = "^4.0.1"
+detect-secrets = "^1.5"
+
+[build-system]
+requires = ["poetry-core>=1.5.0"]
+build-backend = "poetry.core.masonry.api"

src/deep_package_detection/__init__.py

@@ -0,0 +1,3 @@
+""" his is a deep model for detecting different types of packages. """
+
+__version__ = "0.1.0"

src/deep_package_detection/loggings.py

@@ -0,0 +1,38 @@
+"""Logger initialization"""
+
+import logging
+import logging.config
+from typing import Any, Optional
+
+
+def config_logger(loglevel: int, log_file: Optional[str] = None) -> Any:
+    """Initialize a custom logger"""
+    handlers = {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "standard",
+        },
+    }
+    if log_file:
+        handlers["file"] = {
+            "class": "logging.FileHandler",
+            "formatter": "standard",
+            "filename": log_file,
+            "mode": "a",
+        }
+    default_logging_config = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "standard": {
+                "format": "%(asctime)s - [%(levelname)s] [%(name)s.%(funcName)s:%(lineno)d (%(process)d)] | %(message)s",
+                "datefmt": "%Y-%m-%d %H:%M:%S",
+            },
+        },
+        "handlers": handlers,
+        "root": {
+            "handlers": list(handlers.keys()),
+            "level": loglevel,
+        },
+    }
+    logging.config.dictConfig(default_logging_config)

src/deep_package_detection/main.py

@@ -0,0 +1,30 @@
+"""Run the main code for Deep-Package-Detection"""
+
+import logging
+
+import click
+
+from deep_package_detection import __version__
+from deep_package_detection.loggings import config_logger
+
+
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.version_option(version=__version__)
+@click.option("-l", "--log_path", type=str, help="Path to save log file")
+@click.option("-v", "--verbose", count=True, help="Shorthand for info/debug/warning/error loglevel (-v/-vv/-vvv/-vvvv)")
+def deep_package_detection_cli(log_path: str, verbose: int) -> None:
+    """This is a deep model for detecting different types of packages. """
+    if verbose == 1:
+        log_level = 10
+    elif verbose == 2:
+        log_level = 20
+    elif verbose == 3:
+        log_level = 30
+    else:
+        log_level = 40
+    config_logger(log_level, log_path)
+
+    click.echo("Run the main code.")

tests/test_deep_package_detection.py

@@ -0,0 +1,8 @@
+"""Package level tests"""
+
+from deep_package_detection import __version__
+
+
+def test_version() -> None:
+    """Unit test for checking the version of the code"""
+    assert __version__ == "0.1.0"