Initial commit

.github/workflows/ci-cd.yml

@@ -0,0 +1 @@
+# add ci-cd steps here

.gitignore

@@ -0,0 +1,143 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# if using pycharm
+.idea
+
+# if using VScode
+.vscode
+
+# add secret keys or API keys here
+configs/secrets.yaml
+
+# add your env folder here if its there
+mlruns/
+
+

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 SUNNY BHAVEEN CHANDRA
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MLproject

@@ -0,0 +1,7 @@
+name: MLflow-app
+
+conda_env: conda.yaml
+
+entry_points:
+  main:
+    command: "python main.py"

README.md

@@ -0,0 +1,34 @@
+# MLflow-project-template
+MLflow project template
+
+## STEPS -
+
+### STEP 01- Create a repository by using template repository
+
+### STEP 02- Clone the new repository
+
+### STEP 03- Create a conda environment after opening the repository in VSCODE
+
+```bash
+conda create --prefix ./env python=3.7 -y
+```
+
+```bash
+conda activate ./env
+```
+OR
+```bash
+source activate ./env
+```
+
+### STEP 04- install the requirements
+```bash
+pip install -r requirements.txt
+```
+
+### STEP 05 - Create conda.yaml file -
+```bash
+conda env export > conda.yaml
+```
+
+### STEP 06- commit and push the changes to the remote repository

configs/config.yaml

@@ -0,0 +1,20 @@
+# It contains all the configs required in the project
+
+artifacts:
+  ARTIFACTS_DIR: artifacts
+  TRAINED_MODEL_DIR: model
+  BASE_MODEL_DIR: base_model
+  BASE_MODEL_NAME: base_model.h5
+  UPDATED_BASE_MODEL_NAME: updated_base_model.h5
+  CHECKPOINT_DIR: checkpoints
+  BASE_LOG_DIR: base_model_dir
+  TENSORBOARD_ROOT_LOG_DIR: tensorboard_log_dir
+  CALLBACKS_DIR: callbacks
+
+source_download_dirs:
+  - path_to_data_1
+  - path_to_data_1
+
+local_data_dirs:
+  - path_to_local_data_1
+  - path_to_local_data_2

init_setup.sh

@@ -0,0 +1,15 @@
+echo [$(date)]: "START"
+export _VERSION_=3.7
+echo [$(date)]: "creating environment with python ${_VERSION_}"
+conda create --prefix ./env python=${_VERSION_} -y
+echo [$(date)]: "activate environment"
+source activate ./env
+echo [$(date)]: "install requirements"
+pip install -r requirements.txt
+echo [$(date)]: "export conda environment"
+conda env export > conda.yaml
+echo [$(date)]: "END"
+
+# to remove everything -
+# rm -rf env/ .gitignore conda.yaml README.md .git/
+

params.yaml

@@ -0,0 +1 @@
+# This contains params to be used by the stages to train or predict

requirements.txt

@@ -0,0 +1,9 @@
+### dependency
+tqdm
+mlflow
+pandas
+numpy
+PyYAML
+
+### local packages -
+# -e . # << uncomment this to install from setup.py

setup.py

@@ -0,0 +1,26 @@
+from setuptools import setup
+
+with open("README.md", "r", encoding="utf-8") as f:
+    long_description = f.read()
+
+## edit below variables as per your requirements -
+REPO_NAME = "MLflow-project-template"
+AUTHOR_USER_NAME = "c17hawke"
+SRC_REPO = "src"
+LIST_OF_REQUIREMENTS = []
+
+
+setup(
+    name=SRC_REPO,
+    version="0.0.1",
+    author=AUTHOR_USER_NAME,
+    description="A small package for MLflow app",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url=f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}",
+    author_email="sunny.c17hawke@gmail.com",
+    packages=[SRC_REPO],
+    license="MIT",
+    python_requires=">=3.6",
+    install_requires=LIST_OF_REQUIREMENTS
+)

src/__init__.py

@@ -0,0 +1 @@
+

src/stage_00_template.py

@@ -0,0 +1,40 @@
+import argparse
+import os
+import shutil
+from tqdm import tqdm
+import logging
+from src.utils.common import read_yaml, create_directories
+import random
+
+
+STAGE = "STAGE_NAME" ## <<< change stage name 
+
+logging.basicConfig(
+    filename=os.path.join("logs", 'running_logs.log'), 
+    level=logging.INFO, 
+    format="[%(asctime)s: %(levelname)s: %(module)s]: %(message)s",
+    filemode="a"
+    )
+
+
+def main(config_path, params_path):
+    ## read config files
+    config = read_yaml(config_path)
+    params = read_yaml(params_path)
+    pass
+
+
+if __name__ == '__main__':
+    args = argparse.ArgumentParser()
+    args.add_argument("--config", "-c", default="configs/config.yaml")
+    args.add_argument("--params", "-p", default="params.yaml")
+    parsed_args = args.parse_args()
+
+    try:
+        logging.info("\n********************")
+        logging.info(f">>>>> stage {STAGE} started <<<<<")
+        main(config_path=parsed_args.config, params_path=parsed_args.params)
+        logging.info(f">>>>> stage {STAGE} completed!<<<<<\n")
+    except Exception as e:
+        logging.exception(e)
+        raise e

src/utils/__init__.py

@@ -0,0 +1 @@
+

src/utils/common.py

@@ -0,0 +1,24 @@
+import os
+import yaml
+import logging
+import time
+import pandas as pd
+import json
+
+def read_yaml(path_to_yaml: str) -> dict:
+    with open(path_to_yaml) as yaml_file:
+        content = yaml.safe_load(yaml_file)
+    logging.info(f"yaml file: {path_to_yaml} loaded successfully")
+    return content
+
+def create_directories(path_to_directories: list) -> None:
+    for path in path_to_directories:
+        os.makedirs(path, exist_ok=True)
+        logging.info(f"created directory at: {path}")
+
+
+def save_json(path: str, data: dict) -> None:
+    with open(path, "w") as f:
+        json.dump(data, f, indent=4)
+
+    logging.info(f"json file saved at: {path}")