Delete AI

AI/.coveragerc

@@ -1,12 +0,0 @@
-[report]
-exclude_lines =
-    pragma: no cover
-    def __repr__
-    raise NotImplementedError
-    if __name__ == .__main__.:
-    def create_argument_parser
-    if typing.TYPE_CHECKING
-    # Ellipsis are used as placeholders in python 3 that will be overridden
-    \.\.\.
-[run]
-source = rasa

AI/.deepsource.toml

@@ -1,12 +0,0 @@
-version = 1
-
-test_patterns = ["tests/**"]
-
-exclude_patterns = ["docs/**"]
-
-[[analyzers]]
-name = "python"
-enabled = true
-
-  [analyzers.meta]
-  runtime_version = "3.x.x"

AI/.devcontainer/Dockerfile

@@ -1,27 +0,0 @@
-# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
-ARG VARIANT=3-bullseye
-FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
-
-ENV PYTHONFAULTHANDLER=1 \
-  PYTHONUNBUFFERED=1 \
-  PYTHONHASHSEED=random \
-  PIP_NO_CACHE_DIR=off \
-  PIP_DISABLE_PIP_VERSION_CHECK=on \
-  PIP_DEFAULT_TIMEOUT=100
-
-# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
-ARG NODE_VERSION="none"
-RUN if [ "${NODE_VERSION}" != "nne" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
-
-# [Optional] If your requirements rarely change, uncomment this section to add them to the image.
-# COPY requirements.txt /tmp/pip-tmp/
-# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
-#    && rm -rf /tmp/pip-tmp
-RUN pip install poetry==1.1.10 pre-commit
-COPY ../poetry.lock ../pyproject.toml /tmp/pip-tmp/rasa/
-RUN cd /tmp/pip-tmp/rasa && poetry config virtualenvs.create false \
-  && poetry install --no-interaction --no-ansi --no-root
-
-# [Optional] Uncomment this section to install additional OS packages.
-# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-#     && apt-get -y install --no-install-recommends <your-package-list-here>

AI/.devcontainer/devcontacener.json

@@ -1,50 +0,0 @@
-// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
-// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3-postgres
-// Update the VARIANT arg in docker-compose.yml to pick a Python version
-{
-    "name": "Rasa Open Source",
-    "dockerComposeFile": "docker-compose.yml",
-    "service": "app",
-    "workspaceFolder": "/workspaces/rasa",
-    // Set *default* container specific settings.json values on container create.
-    "settings": {
-        "python.defaultInterpreterPath": "/usr/local/bin/python",
-        "python.linting.enabled": true,
-        "python.linting.pylintEnabled": true,
-        "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
-        "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
-        "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
-        "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-        "python.linting.ruffPath": "/usr/local/py-utils/bin/ruff",
-        "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
-        "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
-        "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
-        "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
-        "python.testing.pytestPath": "/usr/local/py-utils/bin/pytest"
-    },
-    // Add the IDs of extensions you want installed when the container is created.
-    "extensions": [
-        "ms-python.python",
-        "ms-python.vscode-pylance"
-    ],
-    // memory is required for frontend build...fails for machines with less than 10g
-    // "hostRequirements": {
-    //    "memory": "12gb"
-    // },
-    // Use 'forwardPorts' to make a list of ports inside the container available locally.
-    // This can be used to network with other containers or the host.
-    "forwardPorts": [
-        5005
-    ],
-    // Use 'postCreateCommand' to run commands after the container is created.
-    "updateContentCommand": "make install && make install-docs && cd / && mkdir example && rasa init --no-prompt --init-dir example",
-    // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
-    //"remoteUser": "vscode",
-    "features": {
-        "docker-in-docker": "20.10",
-        "docker-from-docker": "20.10",
-        "git": "os-provided",
-        "github-cli": "latest",
-        "sshd": "latest"
-    }
-}

AI/.devcontainer/docker-compose.yml

@@ -1,57 +0,0 @@
-version: '3.8'
-
-services:
-  app:
-    build:
-      context: ..
-      dockerfile: .devcontainer/Dockerfile
-      args:
-        # Update 'VARIANT' to pick a version of Python: 3, 3.10, 3.9, 3.8, 3.7, 3.6
-        # Append -bullseye or -buster to pin to an OS version.
-        # Use -bullseye variants on local arm64/Apple Silicon.
-        VARIANT: "3.8"
-        # Optional Node.js version to install
-        NODE_VERSION: "16"
-    environment:
-      DB_DRIVER: "postgresql"
-      DB_USER: "admin"
-      DB_PASSWORD: "postgres"
-
-    volumes:
-      - ..:/workspaces/rasa:cached
-
-    # Overrides default command so things don't shut down after the process ends.
-    command: sleep infinity
-
-    # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
-    network_mode: service:db
-    # Uncomment the next line to use a non-root user for all processes.
-    # user: vscode
-
-  db:
-    image: "bitnami/postgresql:11.15.0"
-    restart: unless-stopped
-    volumes:
-      - postgres-data:/bitnami/postgresql
-    environment:
-      POSTGRESQL_USERNAME: admin
-      POSTGRESQL_DATABASE: rasa
-      POSTGRESQL_PASSWORD: postgres
-
-  duckling:
-    restart: unless-stopped
-    image: "rasa/duckling:0.2.0.2"
-    expose:
-      - "8000"
-    command: ["duckling-example-exe", "--no-access-log", "--no-error-log"]
-
-  redis:
-    restart: unless-stopped
-    image: "bitnami/redis:6.2.7"
-    environment:
-      REDIS_PASSWORD: "redis"
-    expose:
-      - "6379"
-
-volumes:
-  postgres-data: null

AI/.dockerignore

@@ -1,11 +0,0 @@
-docker*
-docs
-.git*
-**/*.pyc
-**/__pycache__
-!docker/configs
-rasa/tests
-rasa/scripts
-data/
-examples/
-docker-data/*

AI/.env

@@ -1 +0,0 @@
-TIMES=2

AI/.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

AI/.github/ISSUE_TEMPLATE.md

@@ -1,2 +0,0 @@
-<!-- IF YOU ARE ASKING A USAGE QUESTION (E.G. "HOW DO I DO XYZ") PLEASE POST
-YOUR QUESTION ON https://forum.rasa.com INSTEAD -->

AI/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,34 +0,0 @@
----
-name: Bug report
-about: Create a report to help us reproduce and fix the issue
----
-
-**Before submitting a bug, please make sure the issue hasn't been already addressed by searching through the [FAQs](https://ai.meta.com/llama/faq/) and [existing/past issues](https://github.com/facebookresearch/llama/issues)**
-
-## Describe the bug
-<Please provide a clear and concise description of what the bug is. If relevant, please include a _minimal_ (least lines of code necessary) _reproducible_ (running this will give us the same result as you get) code snippet. Make sure to include the relevant imports.>
-
-### Minimal reproducible example
-<Remember to wrap the code in ```` ```triple-quotes blocks``` ````>
-
-```python
-# sample code to repro the bug
-```
-
-### Output
-<Remember to wrap the output in ```` ```triple-quotes blocks``` ````>
-
-```
-<paste stacktrace and other outputs here>
-```
-
-## Runtime Environment
-- Model: [eg: `meta-llama-3-8b-instruct`]
-- Using via huggingface?: [yes/no]
-- OS: [eg. Linux/Ubuntu, Windows]
-- GPU VRAM: 
-- Number of GPUs:
-- GPU Make: [eg: Nvidia, AMD, Intel]
-
-**Additional context**
-Add any other context about the problem or environment here.

AI/.github/ISSUE_TEMPLATE/config.yml

@@ -1,11 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Bug Report
-    url: https://rasa-open-source.atlassian.net/browse/OSS
-    about: Create a report to help us improve https://rasa-open-source.atlassian.net/browse/OSS
-  - name: Feature request
-    url: https://rasa-open-source.atlassian.net/browse/OSS
-    about: Suggest an idea on how to improve Rasa https://rasa-open-source.atlassian.net/browse/OSS
-  - name: Ask a question
-    url: https://forum.rasa.com/
-    about: If you have a "How do I?" question please ask in the forum https://forum.rasa.com

AI/.github/PULL_REQUEST_AUTOMATIC_TEMPLATE.md

@@ -1,10 +0,0 @@
-:bulb: This pull request was created automatically to merge a release branch back into the `main` branch.
-
-The changes you see here should have already been reviewed by someone, and shouldn't need an extra
-review. Nonetheless, if you notice something that needs to be addressed, please reach out to the person
-responsible for the original changes. In case additional changes need to be made, they need to target the release branch
-(not this pull request nor `main`).
-
-:auto_rickshaw: This PR should be merged automatically once it has been approved. If it doesn't happen:
-- [ ] Handle merge conflicts
-- [ ] Fix build errors

AI/.github/PULL_REQUEST_TEMPLATE.md

@@ -1,8 +0,0 @@
-**Proposed changes**:
-- ...
-
-**Status (please check what you already did)**:
-- [ ] added some tests for the functionality
-- [ ] updated the documentation
-- [ ] updated the changelog (please check [changelog](https://github.com/RasaHQ/rasa/tree/main/changelog) for instructions)
-- [ ] reformat files using `black` (please check [Readme](https://github.com/RasaHQ/rasa#code-style) for instructions)

AI/.github/change_filters.yml

@@ -1,28 +0,0 @@
-backend:
-  - 'pyproject.toml'
-  - 'poetry.lock'
-  - 'rasa/**/*'
-  - 'tests/**/*'
-  - 'data/**/*'
-  - 'examples/**/*'
-  - 'Makefile'
-  - '.github/workflows/continous-integration.yml'
-  - '.github/workflows/security-scans.yml'
-
-docker:
-  - 'pyproject.toml'
-  - 'poetry.lock'
-  - 'rasa/**/*'
-  - 'docker/**/*'
-  - 'Makefile'
-
-docs:
-  - 'docs/**/*'
-  - 'changelog/*'
-  - 'CHANGELOG.mdx'
-  - 'tests/docs/*'
-  - 'data/**/*'
-  - 'examples/**/*'
-  - 'Makefile'
-  - '.github/workflows/documentation.yml'
-  - '.github/workflows/ci-docs-tests.yml'

AI/.github/configs/mr-test-example.yaml

@@ -1,49 +0,0 @@
-## Example configuration
-#################### syntax #################
-## include:
-##   - dataset: ["<dataset_name>"]
-##     config: ["<configuration_name>"]
-#
-## Example:
-## include:
-##  - dataset: ["Carbon Bot"]
-##    config: ["Sparse + DIET(bow) + ResponseSelector(bow)"]
-#
-## Shortcut:
-## You can use the "all" shortcut to include all available configurations or datasets
-#
-## Example: Use the "Sparse + EmbeddingIntent + ResponseSelector(bow)" configuration
-## for all available datasets
-## include:
-##  - dataset: ["all"]
-##    config: ["Sparse + DIET(bow) + ResponseSelector(bow)"]
-#
-## Example: Use all available configurations for the "Carbon Bot" and "Sara" datasets
-## and for the "Hermit" dataset use the "Sparse + DIET + ResponseSelector(T2T)" and
-## "BERT + DIET + ResponseSelector(T2T)" configurations:
-## include:
-##  - dataset: ["Carbon Bot", "Sara"]
-##    config: ["all"]
-##  - dataset: ["Hermit"]
-##    config: ["Sparse + DIET(seq) + ResponseSelector(t2t)", "BERT + DIET(seq) + ResponseSelector(t2t)"]
-#
-## Example: Define a branch name to check-out for a dataset repository. Default branch is 'main'
-## dataset_branch: "test-branch"
-## include:
-##  - dataset: ["Carbon Bot", "Sara"]
-##    config: ["all"]
-#
-## Example: Define number of repetitions. This will inform how often to repeat all runs defined in the include section. Default is 1
-## num_repetitions: 2
-## include:
-##  - dataset: ["Carbon Bot", "Sara"]
-##    config: ["Sparse + DIET(seq) + ResponseSelector(t2t)"]
-##
-## Shortcuts:
-## You can use the "all" shortcut to include all available configurations or datasets.
-## You can use the "all-nlu" shortcut to include all available NLU configurations or datasets.
-## You can use the "all-core" shortcut to include all available core configurations or datasets.
-
-include:
-  - dataset: ["Carbon Bot"]
-    config: ["Sparse + DIET(bow) + ResponseSelector(bow)"]

AI/.github/configs/mr-test-schedule.json

@@ -1,3 +0,0 @@
-{
-  "body": "```yml\r\ninclude:\r\n  - dataset: [\"all\"]\r\n    config: [\"all\"]\r\n```"
-}

AI/.github/configs/tf-cuda.json

@@ -1,25 +0,0 @@
-{
-  "default_image_tag": "latest",
-  "config": [
-    {
-      "TF": "2.3",
-      "IMAGE_TAG": "cuda-10.1-cudnn7"
-    },
-    {
-      "TF": "2.5",
-      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
-    },
-    {
-      "TF": "2.6",
-      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
-    },
-    {
-      "TF": "2.7",
-      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
-    },
-    {
-      "TF": "2.11",
-      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
-    }
-  ]
-}

AI/.github/dependabot.yml

@@ -1,34 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: pip
-    directory: "/"
-    schedule:
-      interval: weekly
-      time: "13:00"
-    pull-request-branch-name:
-      separator: "-"
-    open-pull-requests-limit: 10
-    labels:
-      - type:dependencies
-      - release:main
-    ignore:
-      - dependency-name: prompt-toolkit
-        versions:
-          - "> 2.0.10"
-      - dependency-name: pytest-asyncio
-        versions:
-          - "> 0.10.0"
-
-  - package-ecosystem: github-actions
-    directory: "/"
-    schedule:
-      interval: weekly
-      day: monday
-      time: "12:00"
-    pull-request-branch-name:
-      separator: "-"
-    open-pull-requests-limit: 10
-    reviewers:
-      - RasaHQ/infrastructure-squad
-    labels:
-      - type:dependencies

AI/.github/matchers/flake8-error-matcher.json

@@ -1,17 +0,0 @@
-{
-  "problemMatcher": [
-    {
-      "owner": "flake8-error",
-      "severity": "error",
-      "pattern": [
-        {
-          "regexp": "^([^:]+):(\\d+):(\\d+):\\s+([DCFNWE]\\d+\\s+.+)$",
-          "file": 1,
-          "line": 2,
-          "column": 3,
-          "message": 4
-        }
-      ]
-    }
-  ]
-}

AI/.github/no-response.yml

@@ -1,12 +0,0 @@
-# Configuration for probot-no-response - https://github.com/probot/no-response
-
-# Number of days of inactivity before an Issue is closed for lack of response
-daysUntilClose: 14
-# Label requiring a response
-responseRequiredLabel: status:more-details-needed
-# Comment to post when closing an Issue for lack of response. Set to `false` to disable
-closeComment: >
-  This issue has been automatically closed because there has been no response
-  to our request for more information from the original author. Without this,
-  we don't have enough information to help you. Please comment below with the
-  requested information if you still need help.

AI/.github/poetry_version.txt

@@ -1,2 +0,0 @@
-# The poetry version is stored in a separate file due to the https://github.com/python-poetry/poetry/issues/3316
-poetry-version=1.4.2

AI/.github/runner/github-runner-deployment.yaml.tmpl

@@ -1,74 +0,0 @@
-# GitHub Runner deployment - uses to deploy a github runner
-# which is used by the CI for model regression tests
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: github-runner-{{getenv "GITHUB_RUN_ID"}}
-  namespace: github-runner
-  labels:
-    app: github-runner
-    pod: github-runner-{{getenv "GITHUB_RUN_ID"}}
-spec:
-  replicas: {{getenv "NUM_REPLICAS" "1"}}
-  selector:
-    matchLabels:
-      app: github-runner
-      pod: github-runner-{{getenv "GITHUB_RUN_ID"}}
-  template:
-    metadata:
-      labels:
-        app: github-runner
-        pod: github-runner-{{getenv "GITHUB_RUN_ID"}}
-    spec:
-      priorityClassName: high-priority
-      automountServiceAccountToken: false
-      terminationGracePeriodSeconds: 720
-      containers:
-      - name: github-runner
-        image: {{getenv "GH_RUNNER_IMAGE"}}:{{getenv "GH_RUNNER_IMAGE_TAG" "latest"}}
-        imagePullPolicy: Always
-        livenessProbe:
-          initialDelaySeconds: 30
-          periodSeconds: 15
-          failureThreshold: 3
-          exec:
-            command:
-            - /bin/bash
-            - -c
-            - "if [[ `curl -sX GET -H \"Authorization: token ${GITHUB_PAT}\" \
-              https://api.github.com/repos/${GITHUB_OWNER}/${GITHUB_REPOSITORY}/actions/runners | \
-              jq -r '.runners[] | select(.name == \"'${POD_NAME}'\") | .status'` == \"offline\" ]]; then \
-              echo \"The GitHub API returns offline status for the ${POD_NAME} runner\" && exit 1; fi"
-        resources:
-          limits:
-            nvidia.com/gpu: 1
-          requests:
-            nvidia.com/gpu: 1
-            memory: 10G
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        # RUNNER_LABELS - defines labels
-        # with which a github-runner will be registered
-        - name: RUNNER_LABELS
-          value: "self-hosted,gpu,kubernetes,{{getenv "GITHUB_RUN_ID"}}"
-        # GITHUB_OWNER - a name of the repository owner
-        - name: GITHUB_OWNER
-          valueFrom:
-            secretKeyRef:
-              name: github-rasa
-              key: owner
-        # GITHUB_REPOSITORY - a name of the repository
-        - name: GITHUB_REPOSITORY
-          valueFrom:
-            secretKeyRef:
-              name: github-rasa
-              key: repository
-        # GITHUB_PAT - Personal Access Token
-        - name: GITHUB_PAT
-          valueFrom:
-            secretKeyRef:
-              name: github-rasa
-              key: pat

AI/.github/scripts/download_pretrained.py

@@ -1,120 +0,0 @@
-import argparse
-import logging
-import time
-from typing import List, NamedTuple, Optional, Text
-
-from transformers import AutoTokenizer, TFAutoModel
-
-import rasa.shared.utils.io
-from rasa.nlu.utils.hugging_face.registry import (
-    model_weights_defaults,
-    model_class_dict,
-)
-
-logger = logging.getLogger(__name__)
-
-COMP_NAME = "LanguageModelFeaturizer"
-DEFAULT_MODEL_NAME = "bert"
-
-
-class LmfSpec(NamedTuple):
-    """Holds information about the LanguageModelFeaturizer."""
-
-    model_name: Text
-    model_weights: Text
-    cache_dir: Optional[Text] = None
-
-
-def get_model_name_and_weights_from_config(
-    config_path: str,
-) -> List[LmfSpec]:
-    config = rasa.shared.utils.io.read_config_file(config_path)
-    logger.info(config)
-    steps = config.get("pipeline", [])
-
-    # Look for LanguageModelFeaturizer steps
-    steps = list(filter(lambda x: x["name"] == COMP_NAME, steps))
-
-    lmf_specs = []
-    for lmfeat_step in steps:
-        if "model_name" not in lmfeat_step:
-            if "model_weights" in lmfeat_step:
-                model_weights = lmfeat_step["model_weights"]
-                raise KeyError(
-                    "When model_name is not given, then model_weights cannot be set. "
-                    f"Here, model_weigths is set to {model_weights}"
-                )
-            model_name = DEFAULT_MODEL_NAME
-            model_weights = model_weights_defaults[DEFAULT_MODEL_NAME]
-        else:
-            model_name = lmfeat_step["model_name"]
-
-            if model_name not in model_class_dict:
-                raise KeyError(
-                    f"'{model_name}' not a valid model name. Choose from "
-                    f"{list(model_class_dict.keys())!s} or create"
-                    f"a new class inheriting from this class to support your model."
-                )
-
-            model_weights = lmfeat_step.get("model_weights")
-            if not model_weights:
-                logger.info(
-                    f"Model weights not specified. Will choose default model "
-                    f"weights: {model_weights_defaults[model_name]}"
-                )
-                model_weights = model_weights_defaults[model_name]
-        cache_dir = lmfeat_step.get("cache_dir", None)
-        lmf_specs.append(LmfSpec(model_name, model_weights, cache_dir))
-
-    return lmf_specs
-
-
-def instantiate_to_download(comp: LmfSpec) -> None:
-    """Instantiates Auto class instances, but only to download."""
-
-    _ = AutoTokenizer.from_pretrained(comp.model_weights, cache_dir=comp.cache_dir)
-    logger.info("Done with AutoTokenizer, now doing TFAutoModel")
-    _ = TFAutoModel.from_pretrained(comp.model_weights, cache_dir=comp.cache_dir)
-
-
-def download(config_path: str):
-    lmf_specs = get_model_name_and_weights_from_config(config_path)
-
-    if not lmf_specs:
-        logger.info(f"No {COMP_NAME} found, therefore, skipping download")
-        return
-
-    for lmf_spec in lmf_specs:
-        logger.info(
-            f"model_name: {lmf_spec.model_name}, "
-            f"model_weights: {lmf_spec.model_weights}, "
-            f"cache_dir: {lmf_spec.cache_dir}"
-        )
-        start = time.time()
-
-        instantiate_to_download(lmf_spec)
-
-        duration_in_sec = time.time() - start
-        logger.info(f"Instantiating Auto classes takes {duration_in_sec:.2f}seconds")
-
-
-def create_argument_parser() -> argparse.ArgumentParser:
-    """Downloads pretrained models, i.e., Huggingface weights."""
-    parser = argparse.ArgumentParser(
-        description="Downloads pretrained models, i.e., Huggingface weights, "
-        "e.g. path to bert_diet_responset2t.yml"
-    )
-    parser.add_argument(
-        "-c",
-        "--config",
-        type=str,
-        required=True,
-        help="The path to the config yaml file.",
-    )
-    return parser
-
-
-if __name__ == "__main__":
-    arg_parser = create_argument_parser()
-    cmdline_args = arg_parser.parse_args()
-    download(cmdline_args.config)

AI/.github/scripts/mr_generate_summary.py

@@ -1,55 +0,0 @@
-# Collect the results of the various model test runs which are done as part of
-# the model regression CI pipeline and dump them as a single file artifact.
-# This artifact will the then be published at the end of the tests.
-from collections import defaultdict
-import json
-import os
-from pathlib import Path
-from typing import Dict, List
-
-
-def combine_result(
-    result1: Dict[str, dict], result2: Dict[str, Dict[str, Dict]]
-) -> Dict[str, Dict[str, List]]:
-    """Combines 2 result dicts to accumulated dict of the same format.
-
-    Args:
-        result1: dict of key: dataset, value: (dict of key: config, value: list of res)
-                 Example: {
-                              "Carbon Bot": {
-                                  "Sparse + DIET(bow) + ResponseSelector(bow)": [{
-                                      "Entity Prediction": {
-                                          "macro avg": {
-                                              "f1-score": 0.88,
-                                          }
-                                      },
-                                      "test_run_time": "47s",
-                                  }]
-                              }
-                          }
-        result2: dict of key: dataset, value: (dict of key: config, value: list of res)
-
-    Returns:
-        dict of key: dataset, and value: (dict of key: config value: list of results)
-    """
-    combined_dict = defaultdict(lambda: defaultdict(list))
-    for new_dict in [result1, result2]:
-        for dataset, results_for_dataset in new_dict.items():
-            for config, res in results_for_dataset.items():
-                for res_dict in res:
-                    combined_dict[dataset][config].append(res_dict)
-    return combined_dict
-
-
-if __name__ == "__main__":
-    data = {}
-    reports_dir = Path(os.environ["REPORTS_DIR"])
-    reports_paths = list(reports_dir.glob("*/report.json"))
-
-    for report_path in reports_paths:
-        report_dict = json.load(open(report_path))
-        data = combine_result(data, report_dict)
-
-    summary_file = os.environ["SUMMARY_FILE"]
-    with open(summary_file, "w") as f:
-        json.dump(data, f, sort_keys=True, indent=2)

AI/.github/scripts/mr_publish_results.py

@@ -1,293 +0,0 @@
-# Send model regression test results to Datadog
-# with a summary of all test results.
-# Also write them into a report file.
-import copy
-import datetime
-import json
-import os
-from typing import Any, Dict, List, Text, Tuple
-
-from datadog_api_client.v1 import ApiClient, Configuration
-from datadog_api_client.v1.api.metrics_api import MetricsApi
-from datadog_api_client.v1.model.metrics_payload import MetricsPayload
-from datadog_api_client.v1.model.point import Point
-from datadog_api_client.v1.model.series import Series
-
-DD_ENV = "rasa-regression-tests"
-DD_SERVICE = "rasa"
-METRIC_RUNTIME_PREFIX = "rasa.perf.benchmark."
-METRIC_ML_PREFIX = "rasa.perf.ml."
-CONFIG_REPOSITORY = "training-data"
-
-TASK_MAPPING = {
-    "intent_report.json": "intent_classification",
-    "CRFEntityExtractor_report.json": "entity_prediction",
-    "DIETClassifier_report.json": "entity_prediction",
-    "response_selection_report.json": "response_selection",
-    "story_report.json": "story_prediction",
-}
-
-METRICS = {
-    "test_run_time": "TEST_RUN_TIME",
-    "train_run_time": "TRAIN_RUN_TIME",
-    "total_run_time": "TOTAL_RUN_TIME",
-}
-
-MAIN_TAGS = {
-    "config": "CONFIG",
-    "dataset": "DATASET_NAME",
-}
-
-OTHER_TAGS = {
-    "config_repository_branch": "DATASET_REPOSITORY_BRANCH",
-    "dataset_commit": "DATASET_COMMIT",
-    "accelerator_type": "ACCELERATOR_TYPE",
-    "type": "TYPE",
-    "index_repetition": "INDEX_REPETITION",
-    "host_name": "HOST_NAME",
-}
-
-GIT_RELATED_TAGS = {
-    "pr_id": "PR_ID",
-    "pr_url": "PR_URL",
-    "github_event": "GITHUB_EVENT_NAME",
-    "github_run_id": "GITHUB_RUN_ID",
-    "github_sha": "GITHUB_SHA",
-    "workflow": "GITHUB_WORKFLOW",
-}
-
-
-def create_dict_of_env(name_to_env: Dict[Text, Text]) -> Dict[Text, Text]:
-    return {name: os.environ[env_var] for name, env_var in name_to_env.items()}
-
-
-def _get_is_external_and_dataset_repository_branch() -> Tuple[bool, Text]:
-    is_external = os.environ["IS_EXTERNAL"]
-    dataset_repository_branch = os.environ["DATASET_REPOSITORY_BRANCH"]
-    if is_external.lower() in ("yes", "true", "t", "1"):
-        is_external_flag = True
-        dataset_repository_branch = os.environ["EXTERNAL_DATASET_REPOSITORY_BRANCH"]
-    else:
-        is_external_flag = False
-    return is_external_flag, dataset_repository_branch
-
-
-def prepare_datasetrepo_and_external_tags() -> Dict[Text, Any]:
-    is_external, dataset_repo_branch = _get_is_external_and_dataset_repository_branch()
-    return {
-        "dataset_repository_branch": dataset_repo_branch,
-        "external_dataset_repository": is_external,
-    }
-
-
-def prepare_dsrepo_and_external_tags_as_str() -> Dict[Text, Text]:
-    return {
-        "dataset_repository_branch": os.environ["DATASET_REPOSITORY_BRANCH"],
-        "external_dataset_repository": os.environ["IS_EXTERNAL"],
-    }
-
-
-def transform_to_seconds(duration: Text) -> float:
-    """Transform string (with hours, minutes, and seconds) to seconds.
-
-    Args:
-        duration: Examples: '1m27s', '1m27.3s', '27s', '1h27s', '1h1m27s'
-
-    Raises:
-        Exception: If the input is not supported.
-
-    Returns:
-        Duration converted in seconds.
-    """
-    h_split = duration.split("h")
-    if len(h_split) == 1:
-        rest = h_split[0]
-        hours = 0
-    else:
-        hours = int(h_split[0])
-        rest = h_split[1]
-    m_split = rest.split("m")
-    if len(m_split) == 2:
-        minutes = int(m_split[0])
-        seconds = float(m_split[1].rstrip("s"))
-    elif len(m_split) == 1:
-        minutes = 0
-        seconds = float(m_split[0].rstrip("s"))
-    else:
-        raise Exception(f"Unsupported duration: {duration}")
-    overall_seconds = hours * 60 * 60 + minutes * 60 + seconds
-    return overall_seconds
-
-
-def prepare_ml_metric(result: Dict[Text, Any]) -> Dict[Text, float]:
-    """Converts a nested result dict into a list of metrics.
-
-    Args:
-        result: Example
-            {'accuracy': 1.0,
-             'weighted avg': {
-                'precision': 1.0, 'recall': 1.0, 'f1-score': 1.0, 'support': 28
-             }
-            }
-
-    Returns:
-        Dict of metric name and metric value
-    """
-    metrics_ml = {}
-    result = copy.deepcopy(result)
-    result.pop("file_name", None)
-    task = result.pop("task", None)
-
-    for metric_name, metric_value in result.items():
-        if isinstance(metric_value, float):
-            metric_full_name = f"{task}.{metric_name}"
-            metrics_ml[metric_full_name] = float(metric_value)
-        elif isinstance(metric_value, dict):
-            for mname, mval in metric_value.items():
-                metric_full_name = f"{task}.{metric_name}.{mname}"
-                metrics_ml[metric_full_name] = float(mval)
-        else:
-            raise Exception(
-                f"metric_value {metric_value} has",
-                f"unexpected type {type(metric_value)}",
-            )
-    return metrics_ml
-
-
-def prepare_ml_metrics(results: List[Dict[Text, Any]]) -> Dict[Text, float]:
-    metrics_ml = {}
-    for result in results:
-        new_metrics_ml = prepare_ml_metric(result)
-        metrics_ml.update(new_metrics_ml)
-
-    return metrics_ml
-
-
-def prepare_datadog_tags() -> List[Text]:
-    tags = {
-        "env": DD_ENV,
-        "service": DD_SERVICE,
-        "branch": os.environ["BRANCH"],
-        "config_repository": CONFIG_REPOSITORY,
-        **prepare_dsrepo_and_external_tags_as_str(),
-        **create_dict_of_env(MAIN_TAGS),
-        **create_dict_of_env(OTHER_TAGS),
-        **create_dict_of_env(GIT_RELATED_TAGS),
-    }
-    tags_list = [f"{k}:{v}" for k, v in tags.items()]
-    return tags_list
-
-
-def send_to_datadog(results: List[Dict[Text, Any]]) -> None:
-    """Sends metrics to datadog."""
-    # Prepare
-    tags_list = prepare_datadog_tags()
-    timestamp = datetime.datetime.now().timestamp()
-    series = []
-
-    # Send metrics about runtime
-    metrics_runtime = create_dict_of_env(METRICS)
-    for metric_name, metric_value in metrics_runtime.items():
-        overall_seconds = transform_to_seconds(metric_value)
-        series.append(
-            Series(
-                metric=f"{METRIC_RUNTIME_PREFIX}{metric_name}.gauge",
-                type="gauge",
-                points=[Point([timestamp, overall_seconds])],
-                tags=tags_list,
-            )
-        )
-
-    # Send metrics about ML model performance
-    metrics_ml = prepare_ml_metrics(results)
-    for metric_name, metric_value in metrics_ml.items():
-        series.append(
-            Series(
-                metric=f"{METRIC_ML_PREFIX}{metric_name}.gauge",
-                type="gauge",
-                points=[Point([timestamp, float(metric_value)])],
-                tags=tags_list,
-            )
-        )
-
-    body = MetricsPayload(series=series)
-    with ApiClient(Configuration()) as api_client:
-        api_instance = MetricsApi(api_client)
-        response = api_instance.submit_metrics(body=body)
-        if response.get("status") != "ok":
-            print(response)
-
-
-def read_results(file: Text) -> Dict[Text, Any]:
-    with open(file) as json_file:
-        data = json.load(json_file)
-
-        keys = [
-            "accuracy",
-            "weighted avg",
-            "macro avg",
-            "micro avg",
-            "conversation_accuracy",
-        ]
-        result = {key: data[key] for key in keys if key in data}
-
-    return result
-
-
-def get_result(file_name: Text, file: Text) -> Dict[Text, Any]:
-    result = read_results(file)
-    result["file_name"] = file_name
-    result["task"] = TASK_MAPPING[file_name]
-    return result
-
-
-def send_all_to_datadog() -> None:
-    results = []
-    for dirpath, dirnames, files in os.walk(os.environ["RESULT_DIR"]):
-        for f in files:
-            if any(f.endswith(valid_name) for valid_name in TASK_MAPPING.keys()):
-                result = get_result(f, os.path.join(dirpath, f))
-                results.append(result)
-    send_to_datadog(results)
-
-
-def generate_json(file: Text, task: Text, data: dict) -> dict:
-    config = os.environ["CONFIG"]
-    dataset = os.environ["DATASET_NAME"]
-
-    if dataset not in data:
-        data = {dataset: {config: []}, **data}
-    elif config not in data[dataset]:
-        data[dataset] = {config: [], **data[dataset]}
-
-    assert len(data[dataset][config]) <= 1
-
-    data[dataset][config] = [
-        {
-            "config_repository": CONFIG_REPOSITORY,
-            **prepare_datasetrepo_and_external_tags(),
-            **create_dict_of_env(METRICS),
-            **create_dict_of_env(OTHER_TAGS),
-            **(data[dataset][config][0] if data[dataset][config] else {}),
-            task: read_results(file),
-        }
-    ]
-    return data
-
-
-def create_report_file() -> None:
-    data = {}
-    for dirpath, dirnames, files in os.walk(os.environ["RESULT_DIR"]):
-        for f in files:
-            if f not in TASK_MAPPING.keys():
-                continue
-
-            data = generate_json(os.path.join(dirpath, f), TASK_MAPPING[f], data)
-
-    with open(os.environ["SUMMARY_FILE"], "w") as f:
-        json.dump(data, f, sort_keys=True, indent=2)
-
-
-if __name__ == "__main__":
-    send_all_to_datadog()
-    create_report_file()

AI/.github/scripts/start_dd_agent.sh

@@ -1,79 +0,0 @@
-#!/bin/bash
-
-DD_API_KEY=$1
-ACCELERATOR_TYPE=$2
-NVML_INTERVAL_IN_SEC=${3:-15}  # 15 seconds are the default interval
-
-# Install Datadog system agent
-DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=$DD_API_KEY DD_SITE="datadoghq.eu" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
-DATADOG_YAML_PATH=/etc/datadog-agent/datadog.yaml
-sudo chmod 666 $DATADOG_YAML_PATH
-
-# Associate metrics with tags and env
-{
-    echo "env: rasa-regression-tests"
-    echo "tags:"
-    echo "- service:rasa"
-    echo "- accelerator_type:${ACCELERATOR_TYPE}"
-    echo "- dataset:${DATASET_NAME}"
-    echo "- config:${CONFIG}"
-    echo "- dataset_commit:${DATASET_COMMIT}"
-    echo "- branch:${BRANCH}"
-    echo "- github_sha:${GITHUB_SHA}"
-    echo "- pr_id:${PR_ID:-schedule}"
-    echo "- pr_url:${PR_URL:-schedule}"
-    echo "- type:${TYPE}"
-    echo "- dataset_repository_branch:${DATASET_REPOSITORY_BRANCH}"
-    echo "- external_dataset_repository:${IS_EXTERNAL:-none}"
-    echo "- config_repository:training-data"
-    echo "- config_repository_branch:${DATASET_REPOSITORY_BRANCH}"
-    echo "- workflow:${GITHUB_WORKFLOW:-none}"
-    echo "- github_run_id:${GITHUB_RUN_ID:-none}"
-    echo "- github_event:${GITHUB_EVENT_NAME:-none}"
-    echo "- index_repetition:${INDEX_REPETITION}"
-    echo "- host_name:${HOST_NAME}"
-    echo ""
-    echo "apm_config:"
-    echo "    enabled: true"
-    echo "process_config:"
-    echo "    enabled: false"
-    echo "use_dogstatsd: true"
-} >> $DATADOG_YAML_PATH
-
-# Enable system_core integration
-sudo mv /etc/datadog-agent/conf.d/system_core.d/conf.yaml.example /etc/datadog-agent/conf.d/system_core.d/conf.yaml
-
-if [[ "${ACCELERATOR_TYPE}" == "GPU" ]]; then
-    # Install and enable NVML integration
-    sudo datadog-agent integration --allow-root install -t datadog-nvml==1.0.1
-    sudo -u dd-agent -H /opt/datadog-agent/embedded/bin/pip3 install grpcio pynvml
-    NVML_CONF_FPATH="/etc/datadog-agent/conf.d/nvml.d/conf.yaml"
-    sudo mv "${NVML_CONF_FPATH}.example" ${NVML_CONF_FPATH}
-    if [[ "${NVML_INTERVAL_IN_SEC}" != 15 ]]; then
-        # Append a line to the NVML config file
-        sudo echo "    min_collection_interval: ${NVML_INTERVAL_IN_SEC}" | sudo tee -a ${NVML_CONF_FPATH} > /dev/null
-    fi
-fi
-
-# Apply changes
-sudo service datadog-agent stop
-
-# Restart agent (such that GPU/NVML metrics are collected)
-# Adusted code from /etc/init/datadog-agent.conf
-INSTALL_DIR="/opt/datadog-agent"
-AGENTPATH="$INSTALL_DIR/bin/agent/agent"
-PIDFILE="$INSTALL_DIR/run/agent.pid"
-AGENT_USER="dd-agent"
-LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64"
-sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $AGENTPATH run -p $PIDFILE"
-
-# Adusted code from /etc/init/datadog-agent-process.conf
-TRACE_AGENTPATH="$INSTALL_DIR/embedded/bin/trace-agent"
-TRACE_PIDFILE="$INSTALL_DIR/run/trace-agent.pid"
-sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $TRACE_PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $TRACE_AGENTPATH --config $DATADOG_YAML_PATH --pid $TRACE_PIDFILE"
-
-# Adusted code from /etc/init/datadog-agent-trace.conf
-PROCESS_AGENTPATH="$INSTALL_DIR/embedded/bin/process-agent"
-PROCESS_PIDFILE="$INSTALL_DIR/run/process-agent.pid"
-SYSTEM_PROBE_YAML="/etc/datadog-agent/system-probe.yaml"
-sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $PROCESS_PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $PROCESS_AGENTPATH --config=$DATADOG_YAML_PATH --sysprobe-config=$SYSTEM_PROBE_YAML --pid=$PROCESS_PIDFILE"

AI/.github/scripts/validate_cpu.py

@@ -1,14 +0,0 @@
-import sys
-
-import tensorflow as tf
-
-
-def check_gpu_not_available():
-    num_gpus = len(tf.config.list_physical_devices("GPU"))
-    print(f"Num GPUs Available: {num_gpus}")
-    if num_gpus > 0:
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    check_gpu_not_available()

AI/.github/scripts/validate_gpus.py

@@ -1,14 +0,0 @@
-import sys
-
-import tensorflow as tf
-
-
-def check_gpu_available():
-    num_gpus = len(tf.config.list_physical_devices("GPU"))
-    print(f"Num GPUs Available: {num_gpus}")
-    if num_gpus <= 0:
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    check_gpu_available()

AI/.github/stale.yml

@@ -1,18 +0,0 @@
-# Number of days of inactivity before an issue becomes stale
-daysUntilStale: 90
-# Label to use when marking an issue as stale
-staleLabel: stale
-
-pulls:
-  # Give more time before closing PRs
-  daysUntilClose: 21
-  # Comment to post when marking a PR as stale. Set to `false` to disable
-  markComment: >
-    This PR has been automatically marked as stale because it has not had
-    recent activity. It will be closed if no further activity occurs. Thank you
-    for your contributions.
-  # Comment to post when closing a stale PR. Set to `false` to disable
-  closeComment: >
-    This PR has been automatically closed due to inactivity. Please reopen
-    this PR or a new one if you plan to follow-up on it. Thank you for your
-    contributions.

AI/.github/templates/README.md

@@ -1,63 +0,0 @@
-# gomplate templates for GitHub Actions
-
-This document describes gomplate templates use for GitHub Actions.
-
-## Requirements
-
-You have to have installed [gomplate](https://docs.gomplate.ca/installing/) tool in order to render a template file.
-
-> gomplate is a template renderer which supports a growing list of datastores, such as: JSON (including EJSON - encrypted JSON), YAML, AWS EC2 metadata, BoltDB, Hashicorp Consul and Hashicorp Vault secrets.
-
-## Templates
-
-Below you can find a list of templates with their description and the commands to render them.
-
-### `configuration_variables.tmpl`
-
-The template maps dataset name and configuration name for the model regression tests into paths where files are located. As a result, the template returns two environment variables `DATASET` and `CONFIG` which contain paths to file/directory.
-
-#### How to run locally
-
-```shell
-gomplate -d mapping=<path_to_json_file_with_mapping> -f .github/templates/configuration_variables.tmpl
-```
-
-### `model_regression_test_config_comment.tmpl`
-
-The template returns a comment message which is used as a help description in a PR. The template reads the `.github/configs/mr-test-example.yaml` file and include it as example content.
-
-The help message is triggered by adding `status:model-regression-tests` label.
-Comment with a help message is added if a PR doesn't contain a comment with a configuration for the model regression tests.
-
-#### How to run locally
-
-```shell
-gomplate -f .github/templates/model_regression_test_config_comment.tmpl
-```
-
-The template uses the `GITHUB_ACTOR` environment variable, you have to export the variable before executing the command.
-
-### `model_regression_test_config_to_json.tmpl`
-
-The template reads an issue/a PR comment and transforms a YAML code block into JSON.
-
-#### How to run locally
-
-```shell
-gomplate -d github=https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ comment-id }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -f .github/templates/model_regression_test_config_to_json.tmpl
-```
-
-### `model_regression_test_results.tmpl`
-
-The template reads a file with a report (the report file is available as an artifact in the model regression tests workflow) and returns markdown table with a summary of tests.
-
-#### How to run locally
-
-```shell
-gomplate -d data=report.json -d results_main=report_main.json -f .github/templates/model_regression_test_results.tmpl
-```
-
-In order to be able to use the `.github/templates/model_regression_test_results.tmpl` template you need the following files:
-
-- `report.json` - the file with a report generated by the `CI - Model Regression` workflow run in a PR. The report is available to download as an artifact in the workflow related to the PR.
-- `report_main.json` - the file with a report generated by the `CI - Model Regression` workflow that is triggered on schedule event. A list of the workflows that you can download an artifact from, can be found [here](https://github.com/RasaHQ/rasa/actions?query=workflow%3A%22CI+-+Model+Regression%22+event%3Aschedule).

AI/.github/templates/configuration_variables.tmpl

@@ -1,43 +0,0 @@
-{{- /*
-
-The template maps dataset name and configuration name for the model
-regression tests into paths where files are located. As a result,
-the template returns two environment variables `DATASET` and `CONFIG`
-which contain paths to file/directory.
-
- */ -}}
-{{- $mapping := (datasource "mapping") -}}
-{{- $dataset := (index $mapping.datasets (getenv "DATASET_NAME")) -}}
-{{- $config := $mapping.configurations -}}
-{{- if has $dataset "repository" }}
-export DATASET="{{ $dataset.repository }}"
-export IS_EXTERNAL="true"
-echo "::add-mask::{{ $dataset.repository }}"
-{{ if has $dataset "repository_branch" }}
-export EXTERNAL_DATASET_REPOSITORY_BRANCH="{{ $dataset.repository_branch }}"
-{{ else }}
-export EXTERNAL_DATASET_REPOSITORY_BRANCH="main"
-{{ end }}
-{{- else if has $dataset "path" }}
-export DATASET="{{ $dataset.path }}"
-export IS_EXTERNAL="false"
-echo "::add-mask::{{ $dataset.path }}"
-{{ end }}
-
-{{- if has $dataset "train" }}
-export TRAIN_DIR="{{ $dataset.train }}"
-{{ end }}
-{{- if has $dataset "test" }}
-export TEST_DIR="{{ $dataset.test }}"
-{{ end }}
-{{- if has $dataset "domain" }}
-export DOMAIN_FILE="{{ $dataset.domain }}"
-{{ end }}
-
-{{- if (has $config.nlu (getenv "CONFIG_NAME")) }}
-export CONFIG="{{ $dataset.language }}/nlu/{{ index $config.nlu (getenv "CONFIG_NAME") }}"
-echo "::add-mask::{{ $dataset.language }}/nlu/{{ index $config.nlu (getenv "CONFIG_NAME") }}"
-{{ else if (has $config.core (getenv "CONFIG_NAME")) }}
-export CONFIG="{{ $dataset.language }}/core/{{ index $config.core (getenv "CONFIG_NAME") }}"
-echo "::add-mask::{{ $dataset.language }}/core/{{ index $config.core (getenv "CONFIG_NAME") }}"
-{{ end -}}

AI/.github/templates/model_regression_test_config_comment.tmpl

@@ -1,45 +0,0 @@
-{{- /*
-
-The template returns a comment message which is used as a help description
-in a PR. The template reads the `.github/configs/mr-test-example.yaml` file
-and include it as example content.
-
-The help message is triggered by adding `status:model-regression-tests` label.
-Comment with a help message is added if a PR doesn't contain a comment
-with a configuration for the model regression tests.
-
- */ -}}
-{{ define "check_available_configuration" -}}
-NLU
-{{- if has .dataset "domain" -}}
-, Core
-{{- end -}}
-{{- end -}}
-Hey @{{ .Env.GITHUB_ACTOR }}! :wave: To run model regression tests, comment with the `/modeltest` command and a configuration.
-
-_Tips :bulb:: The model regression test will be run on `push` events. You can re-run the tests by re-add `status:model-regression-tests` label or use a `Re-run jobs` button in Github Actions workflow._
-
-_Tips :bulb:: Every time when you want to change a configuration you should edit the comment with the previous configuration._
-
-You can copy this in your comment and customize:
-
-> /modeltest
-> ~~~yml
->```yml
->##########
->## Available datasets
->##########
-{{range (coll.Keys (datasource "mapping").datasets)}}># - "{{ . }}" ({{ template "check_available_configuration" (dict "dataset" (index (datasource "mapping").datasets .)) }}){{"\n"}}{{ end -}}
->
->##########
->## Available NLU configurations
->##########
-{{range (coll.Keys (datasource "mapping").configurations.nlu)}}># - "{{.}}"{{"\n"}}{{ end -}}
->
->##########
->## Available Core configurations
->##########
-{{range (coll.Keys (datasource "mapping").configurations.core)}}># - "{{.}}"{{"\n"}}{{ end -}}
->
-{{range split (file.Read ".github/configs/mr-test-example.yaml") "\n"}}>{{.}}{{"\n"}}{{ end -}}
->```

AI/.github/templates/model_regression_test_config_to_json.tmpl

@@ -1,71 +0,0 @@
-{{- /*
-
-The template reads an issue/a PR comment and transforms a YAML code block into JSON.
-
-*/ -}}
-{{ define "check_config_type" -}}
-{{- if has (datasource "mapping").configurations.nlu . -}}
-nlu
-{{- else if has (datasource "mapping").configurations.core . -}}
-core
-{{- end -}}
-{{- end -}}
-{{- $config := ((datasource "github").body | regexp.Find "```(?s)(.*)```" | regexp.ReplaceLiteral "```.*|\r" "" | yaml | toJSON | json) -}}
-{{- $num_repetitions := 1 -}}
-{{- if has $config "num_repetitions" -}}
-{{- $num_repetitions = $config.num_repetitions -}}
-{{- end -}}
-{"include":[
-{{- $inc := coll.Slice -}}
-{{- $dataset := coll.Slice -}}
-{{- range $pair := $config.include -}}
-{{- /* use all available datasets if value is equal to all */ -}}
-{{- if eq (index $pair.dataset 0) "all" -}}
-{{ $dataset = (coll.Keys (datasource "mapping").datasets) }}
-{{- else if eq (index $pair.dataset 0) "all-core" -}}
-{{- range $dataset_name, $dataset_spec := (datasource "mapping").datasets -}}
-{{- if has $dataset_spec "domain" -}}
-{{ $dataset = (coll.Append $dataset_name $dataset) -}}
-{{- end -}}
-{{- end -}}
-{{- else if eq (index $pair.dataset 0) "all-nlu" -}}
-{{- range $dataset_name, $dataset_spec := (datasource "mapping").datasets -}}
-{{- if not (has $dataset_spec "domain") -}}
-{{ $dataset = (coll.Append $dataset_name $dataset) -}}
-{{- end -}}
-{{- end -}}
-{{- else -}}
-{{- $dataset = $pair.dataset -}}
-{{- end -}}
-{{- range $index_dataset, $value_dataset := $dataset -}}
-{{- range $index_config, $value_config := $pair.config -}}
-{{ range $index_repetition, $element := (strings.Repeat $num_repetitions "x " | strings.Trim " " | strings.Split " ") }}
-{{- /* use all available configurations if value is equal to all */ -}}
-{{- if eq $value_config "all" -}}
-{{- range $config_type := (coll.Keys (datasource "mapping").configurations) -}}
-{{- range $config_name, $config_file := (index (datasource "mapping").configurations $config_type ) -}}
-
-{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $config_name "type" $config_type | toJSON) $inc) -}}
-{{- end -}}
-{{- end -}}
-{{- else if eq $value_config "all-core" -}}
-{{- range $config_name, $config_file := (datasource "mapping").configurations.core -}}
-{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $config_name "type" "core" | toJSON) $inc) -}}
-{{- end -}}
-{{- else if eq $value_config "all-nlu" -}}
-{{- range $config_name, $config_file := (datasource "mapping").configurations.nlu -}}
-{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $config_name "type" "nlu" | toJSON) $inc) -}}
-{{- end -}}
-{{- else -}}
-{{- if has (datasource "mapping").configurations.nlu $value_config -}}
-{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $value_config "type" "nlu" | toJSON) $inc) -}}
-{{- else if has (datasource "mapping").configurations.core $value_config -}}
-{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $value_config "type" "core" | toJSON) $inc) -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-{{- join $inc "," -}}
-]}

AI/.github/templates/model_regression_test_read_dataset_branch.tmpl

@@ -1,13 +0,0 @@
-{{- /*
-
-The template reads a PR comment and gets the dataset branch for the training-data
-repository.
-
- */ -}}
-{{- $config := ((datasource "github").body | regexp.Find "```(?s)(.*)```" | regexp.ReplaceLiteral "```.*|\r" "" | yaml | toJSON | json) -}}
-{{- $dataset_branch := "main" -}}
-{{- /* if a branch name for dataset repository is not defined use the main branch */ -}}
-{{- if has $config "dataset_branch" -}}
-{{- $dataset_branch = $config.dataset_branch -}}
-{{- end -}}
-export DATASET_BRANCH="{{ $dataset_branch }}"

AI/.github/templates/model_regression_test_results.tmpl

@@ -1,159 +0,0 @@
-{{- /*
-
-The template reads a file with a report (the report file is available
-as an artifact in the model regression tests workflow) and returns
-a markdown table with a summary of the tests.
-
-*/ -}}
-{{- /*
-
-The print_result_nlu template returns data depends on available fields.
-
-*/ -}}
-{{ define "print_result_nlu" -}}
-{{- if and (has (index .branch "micro avg") "f1-score") (has (index .main "micro avg") "f1-score") -}}
-{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub (index (index .branch "micro avg") "f1-score")) }})
-{{- else if and (has .branch "accuracy") (has .main "accuracy") -}}
-{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" (.main.accuracy | math.Sub .branch.accuracy) }})
-{{- else if and (has .branch "accuracy") (has (index .main "micro avg") "f1-score") -}}
-{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub .branch.accuracy) }})
-{{- else if and (has (index .branch "micro avg") "f1-score") (has .main "accuracy") -}}
-{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" (.main.accuracy | math.Sub (index (index .branch "micro avg") "f1-score")) }})
-{{- else if (has .branch "accuracy") -}}
-{{ printf "%.4f" .branch.accuracy }} (`no data`)
-{{- else if has (index .branch "micro avg") "f1-score" -}}
-{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} (`no data`)
-{{- else -}}
-`no data`
-{{- end -}}
-{{- end -}}
-{{- /*
-
-The print_result_core template returns data depends on available fields.
-
-*/ -}}
-{{ define "print_result_core_micro_avg" -}}
-{{- if and (has (index .branch "micro avg") "f1-score") (has (index .main "micro avg") "f1-score") -}}
-{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub (index (index .branch "micro avg") "f1-score")) }})
-{{- else if and (has .branch "accuracy") (has .main "accuracy") -}}
-{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" (.main.accuracy | math.Sub .branch.accuracy) }})
-{{- else if and (has .branch "accuracy") (has (index .main "micro avg") "f1-score") -}}
-{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub .branch.accuracy) }})
-{{- else if and (has (index .branch "micro avg") "f1-score") (has .main "accuracy") -}}
-{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" (.main.accuracy | math.Sub (index (index .branch "micro avg") "f1-score")) }})
-{{- else if (has .branch "accuracy") -}}
-{{ printf "%.4f" .branch.accuracy }} (`no data`)
-{{- else if has (index .branch "micro avg") "f1-score" -}}
-{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} (`no data`)
-{{- else -}}
-`no data`
-{{- end -}}
-{{- end -}}
-
-{{ define "print_result_core_conversation_accuracy" -}}
-{{- if and (has (index .branch "conversation_accuracy") "accuracy") (has (index .main "conversation_accuracy") "accuracy") -}}
-{{ printf "%.4f" (index (index .branch "conversation_accuracy") "accuracy") }} ({{ printf "%.2f" ((index (index .main "conversation_accuracy") "accuracy") | math.Sub (index (index .branch "conversation_accuracy") "accuracy")) }})
-{{- else if has (index .branch "conversation_accuracy") "accuracy" -}}
-{{ printf "%.4f" (index (index .branch "conversation_accuracy") "accuracy") }} (`no data`)
-{{- else -}}
-`no data`
-{{- end -}}
-{{- end -}}
-
-{{ define "print_table_nlu" }}
-{{- $available_types := (index .results_for_dataset | jsonpath `@..type`) -}}
-{{- if isKind "string" $available_types }}{{- $available_types = (index .results_for_dataset | jsonpath `@..type` | slice) -}}{{- end -}}
-{{- if has $available_types "nlu" -}}
-| Configuration | Intent Classification Micro F1 | Entity Recognition Micro F1 | Response Selection Micro F1 |
-|---------------|-----------------|-----------------|-------------------|
-{{ range $config_name, $config_data_array := .results_for_dataset -}}
-{{ range $config_data := $config_data_array }}
-{{- if eq $config_data.type "nlu" -}}
-| `{{ $config_name }}`<br> test: `{{ $config_data.test_run_time }}`, train: `{{ $config_data.train_run_time }}`, total: `{{ $config_data.total_run_time }}`|
-{{- if has $config_data "intent_classification" -}}
-{{- $intent_class_main := dict -}}
-{{- if has $.results_for_dataset_main $config_name -}}
-{{- $intent_class_main = (index (index $.results_for_dataset_main $config_name) 0).intent_classification -}}
-{{- end -}}
-{{- $intent_class := $config_data.intent_classification -}}
-{{ template "print_result_nlu" (dict "branch" $intent_class "main" $intent_class_main) }}|
-{{- else -}}
-`no data`|
-{{- end -}}
-{{- if has $config_data "entity_prediction" -}}
-{{- $entity_class_main := dict -}}
-{{- if has $.results_for_dataset_main $config_name -}}
-{{- $entity_class_main = (index (index $.results_for_dataset_main $config_name) 0).entity_prediction -}}
-{{- end -}}
-{{- $entity_class := $config_data.entity_prediction -}}
-{{ template "print_result_nlu" (dict "branch" $entity_class "main" $entity_class_main) }}|
-{{- else -}}
-`no data`|
-{{- end -}}
-{{- if has $config_data "response_selection" -}}
-{{- $response_class_main := dict -}}
-{{- if has $.results_for_dataset_main $config_name -}}
-{{- $response_class_main = (index (index $.results_for_dataset_main $config_name) 0).response_selection -}}
-{{- end -}}
-{{- $response_class := $config_data.response_selection -}}
-{{ template "print_result_nlu" (dict "branch" $response_class "main" $response_class_main) }}|
-{{- else -}}
-`no data`|
-{{- end }}
-{{end}}
-{{- end}}
-{{- end}}
-{{- end -}}
-{{- end -}}
-
-{{- define "print_table_core" -}}
-{{- $available_types := (index .results_for_dataset | jsonpath `@..type`) -}}
-{{- if isKind "string" $available_types }}{{- $available_types = (index .results_for_dataset | jsonpath `@..type` | slice) -}}{{- end -}}
-{{- if has $available_types "core" -}}
-| Dialog Policy Configuration | Action Level Micro Avg. F1 | Conversation Level Accuracy | Run Time Train | Run Time Test |
-|---------------|-----------------|-----------------|-------------------|-------------------|
-{{ range $config_name, $config_data_array := .results_for_dataset -}}
-{{ range $config_data := $config_data_array }}
-{{- if eq $config_data.type "core" -}}
-| `{{ $config_name }}` |
-{{- if has $config_data "story_prediction" -}}
-{{- $story_prediction_main := dict -}}
-{{- if has $.results_for_dataset_main $config_name -}}
-{{- $story_prediction_main = (index (index $.results_for_dataset_main $config_name) 0).story_prediction -}}
-{{- end -}}
-{{- $story_prediction := $config_data.story_prediction -}}
-{{ template "print_result_core_micro_avg" (dict "branch" $story_prediction "main" $story_prediction_main) }}|
-{{- else -}}
-`no data`|
-{{- end -}}
-{{- if has $config_data "story_prediction" -}}
-{{- $story_prediction_main := dict -}}
-{{- if has $.results_for_dataset_main $config_name -}}
-{{- $story_prediction_main = (index (index $.results_for_dataset_main $config_name) 0).story_prediction -}}
-{{- end -}}
-{{- $story_prediction := index $config_data.story_prediction -}}
-{{ template "print_result_core_conversation_accuracy" (dict "branch" $story_prediction "main" $story_prediction_main) }}|
-{{- else -}}
-`no data`|
-{{- end -}}
-`{{ $config_data.train_run_time }}`| `{{ $config_data.test_run_time }}`|
-{{ end }}
-{{- end}}
-{{- end}}
-{{- end -}}
-{{- end -}}
-
-{{- $results_main := (datasource "results_main") -}}
-{{ range $dataset, $results_for_dataset := (datasource "data")}}
-{{ $results_for_dataset_main := (index $results_main $dataset) -}}
-{{ $content_dicts := index $results_for_dataset (index (keys $results_for_dataset) 0) -}}
-{{ $one_content_dict := index $content_dicts 0 -}}
-{{- if ($one_content_dict).external_dataset_repository -}}
-Dataset: `{{$dataset}}`, Dataset repository branch: `{{ ($one_content_dict).dataset_repository_branch }}` (external repository), commit: `{{ ($one_content_dict).dataset_commit }}`
-Configuration repository branch: `{{ ($one_content_dict).config_repository_branch }}`
-{{ else -}}
-Dataset: `{{$dataset}}`, Dataset repository branch: `{{ ($one_content_dict).dataset_repository_branch }}`, commit: `{{ ($one_content_dict).dataset_commit }}`
-{{ end -}}
-{{ template "print_table_nlu" (dict "results_for_dataset" $results_for_dataset "results_for_dataset_main" $results_for_dataset_main) }}
-{{ template "print_table_core" (dict "results_for_dataset" $results_for_dataset "results_for_dataset_main" $results_for_dataset_main) }}
-{{- end }}

AI/.github/tests/test_data/bert_diet_response2t.yml

@@ -1,23 +0,0 @@
-# Configuration for Rasa NLU.
-# https://rasa.com/docs/rasa/nlu/components/
-language: en
-pipeline:
-  - name: WhitespaceTokenizer
-  - name: LanguageModelFeaturizer
-    alias: "lmf"
-  - name: RegexFeaturizer
-    alias: "rf"
-  - name: LexicalSyntacticFeaturizer
-    alias: "lsf"
-  - name: DIETClassifier
-    epochs: 50
-    random_seed: 42
-  - name: ResponseSelector
-    epochs: 100
-    num_transformer_layers: 2
-    transformer_size: 256
-    hidden_layers_size:
-      text: []
-      label: []
-    random_seed: 42
-    featurizers: ["lmf"]

AI/.github/tests/test_data/comment_body.json

@@ -1,3 +0,0 @@
-{
-  "body": "/modeltest\r\n\r\n```yml\r\ndataset_branch: \"test_dataset_branch\"\r\ninclude:\r\n - dataset: [\"financial-demo\"]\r\n   config: [\"TEST\"]\r\n ```\r\n\r\n<!-- comment-id:comment_configuration -->"
-}

AI/.github/tests/test_data/comment_body_no_dataset_branch.json

@@ -1,3 +0,0 @@
-{
-  "body": "/modeltest\r\n\r\n```yml\r\ninclude:\r\n - dataset: [\"financial-demo\"]\r\n   config: [\"TEST\"]\r\n ```\r\n\r\n<!-- comment-id:comment_configuration -->"
-}

AI/.github/tests/test_data/intent_report.json

@@ -1,120 +0,0 @@
-{
-	"search_transactions": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 1,
-		"confused_with": {}
-	},
-	"greet": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 2,
-		"confused_with": {}
-	},
-	"out_of_scope": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 1,
-		"confused_with": {}
-	},
-	"thankyou": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 1,
-		"confused_with": {}
-	},
-	"help": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 2,
-		"confused_with": {}
-	},
-	"inform": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 1,
-		"confused_with": {}
-	},
-	"goodbye": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 1,
-		"confused_with": {}
-	},
-	"affirm": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 3,
-		"confused_with": {}
-	},
-	"pay_cc": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 2,
-		"confused_with": {}
-	},
-	"check_balance": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 5,
-		"confused_with": {}
-	},
-	"deny": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 1,
-		"confused_with": {}
-	},
-	"ask_transfer_charge": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 1,
-		"confused_with": {}
-	},
-	"transfer_money": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 3,
-		"confused_with": {}
-	},
-	"check_recipients": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 2,
-		"confused_with": {}
-	},
-	"check_earnings": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 2,
-		"confused_with": {}
-	},
-	"accuracy": 1.0,
-	"macro avg": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 28
-	},
-	"weighted avg": {
-		"precision": 1.0,
-		"recall": 1.0,
-		"f1-score": 1.0,
-		"support": 28
-	}
-}

AI/.github/tests/test_data/report-on-schedule-2022-02-02.json

@@ -1,303 +0,0 @@
-{
-  "RasaHQ/financial-demo": {
-    "BERT + DIET(bow) + ResponseSelector(bow)": [{
-      "accelerator_type": "GPU",
-      "config_repository": "training-data",
-      "config_repository_branch": "main",
-      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
-      "dataset_repository_branch": "fix-model-regression-tests",
-      "entity_prediction": {
-        "macro avg": {
-          "f1-score": 0.7333333333333333,
-          "precision": 0.8,
-          "recall": 0.7,
-          "support": 14
-        },
-        "micro avg": {
-          "f1-score": 0.8333333333333333,
-          "precision": 1.0,
-          "recall": 0.7142857142857143,
-          "support": 14
-        },
-        "weighted avg": {
-          "f1-score": 0.738095238095238,
-          "precision": 0.7857142857142857,
-          "recall": 0.7142857142857143,
-          "support": 14
-        }
-      },
-      "external_dataset_repository": true,
-      "intent_classification": {
-        "accuracy": 1.0,
-        "macro avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 28
-        },
-        "weighted avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 28
-        }
-      },
-      "test_run_time": "35s",
-      "total_run_time": "2m2s",
-      "train_run_time": "1m28s",
-      "type": "nlu"
-    }],
-    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
-      "accelerator_type": "GPU",
-      "config_repository": "training-data",
-      "config_repository_branch": "main",
-      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
-      "dataset_repository_branch": "fix-model-regression-tests",
-      "entity_prediction": {
-        "macro avg": {
-          "f1-score": 0.7333333333333333,
-          "precision": 0.8,
-          "recall": 0.7,
-          "support": 14
-        },
-        "micro avg": {
-          "f1-score": 0.8333333333333333,
-          "precision": 1.0,
-          "recall": 0.7142857142857143,
-          "support": 14
-        },
-        "weighted avg": {
-          "f1-score": 0.738095238095238,
-          "precision": 0.7857142857142857,
-          "recall": 0.7142857142857143,
-          "support": 14
-        }
-      },
-      "external_dataset_repository": true,
-      "intent_classification": {
-        "accuracy": 1.0,
-        "macro avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 28
-        },
-        "weighted avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 28
-        }
-      },
-      "test_run_time": "55s",
-      "total_run_time": "2m8s",
-      "train_run_time": "1m14s",
-      "type": "nlu"
-    }],
-    "Rules + Memo + TED": [{
-      "accelerator_type": "GPU",
-      "config_repository": "training-data",
-      "config_repository_branch": "main",
-      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
-      "dataset_repository_branch": "fix-model-regression-tests",
-      "external_dataset_repository": true,
-      "story_prediction": {
-        "accuracy": 1.0,
-        "conversation_accuracy": {
-          "accuracy": 1.0,
-          "correct": 48,
-          "total": 48,
-          "with_warnings": 0
-        },
-        "macro avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 317
-        },
-        "weighted avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 317
-        }
-      },
-      "test_run_time": "51s",
-      "total_run_time": "8m15s",
-      "train_run_time": "7m24s",
-      "type": "core"
-    }]
-  },
-  "RasaHQ/retail-demo": {
-    "BERT + DIET(bow) + ResponseSelector(bow)": [{
-      "accelerator_type": "GPU",
-      "config_repository": "training-data",
-      "config_repository_branch": "main",
-      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
-      "dataset_repository_branch": "fix-model-regression-tests",
-      "entity_prediction": {
-        "macro avg": {
-          "f1-score": 0.25,
-          "precision": 0.25,
-          "recall": 0.25,
-          "support": 6
-        },
-        "micro avg": {
-          "f1-score": 0.2857142857142857,
-          "precision": 1.0,
-          "recall": 0.16666666666666666,
-          "support": 6
-        },
-        "weighted avg": {
-          "f1-score": 0.16666666666666666,
-          "precision": 0.16666666666666666,
-          "recall": 0.16666666666666666,
-          "support": 6
-        }
-      },
-      "external_dataset_repository": true,
-      "intent_classification": {
-        "macro avg": {
-          "f1-score": 0.8,
-          "precision": 0.8,
-          "recall": 0.85,
-          "support": 16
-        },
-        "micro avg": {
-          "f1-score": 0.8387096774193549,
-          "precision": 0.8666666666666667,
-          "recall": 0.8125,
-          "support": 16
-        },
-        "weighted avg": {
-          "f1-score": 0.8125,
-          "precision": 0.875,
-          "recall": 0.8125,
-          "support": 16
-        }
-      },
-      "test_run_time": "29s",
-      "total_run_time": "1m16s",
-      "train_run_time": "47s",
-      "type": "nlu"
-    }],
-    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
-      "accelerator_type": "GPU",
-      "config_repository": "training-data",
-      "config_repository_branch": "main",
-      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
-      "dataset_repository_branch": "fix-model-regression-tests",
-      "entity_prediction": {
-        "macro avg": {
-          "f1-score": 0.25,
-          "precision": 0.25,
-          "recall": 0.25,
-          "support": 6
-        },
-        "micro avg": {
-          "f1-score": 0.2857142857142857,
-          "precision": 1.0,
-          "recall": 0.16666666666666666,
-          "support": 6
-        },
-        "weighted avg": {
-          "f1-score": 0.16666666666666666,
-          "precision": 0.16666666666666666,
-          "recall": 0.16666666666666666,
-          "support": 6
-        }
-      },
-      "external_dataset_repository": true,
-      "intent_classification": {
-        "accuracy": 0.875,
-        "macro avg": {
-          "f1-score": 0.8300000000000001,
-          "precision": 0.8166666666666667,
-          "recall": 0.85,
-          "support": 16
-        },
-        "weighted avg": {
-          "f1-score": 0.85,
-          "precision": 0.8333333333333333,
-          "recall": 0.875,
-          "support": 16
-        }
-      },
-      "test_run_time": "56s",
-      "total_run_time": "2m2s",
-      "train_run_time": "1m6s",
-      "type": "nlu"
-    }],
-    "Rules + Memo": [{
-      "accelerator_type": "GPU",
-      "config_repository": "training-data",
-      "config_repository_branch": "main",
-      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
-      "dataset_repository_branch": "fix-model-regression-tests",
-      "external_dataset_repository": true,
-      "story_prediction": {
-        "conversation_accuracy": {
-          "accuracy": 0.8888888888888888,
-          "correct": 8,
-          "total": 9,
-          "with_warnings": 0
-        },
-        "macro avg": {
-          "f1-score": 0.9663698541747322,
-          "precision": 1.0,
-          "recall": 0.946007696007696,
-          "support": 67
-        },
-        "micro avg": {
-          "f1-score": 0.9692307692307692,
-          "precision": 1.0,
-          "recall": 0.9402985074626866,
-          "support": 67
-        },
-        "weighted avg": {
-          "f1-score": 0.9656317714563074,
-          "precision": 1.0,
-          "recall": 0.9402985074626866,
-          "support": 67
-        }
-      },
-      "test_run_time": "10s",
-      "total_run_time": "19s",
-      "train_run_time": "10s",
-      "type": "core"
-    }],
-    "Rules + Memo + TED": [{
-      "accelerator_type": "GPU",
-      "config_repository": "training-data",
-      "config_repository_branch": "main",
-      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
-      "dataset_repository_branch": "fix-model-regression-tests",
-      "external_dataset_repository": true,
-      "story_prediction": {
-        "accuracy": 1.0,
-        "conversation_accuracy": {
-          "accuracy": 1.0,
-          "correct": 9,
-          "total": 9,
-          "with_warnings": 0
-        },
-        "macro avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 67
-        },
-        "weighted avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 67
-        }
-      },
-      "test_run_time": "31s",
-      "total_run_time": "4m57s",
-      "train_run_time": "4m27s",
-      "type": "core"
-    }]
-  }
-}

AI/.github/tests/test_data/report_listformat_core.json

@@ -1,70 +0,0 @@
-{
-  "RasaHQ/retail-demo": {
-    "Rules + Memo + TED": [{
-      "accelerator_type": "GPU",
-      "config_repository": "training-data",
-      "config_repository_branch": "main",
-      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
-      "dataset_repository_branch": "fix-model-regression-tests",
-      "external_dataset_repository": true,
-      "story_prediction": {
-        "accuracy": 1.0,
-        "conversation_accuracy": {
-          "accuracy": 1.0,
-          "correct": 9,
-          "total": 9,
-          "with_warnings": 0
-        },
-        "macro avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 67
-        },
-        "weighted avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 67
-        }
-      },
-      "test_run_time": "31s",
-      "total_run_time": "4m57s",
-      "train_run_time": "4m27s",
-      "type": "core"
-    },
-    {
-      "accelerator_type": "GPU",
-      "config_repository": "training-data",
-      "config_repository_branch": "main",
-      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
-      "dataset_repository_branch": "fix-model-regression-tests",
-      "external_dataset_repository": true,
-      "story_prediction": {
-        "accuracy": 1.0,
-        "conversation_accuracy": {
-          "accuracy": 1.0,
-          "correct": 9,
-          "total": 9,
-          "with_warnings": 0
-        },
-        "macro avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 67
-        },
-        "weighted avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 67
-        }
-      },
-      "test_run_time": "41s",
-      "total_run_time": "5m57s",
-      "train_run_time": "5m27s",
-      "type": "core"
-    }]
-  }
-}

AI/.github/tests/test_data/report_listformat_nlu.json

@@ -1,98 +0,0 @@
-{
-  "RasaHQ/financial-demo": {
-    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
-      "accelerator_type": "CPU",
-      "config_repository": "training-data",
-      "config_repository_branch": "main",
-      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
-      "dataset_repository_branch": "fix-model-regression-tests",
-      "entity_prediction": {
-        "macro avg": {
-          "f1-score": 0.7333333333333333,
-          "precision": 0.8,
-          "recall": 0.7,
-          "support": 14
-        },
-        "micro avg": {
-          "f1-score": 0.8333333333333333,
-          "precision": 1.0,
-          "recall": 0.7142857142857143,
-          "support": 14
-        },
-        "weighted avg": {
-          "f1-score": 0.738095238095238,
-          "precision": 0.7857142857142857,
-          "recall": 0.7142857142857143,
-          "support": 14
-        }
-      },
-      "external_dataset_repository": true,
-      "intent_classification": {
-        "accuracy": 1.0,
-        "macro avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 28
-        },
-        "weighted avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 28
-        }
-      },
-      "test_run_time": "1m29s",
-      "total_run_time": "4m24s",
-      "train_run_time": "2m55s",
-      "type": "nlu"
-    },
-    {
-      "accelerator_type": "CPU",
-      "config_repository": "training-data",
-      "config_repository_branch": "main",
-      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
-      "dataset_repository_branch": "fix-model-regression-tests",
-      "entity_prediction": {
-        "macro avg": {
-          "f1-score": 0.7333333333333333,
-          "precision": 0.8,
-          "recall": 0.7,
-          "support": 14
-        },
-        "micro avg": {
-          "f1-score": 0.8333333333333333,
-          "precision": 1.0,
-          "recall": 0.7142857142857143,
-          "support": 14
-        },
-        "weighted avg": {
-          "f1-score": 0.738095238095238,
-          "precision": 0.7857142857142857,
-          "recall": 0.7142857142857143,
-          "support": 14
-        }
-      },
-      "external_dataset_repository": true,
-      "intent_classification": {
-        "accuracy": 1.0,
-        "macro avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 28
-        },
-        "weighted avg": {
-          "f1-score": 1.0,
-          "precision": 1.0,
-          "recall": 1.0,
-          "support": 28
-        }
-      },
-      "test_run_time": "2m29s",
-      "total_run_time": "5m24s",
-      "train_run_time": "3m55s",
-      "type": "nlu"
-    }]
-  }
-}

AI/.github/tests/test_download_pretrained.py

@@ -1,111 +0,0 @@
-from copy import deepcopy
-import sys
-import tempfile
-from pathlib import Path
-
-import pytest
-from ruamel.yaml import YAML
-
-sys.path.append(".github/scripts")
-import download_pretrained  # noqa: E402
-
-CONFIG_FPATH = Path(__file__).parent / "test_data" / "bert_diet_response2t.yml"
-
-
-def test_download_pretrained_lmf_exists_no_params():
-    lmf_specs = download_pretrained.get_model_name_and_weights_from_config(CONFIG_FPATH)
-    assert lmf_specs[0].model_name == "bert"
-    assert lmf_specs[0].model_weights == "rasa/LaBSE"
-
-
-def test_download_pretrained_lmf_exists_with_model_name():
-    yaml = YAML(typ="safe")
-    config = yaml.load(CONFIG_FPATH)
-
-    steps = config.get("pipeline", [])
-    step = list(  # noqa: RUF015
-        filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps)
-    )[0]
-    step["model_name"] = "roberta"
-    step["cache_dir"] = "/this/dir"
-
-    with tempfile.NamedTemporaryFile("w+") as fp:
-        yaml.dump(config, fp)
-        fp.seek(0)
-        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
-    assert lmf_specs[0].model_name == "roberta"
-    assert lmf_specs[0].model_weights == "roberta-base"
-    assert lmf_specs[0].cache_dir == "/this/dir"
-
-
-def test_download_pretrained_unknown_model_name():
-    yaml = YAML(typ="safe")
-    config = yaml.load(CONFIG_FPATH)
-
-    steps = config.get("pipeline", [])
-    step = list(  # noqa: RUF015
-        filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps)
-    )[0]
-    step["model_name"] = "unknown"
-
-    with tempfile.NamedTemporaryFile("w+") as fp:
-        yaml.dump(config, fp)
-        fp.seek(0)
-        with pytest.raises(KeyError):
-            download_pretrained.get_model_name_and_weights_from_config(fp.name)
-
-
-def test_download_pretrained_multiple_model_names():
-    yaml = YAML(typ="safe")
-    config = yaml.load(CONFIG_FPATH)
-
-    steps = config.get("pipeline", [])
-    step = list(  # noqa: RUF015
-        filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps)
-    )[0]
-    step_new = deepcopy(step)
-    step_new["model_name"] = "roberta"
-    steps.append(step_new)
-
-    with tempfile.NamedTemporaryFile("w+") as fp:
-        yaml.dump(config, fp)
-        fp.seek(0)
-        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
-    assert len(lmf_specs) == 2
-    assert lmf_specs[1].model_name == "roberta"
-
-
-def test_download_pretrained_with_model_name_and_nondefault_weight():
-    yaml = YAML(typ="safe")
-    config = yaml.load(CONFIG_FPATH)
-
-    steps = config.get("pipeline", [])
-    step = list(  # noqa: RUF015
-        filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps)
-    )[0]
-    step["model_name"] = "bert"
-    step["model_weights"] = "bert-base-uncased"
-
-    with tempfile.NamedTemporaryFile("w+") as fp:
-        yaml.dump(config, fp)
-        fp.seek(0)
-        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
-    assert lmf_specs[0].model_name == "bert"
-    assert lmf_specs[0].model_weights == "bert-base-uncased"
-
-
-def test_download_pretrained_lmf_doesnt_exists():
-    yaml = YAML(typ="safe")
-    config = yaml.load(CONFIG_FPATH)
-
-    steps = config.get("pipeline", [])
-    step = list(  # noqa: RUF015
-        filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps)
-    )[0]
-    steps.remove(step)
-
-    with tempfile.NamedTemporaryFile("w+") as fp:
-        yaml.dump(config, fp)
-        fp.seek(0)
-        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
-    assert len(lmf_specs) == 0

AI/.github/tests/test_model_regression_test_read_dataset_branch_tmpl.py

@@ -1,27 +0,0 @@
-import pathlib
-import subprocess
-import pytest
-from typing import Text
-
-TEMPLATE_FPATH = ".github/templates/model_regression_test_read_dataset_branch.tmpl"
-REPO_DIR = pathlib.Path("").absolute()
-TEST_DATA_DIR = str(pathlib.Path(__file__).parent / "test_data")
-DEFAULT_DATASET_BRANCH = "main"
-
-
-@pytest.mark.parametrize(
-    "comment_body_file,expected_dataset_branch",
-    [
-        ("comment_body.json", "test_dataset_branch"),
-        ("comment_body_no_dataset_branch.json", DEFAULT_DATASET_BRANCH),
-    ],
-)
-def test_read_dataset_branch(comment_body_file: Text, expected_dataset_branch: Text):
-    cmd = (
-        "gomplate "
-        f"-d github={TEST_DATA_DIR}/{comment_body_file} "
-        f"-f {TEMPLATE_FPATH}"
-    )
-    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
-    output = output.decode("utf-8").strip()
-    assert output == f'export DATASET_BRANCH="{expected_dataset_branch}"'

AI/.github/tests/test_model_regression_test_results_tmpl.py

@@ -1,50 +0,0 @@
-import pathlib
-import subprocess
-
-TEMPLATE_FPATH = ".github/templates/model_regression_test_results.tmpl"
-REPO_DIR = pathlib.Path("").absolute()
-TEST_DATA_DIR = str(pathlib.Path(__file__).parent / "test_data")
-
-
-def test_comment_nlu():
-    cmd = (
-        "gomplate "
-        f"-d data={TEST_DATA_DIR}/report_listformat_nlu.json "
-        f"-d results_main={TEST_DATA_DIR}/report-on-schedule-2022-02-02.json "
-        f"-f {TEMPLATE_FPATH}"
-    )
-    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
-    output = output.decode("utf-8")
-    expected_output = """
-Dataset: `RasaHQ/financial-demo`, Dataset repository branch: `fix-model-regression-tests` (external repository), commit: `52a3ad3eb5292d56542687e23b06703431f15ead`
-Configuration repository branch: `main`
-| Configuration | Intent Classification Micro F1 | Entity Recognition Micro F1 | Response Selection Micro F1 |
-|---------------|-----------------|-----------------|-------------------|
-| `BERT + DIET(seq) + ResponseSelector(t2t)`<br> test: `1m29s`, train: `2m55s`, total: `4m24s`|1.0000 (0.00)|0.8333 (0.00)|`no data`|
-| `BERT + DIET(seq) + ResponseSelector(t2t)`<br> test: `2m29s`, train: `3m55s`, total: `5m24s`|1.0000 (0.00)|0.8333 (0.00)|`no data`|
-
-
-"""  # noqa E501
-    assert output == expected_output
-
-
-def test_comment_core():
-    cmd = (
-        "gomplate "
-        f"-d data={TEST_DATA_DIR}/report_listformat_core.json "
-        f"-d results_main={TEST_DATA_DIR}/report-on-schedule-2022-02-02.json "
-        f"-f {TEMPLATE_FPATH}"
-    )
-    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
-    output = output.decode("utf-8")
-    expected_output = """
-Dataset: `RasaHQ/retail-demo`, Dataset repository branch: `fix-model-regression-tests` (external repository), commit: `8226b51b4312aa4d3723098cf6d4028feea040b4`
-Configuration repository branch: `main`
-
-| Dialog Policy Configuration | Action Level Micro Avg. F1 | Conversation Level Accuracy | Run Time Train | Run Time Test |
-|---------------|-----------------|-----------------|-------------------|-------------------|
-| `Rules + Memo + TED` |1.0000 (0.00)|1.0000 (0.00)|`4m27s`| `31s`|
-| `Rules + Memo + TED` |1.0000 (0.00)|1.0000 (0.00)|`5m27s`| `41s`|
-
-"""  # noqa E501
-    assert output == expected_output

AI/.github/tests/test_mr_generate_summary.py

@@ -1,208 +0,0 @@
-import sys
-
-sys.path.append(".github/scripts")
-from mr_generate_summary import combine_result  # noqa: E402
-
-
-RESULT1 = {
-    "financial-demo": {
-        "BERT + DIET(bow) + ResponseSelector(bow)": [
-            {
-                "Entity Prediction": {
-                    "macro avg": {
-                        "f1-score": 0.7333333333333333,
-                    }
-                },
-                "test_run_time": "47s",
-            }
-        ]
-    }
-}
-
-
-def test_same_ds_different_config():
-    result2 = {
-        "financial-demo": {
-            "Sparse + DIET(bow) + ResponseSelector(bow)": [
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.88,
-                        }
-                    },
-                    "test_run_time": "47s",
-                }
-            ]
-        }
-    }
-    expected_combined = {
-        "financial-demo": {
-            "BERT + DIET(bow) + ResponseSelector(bow)": [
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.7333333333333333,
-                        }
-                    },
-                    "test_run_time": "47s",
-                }
-            ],
-            "Sparse + DIET(bow) + ResponseSelector(bow)": [
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.88,
-                        }
-                    },
-                    "test_run_time": "47s",
-                }
-            ],
-        }
-    }
-
-    actual_combined = combine_result(RESULT1, result2)
-    assert actual_combined == expected_combined
-
-    actual_combined = combine_result(result2, RESULT1)
-    assert actual_combined == expected_combined
-
-
-def test_different_ds_same_config():
-    result2 = {
-        "Carbon Bot": {
-            "Sparse + DIET(bow) + ResponseSelector(bow)": [
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.88,
-                        }
-                    },
-                    "test_run_time": "47s",
-                }
-            ]
-        }
-    }
-    expected_combined = {
-        "financial-demo": {
-            "BERT + DIET(bow) + ResponseSelector(bow)": [
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.7333333333333333,
-                        }
-                    },
-                    "test_run_time": "47s",
-                }
-            ],
-        },
-        "Carbon Bot": {
-            "Sparse + DIET(bow) + ResponseSelector(bow)": [
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.88,
-                        }
-                    },
-                    "test_run_time": "47s",
-                }
-            ]
-        },
-    }
-
-    actual_combined = combine_result(RESULT1, result2)
-    assert actual_combined == expected_combined
-
-    actual_combined = combine_result(result2, RESULT1)
-    assert actual_combined == expected_combined
-
-
-def test_start_empty():
-    result2 = {}
-    expected_combined = {
-        "financial-demo": {
-            "BERT + DIET(bow) + ResponseSelector(bow)": [
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.7333333333333333,
-                        }
-                    },
-                    "test_run_time": "47s",
-                }
-            ]
-        }
-    }
-
-    actual_combined = combine_result(RESULT1, result2)
-    assert actual_combined == expected_combined
-
-    actual_combined = combine_result(result2, RESULT1)
-    assert actual_combined == expected_combined
-
-
-def test_combine_result_repetition():
-    expected_combined = {
-        "financial-demo": {
-            "BERT + DIET(bow) + ResponseSelector(bow)": [
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.7333333333333333,
-                        }
-                    },
-                    "test_run_time": "47s",
-                },
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.7333333333333333,
-                        }
-                    },
-                    "test_run_time": "47s",
-                },
-            ]
-        }
-    }
-
-    actual_combined = combine_result(RESULT1, RESULT1)
-    assert actual_combined == expected_combined
-
-
-def test_combine_result_repetition_3times():
-    expected_combined = {
-        "financial-demo": {
-            "BERT + DIET(bow) + ResponseSelector(bow)": [
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.7333333333333333,
-                        }
-                    },
-                    "test_run_time": "47s",
-                },
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.7333333333333333,
-                        }
-                    },
-                    "test_run_time": "47s",
-                },
-                {
-                    "Entity Prediction": {
-                        "macro avg": {
-                            "f1-score": 0.7333333333333333,
-                        }
-                    },
-                    "test_run_time": "47s",
-                },
-            ]
-        }
-    }
-
-    tmp_combined = combine_result(RESULT1, RESULT1)
-    actual_combined = combine_result(tmp_combined, RESULT1)
-    assert actual_combined == expected_combined
-
-    actual_combined = combine_result(RESULT1, tmp_combined)
-    assert actual_combined == expected_combined

AI/.github/tests/test_mr_publish_results.py

@@ -1,132 +0,0 @@
-import os
-from pathlib import Path
-import sys
-from unittest import mock
-
-sys.path.append(".github/scripts")
-from mr_publish_results import (  # noqa: E402
-    prepare_ml_metric,
-    prepare_ml_metrics,
-    transform_to_seconds,
-    generate_json,
-    prepare_datadog_tags,
-)
-
-EXAMPLE_CONFIG = "Sparse + BERT + DIET(seq) + ResponseSelector(t2t)"
-EXAMPLE_DATASET_NAME = "financial-demo"
-
-ENV_VARS = {
-    "BRANCH": "my-branch",
-    "PR_ID": "10927",
-    "PR_URL": "https://github.com/RasaHQ/rasa/pull/10856/",
-    "GITHUB_EVENT_NAME": "pull_request",
-    "GITHUB_RUN_ID": "1882718340",
-    "GITHUB_SHA": "abc",
-    "GITHUB_WORKFLOW": "CI - Model Regression",
-    "IS_EXTERNAL": "false",
-    "DATASET_REPOSITORY_BRANCH": "main",
-    "CONFIG": EXAMPLE_CONFIG,
-    "DATASET_NAME": EXAMPLE_DATASET_NAME,
-    "CONFIG_REPOSITORY_BRANCH": "main",
-    "DATASET_COMMIT": "52a3ad3eb5292d56542687e23b06703431f15ead",
-    "ACCELERATOR_TYPE": "CPU",
-    "TEST_RUN_TIME": "1m54s",
-    "TRAIN_RUN_TIME": "4m4s",
-    "TOTAL_RUN_TIME": "5m58s",
-    "TYPE": "nlu",
-    "INDEX_REPETITION": "0",
-    "HOST_NAME": "github-runner-2223039222-22df222fcd-2cn7d",
-}
-
-
-@mock.patch.dict(os.environ, ENV_VARS, clear=True)
-def test_generate_json():
-    f = Path(__file__).parent / "test_data" / "intent_report.json"
-    result = generate_json(f, task="intent_classification", data={})
-    assert isinstance(result[EXAMPLE_DATASET_NAME][EXAMPLE_CONFIG], list)
-
-    actual = result[EXAMPLE_DATASET_NAME][EXAMPLE_CONFIG][0]["intent_classification"]
-    expected = {
-        "accuracy": 1.0,
-        "weighted avg": {
-            "precision": 1.0,
-            "recall": 1.0,
-            "f1-score": 1.0,
-            "support": 28,
-        },
-        "macro avg": {"precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 28},
-    }
-    assert expected == actual
-
-
-def test_transform_to_seconds():
-    assert 87.0 == transform_to_seconds("1m27s")
-    assert 87.3 == transform_to_seconds("1m27.3s")
-    assert 27.0 == transform_to_seconds("27s")
-    assert 3627.0 == transform_to_seconds("1h27s")
-    assert 3687.0 == transform_to_seconds("1h1m27s")
-
-
-def test_prepare_ml_model_perf_metrics():
-    results = [
-        {
-            "macro avg": {
-                "precision": 0.8,
-                "recall": 0.8,
-                "f1-score": 0.8,
-                "support": 14,
-            },
-            "micro avg": {
-                "precision": 1.0,
-                "recall": 0.7857142857142857,
-                "f1-score": 0.88,
-                "support": 14,
-            },
-            "file_name": "DIETClassifier_report.json",
-            "task": "Entity Prediction",
-        },
-        {
-            "accuracy": 1.0,
-            "weighted avg": {
-                "precision": 1.0,
-                "recall": 1.0,
-                "f1-score": 1.0,
-                "support": 28,
-            },
-            "macro avg": {
-                "precision": 1.0,
-                "recall": 1.0,
-                "f1-score": 1.0,
-                "support": 28,
-            },
-            "file_name": "intent_report.json",
-            "task": "Intent Classification",
-        },
-    ]
-    metrics_ml = prepare_ml_metrics(results)
-    assert len(metrics_ml) == 17
-
-
-def test_prepare_ml_model_perf_metrics_simple():
-    result = {
-        "accuracy": 1.0,
-        "weighted avg": {"precision": 1, "recall": 1.0, "f1-score": 1, "support": 28},
-        "task": "Intent Classification",
-    }
-    metrics_ml = prepare_ml_metric(result)
-    assert len(metrics_ml) == 5
-
-    for _, v in metrics_ml.items():
-        assert isinstance(v, float)
-
-    key, value = "Intent Classification.accuracy", 1.0
-    assert key in metrics_ml and value == metrics_ml[key]
-
-    key, value = "Intent Classification.weighted avg.f1-score", 1.0
-    assert key in metrics_ml and value == metrics_ml[key]
-
-
-@mock.patch.dict(os.environ, ENV_VARS, clear=True)
-def test_prepare_datadog_tags():
-    tags_list = prepare_datadog_tags()
-    assert "dataset:financial-demo" in tags_list

AI/.github/tests/test_validate_gpus.py

@@ -1,27 +0,0 @@
-import os
-import sys
-from unittest import mock
-
-import pytest
-
-sys.path.append(".github/scripts")
-import validate_cpu  # noqa: E402
-import validate_gpus  # noqa: E402
-
-ENV_VARS = {
-    "CUDA_VISIBLE_DEVICES": "-1",
-}
-
-
-@mock.patch.dict(os.environ, ENV_VARS, clear=True)
-def test_validate_cpu_succeeds_when_there_are_no_gpus():
-    validate_cpu.check_gpu_not_available()
-
-
-@mock.patch.dict(os.environ, ENV_VARS, clear=True)
-def test_validate_gpus_exits_when_there_are_no_gpus():
-    # This unit test assumes that unit tests are run on a CPU
-    with pytest.raises(SystemExit) as pytest_wrapped_e:
-        validate_gpus.check_gpu_available()
-    assert pytest_wrapped_e.type == SystemExit
-    assert pytest_wrapped_e.value.code == 1

AI/.github/workflows/automatic-pr-update.yml

@@ -1,22 +0,0 @@
-name: Automatic PR Merger
-
-on:
-  push: {} # update PR when base branch is updated
-
-jobs:
-  # thats's all. single step is needed - if PR is mergeable according to
-  # branch protection rules it will be merged automatically
-  mergepal:
-    runs-on: ubuntu-22.04
-    if: github.repository == 'RasaHQ/rasa'
-
-    steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
-      - uses: rasahq/update-pr-branch@f7012036a6d5659cfbc37f180716963511e81f95
-        with:
-          token: ${{ secrets.UPDATE_BRANCH_PAT }}
-          # required parameter by original action -
-          # check is already done through protected branches so not needed for us
-          required_approval_count: 0
-          # update branch despite failing check runs
-          require_passed_checks: false

AI/.github/workflows/ci-docs-tests.yml

@@ -1,164 +0,0 @@
-name: Docs Tests
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    types: [opened, synchronize, labeled]
-
-concurrency:
-  group: ci-docs-tests-${{ github.ref }} # branch name
-  cancel-in-progress: true
-
-env:
-  DEFAULT_PYTHON_VERSION: "3.10"
-
-jobs:
-  changes:
-    name: Check for file changes
-    runs-on: ubuntu-22.04
-    outputs:
-      docs: ${{ steps.filter.outputs.docs }}
-    steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
-      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50
-        id: filter
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          filters: .github/change_filters.yml
-
-  test_documentation:
-    name: Test Documentation
-    runs-on: ubuntu-22.04
-    needs: [changes]
-    if: needs.changes.outputs.docs == 'true'
-
-    steps:
-      - name: Checkout git repository 🕝
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
-
-      - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
-        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
-        with:
-          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
-
-      - name: Set up Node 12.x 🦙
-        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c
-        with:
-          node-version: "12.x"
-
-      - name: Read Poetry Version 🔢
-        run: |
-          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
-        shell: bash
-
-      - name: Install poetry 🦄
-        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
-        with:
-          poetry-version: ${{ env.POETRY_VERSION }}
-
-      - name: Load Poetry Cached Libraries ⬇
-        id: cache-poetry
-        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
-        with:
-          path: .venv
-          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
-          restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
-
-      - name: Clear Poetry cache
-        if: steps.cache-poetry.outputs.cache-hit == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-docs-tests')
-        run: rm -r .venv
-
-      - name: Create virtual environment
-        if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-docs-tests'))
-        run: python -m venv create .venv
-
-      - name: Set up virtual environment
-        if: needs.changes.outputs.docs == 'true'
-        run: poetry config virtualenvs.in-project true
-
-      - name: Load Yarn Cached Packages ⬇
-        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
-        with:
-          path: docs/node_modules
-          key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }}
-          restore-keys: ${{ runner.os }}-yarn-12.x
-
-      - name: Install Dependencies 📦
-        run: |
-          sudo apt-get -y install libpq-dev
-          make install-full install-docs
-
-      - name: Run Swagger 🕵️‍♀️
-        run: |
-          npm install -g swagger-cli
-          swagger-cli validate docs/static/spec/action-server.yml
-          swagger-cli validate docs/static/spec/rasa.yml
-
-      - name: Test Docs 🕸
-        run: make test-docs
-
-  documentation_lint:
-    name: Documentation Linting Checks
-    runs-on: ubuntu-22.04
-    needs: [changes]
-    if: needs.changes.outputs.docs == 'true'
-
-    steps:
-      - name: Checkout git repository 🕝
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
-
-      - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
-        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
-        with:
-          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
-
-      - name: Set up Node 12.x 🦙
-        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c
-        with:
-          node-version: "12.x"
-
-      - name: Read Poetry Version 🔢
-        run: |
-          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
-        shell: bash
-
-      - name: Install poetry 🦄
-        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
-        with:
-          poetry-version: ${{ env.POETRY_VERSION }}
-
-      - name: Load Poetry Cached Libraries ⬇
-        id: cache-poetry
-        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
-        with:
-          path: .venv
-          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
-          restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
-
-      - name: Clear Poetry cache
-        if: steps.cache-poetry.outputs.cache-hit == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-docs-tests')
-        run: rm -r .venv
-
-      - name: Create virtual environment
-        if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-docs-tests'))
-        run: python -m venv create .venv
-
-      - name: Set up virtual environment
-        if: needs.changes.outputs.docs == 'true'
-        run: poetry config virtualenvs.in-project true
-
-      - name: Load Yarn Cached Packages ⬇
-        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
-        with:
-          path: docs/node_modules
-          key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }}
-          restore-keys: ${{ runner.os }}-yarn-12.x
-
-      - name: Install Dependencies 📦
-        run: |
-          sudo apt-get -y install libpq-dev
-          make install-full install-docs
-
-      - name: Docs Linting Checks 🕸
-        run: make lint-docs