Upload 87 files

bitagent_subnet-main/.circleci/config.yml

@@ -0,0 +1,168 @@
+version: 2.1
+
+orbs:
+  python: circleci/python@2.1.1
+  python-lib: dialogue/python-lib@0.1.55
+  # coveralls: coveralls/coveralls@1.0.6
+
+jobs:
+  black:
+    resource_class: small
+    parameters:
+      python-version:
+        type: string
+    docker:
+      - image: cimg/python:<< parameters.python-version >>
+
+    steps:
+      - checkout
+
+      - restore_cache:
+          name: Restore cached black venv
+          keys:
+            - v1-pypi-py-black-<< parameters.python-version >>
+
+      - run:
+          name: Update & Activate black venv
+          command: |
+            python -m venv env/
+            . env/bin/activate
+            python -m pip install --upgrade pip
+            pip install black
+
+      - save_cache:
+          name: Save cached black venv
+          paths:
+            - "env/"
+          key: v1-pypi-py-black-<< parameters.python-version >>
+
+      - run:
+          name: Black format check
+          command: |
+            . env/bin/activate
+            black --line-length 79 --exclude '(env|venv|.eggs)' --check .
+
+  pylint:
+    resource_class: small
+    parameters:
+      python-version:
+        type: string
+    docker:
+      - image: cimg/python:<< parameters.python-version >>
+
+    steps:
+      - checkout
+
+      - run:
+          name: Install Pylint
+          command: |
+            python -m venv env/
+            . env/bin/activate
+            pip install pylint
+
+      - run:
+          name: Pylint check
+          command: |
+            . env/bin/activate
+            pylint --fail-on=W,E,F --exit-zero  ./
+
+  check_compatibility:
+    parameters:
+      python_version:
+        type: string
+    docker:
+      - image: cimg/python:3.10
+    steps:
+      - checkout
+      - run:
+          name: Check if requirements files have changed
+          command: ./scripts/check_requirements_changes.sh
+      - run:
+          name: Install dependencies and Check compatibility
+          command: |
+            if [ "$REQUIREMENTS_CHANGED" == "true" ]; then
+              sudo apt-get update
+              sudo apt-get install -y jq curl
+              ./scripts/check_compatibility.sh << parameters.python_version >>
+            else
+              echo "Skipping compatibility checks..."
+            fi
+
+  build:
+    resource_class: medium
+    parallelism: 2
+    parameters:
+      python-version:
+        type: string
+    docker:
+      - image: cimg/python:<< parameters.python-version >>
+
+    steps:
+      - checkout
+
+      - restore_cache:
+          name: Restore cached venv
+          keys:
+            - v1-pypi-py<< parameters.python-version >>-{{ checksum "requirements.txt" }}
+            - v1-pypi-py<< parameters.python-version >>
+
+      - run:
+          name: Update & Activate venv
+          command: |
+            python -m venv env/
+            . env/bin/activate
+            python -m pip install --upgrade pip
+
+      - save_cache:
+          name: Save cached venv
+          paths:
+            - "env/"
+          key: v1-pypi-py<< parameters.python-version >>-{{ checksum "requirements.txt" }}
+
+      - run:
+          name: Install Bittensor Subnet Template
+          command: |
+            . env/bin/activate
+            pip install -e .
+
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
+  coveralls:
+    docker:
+      - image: cimg/python:3.10
+    steps:
+      - run:
+          name: Combine Coverage
+          command: |
+            pip3 install --upgrade coveralls
+            coveralls --finish --rcfile .coveragerc || echo "Failed to upload coverage"
+
+workflows:
+  compatibility_checks:
+    jobs:
+      - check_compatibility:
+          python_version: "3.8"
+          name: check-compatibility-3.8
+      - check_compatibility:
+          python_version: "3.9"
+          name: check-compatibility-3.9
+      - check_compatibility:
+          python_version: "3.10"
+          name: check-compatibility-3.10
+      - check_compatibility:
+          python_version: "3.11"
+          name: check-compatibility-3.11
+
+  pr-requirements:
+    jobs:
+      - black:
+          python-version: "3.8.12"
+      - pylint:
+          python-version: "3.8.12"
+      - build:
+          matrix:
+            parameters:
+              python-version: ["3.9.13", "3.10.6", "3.11.4"]

bitagent_subnet-main/.gitignore

@@ -0,0 +1,184 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+.venvsglang
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+testing/
+env/
+app.config.js
+mnemonics.txt
+run_it_all.sh
+
+wandb
+TODO
+bitagent.data*
+.vscode
+repo
+.cometml-runs
+old.*
+
+node_modules
+package-lock.json
+package.json
+*net.py
+*.old
+
+notebooks
+Notebooks

bitagent_subnet-main/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2023 Opentensor
+Copyright (c) 2023 RogueTensor
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bitagent_subnet-main/README.md

@@ -0,0 +1,474 @@
+<div align="center">
+
+# **BitAgent Subnet (#20) on Bittensor** <!-- omit in toc -->
+[![Discord Chat](https://img.shields.io/discord/308323056592486420.svg)](https://discord.com/channels/799672011265015819/1175085112703078400)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 
+
+---
+
+## Agency for Your World Through Natural Language <!-- omit in toc -->
+
+**Communications:** [BitAgent Discord](https://discord.com/channels/799672011265015819/1194736998250975332)\
+**Downstream Applications:** [GoGoAgent](https://gogoagent.ai/) • [MSP Tech](https://msptech.ai)
+</div>
+
+---
+- [Introduction](#introduction)
+- [Get Running](#get-running)
+  - [BitAgent](#bitagent)
+  - [Validator](#validator)
+    - [Dependencies](#dependencies)
+    - [Installation](#installation)
+    - [vLLM Setup for Validators](#vllm-setup-for-validators)
+    - [sglang Setup for Validators](#sglang-setup-for-validators)
+    - [Recommended Startup](#recommended-startup)
+    - [Alternative Startup](#alternative-startup)
+    - [Verify Validator is Working](#verify-validator-is-working)
+    - [Hardware Requirements](#validator-hardware-requirements)
+  - [Miner](#miner)
+    - [Hardware Requirements](#miner-hardware-requirements)
+    - [Default Miner](#default-miner)
+    - [Miner Emissions](#miner-emissions)
+    - [Miner Considerations](#miner-considerations)
+    - [Example Task](#example-task)
+    - [Miner Feedback](#miner-feedback)
+  - [Advanced](#advanced)
+- [FAQ](#faq)
+- [License](#license)
+
+## Introduction
+
+**Quick Pitch**: BitAgent revolutionizes how you manage tasks and workflows across platforms, merging the capabilities of large language models (LLMs) with the convenience of your favorite apps such as web browsers, Discord, and custom integrations. BitAgent empowers users to seamlessly integrate intelligent agents, providing personalized assistance and integrated task automation.
+
+**Key Objective** - provide intelligent agency to simplify and automate tasks in your day-to-day
+
+**GoGoAgent - Our Application** - [https://gogoagent.ai](https://gogoagent.ai) \
+**MSPTech - Real world business case** - [https://MSPTech.ai](https://msptech.ai)
+
+**Key Features**
+- Working our way up the [Berkeley Function Calling Leaderboard](https://gorilla.cs.berkeley.edu/leaderboard.html#leaderboard) (BFCL)
+- No API / subscription requirements
+- Run light models (8B parameter) for huge impact
+- FINETUNED MODEL evaluation of tool calling language model fine tunes
+- MINER HOSTED evaluation of miners running tool calling language models allowing applications to scale on top of SN20
+- Miner's receive [transparent feedback](#miner-feedback)
+- And a BONUS for getting this far - are you tired of waiting for registration slots?  Check out [register.sh](./scripts/register.sh)
+
+---
+
+## Get Running
+
+- BitAgent is a competitive subnet, meaning miners succeed and fail based on how well they perform on tasks.
+- **Make sure to test your miner on Testnet 76 before ever considering registering for Subnet 20.**
+- Newly registered miners will start at the median score per validator and go up or down depending on their performance.
+- Before getting too far, please make sure you've looked over the [Bittensor documentation](https://docs.bittensor.com/) for your needs.
+- The min compute requirements are [noted below for Validators](#hardware-requirements).
+- See [FAQ](#faq) for a few more details related to computing requirements for validators and miners.
+- The minimum requirements for a miner are determined by the resources needed to run a competitive and performant tool calling LLM.
+
+### BitAgent
+This repository requires python 3.10 or higher. 
+To install and get running, simply clone this repository and install the requirements.
+```bash
+git clone https://github.com/RogueTensor/bitagent_subnet
+cd bitagent_subnet
+# at this point, it's recommended that you use a venv, but not required; the next two lines are venv specific
+python -m venv .venv #replace .venv with the name you'd like to use for your primary venv
+source ./.venv/bin/activate
+python -m pip install -e .
+```
+
+Then make sure to register your intended wallet (coldkey, hotkey) to Subnet 20:
+```bash
+btcli subnet register --wallet.path <YOUR PATH: e.g., ~/.bittensor/wallets> --wallet.name $coldkey --wallet.hotkey $hotkey --subtensor.network finney --netuid 20
+```
+
+### Validator
+
+#### Dependencies
+
+You must have the following things:
+
+- System with at least 48gb of VRAM
+- Python >=3.10
+- Docker with [gpu support](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)
+
+#### Installation
+
+Ensure that you have Docker with GPU support, you can choose to follow either of the instructions:
+
+- [Official Guide](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) 
+- [Quick and Dirty Stack Overflow Guide](https://stackoverflow.com/questions/75118992/docker-error-response-from-daemon-could-not-select-device-driver-with-capab)
+
+
+Install [PM2](https://pm2.io/docs/runtime/guide/installation/) and the [`jq` package](https://jqlang.github.io/jq/) on your system.\
+   **On Linux**:
+   ```bash
+   sudo apt update && sudo apt install jq && sudo apt install npm && sudo npm install pm2 -g && pm2 update
+   ``` 
+   **On Mac OS**
+   ```bash
+   brew update && brew install jq && brew install npm && sudo npm install pm2 -g && pm2 update
+   ```
+
+#### vLLM Setup for Validators
+
+Validators must spin-up their own LLM (specifically mistral 7B).
+Note: Previously we ran the LLM's inside the validator code with the transformer package, however we pivoted away from that due to the inefficiency of running the model using vanilla transformers. Hosting the models using llama.cpp, oobabooga, vllm, TGI, are much better options as they provide additional functionality.  
+
+To run with vLLM you can do the following:
+
+```bash
+sudo docker run -d -p 8000:8000  --gpus all --ipc host --name mistral-instruct docker.io/vllm/vllm-openai:latest --model thesven/Mistral-7B-Instruct-v0.3-GPTQ --max-model-len 8912 --quantization gptq --dtype half --gpu-memory-utilization 0.45
+```
+
+This will run the LLM on port 8000. To change the port, change the host port for this parameter up above `-p <host port>:<container port>`. And use `--openai-api-base http://localhost:<new_port>/v1` in your params to point to the vLLM model for SN20.
+
+#### sglang Setup for Validators
+
+You'll need to create a virtual env and install the requirements for sglang:
+```bash
+python3 -m venv .venvsglang
+# note to change cu121 in this path according to this page: https://docs.flashinfer.ai/installation.html
+./.venvsglang/bin/pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.4/ 
+./.venvsglang/bin/pip install -r requirements.sglang.txt
+```
+
+**Test that it's working with:**
+```
+.venvsglang/bin/python -m sglang.launch_server --model-path Salesforce/xLAM-7b-r --port 8028 --host 0.0.0.0 --mem-fraction-static 0.40
+```
+
+You should not run out of memory and it should eventually show that the Salesforce model loaded correclty.
+
+#### Recommended Startup
+
+Make sure you do the [vLLM setup](#vllm-setup-for-validators) above and the [sglang setup](#sglang-setup-for-validators) above.
+
+```bash
+# for mainnet with AUTO UPDATES (recommended)
+pm2 start run.sh --name bitagent_validators_autoupdate -- --wallet.path <YOUR PATH: e.g., ~/.bittensor/wallets> --wallet.name <your-wallet-name> --wallet.hotkey <your-wallet-hot-key> --netuid 20
+```
+
+Double check everything is working by following [these steps](#verify-validator-is-working).
+
+#### Alternative Startup
+
+Make sure you do the [vLLM setup](#vllm-setup-for-validators) above and the [sglang setup](#sglang-setup-for-validators) above.
+
+```bash
+# for testnet
+python3 neurons/validator.py --netuid 76 --subtensor.network test --wallet.path <YOUR PATH: e.g., ~/.bittensor/wallets> --wallet.name <COLDKEY> --wallet.hotkey <HOTKEY>
+
+# for mainnet
+pm2 start neurons/validator.py --interpreter python3 -- --netuid 20 --subtensor.network <LOCAL/FINNEY/TEST> --wallet.path <YOUR PATH: e.g., ~/.bittensor/wallets> --wallet.name <COLDKEY> --wallet.hotkey <HOTKEY> --axon.port <PORT>
+```
+
+Double check everything is working by following [these steps](#verify-validator-is-working).
+
+#### Verify Validator is Working
+
+After you've launched and pm2 is running, here's what to expect:\
+- You'll see a LOT (one per mind) of IsAlive() queries like this:\
+  ```bash
+  1|bitagent | 2024-11-17 23:25:59.156 |      TRACE       | bittensor:loggingmachine.py:432 | dendrite | <-- | 3354 B | IsAlive | 5GbnkQJ6zfsWa9iX4ZtwKccXZv4s8MTt2LSQFmS8CMgjkSgx | 213.180.0.45:20019 | 200 | Success
+  1|bitagent | 2024-11-17 23:26:04.135 |      TRACE       | bittensor:loggingmachine.py:432 | dendrite | <-- | 3327 B | IsAlive | 5E7eqUChR4WUnRwNAUXRNUZhhjEzTfdeGAvDyf99aygVGYBJ | 176.55.1.98:8091 | 408 | Request timeout after 5.0 seconds
+  1|bitagent | 2024-11-17 23:26:04.180 |      TRACE       | bittensor:loggingmachine.py:432 | dendrite | <-- | 3331 B | IsAlive | 5EHQoRqwMHG3QVVpsSZBPHJD87SEwGn6FhTSR3LCj8XiHVUC | 109.206.196.130:8888 | 408 | Request timeout after 5.0 seconds
+  ```
+- After the IsAlive() queries, you'll start to see QueryTask queries followed by QueryResult queries, like these:\
+  ```bash
+  1|bitagent | 2024-11-17 23:53:20.322 |      ERROR       | bittensor:loggingmachine.py:457 |  - ContentTypeError#aefadd84-8586-4faa-9206-e048c2b85114: 404, message='Attempt to decode JSON with unexpected mimetype: text/html', url='http://52.220.128.145:32222/QueryTask' - 
+  1|bitagent | 2024-11-17 23:53:20.323 |      TRACE       | bittensor:loggingmachine.py:432 | dendrite | <-- | 27205 B | QueryTask | 5GjGiziPatj7mf4is5JaDPJq4jbPnagoeiSHe4TfFERafM7X | 52.220.128.145:32222 | 422 | Failed to parse response: 404, message='Attempt to decode JSON with unexpected mimetype: text/html', url='http://52.220.128.145:32222/QueryTask'
+  1|bitagent | 2024-11-17 23:53:21.708 |      TRACE       | bittensor:loggingmachine.py:432 | dendrite | <-- | 27522 B | QueryTask | 5GbnkQJ6zfsWa9iX4ZtwKccXZv4s8MTt2LSQFmS8CMgjkSgx | 213.180.0.45:20019 | 500 | Internal Server Error #b36dc761-1035-44d0-b88d-24fe9ccc7e1e
+  1|bitagent | 2024-11-17 23:53:21.806 |      TRACE       | bittensor:loggingmachine.py:432 | dendrite | --> | 5418 B | QueryResult | 5GbnkQJ6zfsWa9iX4ZtwKccXZv4s8MTt2LSQFmS8CMgjkSgx | 213.180.0.45:20019 | 0 | Success
+  1|bitagent | 2024-11-17 23:53:23.200 |      TRACE       | bittensor:loggingmachine.py:432 | dendrite | <-- | 5578 B | QueryResult | 5GbnkQJ6zfsWa9iX4ZtwKccXZv4s8MTt2LSQFmS8CMgjkSgx | 213.180.0.45:20019 | 200 | Success
+
+  ```
+- These logs above let you know that the ONLINE / MINER HOSTED querying is working.
+- Finally, you'll want to check the miners' HF (hugging face) models are being evaluated OFFLINE.
+- You'll want to check your `pm2 log <ID> | grep OFFLINE` output for lines like these (from testnet):\
+  ```bash
+  1|bitagent | 2024-11-17 23:26:07.154 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Starting offline mode for competition 1-1
+  1|bitagent | 2024-11-17 23:26:08.831 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Starting offline task
+  1|bitagent | 2024-11-17 23:26:12.529 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Miner HF model names: [None, 'Salesforce/xLAM-7b-r']
+  1|bitagent | 2024-11-17 23:26:12.529 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Unique miner HF model names: ['Salesforce/xLAM-7b-r']
+  1|bitagent | 2024-11-17 23:26:12.529 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Generating tasks
+  1|bitagent | 2024-11-17 23:28:21.793 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Generated 1000 tasks of 1000 total
+  1|bitagent | 2024-11-17 23:28:21.793 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Running tasks for model Salesforce/xLAM-7b-r
+  1|bitagent | 2024-11-17 23:28:21.939 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Starting server for model Salesforce/xLAM-7b-r
+  1|bitagent | 2024-11-17 23:28:21.941 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Started server for model Salesforce/xLAM-7b-r, waiting for it to start on port 8028 (could take several minutes)
+  1|bitagent | 2024-11-17 23:29:25.469 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Server for model Salesforce/xLAM-7b-r started
+  1|bitagent | 2024-11-17 23:29:25.470 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Getting LLM responses for model Salesforce/xLAM-7b-r
+  1|bitagent | 2024-11-17 23:38:54.257 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Got 1000 LLM responses for model: Salesforce/xLAM-7b-r
+  1|bitagent | 2024-11-17 23:38:54.258 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Terminating server for model: Salesforce/xLAM-7b-r
+  1|bitagent | 2024-11-17 23:38:54.965 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Terminated server for model: Salesforce/xLAM-7b-r
+  1|bitagent | 2024-11-17 23:38:55.030 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Processing rewards for model: Salesforce/xLAM-7b-r, for miners: [160]
+  1|bitagent | 2024-11-17 23:38:58.530 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE Scattered rewards: [np.float64(0.16442660138718893)]
+  1|bitagent | 2024-11-17 23:38:58.531 |      DEBUG       | bittensor:loggingmachine.py:437 | Updated moving avg OFFLINE scores for Competition 1-1: [-0.5       -0.5       -0.5       -0.5       -0.5       -0.5
+  1|bitagent | 2024-11-17 23:38:58.533 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Deleting model from HF cache: Salesforce/xLAM-7b-r
+  1|bitagent | 2024-11-17 23:39:01.198 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Model 'Salesforce/xLAM-7b-r' has been removed from the cache.
+  1|bitagent | 2024-11-17 23:39:01.199 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Finished processing offline tasks
+  1|bitagent | 2024-11-17 23:39:02.765 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Starting offline mode for competition 1-1
+  1|bitagent | 2024-11-17 23:39:03.147 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Starting offline task
+  1|bitagent | 2024-11-17 23:39:03.638 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: Miner HF model names: [None]
+  1|bitagent | 2024-11-17 23:39:03.638 |      DEBUG       | bittensor:loggingmachine.py:437 | OFFLINE: No unique miner HF model names to evaluate in OFFLINE mode
+  ```
+- If you're seeing all of this output, your validator is working!
+
+#### Validator Hardware Requirements
+
+Validators have hardware requirements. Two LLMS are needed to be run simultaneously:
+  - 1st LLM `thesven/Mistral-7B-Instruct-v0.3-GPTQ` can run off of 10GB to 20GB of VRAM - this model is used to alter tasks before going out to miners.
+  - 2nd LLM is each miner's tool calling model fetched from Hugging Face, one at a time to be evaluated OFFLINE for FINETUNED SUBMISSION and takes up 20GB to 30GB of VRAM.
+
+### Miner
+If you just want to run the miner without the [script](./scripts/setup_and_run.sh) or are connecting to mainnet:
+```bash
+# for testing (use testnet 76)
+python3 neurons/miner.py --netuid 76 --subtensor.network test --wallet.path <YOUR PATH: e.g., ~/.bittensor/wallets> --wallet.name <COLDKEY> --wallet.hotkey <HOTKEY>
+# for mainnet
+pm2 start neurons/miner.py --interpreter python3 --
+    --netuid 20
+    --subtensor.network <finney/local/test>
+    --neuron.device cuda # could be cuda:0, cuda:1 depending on which GPU device
+    --wallet.path <YOUR PATH: e.g., ~/.bittensor/wallets> # 8.2.0 has a bug that requires wallet path to be provided
+    --wallet.name <your wallet> # Must be created using the bittensor-cli
+    --wallet.hotkey <your hotkey> # Must be created using the bittensor-cli
+    --miner-hf-model-name-to-submit Salesforce/xLAM-7b-r # submit your own fine tune with this param
+    --hf-model-name-to-run Salesforce/xLAM-7b-r # run the best tool calling LLM you can
+    --openai-api-base http://localhost:8000/v1 # point to your vllm instance of the model you are running
+    --logging.debug # Run in debug mode, alternatively --logging.trace for trace mode
+    --log_level trace # for trace logs
+    --axon.port # VERY IMPORTANT: set the port to be one of the open TCP ports on your machine
+
+```
+#### Miner Hardware Requirements
+Miners will need to run a top tool calling LLM or a fine-tune of their own, needing a GPU with 20GB to 30GB of VRAM. 
+
+#### Default Miner
+The default miner is all you need with these modifications:
+1) `--miner-hf-model-name-to-submit` - set this to the HF model path and repo name from Hugging Face (HF).  \
+   Example: `--miner-hf-model-name-to-submit Salesforce/xLAM-7b-r`
+2) `--hf-model-name-to-run` - this is the model the miner is running to respond to queries that are sent to the miner. \
+   Example: `--hf-model-name-to-run Salesforce/xLAM-7b-r`
+3) `--openai-api-base` - this sets the vLLM endpoint that's running your local model. \
+   Example: `--openai-api-base http://localhost:8000/v1`
+
+See [Miner Configuration Considerations](#miner-configuration-considerations) for common areas miners should look to improve.
+
+#### Miner Emissions
+
+Miner emissions are composed of both MINER-HOSTED and FINETUNED SUBMISSION evaluation:
+- 20% of the miner's score is determined by the persistent availability of the miners and their response to on-demand queries  This is MINER-HOSTED evaluation of the miner.
+- 80% is determined by bi-weekly challenges in which the miner submits their latest huggingface model and Validators load the model on their machine to evaluate. This is FINETUNED SUBMISSION evaluation. This 80% portion serves as a delayed incentive mechanism, meaning it is always based on miner/model performance from the PREVIOUS competition.
+
+Both MINER-HOSTED and FINETUNED SUBMISSION tasks are evaluated against modifications of these datasets:
+- Berkeley Function Calling tasks
+- Glaive Function Calling tasks
+- BitAgent Function calling tasks
+
+The Bi-weekly challenge is to finetune an 8B model (or less) to perform well on the tool calling tasks and perform well on the [BFCL Leaderboard](https://gorilla.cs.berkeley.edu/leaderboard.html). Miners must publish their model to HuggingFace and update their `--miner-hf-model-name-to-subnet` parameter when starting/restarting their miner - see [Default Miner](#default-miner)
+
+#### Miner Registration Considerations
+
+Due to the delayed incentive mechanism of finetuned model evaluation, miners are not recommended to register during the middle of a competition. This is because miners registering mid-competition will not have a score from the prior competition, making them unable to benefit from the 80% incentive calculation.
+- It is recommended that miners register on the day a competition ends (prior to the actual time of competition close). The competitions end on the midnight (00:00) UTC between Monday and Tuesday every two weeks starting from 11-5-2024.
+- Registering on the competition end date (within 16 hours of the deadline) ensures that the miner's 16-hour immunity will be used for model submission grading and scoring in the current competition. Additionally, while the miner is still immune, the competition will roll over into the next cycle, and the miner's score will be finalized for the incentive calculation for the entire next competition cycle.
+- On the day of the competition's end, registration slots are expected to be extremely competitive. Due to substrate constraints, at most three miners can register per hour. If you're receiving error messages while registering, this is why and you will need to keep trying.
+
+#### Miner Configuration Considerations
+The default miner is all you need, just make sure you update the parameters described in [Default Miner](#default-miner).  
+For your consideration:
+1) Use vLLM as a fast inference runner for your tool calling LLM. Check [this](https://docs.vllm.ai/en/v0.6.0/getting_started/quickstart.html#openai-compatible-server) out to stand up an openAI compliant vLLM instance.
+2) Use pm2 to launch your miner for easy management and reconfiguration as needed.
+3) We use [SGLang](https://sgl-project.github.io/start/install.html) to run your hugging face models, please make sure your model loads with SGLang.
+4) Don't make it obvious to other miners where your HuggingFace submission is, manage this discretely.
+
+
+#### Example Task
+Here's an example task you can expect your model to see in FINETUNED SUBMISSION mode as well as your local miner to see in MINER-HOSTED mode:
+
+You'll receive messages like this:
+```baseh
+[{"content":"What is the discounted price of the jacket, given it was originally $200 and there is a 20% reduction?","role":"user"}]
+```
+and Tools like this:
+```bash
+[{"arguments":{"discount_percentage":{"required":true,"type":"number","description":"The percentage discount to be applied"},
+"original_price":{"description":"The original price of the item","required":true,"type":"number"}},
+"description":"Calculate the discounted price of an item based on the original price and discount percentage","name":"calculate_discount"},
+{"arguments":{"pod_name":{"description":"The name of the pod to be restarted","required":true,"type":"str"}},
+"description":"A function to restart a given pod, useful for deployment and testing.","name":"restart_pod"},...]
+```
+
+In response your model should return the function call like this:\
+`calculate_discount(discount_percentation=..., original_price=...)`
+
+The model is responsible for returning a function call like above with the right function name, the correct function argument names and values, being sure to set any required arguments appropriately.
+
+#### Miner Feedback
+As a miner, you receive tasks, you get rewarded, but on most subnets, you do not know what you're being graded on.
+BitAgent (SN20) offers transparent feedback (in debug logging mode), so you know what you're up against.
+
+Here's an example of a well performed task:
+![miner feedback - good example](./docs/examples/output_to_miner.png)
+
+Here's an example of a poorly performed task:
+![miner feedback - bad example](./docs/examples/bad_output_to_miner.png)
+
+Additionally, we send all queries and results to Wandb:
+- WandB Testnet - https://wandb.ai/bitagentsn20/testnet
+- WandB Mainnet - https://wandb.ai/bitagentsn20/mainnet
+
+### Advanced
+If you have a need to create and fund wallets for your own testing ...
+
+After getting the [subtensor package started and a subnet up and running](./docs/running_on_staging.md) (for staging/local) - you can use this [script](./scripts/setup_and_run.sh) to:
+- create wallets (for owner, validators, miners),
+- fund those wallets with the right amount of tao,
+- register wallets on the local subnet,
+- start miners and validators
+
+```bash
+./scripts/setup_and_run.sh
+```
+You can use several flags to configure:
+- the number of miners or validators it sets up,
+- whether it funds wallets,
+- or if it registers wallets,
+- or just launches a miner
+```bash
+bitagent_subnet$ ./scripts/setup_and_run.sh --help
+
+Creates wallets for the subnet (owner, validators, miners), funds them, registers them, then starts them.
+
+usage: ./scripts/setup_and_run.sh --num_validators num --num_miners num --subnet_prefix string
+
+  --num_validators num     number of validators to launch
+                           (default: 1)
+  --num_miners     num     number of miners to launch
+                           (default: 2)
+  --subnet_prefix  string  the prefix of the subnet wallets
+                           (default: local_subnet_testing_bitagent)
+  --skip-wallet            skip wallet creation
+                           (default: run wallet creation)
+  --skip-faucet            skip wallet funding
+                           (default: fund wallets)
+  --skip-subnet            skip subnet creation
+                           (default: create subnet)
+  --skip-reg               skip all registration to the subnet
+                           (default: register wallets)
+  --skip-val-reg           skip validator registration to the subnet
+                           (default: register validator)
+  --skip-miner-reg         skip miner registration to the subnet
+                           (default: register miner)
+  --skip-launch            skip validator and miner launching on the subnet
+                           (default: launch validators and miners)
+  --skip-launch_v          skip validator launching on the subnet
+                           (default: launch validators)
+  --only-launch            skip everything but launching
+                           (default: do everything)
+  --test-net               do the same things, but for testnet
+                           (default: false, local)
+  --main-net               do the same things, but for mainnet
+                           (default: false, local)
+  --netuid                 the netuid to work with
+                           (default: 1 for local, change if main or test)
+
+Example: ./scripts/setup_and_run.sh --only-launch
+This will skip everything and just launch the already registered and funded validators and miners
+```
+
+---
+
+## FAQ
+**Q: How much GPU (VRAM) and RAM do I need to run a validator and/or miner?** \
+A: Validators need a GPU and require a minimum of 48 GBs of VRAM with performant CPU.  Miners are left to their own setup, but should be aware that the more capable tool calling LLMs require a decent amount of VRAM (common configurations: a 3090 (with 24GB VRAM) is capable enough for the smaller (~8B params) models we require).
+
+**Q: Are there any required subscriptions or paid APIs?** \
+A: No - no subs, no external companies, in fact we'd rather the community build amazing AI capabilities than relying on corporations.
+
+**Q: What LLM should I use?** \
+A: This is where the miner needs to experiment some and test and fine-tune different LLM models to find what accomplishes the tasks most successfully.  Have a look at models in the Salesforce xLAM family as good starting points.
+
+**Q: Validators are running miner-submitted HF models, will validators require `trust_remote_code`?** \
+A: No, we require that no setup scripts or any code be necessary for running the models.
+
+**Q: I started my miner and I am not receiving any tasks.** \
+A: There are a few things to check:
+- Is your axon port, as reported on the metagraph correct (you can check taostats or metagraph)?
+- Is your axon port open and reachable from a system in the real world (like where the validators are)?
+- Do you have Trace logging on to see the dendrite requests and Debug logging on to see the task results?
+- Make sure your IsAlive() forward is returning True and wait an hour for that to update in the validator's cache.
+- Make sure there isn't a stale process that is preventing your new miner process from starting up on the intended port.
+
+**Q: What about model copying?** \
+A: https://discord.com/channels/799672011265015819/1194736998250975332/1302870011362279514
+
+**Q: My model is not being evaluated OFFLINE for FINETUNED SUBMISSION and is receiving a score of 0.** \
+A: There are a few things to check:
+- Is your model licensed under the apache-2.0 license?
+- Is your model size less than 10B parameters? We are looking for 8B params or less models.
+- Is your model name properly set in the Hugging Face?
+
+**Q: I'm getting a wallet path error, like: `KeyFileError: Keyfile at: ${HOME}/~/.bittensor/wallets/...`** \
+A: There is a bug in 8.2.0 that is setting the wallet path incorrectly, so you may need to fix this by adding this parameter to your start command: \
+  `--wallet.path ~/.bittensor/wallets`
+
+**Q: I have a complicated CUDA Device setup and need to use a specific GPU device as a validator running the FINETUNED models:** \
+A: We provide two parameters for this: \
+  `--neuron.visible_devices`\
+  `--neuron.device`\
+Example usage: To use the 2nd CUDA Device, you would add these to your parameters: \
+  `--neuron.visible_devices 1 --neuron.device cuda:0`
+
+**Q: My validator is running out of GPU memory when loading OFFLINE models via sglang.** \
+A: You can use this parameter: `--validator-hf-server-mem-fraction-static` to increase or decrease the amount of the GPU VRAM to use.\
+It defaults to 0.55, just over half of the VRAM.
+
+**Q: My vLLM or other inference instance is not served on 8000, how do I change this?**\
+A: We provide a parameter `--openai-api-base`\
+It defaults to this: `http://localhost:8000/v1`, updated as needed by passing the `--openai-api-base` parameter to your start command.
+
+**Q: My vTrust is low and it looks like I'm not setting OFFLINE weights.**\
+A: Please test your sglang setup - check [here](#sglang-setup-for-validators).
+
+**Q: I'm validating and seeing errors like:**
+- TimeoutError
+- ClientConnectorError \
+
+A: These are responses likely during the IsAlive() query, they are just letting you know that the miner is not responding or connecting in time.
+
+**Q: My validator is hanging, just printing out "Validator running ..."**\
+A: There are a few things to check:\
+- Make sure your vLLM is running with the required LLM from [vLLM Setup](#vllm-setup-for-validators)
+- You may not see much unless you turn on some logging, you can add this to your params to see more details:\
+  `--log_level trace --logging.trace --logging.debug`
+- Check your storage, make sure you didn't run out:\
+  `df -h`
+- If all else fails, [reach out](https://discord.com/channels/799672011265015819/1194736998250975332)
+
+
+---
+
+## License
+This repository is licensed under the MIT License.
+```text
+# The MIT License (MIT)
+# Copyright © 2023 Yuma Rao
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+```

bitagent_subnet-main/bitagent/__init__.py

@@ -0,0 +1,11 @@
+# not used for weight versioning
+__version__ = "1.0.8"
+version_split = __version__.split(".")
+__spec_version__ = (
+    (1000 * int(version_split[0]))
+    + (10 * int(version_split[1]))
+    + (1 * int(version_split[2]))
+)
+
+# Import all submodules.
+from . import protocol

bitagent_subnet-main/bitagent/criteria/__init__.py

@@ -0,0 +1 @@
+from .criterion import *

bitagent_subnet-main/bitagent/criteria/criterion.py

@@ -0,0 +1,95 @@
+# The MIT License (MIT)
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import ast
+import bittensor as bt
+from pprint import pformat
+from typing import Callable, List, Tuple
+from bitagent.criteria.utils import bad_message
+from bitagent.criteria.default_criteria import *
+from bitagent.criteria.tool_call_criteria import *
+
+# building block for the criteria used to evaluate the miner's response
+class Criterion():
+    name: str
+    desc: str
+    eval_fx: Callable
+
+    def __init__(self, name: str, desc: str, eval_fx: Callable, eval_args=[]) -> None:
+        self.name = name
+        self.desc = desc
+        self.eval_fx = eval_fx
+        self.eval_args = eval_args
+
+    def clean_response(self,response):
+        # TODO check multiple functions for parallel, when the response is a list [fx1(), fx2()]
+        response = response.strip()
+        if "[" in response[0] and "]" in response[-1]:
+            response = response[1:-1]
+
+        try:
+            ast.parse(response.strip())
+        except:
+            # if it not a parsable function, then it's potentially an irrelevance call
+            response = ""
+
+        return response.strip()
+
+    def evaluate(self, task, validator, synapse: bt.Synapse) -> Tuple[float, float, str]:
+        try:
+            # make sure the tool response converts nicely to an ast
+            synapse.response = self.clean_response(synapse.response)
+            try:
+                ast.parse(synapse.response)
+            except:
+                reward = -0.5
+                max_reward = 1.0
+                feedback = bad_message(f"Your response: {synapse.response} was not parsable")
+                return reward, max_reward, feedback
+
+            # actually do the evaluation 
+            reward, max_reward, feedback = self.eval_fx(task, validator, synapse, *self.eval_args)
+        except Exception as e:
+            #bt.logging.error(f"Exception was raised during criteria evaluation: {e}")
+            reward = -0.5
+            max_reward = 1.0
+            feedback = bad_message(f"Exception while processing your response, please check format per protocol - {e}")
+        feedback = f"[bold blue]{self.name}[/bold blue]\n" + feedback
+        return reward, max_reward, feedback
+
+    def __repr__(self):
+        return pformat(vars(self), indent=4, width=1)
+
+# Function Call
+def tool_call_criteria(expected_response: dict) -> List[Criterion]:
+    return [
+        Criterion(name="Return correct function format", desc="", eval_fx=correct_tool_call_function_format),
+        Criterion(name="Return correct function name", desc="", eval_fx=correct_tool_call_function_name, eval_args=[expected_response]),
+        Criterion(name="Return function with correct argument names", desc="", eval_fx=correct_tool_argument_names, eval_args=[expected_response]),
+        Criterion(name="Return function with correct argument values", desc="", eval_fx=correct_tool_argument_values, eval_args=[expected_response]),
+    ]
+
+def irrelevant_tool_call_criteria() -> List[Criterion]:
+    return [
+        Criterion(name="Return valid function call for irrelevant tool", desc="", eval_fx=correct_irrelevant_tool_call),
+    ]
+
+# simple, defaults
+default_criteria = [
+    Criterion(name="Does not error", desc="", eval_fx=does_not_error), 
+    Criterion(name="Does not take a long time", desc="", eval_fx=does_not_take_a_long_time),
+]

bitagent_subnet-main/bitagent/criteria/default_criteria.py

@@ -0,0 +1,59 @@
+# The MIT License (MIT)
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import bittensor as bt
+from typing import Tuple
+from bitagent.criteria.utils import good_message, bad_message, received_reward_template
+
+# CRITERION: successful call to miner
+def does_not_error(task, validator, synapse: bt.Synapse) -> Tuple[float, float, str]:
+    max_reward = 0.25
+    a_status_code = synapse.axon.status_code
+    d_status_code = synapse.dendrite.status_code
+    reward = 0.0
+    if a_status_code == 200 and d_status_code == 200:
+        reward = max_reward
+        feedback = good_message("You successfully responded to the request.")
+    else:
+        feedback = bad_message("You failed to respond correctly to the request.")
+        if d_status_code == 408:
+            feedback += "You timed out and will fail the remainder of the criteria."
+        feedback += f" Status Code: {a_status_code}/{d_status_code}"
+        
+    return reward, max_reward, feedback + received_reward_template.format(reward, max_reward)
+
+# CRITERION: reward speedy response
+def does_not_take_a_long_time(task, validator, synapse: bt.Synapse) -> Tuple[float, float, str]:
+    max_reward = 0.5
+    process_time = synapse.dendrite.process_time
+    if not process_time:
+        feedback = f"You likely ran into an error processing this task and failed to respond appropriately."
+        reward = 0
+        return reward, max_reward, bad_message(feedback) + received_reward_template.format(reward,max_reward)
+
+    feedback = f"You responded to the request in {process_time}."
+    reward = 0.0
+    if process_time <= task.timeout/1.75:
+        reward = max_reward
+        return reward, max_reward, good_message(feedback) + received_reward_template.format(reward,max_reward)
+    if process_time <= task.timeout/1.25:
+        reward = max_reward/2
+        return reward, max_reward, good_message(feedback, color="yellow") + received_reward_template.format(reward,max_reward)
+    if process_time <= task.timeout:
+        reward = max_reward/5
+        return reward, max_reward, bad_message(feedback, color="yellow") + received_reward_template.format(reward,max_reward)
+    return reward, max_reward, bad_message(feedback) + received_reward_template.format(reward,max_reward)

bitagent_subnet-main/bitagent/criteria/tool_call_criteria.py

@@ -0,0 +1,422 @@
+# The MIT License (MIT)
+# Copyright 2024 RogueTensor
+# Copyright 2024 TheIntern
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+import ast
+import bittensor as bt
+from typing import Tuple
+from bitagent.criteria.utils import good_message, bad_message, received_reward_template
+from functools import lru_cache
+
+# just checking if the function can be parsed by ast
+def correct_tool_call_function_format(task, validator, synapse: bt.Synapse) -> Tuple[float, float, str]:
+    max_reward = 1.0
+    reward = 1.0
+
+    try:
+        ast.parse(synapse.response)
+    except Exception as e:
+        reward = -1.0
+        feedback = bad_message(f"Your response was not in the correct format - {e}")
+        return reward, max_reward, feedback+received_reward_template.format(reward, max_reward)
+    
+    feedback = good_message(f"Your response was in the correct format.")
+    return reward, max_reward, feedback+received_reward_template.format(reward, max_reward)
+
+@lru_cache(maxsize=128)
+def extract_function_name_and_params(response: str):
+    if response == "":
+        return "", [], {}
+
+    node = ast.parse(response , mode="eval")
+
+    # Walk through the AST to extract the function name
+    class FunctionNameExtractor(ast.NodeVisitor):
+        def __init__(self):
+            self.function_name = None
+
+        def visit_Call(self, node):
+            # Check if the node is a function call
+            if isinstance(node.func, ast.Attribute):  # Handles dot notation (e.g., module.function)
+                parts = []
+                current = node.func
+                while isinstance(current, ast.Attribute):
+                    parts.append(current.attr)
+                    current = current.value
+                if isinstance(current, ast.Name):
+                    parts.append(current.id)
+                # Join the parts in reverse to get the full function name
+                self.function_name = '.'.join(reversed(parts))
+            elif isinstance(node.func, ast.Name):  # Handles simple function names (e.g., functionName)
+                self.function_name = node.func.id
+            # No need to visit further
+            self.generic_visit(node)
+
+    extractor = FunctionNameExtractor()
+    extractor.visit(node)
+    function_name = extractor.function_name
+
+    param_names = [kw.arg for kw in node.body.keywords]
+    if param_names: 
+        param_values = [ast.literal_eval(kw.value) for kw in node.body.keywords]
+    else:
+        param_values = []
+
+    param_values_dict = {}
+    for i,param_name in enumerate(param_names):
+        param_values_dict[param_name] = param_values[i]
+
+    return function_name, param_names, param_values_dict
+
+# just checking if the function name is correct
+def correct_tool_call_function_name(task, validator, synapse: bt.Synapse, expected_response: dict) -> Tuple[float, float, str]:
+    max_reward = 3.0
+    reward = 3.0    
+
+    function_name, _, _ = extract_function_name_and_params(synapse.response)
+    expected_function_name = expected_response['name']
+
+    if function_name.strip() == expected_function_name.strip():
+        feedback = good_message(f"Your function name matches the expected function name.")
+        return reward, max_reward, feedback+received_reward_template.format(reward, max_reward)
+    else:
+        reward = -0.5
+        feedback = bad_message(f"Your function name does not match the expected function name.")
+        return reward, max_reward, feedback+received_reward_template.format(reward, max_reward)
+
+# comparing just the argument names
+# looking for required arguments and that they are present
+def correct_tool_argument_names(task, validator, synapse: bt.Synapse, expected_response: dict) -> Tuple[float, float, str]:
+    max_reward = 1.0
+    reward = max_reward
+    feedback_parts = []
+
+    # MINER response
+    function_name, function_args, _ = extract_function_name_and_params(synapse.response)
+    expected_args = set(expected_response['arguments'].keys())
+    function_args_set = set(function_args)
+
+    # no args
+    if not expected_args and not function_args and function_name:
+        feedback_parts.append(good_message("Function has no arguments, good job"))
+        return reward, max_reward, ''.join(feedback_parts) + received_reward_template.format(reward, max_reward)
+
+    required_args = get_required_args(task, expected_response)
+    
+    for arg in required_args:
+        if arg in function_args_set:
+            feedback_parts.append(good_message(f"Your function has the required argument: {arg}"))
+        else:
+            reward -= max_reward/len(required_args)
+            feedback_parts.append(bad_message(f"Your function is missing the required argument: {arg}"))
+
+    return reward, max_reward, '\n'.join(feedback_parts) + received_reward_template.format(reward, max_reward)
+
+def correct_tool_argument_values(task, validator, synapse: bt.Synapse, expected_response: dict) -> Tuple[float, float, str]:
+    max_reward = 3.0
+    reward = 0.0        
+    feedback = "" 
+
+    # MINER response
+    function_name, function_args, function_values = extract_function_name_and_params(synapse.response)
+    expected_args = set(expected_response['arguments'].keys())  # Convert to set for O(1) lookups
+    function_args_set = set(function_args)  # Convert to set for O(1) lookups
+
+    # no args
+    if not expected_args and not function_args and function_name:
+        reward = max_reward
+        feedback = good_message("Function has no arguments, good job")
+        return reward, max_reward, feedback+received_reward_template.format(reward, max_reward)
+
+    required_args = get_required_args(task, expected_response)
+    is_flipped = is_distance_calculation_with_flipped_args(function_name, task.synapse.messages)
+
+    # Check if this is a distance calculation with flipped arguments
+    for arg in required_args:
+        if arg in function_args_set:
+            correct_values = get_correct_values_for_arg(arg, expected_response, is_flipped)
+            
+            if "is_ground_truth" in expected_response and function_values[arg] in correct_values:
+                reward += max_reward/max(len(function_args),len(expected_args))
+                feedback += good_message(f"Your function has the required value for argument: {arg}") + "\n"
+            elif function_values[arg] == correct_values:
+                reward += max_reward/max(len(function_args),len(expected_args))
+                feedback += good_message(f"Your function has the required value for argument: {arg}") + "\n"
+            else:
+                reward -= 0
+                feedback += bad_message(f"Your function has the incorrect value for argument: {arg}") + "\n"
+        else:
+            reward -= max_reward/len(required_args)
+            feedback += bad_message(f"Your function is missing the required argument: {arg}") + "\n"
+
+    return reward, max_reward, feedback+received_reward_template.format(reward, max_reward)
+
+def correct_irrelevant_tool_call(task, validator, synapse: bt.Synapse) -> Tuple[float, float, str]:
+    max_reward = 3.0
+    reward = 3.0
+    
+    if synapse.response.strip() != "":
+        reward = -0.5
+        feedback = bad_message(f"Your response (`{synapse.response}`) was not empty, expected an empty response to be returned.")
+        return reward, max_reward, feedback+received_reward_template.format(reward, max_reward)
+
+    feedback = good_message(f"You responded with the expected response.")
+    return reward, max_reward, feedback+received_reward_template.format(reward, max_reward)
+
+
+# Examples:
+synapse_response1 = 'calculate_gpa(grades=["A", "B", "A", "C"], credit_hours=[3, 4, 3, 2])'
+synapse_response2 = 'calculate_gpa(credit_hours=[3, 4, 3, 2], grades=["A", "B", "A", "C"])'
+expected_response = {'name': 'calculate_gpa', 'arguments': {'grades': ['A', 'B', 'A', 'C'], 'credit_hours': [3, 4, 3, 2]}}
+
+import unittest
+from typing import List
+
+class MockSynapse:
+    from bitagent.schemas.tool import Tool
+    response: str
+    tools: List[Tool] = [Tool(name="calculate_gpa", description="Calculate the GPA of a student", arguments={"grades": {"type": "list", "required": True}, "credit_hours": {"type": "list", "required": True}})]
+
+    def __init__(self, response: str):
+        self.response = response
+
+class MockTask:
+    synapse: MockSynapse
+
+    def __init__(self, synapse: MockSynapse):
+        self.synapse = synapse
+
+class TestToolCallCriteria(unittest.TestCase):
+
+    def setUp(self):
+        self.validator = ""
+
+    def test_correct_tool_call_function_format(self):
+        # Test valid function format
+        synapse = MockSynapse(response="calculate_gpa(grades=['A'], credit_hours=[3])")
+        task = MockTask(synapse=synapse)
+        reward, max_reward, feedback = correct_tool_call_function_format(task, self.validator, synapse)
+        self.assertEqual(reward, 1.0)
+        self.assertEqual(max_reward, 1.0)
+        self.assertTrue("was in the correct format" in feedback.lower())
+
+        # Test invalid function format
+        synapse = MockSynapse(response="invalid(function syntax")
+        task = MockTask(synapse=synapse)
+        reward, max_reward, feedback = correct_tool_call_function_format(task, self.validator, synapse)
+        self.assertEqual(reward, -1.0)
+        self.assertEqual(max_reward, 1.0)
+        self.assertTrue("not in the correct format" in feedback.lower())
+
+        # Test json response
+        synapse = MockSynapse(response='{"name": "calculate_gpa", "arguments": {"grades": ["A"], "credit_hours": [3]}}')
+        task = MockTask(synapse=synapse)
+        reward, max_reward, feedback = correct_tool_call_function_format(task, self.validator, synapse)
+        self.assertEqual(reward, 1.0)
+        self.assertEqual(max_reward, 1.0)
+        self.assertTrue("was in the correct format" in feedback.lower())
+
+    def test_extract_function_name_and_params(self):
+        # Test basic function extraction
+        response = "calculate_gpa(grades=['A'], credit_hours=[3])"
+        name, params, values = extract_function_name_and_params(response)
+        self.assertEqual(name, "calculate_gpa")
+        self.assertEqual(params, ["grades", "credit_hours"])
+        self.assertEqual(values, {"grades": ["A"], "credit_hours": [3]})
+
+        # Test empty response
+        name, params, values = extract_function_name_and_params("")
+        self.assertEqual(name, "")
+        self.assertEqual(params, [])
+        self.assertEqual(values, {})
+
+        # Test function with dot notation
+        response = "math.sqrt(value=16)"
+        name, params, values = extract_function_name_and_params(response)
+        self.assertEqual(name, "math.sqrt")
+        self.assertEqual(params, ["value"])
+        self.assertEqual(values, {"value": 16})
+
+        # Test function with dot notation and no value
+        response = "math.sqrt()"
+        name, params, values = extract_function_name_and_params(response)
+        self.assertEqual(name, "math.sqrt")
+        self.assertEqual(params, [])
+        self.assertEqual(values, {})
+
+    def test_correct_irrelevant_tool_call(self):
+        # Test empty response (correct)
+        synapse = MockSynapse(response="")
+        task = MockTask(synapse=synapse)
+        reward, max_reward, feedback = correct_irrelevant_tool_call(task, self.validator, synapse)
+        self.assertEqual(reward, 3.0)
+        self.assertEqual(max_reward, 3.0)
+        self.assertTrue("expected response" in feedback.lower())
+
+        # Test non-empty response (incorrect)
+        synapse = MockSynapse(response="some_function()")
+        task = MockTask(synapse=synapse)
+        reward, max_reward, feedback = correct_irrelevant_tool_call(task, self.validator, synapse)
+        self.assertEqual(reward, -0.5)
+        self.assertEqual(max_reward, 3.0)
+        self.assertTrue("not empty" in feedback.lower())
+
+    def test_correct_tool_call_function_name(self):
+        # Test correct function name
+        synapse = MockSynapse(response="calculate_gpa(grades=['A'])")
+        task = MockTask(synapse=synapse)
+        expected = {"name": "calculate_gpa", "arguments": {"grades": ["A"]}}
+        reward, max_reward, feedback = correct_tool_call_function_name(task, self.validator, synapse, expected)
+        self.assertEqual(reward, 3.0)
+        self.assertEqual(max_reward, 3.0)
+        self.assertTrue("matches the expected function name" in feedback.lower())
+
+        # Test incorrect function name
+        synapse = MockSynapse(response="wrong_function(grades=['A'])")
+        task = MockTask(synapse=synapse)
+        reward, max_reward, feedback = correct_tool_call_function_name(task, self.validator, synapse, expected)
+        self.assertEqual(reward, -0.5)
+        self.assertEqual(max_reward, 3.0)
+        self.assertTrue("not match" in feedback.lower())
+
+    def test_correct_tool_argument_names(self):
+        # Test no expected arguments
+        synapse = MockSynapse(response="calculate_gpa()")
+        task = MockTask(synapse=synapse)
+        expected = {"name": "calculate_gpa", "arguments": {}}
+        reward, max_reward, feedback = correct_tool_argument_names(task, self.validator, synapse, expected)
+        self.assertEqual(reward, 1.0)
+        self.assertEqual(max_reward, 1.0)
+        self.assertTrue("no arguments, good job" in feedback.lower())
+
+        # Test no expected arguments, but pass in arguments anyway
+        synapse = MockSynapse(response="calculate_gpa(grades=['A'])")
+        task = MockTask(synapse=synapse)
+        expected = {"name": "calculate_gpa", "arguments": {}}
+        reward, max_reward, feedback = correct_tool_argument_names(task, self.validator, synapse, expected)
+        self.assertEqual(reward, 0.0)
+        self.assertEqual(max_reward, 1.0)
+        self.assertTrue("expects no arguments" in feedback.lower())
+
+        # Test correct argument names
+        synapse = MockSynapse(response="calculate_gpa(grades=['A'], credit_hours=[3])")
+        task = MockTask(synapse=synapse)
+        expected = {"name": "calculate_gpa", "arguments": {"grades": ["A"], "credit_hours": [3]}}
+        reward, max_reward, feedback = correct_tool_argument_names(task, self.validator, synapse, expected)
+        self.assertEqual(reward, 1.0)
+        self.assertEqual(max_reward, 1.0)
+        self.assertEqual(feedback.lower().count("has the required argument"), 2)
+
+        # Test correct argument names plus an incorrect argument
+        synapse = MockSynapse(response="calculate_gpa(grades=['A'], credit_hours=[3], extra_arg=1)")
+        task = MockTask(synapse=synapse)
+        expected = {"name": "calculate_gpa", "arguments": {"grades": ["A"], "credit_hours": [3]}}
+        reward, max_reward, feedback = correct_tool_argument_names(task, self.validator, synapse, expected)
+        self.assertEqual(reward, 1.0)
+        self.assertEqual(max_reward, 1.0)
+        self.assertEqual(feedback.lower().count("has the required argument"), 2)
+
+        # Test correct argument names out of order
+        synapse = MockSynapse(response="calculate_gpa(credit_hours=[3], grades=['A'])")
+        task = MockTask(synapse=synapse)
+        expected = {"name": "calculate_gpa", "arguments": {"grades": ["A"], "credit_hours": [3]}}
+        reward, max_reward, feedback = correct_tool_argument_names(task, self.validator, synapse, expected)
+        self.assertEqual(reward, 1.0)
+        self.assertEqual(max_reward, 1.0)
+        self.assertEqual(feedback.lower().count("has the required argument"), 2)
+
+        # Test missing argument
+        synapse = MockSynapse(response="calculate_gpa(grades=['A'])")
+        task = MockTask(synapse=synapse)
+        reward, max_reward, feedback = correct_tool_argument_names(task, self.validator, synapse, expected)
+        self.assertEqual(reward, 0.0)
+        self.assertEqual(max_reward, 1.0)
+        self.assertEqual(feedback.lower().count("has the required argument"), 1)
+        self.assertEqual(feedback.lower().count("missing the required argument"), 1)
+
+    def test_correct_tool_argument_values(self):
+        # Test correct argument values
+        synapse = MockSynapse(response="calculate_gpa(grades=['A'], credit_hours=[3])")
+        task = MockTask(synapse=synapse)
+        expected = {"name": "calculate_gpa", "arguments": {"grades": ["A"], "credit_hours": [3]}}
+        reward, max_reward, feedback = correct_tool_argument_values(task, self.validator, synapse, expected)
+        self.assertEqual(reward, 3.0)
+        self.assertEqual(max_reward, 3.0)
+        self.assertEqual(feedback.lower().count("has the required value for argument"), 2)
+
+        # Test correct argument values out of order
+        synapse = MockSynapse(response="calculate_gpa(credit_hours=[3], grades=['A'])")
+        task = MockTask(synapse=synapse)
+        expected = {"name": "calculate_gpa", "arguments": {"grades": ["A"], "credit_hours": [3]}}
+        reward, max_reward, feedback = correct_tool_argument_values(task, self.validator, synapse, expected)
+        self.assertEqual(reward, 3.0)
+        self.assertEqual(max_reward, 3.0)
+        self.assertEqual(feedback.lower().count("has the required value for argument"), 2)
+
+        # Test incorrect argument values
+        synapse = MockSynapse(response="calculate_gpa(grades=['B'], credit_hours=[4])")
+        task = MockTask(synapse=synapse)
+        reward, max_reward, feedback = correct_tool_argument_values(task, self.validator, synapse, expected)
+        self.assertEqual(reward, -3.0)
+        self.assertEqual(max_reward, 3.0)
+        self.assertEqual(feedback.lower().count("has the incorrect value for argument"), 2)
+
+        # Test incorrect argument values out of order
+        synapse = MockSynapse(response="calculate_gpa(credit_hours=[4], grades=['B'])")
+        task = MockTask(synapse=synapse)
+        reward, max_reward, feedback = correct_tool_argument_values(task, self.validator, synapse, expected)
+        self.assertEqual(reward, -3.0)
+        self.assertEqual(max_reward, 3.0)
+        self.assertEqual(feedback.lower().count("has the incorrect value for argument"), 2)
+
+        # Test incorrect value types
+        synapse = MockSynapse(response="calculate_gpa(grades='A', credit_hours=3)")
+        task = MockTask(synapse=synapse)
+        reward, max_reward, feedback = correct_tool_argument_values(task, self.validator, synapse, expected)
+        self.assertEqual(reward, -3.0)
+        self.assertEqual(max_reward, 3.0)
+        self.assertEqual(feedback.lower().count("has the incorrect value for argument"), 2)
+
+if __name__ == '__main__':
+    # Run all tests in this file
+    # You can run this file directly with: python -m bitagent.criteria.tool_call_criteria
+    # Or run all tests with: python -m pytest bitagent/criteria/tool_call_criteria.py
+    unittest.main(verbosity=2)
+
+def get_required_args(task, expected_response: dict) -> set:
+    """Helper function to get required arguments."""
+    if "is_ground_truth" in expected_response:
+        return {arg for arg in expected_response['arguments'] if expected_response['arguments'][arg] != [""]}
+    
+    expected_tool = next((tool for tool in task.synapse.tools if tool.name == expected_response['name']), None)
+    if not expected_tool:
+        return set()
+    return {k for k in expected_tool.arguments if expected_tool.arguments[k].get('required', False)}
+
+def is_distance_calculation_with_flipped_args(function_name: str, messages) -> bool:
+    """Helper function to check if this is a distance calculation with flipped arguments."""
+    return function_name == "calculate_distance" and "flipped" in str(messages).lower()
+
+def get_correct_values_for_arg(arg: str, expected_response: dict, is_flipped: bool) -> list:
+    """Helper function to get correct values for an argument, handling flipped cases."""
+    if not is_flipped:
+        return expected_response['arguments'][arg]
+    
+    if arg == "origin":
+        return expected_response['arguments'].get("destination", expected_response['arguments'][arg])
+    elif arg == "destination":
+        return expected_response['arguments'].get("origin", expected_response['arguments'][arg])
+    return expected_response['arguments'][arg]

bitagent_subnet-main/bitagent/criteria/utils.py

@@ -0,0 +1,25 @@
+# The MIT License (MIT)
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+received_reward_template="\nYou received {} of {} reward."
+
+def bad_message(text: str, color: str="red") -> str:
+    return f":cross_mark: [{color}]{text}[/{color}]"
+
+def good_message(text: str, color: str="green") -> str:
+    return f":white_heavy_check_mark: [{color}]{text}[/{color}]"
+

bitagent_subnet-main/bitagent/datasources/__init__.py

@@ -0,0 +1 @@
+from .tools import *

bitagent_subnet-main/bitagent/datasources/loaders.py

@@ -0,0 +1,63 @@
+import os
+import pandas as pd
+import bittensor as bt
+from datasets import load_dataset, load_from_disk
+from huggingface_hub import snapshot_download
+
+class ShuffledJSONDatasetIterator:
+    def __init__(self):
+        dataframes = []
+
+        # TODO - other BFCL task types:
+        # irrelevance and live_irrelevance - answer is NOTHING
+        # exec_* (simple, multiple, parallel, parallel_multiple) - answer in the file itself
+        # multi_turn_* - answer in the file itself
+        # parallel* - answer in the file itself
+        # rest - maybe later - calls to API that the validator would need to setup
+
+        for filename in ["java", "javascript", "simple", "multiple", "sql", "live_simple", "live_multiple"]:
+            bfcl_path = "bitagent.data/bfcl/BFCL_v3_{filename}.json"
+            bfcl_answer_path = "bitagent.data/bfcl/possible_answer/BFCL_v3_{filename}.json"
+            file_path = bfcl_path.format(filename=filename)
+            answer_path = bfcl_answer_path.format(filename=filename)
+            df_data = pd.read_json(file_path, lines=True)
+            df_answer = pd.read_json(answer_path, lines=True)
+            df_data['ground_truth'] = df_answer['ground_truth']
+            dataframes.append(df_data[['id','question','function','ground_truth']])
+        self.all_data = pd.concat(dataframes)
+        self._shuffle_data()
+
+    def _shuffle_data(self):
+        self.shuffled_data = self.all_data.sample(frac=1).reset_index(drop=True)
+        self.index = 0
+
+    def __iter__(self):
+        self.index = 0
+        return self
+
+    def __next__(self):
+        if self.index < len(self.shuffled_data):
+            row = self.shuffled_data.iloc[self.index]
+            self.index += 1
+            return row
+        else:
+            self._shuffle_data()  # Shuffle and reset index if end is reached
+            return self.__next__()
+
+def huggingface_loader(dataset_name, root_data_dir="bitagent.data", split="train", name=None):
+    bt.logging.debug(f"Loading {dataset_name}")
+    dataset_dir = f"{root_data_dir}/{dataset_name.replace('/','_')}"
+    if os.path.exists(f"{dataset_dir}/state.json"):
+        bt.logging.debug(f"Loading from disk ({dataset_dir}) ...")
+        ds = load_from_disk(dataset_dir)
+    else:
+        bt.logging.debug("Loading from web ...") 
+        ds = load_dataset(dataset_name, split=split, name=name, token=os.getenv("HF_TOKEN", None))
+        ds.save_to_disk(dataset_dir)
+    bt.logging.debug("Loaded.")
+    return ds
+
+def load_bfcl_dataset(dataset_name, root_data_dir="bitagent.data", split="train", name=None):
+    snapshot_download(repo_id=dataset_name, allow_patterns="*.json", repo_type="dataset", local_dir="bitagent.data/bfcl/")
+
+    return ShuffledJSONDatasetIterator()

bitagent_subnet-main/bitagent/datasources/tools.py

@@ -0,0 +1,194 @@
+import re
+import json
+import random
+import bittensor as bt
+from pydantic import BaseModel
+from typing import List, Dict, Any
+from collections.abc import Iterator
+from bitagent.schemas.tool import Tool
+from bitagent.schemas.chat import ChatMessage, messages_from_list
+from bitagent.datasources.loaders import huggingface_loader, load_bfcl_dataset
+from bitagent.helpers.string_parse import parse_multiple_space_sep_json
+
+
+def split_dialogue(text) -> List[ChatMessage]:
+    # Define a pattern to match the roles and capture messages
+    pattern = r"(USER|ASSISTANT|TOOL CALL|TOOl RESPONSE): (.*?)(?=\s*(USER|ASSISTANT|TOOL CALL|TOOL RESPONSE):|$)"
+
+    # Find all matches in the text using the pattern
+    matches = re.findall(pattern, text, re.DOTALL)
+
+    # Create a list of dictionaries based on the matches
+    dialogue_list = [{"role": role.lower(), "content": message.strip().replace('\'','')} for role, message, _ in matches]
+    
+    for message in dialogue_list:
+        if not message['role']:
+            raise ValueError("There is a message with no role.")
+     
+    return messages_from_list(dialogue_list)
+
+
+def clean_text(text):
+    text = text.replace("<|endoftext|>", "")
+    text = text.replace("ASSISTANT: <functioncall>", "TOOL CALL: ")
+    text = text.replace("FUNCTION RESPONSE", "TOOL RESPONSE")
+    text = text.replace("  ", " ")
+    return text.strip()
+
+def custom_json_schema_to_pydantic_tool(schema: dict) -> Tool:
+    tool_name = schema.get("name", "")
+    tool_description = schema.get("description", "")
+
+    schema_arguments = schema.get("arguments", {})
+    parameters = {}
+    for param_name, param_info in schema_arguments.items():
+        parameters[param_name] = {
+            "required": param_info.get("required", False),
+            "type": param_info.get("type", ""),
+            "description": param_info.get("description", ""),
+        }
+
+    return Tool(name=tool_name, description=tool_description, arguments=parameters)
+
+def json_schema_to_pydantic_tool(schema: dict) -> Tool:
+    tool_name = schema.get("name", "")
+    tool_description = schema.get("description", "")
+
+    schema_parameters = schema.get("parameters", {})
+    if not schema_parameters:
+        schema_parameters = schema.get("arguments", {})
+    properties = schema_parameters.get("properties", {})
+    required_params = schema_parameters.get("required", [])
+    if isinstance(required_params, bool):
+        required_params = list(properties.keys()) if required_params else []
+    elif not isinstance(required_params, list):
+        required_params = []
+    parameters = {}
+    for param_name, param_info in properties.items():
+        if param_name == "required":
+            continue
+        parameters[param_name] = {
+            "required": param_name in required_params,
+            "type": param_info.get("type", ""),
+            "description": param_info.get("description", ""),
+        }
+    return Tool(name=tool_name, description=tool_description, arguments=parameters)
+
+class ToolCallData(BaseModel):
+    messages: List[ChatMessage]
+    tools: list[Tool]
+
+TYPES = ["str", "int", "dict", "list", "float", "bool", "string", "integer", "number", "boolean", "dictionary", "object"]
+
+def detect_type(value: Any) -> str:
+    type_mapping = {
+        int: 'integer',
+        float: 'number',
+        str: 'string',
+        bool: 'boolean',
+        list: 'array',
+        dict: 'object'
+    }
+    return type_mapping.get(type(value), 'string')
+
+def add_extra_arguments(tool_call: Dict[str, Any], tools: List[Tool]):
+    # Find the tool in the list
+    tool_name = tool_call['name']
+    arguments = tool_call.get('arguments', {})
+    
+    for tool in tools:
+        if tool.name == tool_name:
+            for arg_name, arg_value in arguments.items():
+                if arg_name not in tool.arguments:
+                    # Detect the type of the argument
+                    arg_type = detect_type(arg_value)
+                    # Add the new argument to the tool's schema
+                    tool.arguments[arg_name] = {
+                        'required': False, # assume false
+                        'type': arg_type,
+                        'description': arg_name
+                    }
+            break
+
+class ToolDataset(Iterator):
+    def __init__(self):
+        super().__init__()
+        seed = random.randint(0, 10000)
+        glaive_ds = huggingface_loader("glaiveai/glaive-function-calling-v2")
+        bitagent_ds = huggingface_loader("BitAgent/tool_calling")
+        bfcl_ds = load_bfcl_dataset("gorilla-llm/Berkeley-Function-Calling-Leaderboard")
+
+        self.datasets = {
+            "glaive": iter(glaive_ds.shuffle(seed=seed)),
+            "bitagent": iter(bitagent_ds.shuffle(seed=seed)),
+            "bfcl": iter(bfcl_ds),
+        }
+
+    def __next__(self) -> ToolCallData:
+        #bt.logging.debug("Retrieving function call data from dataset...")
+        count = 0
+        while count < 25:
+            count += 1
+            try:
+                dname, ds = random.choices(list(self.datasets.items()), [5, 5, 10])[0]
+                data = next(ds)
+                if dname == "glaive":
+                    system_prompt = data["system"].replace("SYSTEM: ", "")
+                    if "following functions" not in system_prompt:
+                        continue
+
+                    chat_history = clean_text(data["chat"])
+                    tools = parse_multiple_space_sep_json(
+                        system_prompt.replace(
+                            "You are a helpful assistant with access to the following functions. Use them if required - ",
+                            "",
+                        )
+                    )
+                    tools = [json_schema_to_pydantic_tool(tool) for tool in tools]
+                    messages = split_dialogue(chat_history)
+
+                    # Add arguments that werent defined in schema to the tool
+                    for msg in messages:
+                        if msg.role == "tool call":
+                            tool_call = None
+                            if isinstance(msg.content, str):
+                                tool_call = json.loads(msg.content)
+                            else:
+                                tool_call = msg.content
+                            
+                            add_extra_arguments(tool_call, tools) 
+
+                    
+                    return ToolCallData(messages=messages, tools=tools)
+                elif dname == "bitagent":
+                    for key, value in data.items():
+                        if isinstance(value, str):
+                            data[key] = json.loads(value)
+                    messages = messages_from_list(data["conversation"])
+                    if isinstance(data["tools"], str):
+                        tools = [
+                            json_schema_to_pydantic_tool(tool)
+                            for tool in json.loads(data["tools"])
+                        ]
+                    elif isinstance(data["tools"], list):
+                        tools = [Tool(**tool) for tool in data["tools"]]
+                    else:
+                        raise ValueError(f"Invalid format for tools: {data['tools']}")
+                    for tool in tools:
+                        for arg_name, arg_value in tool.arguments.items():
+                            if arg_value["type"] not in TYPES:
+                                raise ValueError(f"Inavlid type used type: {arg_value['type']}")
+                    return ToolCallData(messages=messages, tools=tools)
+                elif dname == "bfcl":
+                    messages = messages_from_list(data["question"][0])
+                    ground_truth = data['ground_truth'][0]
+                    messages.append(ChatMessage(role="tool call", 
+                                                content={"is_ground_truth": True, 
+                                                         "name": list(ground_truth.keys())[0], 
+                                                         "arguments": list(ground_truth.values())[0]}))
+                    tools = [json_schema_to_pydantic_tool(tool) for tool in data["function"]]
+                    return ToolCallData(messages=messages, tools=tools)
+                    
+            except Exception as e:
+                #bt.logging.debug(f"Issue getting tool call from dataset ... {e}")
+                pass

bitagent_subnet-main/bitagent/helpers/dockers.py

@@ -0,0 +1,44 @@
+import os
+import docker
+import bittensor as bt
+
+def create_container(container_name, model_name, docker_vllm_port):
+    container_to_run = "docker.io/vllm/vllm-openai:latest"
+
+    dclient = docker.from_env()
+
+    try:
+        dclient.containers.get(container_name).remove(force=True)
+    except:
+        pass
+
+    # get home directory
+    home_dir = os.path.expanduser('~')
+
+    bt.logging.debug('starting container')
+    dclient.containers.run(container_to_run, 
+        f"--model {model_name} --max-model-len 8912 --gpu-memory-utilization 0.9",
+        name=container_name,
+        device_requests=[docker.types.DeviceRequest(count=1, capabilities=[["gpu"]])],
+        detach=True,
+        volumes={f'{home_dir}/.cache/huggingface': {'bind': '/root/.cache/huggingface', 'mode': 'rw'}},
+        ports={'8000/tcp': docker_vllm_port})
+    bt.logging.debug('started container')
+
+    return dclient.containers.get(container_name)
+
+def wait_for_container(openai_client, model_name):
+    bt.logging.debug('waiting for container')
+    while True:
+        try:
+            openai_client.chat.completions.create(
+                model=model_name,
+                messages=[{"role": "user", "content": "Hello!"}]
+            )
+            break
+        except Exception as e:
+            #bt.logging.debug(e)
+            import time
+            time.sleep(1)
+            pass
+    bt.logging.debug('container ready')

bitagent_subnet-main/bitagent/helpers/llms.py

@@ -0,0 +1,85 @@
+# The MIT License (MIT)
+# Copyright 2024 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import bittensor as bt
+from openai import OpenAI
+
+# specifically for the validator
+def get_openai_llm(self, hugging_face=False):
+    if "validator" in self.__class__.__name__.lower() and hugging_face and self.config.validator_hf_server_port:
+        # stand up a vLLM server on this port for the OFFLINE HF model evals
+        base_url = f'http://localhost:{self.config.validator_hf_server_port}/v1'
+    else:
+        base_url = self.config.openai_api_base
+
+    return OpenAI(
+        api_key=self.config.openai_api_key,
+        base_url=base_url
+    )
+
+def system_prompt(tools):
+    prompt = """You are an expert in composing functions. You are given a question and a set of possible functions. Based on the question, you will need to make one or more function/tool calls to achieve the purpose.
+    If none of the function can be used, point it out. If the given question lacks the parameters required by the function, also point it out.
+    You should only return the function call in tools call sections.
+
+    For the calculate_distance function:
+    When asking for distance FROM A TO B and parameters are flipped:
+    - Set origin=B (the endpoint)
+    - Set destination=A (the starting point)
+    Example: For "distance from Los Angeles TO New York":
+    - Use origin="New York" (B/endpoint)
+    - Use destination="Los Angeles" (A/starting point)
+
+    If you decide to invoke any of the function(s), you MUST put it in the format of [func_name1(params_name1="params_string_value1", params_name2=params_value2...), func_name2(params)]
+    Notice that any values that are strings must be put in quotes like this: "params_string_value1"
+    You SHOULD NOT include any other text in the response.
+    Here is a list of functions in JSON format that you can invoke.\n{functions}\n
+    """
+
+    return prompt.format(functions=tools)
+
+
+def llm(self, messages, tools, model_name, hugging_face=False,max_new_tokens = 160, temperature=0.7):
+    prompt = system_prompt(tools)
+
+    try:
+        #try:
+        #    new_messages = [{"role":"system", "content":prompt}] + messages
+        #    response = get_openai_llm(self, hugging_face).chat.completions.create(
+        #        messages=new_messages,
+        #        max_tokens=max_new_tokens,
+        #        model=model_name,
+        #        temperature=temperature
+        #    )
+        #except Exception as e:
+            # errored b/c the model does not allow system prompts
+        messages[0].content = prompt + "\n\n" + messages[0].content
+        response = get_openai_llm(self, hugging_face).chat.completions.create(
+            messages=messages,
+            max_tokens=max_new_tokens,
+            model=model_name,
+            temperature=temperature
+        )
+
+    except Exception as e:
+        bt.logging.error(f"Error calling to LLM: {e}")
+        return ""
+
+    if hugging_face:
+        return response.choices[0].message.content.strip(), response.choices[0].finish_reason
+    else:
+        return response.choices[0].message.content.strip()

bitagent_subnet-main/bitagent/helpers/logging.py

@@ -0,0 +1,38 @@
+import bittensor as bt
+from contextlib import contextmanager
+@contextmanager
+def temporary_logging_state(new_state):
+    """
+    A context manager to temporarily set Bittensor's logging state.
+    """
+    # Cache the current logging state
+    current_state = bt.logging.current_state
+    bt.logging.info(f"OFFLINE: Caching current logging state: {current_state}")
+
+    # Set the new logging state
+    if new_state == 'Debug':
+        bt.logging.set_debug()
+    elif new_state == 'Trace':
+        bt.logging.set_trace()
+    elif new_state == 'Warning':
+        bt.logging.set_warning()
+    elif new_state == 'Info':
+        bt.logging.set_info()
+    else:
+        bt.logging.set_default()
+
+    try:
+        yield
+    finally:
+        # Restore the original logging state
+        if current_state.value == 'Debug':
+            bt.logging.set_debug()
+        elif current_state.value == 'Trace':
+            bt.logging.set_trace()
+        elif current_state.value == 'Warning':
+            bt.logging.set_warning()
+        elif current_state.value == 'Info':
+            bt.logging.set_info()
+        else:
+            bt.logging.set_default()
+        bt.logging.info(f"OFFLINE: Restored logging state to: {current_state}")

bitagent_subnet-main/bitagent/helpers/sbert.py

@@ -0,0 +1,45 @@
+from sentence_transformers import SentenceTransformer
+import numpy as np
+import torch
+
+class CachedSentenceTransformer(SentenceTransformer):
+    def __init__(self, model_name_or_path: str):
+        super().__init__(model_name_or_path)
+        self.cache = {}  # Initialize an empty cache
+
+    def encode(self, sentences, convert_to_tensor=False, **kwargs):
+        if isinstance(sentences, str):
+            sentences = [sentences]
+
+        results = []
+        sentences_to_encode = []
+        original_positions = []
+
+        cache_key_suffix = "_tensor" if convert_to_tensor else "_array"
+
+        for i, sentence in enumerate(sentences):
+            cache_key = f"{sentence}" + cache_key_suffix
+            if cache_key in self.cache:
+                results.append(self.cache[cache_key])
+            else:
+                sentences_to_encode.append(sentence)
+                original_positions.append(i)
+                results.append(None)  # Placeholder
+
+        if sentences_to_encode:
+            encoded = super().encode(sentences_to_encode, convert_to_tensor=convert_to_tensor, **kwargs)
+            if not isinstance(encoded, list):
+                encoded = [encoded[i] for i in range(len(sentences_to_encode))]
+            
+            for original_pos, sentence, emb in zip(original_positions, sentences_to_encode, encoded):
+                cache_key = sentence + cache_key_suffix
+                self.cache[cache_key] = emb
+                results[original_pos] = emb
+
+        if len(results) == 1:
+            return results[0]
+
+        if convert_to_tensor:
+            return torch.stack(results)
+        else:
+            return np.array(results)

bitagent_subnet-main/bitagent/helpers/string_parse.py

@@ -0,0 +1,34 @@
+import re
+import json
+
+def extract_text_inside_quotes(s):
+    match = re.search(r'"(.*?)"', s)
+    if match:
+        return match.group(1)  # Returns the text inside the first pair of double quotes
+    else:
+        return s  # Returns the original string if no double quotes are found
+
+def parse_multiple_space_sep_json(json_str):
+    """
+    Parses a string containing multiple JSON objects separated by whitespace.
+    
+    {} {} -> [{},{}]
+    """
+    results = []
+    start = 0
+    json_str = json_str.strip()  # Remove leading and trailing whitespace
+    while start < len(json_str):
+        # Find the start of a JSON object
+        start = json_str.find('{', start)
+        if start == -1:  # No more JSON object
+            break
+        try:
+            obj, index = json.JSONDecoder().raw_decode(json_str[start:])
+            results.append(obj)
+            start += index
+            while start < len(json_str) and json_str[start] in ' \t\n\r':  # Skip whitespace
+                start += 1
+        except json.JSONDecodeError:
+            # Move start forward and try again
+            start += 1
+    return results

bitagent_subnet-main/bitagent/helpers/tool_parsing.py

@@ -0,0 +1,92 @@
+import bittensor as bt
+from typing import Dict, Any, List
+from pydantic import ValidationError
+from bitagent.schemas.tool import Tool, ToolCall
+from bitagent.schemas.chat import ChatMessage, messages_to_list
+
+# Mapping from type strings to Python types
+type_mapping = {
+    "str": str,
+    "int": int,
+    "dict": Dict,
+    "list": List,
+    "float": float,
+    "bool": bool,
+    "string": str,
+    "integer": int,
+    "number": (int, float),  # Allow both int and float for 'number'
+    "boolean": bool,
+    "array": List,
+    "dictionary": Dict,
+    "object": Dict,  # Handle nested objects as dictionaries
+}
+
+def validate_tool_call(tool: Tool, tool_call: Dict[str, Any]) -> bool:
+    try:
+        # Validate the tool call structure
+        tool_call_validated = ToolCall(**tool_call)
+
+        # Check if the tool call name matches the tool name
+        if tool_call_validated.name != tool.name:
+            #bt.logging.warning(f"Tool name mismatch: {tool_call_validated.name} != {tool.name}")
+            return False
+        
+        if len(tool_call_validated.arguments.keys()) < len([argname for argname, argdict in tool.arguments.items() if argdict['required']]) or len(tool_call_validated.arguments.keys()) > len([argname for argname, argdict in tool.arguments.items()]):
+            #bt.logging.warning(f"Argument length mismatch")
+            return False
+
+        # Check arguments
+        for arg_name, arg_schema in tool.arguments.items():
+            if arg_schema['required'] and arg_name not in tool_call_validated.arguments:
+                #bt.logging.warning(f"Missing required argument: {arg_name}")
+                return False
+            if arg_name in tool_call_validated.arguments:
+                expected_type = type_mapping.get(arg_schema['type'])
+                if expected_type is None:
+                    #bt.logging.warning(f"Unknown type for argument {arg_name}: {arg_schema['type']}")
+                    return False
+                
+                # Handle nested objects
+                if 'is_ground_truth' in list(tool_call.keys()) and tool_call['is_ground_truth']:
+                    arg_value = tool_call_validated.arguments[arg_name][-1]
+                else:
+                    arg_value = tool_call_validated.arguments[arg_name]
+                # convert int to float if expected type is float
+                if expected_type == float and type(arg_value) == int:
+                    arg_value = float(arg_value)
+                # convert str to float if expected type is float
+                if expected_type == float and type(arg_value) == str:
+                    arg_value = float(arg_value)
+                # convert str to int if expected type is int
+                if expected_type == int and type(arg_value) == str:
+                    arg_value = int(arg_value)
+
+                if expected_type == dict:
+                    if not isinstance(arg_value, dict):
+                        #bt.logging.warning(f"""Argument {arg_name} has incorrect type. 
+                        #                    Expected {expected_type}, got {type(arg_value)}""")
+                        return False
+                else:
+                    if not isinstance(arg_value, expected_type):
+                        #bt.logging.warning(f"""Argument {arg_name} has incorrect type. 
+                        #                    Expected {expected_type}, got {type(arg_value)}""")
+                        return False
+        
+        # All checks passed
+        return True
+    except ValidationError as e:
+        #bt.logging.warning(f"Validation error: {e}")
+        return False
+
+def find_first_tool_call(messages: List[ChatMessage]):
+    for msg in messages:
+        if msg.role == 'tool call':
+            return msg
+    
+def find_msgs_before_tool_call(messages: List[ChatMessage]):
+    result = []
+    for msg in messages:
+        if msg.role == 'tool call':
+            break
+        result.append(msg)
+    return result

bitagent_subnet-main/bitagent/miners/__init__.py

@@ -0,0 +1,9 @@
+__version__ = "1.0.0"
+version_split = __version__.split(".")
+__spec_version__ = (
+    (1000 * int(version_split[0]))
+    + (10 * int(version_split[1]))
+    + (1 * int(version_split[2]))
+)
+from . import mock_miner
+from . import default_miner

bitagent_subnet-main/bitagent/miners/default_miner.py

@@ -0,0 +1,30 @@
+# The MIT License (MIT)
+# Copyright © 2024 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import bitagent
+from bitagent.helpers.llms import llm
+
+def miner_init(self, config=None):
+    self.model_name = self.config.hf_model_name_to_run
+    self.llm = llm
+
+def miner_process(self, synapse: bitagent.protocol.QueryTask) -> bitagent.protocol.QueryTask:
+    llm_response = self.llm(self, synapse.messages, synapse.tools, self.model_name)
+    synapse.response = llm_response
+    synapse.hf_run_model_name = self.model_name
+
+    return synapse

bitagent_subnet-main/bitagent/miners/mock_miner.py

@@ -0,0 +1,32 @@
+# The MIT License (MIT)
+# Copyright © 2023 Yuma Rao
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import bitagent
+
+def miner_init(self, config=None):
+
+    def llm():
+        return {"role": "assistant", "content": "I'm the LLM response, b/c I'm mock miner - rahhh!"}
+
+    self.llm = llm
+
+def miner_process(self, synapse: bitagent.protocol.QueryTask) -> bitagent.protocol.QueryTask:
+    llm_response = self.llm()
+    synapse.response = llm_response
+
+    return synapse

bitagent_subnet-main/bitagent/protocol.py

@@ -0,0 +1,66 @@
+# The MIT License (MIT)
+# Copyright © 2023 Yuma Rao
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+from typing import Optional, List
+import bittensor as bt
+from bitagent.schemas.chat import ChatMessage
+from bitagent.schemas.tool import Tool
+
+class QueryTask(bt.Synapse):
+    """
+    A simple BitAgent protocol representation which uses bt.Synapse as its base.
+    This protocol helps in handling validator request and miner response communication
+
+    Attributes:
+    - messages: a list of ChatMessage (see bitagent/schemas) - will be used for every task except Tool Gen
+    - tools: list of tools {name, description, arguments } in a List of dicts
+    - repsonse: string (e.g., tool_name(arg1=value1, arg2=value2))
+    - hf_run_model_name: string representing the HF model the miner is running
+    """
+
+    # Required request input, filled by sending dendrite caller.
+    tools: List[Tool] = []
+    messages: List[ChatMessage] = []
+
+    # Optional request output, filled by recieving axon.
+    response: str = ""
+    hf_run_model_name: str = "N/A"
+    competition_version: Optional[str] = None
+
+class QueryResult(bt.Synapse):
+    """
+    Provide feedback on last task request from validator to inform Miner of performance.
+    This is a one-way request does not require a response.
+    Attributes:
+    - results: string of results to be printed to the logs
+    """
+    results: str
+
+class IsAlive(bt.Synapse):
+    response: bool = False
+
+# Validator calls this to get the HF model name that this miner hosts on HF
+class GetHFModelName(bt.Synapse):
+    hf_model_name: Optional[str] = None
+
+# Validator calls this to have the miner set the TOP HF model for this miner to run
+class SetHFModelName(bt.Synapse):
+    hf_model_name: str
+
+class GetHFRunModelName(bt.Synapse):
+    hf_run_model_name: Optional[str] = None

bitagent_subnet-main/bitagent/schemas/chat.py

@@ -0,0 +1,39 @@
+from strenum import StrEnum
+from typing import Dict, List
+from pydantic import BaseModel, Field
+
+class ChatRole(StrEnum):
+    """One of ASSISTANT|USER to identify who the message is coming from."""
+
+    SYSTEM = "system"
+    ASSISTANT = "assistant"
+    USER = "user"
+    TOOL_CALL = "tool call"
+    TOOL_RESPONSE = "tool response"
+
+
+class ChatMessage(BaseModel):
+    """A list of previous messages between the user and the model, meant to give the model conversational context for responding to the user's message."""
+
+    role: ChatRole = Field(
+        title="One of the ChatRole's to identify who the message is coming from.",
+    )
+    content: str | dict | list = Field( # TODO the dict/list was added to support json loading the function calls. this should maybe be done inside  a ToolMessage type
+        title="Contents of the chat message.",
+    )
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, str]):
+        """Create a ChatMessage object from a dictionary."""
+        return cls(role=ChatRole(data['role']), content=data['content'])
+    
+    def to_dict(self) -> Dict[str, str]:
+        return {"role": self.role.value, "content": self.content}
+    
+
+def messages_from_list(data_list: List[Dict[str, str]]):
+    messages = [ChatMessage.from_dict(item) for item in data_list]
+    return messages
+
+def messages_to_list(messages: List[ChatMessage]):
+    return [msg.to_dict() for msg in messages]

bitagent_subnet-main/bitagent/schemas/tool.py

@@ -0,0 +1,21 @@
+from pydantic import BaseModel
+from typing import Dict, Any, List
+
+class Tool(BaseModel):
+    """
+    Attributes:
+    - name: str
+    - description: str
+    - arguments: dict where the key is the name of the argument and the value is a dict containing the keys (required:bool, type:str, description:str)
+    """
+    name: str
+    description: str
+    arguments: Dict[str, Any]
+    
+    def to_dict(self):
+        return self.dict()
+    
+    
+class ToolCall(BaseModel):
+    name: str
+    arguments: Dict[str, Any]

bitagent_subnet-main/bitagent/tasks/__init__.py

@@ -0,0 +1,3 @@
+from .constants import * 
+from .task import *
+from .tool_call_task import *

bitagent_subnet-main/bitagent/tasks/constants.py

@@ -0,0 +1,7 @@
+TASK_FREQUENCY = {
+    "tool_call": 1,
+}
+
+TASK_WEIGHTS = {
+    "tool_call": 0.05,
+}

bitagent_subnet-main/bitagent/tasks/task.py

@@ -0,0 +1,105 @@
+# The MIT License (MIT)
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+import random
+import bittensor as bt
+from pprint import pformat
+from typing import List, Tuple
+from bitagent.protocol import QueryTask
+from bitagent.schemas.tool import Tool
+from bitagent.tasks import TASK_FREQUENCY
+from bitagent.criteria import Criterion, default_criteria
+from bitagent.schemas.chat import ChatMessage, messages_to_list
+
+# Task()
+# combines criterion/criteria with the QueryTask (messages,tools) for eval to form a task for the miner
+class Task():
+    criteria: List[Criterion]
+    synapse: QueryTask
+
+    def __init__(self, 
+                 name: str, 
+                 weight: int = 0.05,
+                 desc: str = "", 
+                 timeout: int = 12,
+                 tools: List[Tool] = [],
+                 messages: List[ChatMessage] = [],
+                 criteria: List[Criterion] = default_criteria,
+                 correct_answer: str=None
+                        ) -> None:
+
+        self.name=name
+        self.mode = "online"
+        self.weight = weight
+        self.desc=desc
+        self.timeout=timeout
+        self.criteria=criteria
+        self.messages = messages
+        self.synapse = QueryTask(messages=messages, tools=tools)
+        self.correct_answer = correct_answer
+
+    def reward(self, validator, synapse: QueryTask) -> Tuple[float, float, List[str]]:
+        total_score = 0.0
+        total_possible = 0.0
+        results = []
+        for criterion in self.criteria:
+            score, max_score, result = criterion.evaluate(self, validator, synapse)
+            total_score += score
+            total_possible += max_score
+            results.append(result)
+        if self.correct_answer:
+            correct_answer = self.correct_answer
+        else:
+            correct_answer = "N/A"
+        return [total_score, total_possible, results, correct_answer]
+
+    def __repr__(self):
+        return pformat(vars(self), indent=4, width=1)
+    
+    def toJSON(self):
+        return {
+            "weight": self.weight,
+            "name": self.name,
+            "mode": self.mode,
+            "desc": self.desc,
+            "messages": messages_to_list(self.messages) if isinstance(self.messages, list) else [], 
+            "tools": [tool.to_dict() for tool in self.synapse.tools],
+            "timeout": self.timeout,
+        }
+
+# evaluate task
+def evaluate_task(validator, task:Task, synapse:bt.Synapse) -> Tuple[float, float, List[str]]:
+    return task.reward(validator, synapse)
+
+# get random task
+def get_random_task(validator, offline=False) -> Task:
+    from bitagent.tasks.tool_call_task import ToolCallTask
+    task_names = list(TASK_FREQUENCY.keys())
+    task_frequencies = list(TASK_FREQUENCY.values())
+    choice = random.choices(task_names, weights=task_frequencies)[0]
+    
+    for _ in range(100):
+        try:
+            match choice:
+                case "tool_call":
+                    return ToolCallTask(validator=validator, name="Responds with correct function call", offline=offline)
+
+        except Exception as e:
+            #bt.logging.warning(f'Error getting task (name {choice}): ', e)
+            #bt.logging.warning(traceback.format_exc())
+            pass
+
+    raise Exception("Failed to get task after 100 attempts")

bitagent_subnet-main/bitagent/tasks/tool_call_task.py

@@ -0,0 +1,191 @@
+# The MIT License (MIT)
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+import ast
+import json
+import random
+import bittensor as bt
+from bitagent.protocol import QueryTask
+from bitagent.tasks import Task
+from bitagent.tasks import TASK_WEIGHTS
+from bitagent.schemas.chat import messages_to_list
+from bitagent.datasources.tools import ToolCallData
+from bitagent.helpers.tool_parsing import validate_tool_call, find_msgs_before_tool_call, find_first_tool_call
+from bitagent.criteria import default_criteria, tool_call_criteria, irrelevant_tool_call_criteria
+
+REWRITE_TOOL_USER_PROMPT = """You rewrite questions to make sense when paired with a function call. 
+The rewritten question will need to be changed to match the argument parameters and values relative to the function name.
+You should change the phrasing of the question to be different and keeping aligned with the function name and arguments. 
+The capitalization of your user prompt rephrasasl should match the exact case of what is expected in the function call.
+Your response should be the rewritten question only.\n
+Function call:\n`{tool_call}`\n 
+Question: {user}\n 
+Modified Question: """
+
+class ToolCallTask(Task):
+    def __init__(
+        self,
+        validator,
+        name: str,
+        desc: str = "",
+        offline: bool = False,
+    ):
+        super().__init__(name=name, desc=desc)
+        self.validator = validator
+        self.timeout = 15.0
+        self.name += " - Tool Call"
+        self.weight = TASK_WEIGHTS["tool_call"]
+
+        if offline:
+            self.mode = "offline"
+        messages = None
+        for _ in range(10):
+            try:
+                messages, tools, data = self.generate_task_data()
+                expected_messages = messages_to_list(data.messages)
+                expected_tool_call_messages = [em for em in expected_messages if em['role'] == 'tool call']
+                if messages[0].role == 'system':
+                    # try again - skip tasks with system prompts
+                    continue
+                if len(expected_tool_call_messages) > 0:
+                    expected_tool_call_message = expected_tool_call_messages[0]['content']
+                else:
+                    #bt.logging.debug(f"Skipping - no tool call message found in expected messages: {expected_messages}")
+                    continue
+
+                if type(expected_tool_call_message) == str:
+                    expected_tool_call = json.loads(expected_tool_call_message)
+                else:
+                    expected_tool_call = expected_tool_call_message
+                self.criteria = default_criteria + tool_call_criteria(expected_response=expected_tool_call)
+
+                # 75% of the time do a tool call task with a relevant tool, other times do a tool call with no valid tool option
+                # irrelevant tool call
+                if "is_ground_truth" not in expected_tool_call_message and bool(random.random() < 0.25) and len(tools) > 1:
+                    # remove the real tool
+                    expected_tool_call_message_json = json.loads(expected_tool_call_message)
+                    if isinstance(expected_tool_call_message_json, str):
+                        expected_tool_call_message_json = json.loads(expected_tool_call_message_json)
+                    tools = [t for t in tools if t.name != expected_tool_call_message_json['name']]
+                    self.criteria = default_criteria + irrelevant_tool_call_criteria()
+
+                break
+
+            except Exception as e:
+                bt.logging.debug(f'Exception getting new task - {e} - you may need to CHECK YOUR vLLM docker instance')
+                pass
+        if not messages:
+            raise Exception(f"Failed to generate task data 10 times")
+        self.messages = messages
+        self.synapse = QueryTask(messages=messages, tools=tools)
+    
+    def generate_task_data(self) -> ToolCallData:
+        data: ToolCallData = next(self.validator.tool_dataset)
+
+        tool_call = find_first_tool_call(data.messages)
+        if not tool_call:
+            # no tool call in the messages, so skip
+            raise Exception(f"Skipping - no tool call in the messages: {data.messages}")
+
+        # increase number of tools
+        for _ in range(random.randint(2,4)):
+            # filter out the tools by name that are already in the data.tools
+            new_tools = [t for t in next(self.validator.tool_dataset).tools if t.name not in [dt.name for dt in data.tools]]
+            data.tools = data.tools + new_tools
+        
+        # remove all the messages after the first tool call, keeping the assistant
+        # this reduces the number of messages needing rewording
+        messages = data.messages
+        filtered_msgs = []
+        seen_tool_call = False
+        for msg in messages:
+            filtered_msgs.append(msg)
+            if seen_tool_call: # want to do break after to include the assistant response
+                break
+            if msg.role == 'tool call':
+                seen_tool_call = True
+        data.messages = filtered_msgs
+
+        user = data.messages[0].content
+
+        count = 0
+        while count < 10:
+            count += 1
+            if find_first_tool_call(data.messages):
+                tool_call = find_first_tool_call(data.messages).content
+                try: # check that the tool call can be loaded, and that it's valid
+                    try:
+                        if isinstance(tool_call, str):
+                            new_tool_call = json.dumps(json.loads(tool_call))
+                            tool_call_dict = json.loads(new_tool_call)
+                        elif isinstance(tool_call, dict):
+                            new_tool_call = tool_call
+                            tool_call_dict = tool_call
+                        else:
+                            raise Exception(f'tool call is not a string or dict: {tool_call}')
+
+                    except Exception as e:
+                        # this usually happens when the json is not valid (single vs double quotes)
+                        new_tool_call = json.dumps(ast.literal_eval(tool_call))
+                        tool_call_dict = ast.literal_eval(tool_call)
+                    # check through all the tools that will be passed to the miner
+                    # find the tool that is THE tool that is expected to be returned
+                    # since it has been rewritten, validate that the tool call is valid/comparable still
+                    #for tool in data.tools:
+                    #    if tool.name == tool_call_dict['name']:
+                    #        if not validate_tool_call(tool, tool_call_dict):
+                    #            raise Exception('The rewritten tool call is not valid')
+                    #bt.logging.debug(f'finished validating tool call: {tool_call_dict}')
+                except Exception as e:
+                    bt.logging.error(f'An error occured while rewriting the tool call {e} - you may need to CHECK YOUR vLLM docker instance')
+                    count = 11
+                    continue
+
+                rw_prompt = REWRITE_TOOL_USER_PROMPT.format(tool_call=new_tool_call, user=user)
+                new_user = self.validator.llm([{"role": "user", "content": rw_prompt}], max_new_tokens=1000, temperature=1)
+                if not self.check_rewrite_alignment(new_user, user):
+                    raise Exception(f"User rewrite is not in alignment\nOriginal: {user}\n Rewrite: {new_user}")
+                
+                data.messages[0].content = new_user
+
+                data = ToolCallData(messages=data.messages, tools=data.tools)
+                messages_before_call = find_msgs_before_tool_call(data.messages)
+                
+            else:
+                # no tool call in the messages, so skip
+                raise Exception(f"Skipping - guess there was no tool call in the messages: {data.messages}")
+                
+            all_tools = data.tools
+            random.shuffle(all_tools)
+            return messages_before_call, all_tools, data
+        
+        raise Exception("Skipping - while loop ended without a tool call task")
+
+    def check_rewrite_alignment(self, original: str, rewrite: str) -> bool:
+        score = self.validator.measure_relevance_of_texts(original, rewrite)
+        
+        if score > 0.98:
+            return False
+        
+        if score < 0.2:
+            return False
+
+        if len(rewrite) > 2 * len(rewrite):
+            return False
+        
+        if len(rewrite) < 0.25 * len(rewrite):
+            return False
+        
+        return True

bitagent_subnet-main/bitagent/validator/__init__.py

@@ -0,0 +1,10 @@
+__version__ = "1.0.15"
+version_split = __version__.split(".")
+__spec_version__ = (
+    (1000 * int(version_split[0]))
+    + (10 * int(version_split[1]))
+    + (1 * int(version_split[2]))
+)
+from . import reward
+from .forward import forward
+from .initiation import initiate_validator

bitagent_subnet-main/bitagent/validator/constants.py

@@ -0,0 +1,5 @@
+COMPETITION_PREVIOUS_PREFIX = 1 # This is the previous competition prefix for when we swap to a new competition prefix and need to keep track of the old scores
+COMPETITION_PREFIX = 1
+DEPLOYED_DATE = "2024-12-10"
+COMPETITION_LENGTH_DAYS = 7
+TESTNET_COMPETITION_LENGTH_DAYS = 1.0/24.0 # every hour

bitagent_subnet-main/bitagent/validator/forward.py

@@ -0,0 +1,129 @@
+# The MIT License (MIT)
+# Copyright © 2023 Yuma Rao
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import asyncio
+import numpy as np
+import bittensor as bt
+from bitagent.protocol import QueryTask
+from common.utils.uids import get_alive_uids
+from common.utils.uids import get_random_uids
+from bitagent.tasks.task import get_random_task
+from bitagent.validator.offline_task import offline_task
+from bitagent.validator.reward import process_rewards_update_scores_and_send_feedback
+
+async def forward(self, synapse: QueryTask=None) -> QueryTask:
+    """
+    The forward function is called by the validator every time step.
+    It is responsible for querying the network and scoring the responses.
+
+    Args:
+        self (:obj:`bittensor.neuron.Neuron`): The neuron object which contains all the necessary state for the validator.
+
+    """
+    # complete this first so it's cached for both ONLINE and OFFLINE
+    get_alive_uids(self)
+
+    # ###########################################################
+    # OFFLINE TASKING
+    # ###########################################################
+    # if all miners have been processed for this competition, then don't run offline mode
+    self.update_competition_numbers()
+
+    wandb_data = {
+        "task_name": "offline_model_check",
+        "task_mode": "offline",
+        "validator_uid": self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address),
+        "val_spec_version": self.spec_version,
+        "highest_score_for_miners_with_this_validator": self.scores.max(),
+        "median_score_for_miners_with_this_validator": np.median(self.scores),
+        "highest_offline_score_for_miners_with_this_validator": self.offline_scores[self.competition_version].max(),
+        "median_offline_score_for_miners_with_this_validator": np.median(self.offline_scores[self.competition_version]),
+        "average_offline_score_for_miners_with_this_validator": np.mean(self.offline_scores[self.competition_version]),
+        "prior_highest_offline_score_for_miners_with_this_validator": self.offline_scores[self.previous_competition_version].max(),
+        "prior_median_offline_score_for_miners_with_this_validator": np.median(self.offline_scores[self.previous_competition_version]),
+        "prior_average_offline_score_for_miners_with_this_validator": np.mean(self.offline_scores[self.previous_competition_version]),
+        "competition_version": self.competition_version,
+    }
+
+    if self.config.subtensor.network != "test":
+        if len(self.miners_left_to_score) == 0:
+            if self.offline_status != "complete":
+                self.offline_status = "complete"
+                wandb_data['offline_status'] = self.offline_status
+                wandb_data['num_miners_left_to_score'] = len(self.miners_left_to_score)
+                self.log_event(wandb_data)
+                wandb_data.pop('offline_status')
+                wandb_data.pop('num_miners_left_to_score')
+            self.running_offline_mode = False
+            #bt.logging.debug(f"OFFLINE: No miners left to score for competition {self.competition_version}")
+            pass
+        elif not self.running_offline_mode:
+            bt.logging.debug(f"OFFLINE: Starting offline mode for competition {self.competition_version}")
+            #bt.logging.debug(f"OFFLINE: Miners left to score: {self.miners_left_to_score}")
+            self.running_offline_mode = True
+            self.offline_status = "starting"
+            wandb_data['offline_status'] = self.offline_status
+            wandb_data['num_miners_left_to_score'] = len(self.miners_left_to_score)
+            wandb_data['miners_left_to_score'] = self.miners_left_to_score
+            self.log_event(wandb_data)
+            wandb_data.pop('num_miners_left_to_score')
+            wandb_data.pop('miners_left_to_score')
+            wandb_data.pop('offline_status')
+            asyncio.create_task(offline_task(self, wandb_data))
+            self.running_offline_mode = False
+        elif self.running_offline_mode:
+            #bt.logging.debug(f"OFFLINE: Already running offline mode for competition {self.competition_version}")
+            #bt.logging.debug(f"OFFLINE: Miners left to score: {self.miners_left_to_score}")
+            if self.offline_status != "running":
+                self.offline_status = "running"
+                wandb_data['offline_status'] = self.offline_status
+                wandb_data['num_miners_left_to_score'] = len(self.miners_left_to_score)
+                self.log_event(wandb_data)
+                wandb_data.pop('num_miners_left_to_score')
+                wandb_data.pop('offline_status')
+            pass
+    else:
+        bt.logging.debug("OFFLINE: Skipping offline for testnet")
+
+    # ###########################################################
+    # ONLINE TASKING
+    # ###########################################################
+    try:
+        bt.logging.debug(f"ONLINE: Starting online run")
+        # check a random sample of miners in online mode
+        bt.logging.debug(f"ONLINE: Getting random miner uids")
+        miner_uids = get_random_uids(self, min(self.config.neuron.sample_size, self.metagraph.n.item()))
+        bt.logging.debug(f"ONLINE: Getting random task")
+        task = get_random_task(self)
+        task.mode = "online"
+
+        # send the task to the miners
+        bt.logging.debug(f"ONLINE: Sending task to miners")
+        responses = self.dendrite.query(
+            axons=[self.metagraph.axons[uid] for uid in miner_uids],
+            synapse=task.synapse,
+            deserialize=False,
+            timeout=task.timeout,
+        )
+
+        bt.logging.debug(f"ONLINE: Evaluating responses")
+        await asyncio.create_task(process_rewards_update_scores_and_send_feedback(self, task=task, responses=responses, miner_uids=miner_uids))
+        bt.logging.debug(f"ONLINE: Evaluation complete")
+
+    except Exception as e:
+        bt.logging.debug(f"Error in forward: {e}")

bitagent_subnet-main/bitagent/validator/initiation.py

@@ -0,0 +1,151 @@
+# The MIT License (MIT)
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import os
+import copy
+import wandb
+import shutil
+import bittensor as bt
+from datetime import datetime
+from bitagent.datasources import ToolDataset
+from langchain_openai import ChatOpenAI
+from sentence_transformers import util
+from bitagent.helpers.sbert import CachedSentenceTransformer
+
+# setup validator with wandb
+# clear out the old wandb dirs if possible
+def initiate_validator(self):
+    
+    def init_wandb(self, reinit=False):
+        uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address)
+        spec_version = self.spec_version
+
+        """Starts a new wandb run."""
+        tags = [
+            self.wallet.hotkey.ss58_address,
+            str(spec_version),
+            f"netuid_{self.config.netuid}",
+        ]
+
+        wandb_config = {
+            key: copy.deepcopy(self.config.get(key, None))
+            for key in ("neuron", "reward", "netuid", "wandb")
+        }
+        wandb_config["neuron"].pop("full_path", None)
+        wandb_config["validator_uid"] = uid
+
+        if self.config.netuid == 20:
+            project_name = "mainnet"
+        elif self.config.netuid == 76:
+            project_name = "testnet" # for TN76
+        else:
+            self.wandb = "errored"
+            return # must be using a local netuid, no need to log to wandb
+
+        try:
+            self.wandb = wandb.init(
+                anonymous="allow",
+                reinit=reinit,
+                entity='bitagentsn20',
+                project=project_name,
+                config=wandb_config,
+                dir=self.config.neuron.full_path,
+                tags=tags,
+                resume='allow',
+                name=f"{uid}-{spec_version}-{datetime.today().strftime('%Y-%m-%d')}",
+            )
+            bt.logging.success(f"Started a new wandb run <blue> {self.wandb.name} </blue>")
+        except Exception as e:
+            self.wandb = "errored"
+            bt.logging.error("Could not connect to wandb ... continuing without it ...")
+            bt.logging.error(f"WANDB Error: {e}")
+
+    def log_event(event):
+        #bt.logging.debug("Writing to WandB ....")
+
+        if not self.config.wandb.on:
+            return
+
+        if not getattr(self, "wandb", None):
+            clear_wandb_dir(self)
+            init_wandb(self)
+
+        if self.wandb == "errored":
+            return
+
+        # Log the event to wandb.
+        self.wandb.log(event)
+        #bt.logging.debug("Logged event to WandB ....")
+
+    self.log_event = log_event
+
+    initiate_validator_local(self)
+
+def clear_wandb_dir(self):
+    wandb_path = os.path.join(self.config.neuron.full_path, "wandb")
+    if os.path.exists(wandb_path):
+        bt.logging.info(f"Clearing WandB directory of old runs")
+        for item in os.listdir(wandb_path):
+            item_path = os.path.join(wandb_path, item)
+            try:
+                if os.path.islink(item_path):  # If it's a symbolic link
+                    os.unlink(item_path)  # Remove the symlink
+                elif os.path.isfile(item_path):  # If it's a regular file
+                    os.remove(item_path)
+                elif os.path.isdir(item_path):  # If it's a directory
+                    shutil.rmtree(item_path)
+            except Exception as e:
+                bt.logging.warning(f"Failed to remove {item_path}: {e}")
+        bt.logging.info(f"Cleared WandB directory of old runs")
+
+# provide some capabilities to the task API (LLM, cossim)
+def initiate_validator_local(self):
+    #bt.logging.info("Initializing Validator - this may take a while (downloading data and models).")
+    self.tool_dataset = ToolDataset()
+    #bt.logging.debug("Initializing Validator - this may take a while (downloading data and models) - loading model ...")
+    self.sentence_transformer = CachedSentenceTransformer('BAAI/bge-small-en-v1.5')
+
+    def llm(messages, max_new_tokens = 160, temperature=0.7):
+        if isinstance(messages, str):
+            messages = [{"role":"user","content":messages}]
+        llm = ChatOpenAI(
+            openai_api_key=self.config.openai_api_key,
+            openai_api_base=self.config.openai_api_base,
+            model_name=self.config.validator_model_name,
+            max_tokens = max_new_tokens,
+            temperature = temperature,
+        )
+        return llm.invoke(messages).content.strip()
+    
+    self.llm = llm
+    
+    #bt.logging.debug("Initializing Validator - this may take a while (downloading data and models) - finished loading model")
+
+    # code to measure the relevance of the response to the question
+    def measure_relevance_of_texts(text1, text2): 
+        # Encode the texts to get the embeddings
+        if type(text2) == list:
+            embeddings = self.sentence_transformer.encode([text1,*text2], convert_to_tensor=True, show_progress_bar=False)
+        else:
+            embeddings = self.sentence_transformer.encode([text1,text2], convert_to_tensor=True, show_progress_bar=False)
+        # Compute the cosine similarity between the embeddings
+        if type(text2) == list:
+            return util.pytorch_cos_sim(embeddings[0], embeddings[1:])[0]
+        else:
+            return float(util.pytorch_cos_sim(embeddings[0], embeddings[1:])[0][0])
+
+    self.measure_relevance_of_texts = measure_relevance_of_texts

bitagent_subnet-main/bitagent/validator/offline_task.py

@@ -0,0 +1,425 @@
+import os
+import time
+import shutil
+import psutil
+import asyncio
+import requests
+import bittensor as bt
+
+from sglang.utils import (
+    terminate_process)
+from bitagent.helpers.llms import llm
+from huggingface_hub import model_info
+from common.utils.uids import get_alive_uids
+from bitagent.protocol import GetHFModelName
+from bitagent.tasks.task import get_random_task
+from common.utils.shell import execute_shell_command
+from bitagent.helpers.logging import temporary_logging_state
+from bitagent.validator.reward import process_rewards_update_scores_for_many_tasks_and_many_miners
+
+# TODO overall for tracking, would be nice to track based on hotkey instead of UID
+# it's currently handled for uid and new hotkeys taking over a uid, but might be cleaner
+
+# Delete the model from the huggingface cache when we're done serving it so we don't run out of disk space
+def delete_model_from_hf_cache(self, model_name: str):
+    # Determine the cache directory
+    cache_dir = os.path.expanduser(self.config.validator_hf_cache_dir)
+    
+    # Format the directory name based on the model name
+    model_cache_dir = os.path.join(cache_dir, f"models--{model_name.replace('/', '--')}")
+    
+    # Check if the directory exists and delete it
+    if os.path.exists(model_cache_dir):
+        try:
+            shutil.rmtree(model_cache_dir)
+            bt.logging.debug(f"OFFLINE: Model has been removed from the HF cache.")
+        except Exception as e:
+            bt.logging.error(f"OFFLINE: Error deleting model: from HF cache: {e}")
+    else:
+        bt.logging.debug(f"OFFLINE: Model not found in the cache, could not delete")
+
+# added our own wait for server to check the process itself 
+# this will check to see if the sglang process crashes due to limited VRAM
+def wait_for_server(base_url: str, server_process, timeout: int = None) -> None:
+    """Wait for the server to be ready by polling the /v1/models endpoint.
+
+    Args:
+        base_url: The base URL of the server
+        server_process: The process to terminate if the server is ready
+        timeout: Maximum time to wait in seconds. None means wait forever.
+    """
+    start_time = time.time()
+    procutil = psutil.Process(int(server_process.pid))
+    while True:
+        try:
+            if timeout and time.time() - start_time > timeout:
+                bt.logging.error(f"OFFLINE: Server did not become ready within timeout period")
+                raise TimeoutError("Server did not become ready within timeout period")
+
+            # Use psutil to monitor the process
+            if not procutil.is_running():  # Check if process is still running
+                bt.logging.error(f"OFFLINE: Server process terminated unexpectedly, check VRAM usage")
+                raise Exception("Server process terminated unexpectedly, potentially VRAM usage issue")
+            if server_process.poll() is not None:
+                bt.logging.error(f"OFFLINE: Server process terminated with code {server_process.poll()}")
+                raise Exception(f"Server process terminated with code {server_process.poll()}")
+
+            response = requests.get(
+                f"{base_url}/v1/models",
+                headers={"Authorization": "Bearer None"},
+            )
+            if response.status_code == 200:
+                time.sleep(5)
+                break
+
+        except requests.exceptions.RequestException:
+            time.sleep(1)
+
+
+# ###########################################################
+# OFFLINE TASKING
+# ###########################################################
+
+# TODO also run the bfcl suite on the validator - but skip the API calls, don't use those at first
+# TODO store TOP score from last round and all-time in validator state
+
+async def offline_task(self, wandb_data):
+    bt.logging.debug("OFFLINE: Starting offline task")
+    self.running_offline_mode = True
+    wandb_data['event_name'] = "offline_task_started"
+    self.log_event(wandb_data)
+
+    # get all alive miner UIDs to compare against the top scores from the last round
+    miner_uids = self.miners_left_to_score
+
+    # TODO potentially fetch prompt template from miner too
+    # Grab all the models that the miners submitted
+    responses = await self.dendrite.forward(
+        axons=[self.metagraph.axons[miner_uid] for miner_uid in miner_uids],
+        synapse=GetHFModelName(),
+        deserialize=False,
+        timeout=15.0,
+    )
+
+    wandb_data['event_name'] = "GetHFModelName Responses Fetched"
+    self.log_event(wandb_data)
+
+    # get all the HF model names from the responses
+    miner_hf_model_names = [response.hf_model_name for response in responses]
+    bt.logging.debug(f"OFFLINE: Miner HF model names: {len(miner_hf_model_names)}")
+
+    try:
+        hf_model_name_to_miner_uids = {}
+        for i,miner_uid in enumerate(miner_uids):
+            self.offline_model_names[self.competition_version][miner_uid] = responses[i].hf_model_name
+            if responses[i].hf_model_name is not None:
+                if responses[i].hf_model_name not in hf_model_name_to_miner_uids:
+                    hf_model_name_to_miner_uids[responses[i].hf_model_name] = []
+                hf_model_name_to_miner_uids[responses[i].hf_model_name].append(int(miner_uid))
+
+        # Group all the models together uniquely and share the same inference server
+        unique_miner_hf_model_names = [m for m in list(set(miner_hf_model_names)) if m not in [None, ""]]
+        if len(unique_miner_hf_model_names) == 0:
+            bt.logging.debug(f"OFFLINE: No unique miner HF model names to evaluate in OFFLINE mode")
+            for miner_uid in miner_uids:
+                self.offline_scores[self.competition_version][miner_uid] = 0.0
+            wandb_data['event_name'] = "No Unique HF Models"
+            wandb_data['miners_left_to_score'] = miner_uids
+            self.log_event(wandb_data)
+            wandb_data.pop('miners_left_to_score')
+            self.running_offline_mode = False
+            return
+    except Exception as e:
+        bt.logging.error(f"OFFLINE: Error getting unique miner HF model names: {e}")
+        wandb_data['event_name'] = "Error Getting Unique HF Models"
+        wandb_data['error'] = f"{e}"
+        self.log_event(wandb_data)
+        wandb_data.pop('error')
+        self.running_offline_mode = False
+        return
+
+    bt.logging.debug(f"OFFLINE: Unique miner HF model names: {len(unique_miner_hf_model_names)}")
+    wandb_data['event_name'] = "Unique HF Model Fetched"
+    wandb_data['num_unique_hf_models'] = len(unique_miner_hf_model_names)
+    self.log_event(wandb_data)
+    wandb_data.pop('num_unique_hf_models')
+
+    # no need to regrade if score exists for the same model
+    models_to_skip = []
+    
+    for hfmn in unique_miner_hf_model_names:
+        uids_with_same_model = []
+        scores_with_same_model = []
+        for k, model_name in self.offline_model_names[self.competition_version].items():
+            if model_name == hfmn:
+                uids_with_same_model.append(k)
+                scores_with_same_model.append(self.offline_scores[self.competition_version][k])
+        
+        if len(uids_with_same_model) > 0:
+            max_score_for_model = max(scores_with_same_model)  # Calculate max score once
+
+            if max_score_for_model <= 0:
+                # Skip adding to models_to_skip if max score is zero
+                continue
+            
+            models_to_skip.append(hfmn)  # Add only if max_score > 0
+            
+            # Process the miners
+            the_uids = hf_model_name_to_miner_uids[hfmn]
+            bt.logging.debug(f"OFFLINE: Found miner with same model, using existing score")
+            for uid in the_uids:
+                self.offline_scores[self.competition_version][uid] = max_score_for_model
+            self.update_offline_scores([max_score_for_model] * len(the_uids), the_uids)
+
+    # skip the models we already have scores for
+    unique_miner_hf_model_names = [m for m in unique_miner_hf_model_names if m not in models_to_skip]
+
+    if len(unique_miner_hf_model_names) > 0:
+        bt.logging.debug(f"OFFLINE: Generating tasks")
+        # Generate a set of tasks to run on all the offline models
+        num_tasks = 1000
+        batch_size = 100
+        wandb_data['event_name'] = "Generating Tasks"
+        self.log_event(wandb_data)
+        tasks = []
+        for i,_ in enumerate(range(0, num_tasks, batch_size)):
+            #bt.logging.debug(f"OFFLINE: Generating tasks batch {i+1} of {num_tasks // batch_size}")
+            tasks.extend(await asyncio.gather(*[asyncio.to_thread(get_random_task, self, offline=True) for _ in range(batch_size)]))
+            #bt.logging.debug(f"OFFLINE: Generated tasks batch {i+1} of {num_tasks // batch_size}")
+        bt.logging.debug(f"OFFLINE: Generated {len(tasks)} tasks of {num_tasks} total")
+        wandb_data['event_name'] = "Generated Tasks"
+        wandb_data['num_tasks'] = len(tasks)
+        self.log_event(wandb_data)
+        wandb_data.pop('num_tasks')
+
+    for i,hf_model_name in enumerate(unique_miner_hf_model_names):
+        bt.logging.debug(f"OFFLINE: Running tasks for model {i+1} of {len(unique_miner_hf_model_names)}")
+        wandb_data['event_name'] = "Running HF Model"
+        wandb_data['num_hf_model'] = i
+        wandb_data['miner_uids'] = hf_model_name_to_miner_uids[hf_model_name]
+        self.log_event(wandb_data)
+        wandb_data.pop('miner_uids')
+
+        if hf_model_name is None or hf_model_name == "" or hf_model_name.lower() == "none":
+            bt.logging.debug(f"OFFLINE: Miner returned empty HF model name ... skipping")
+            for miner_uid in hf_model_name_to_miner_uids[hf_model_name]:
+                self.offline_scores[self.competition_version][miner_uid] = 0.0
+            wandb_data['event_name'] = "Skipping Empty HF Model"
+            wandb_data['miner_uids'] = hf_model_name_to_miner_uids[hf_model_name]
+            self.log_event(wandb_data)
+            wandb_data.pop('miner_uids')
+            continue # skip this model
+
+        # Extract the model card data for the model from HF
+        # ensure logger doesn't print the model name publicly, so restrict to only HF warnings
+        # Temporarily set logging to WARNING within the context manager
+        with temporary_logging_state('Warning'):
+            info = model_info(hf_model_name)
+            total_size = info.safetensors.total
+            try:
+                license = info.card_data['license']
+            except Exception:
+                bt.logging.debug("OFFLINE: No license found for model")
+                license = 'No license available'
+
+        # confirm model license is apache-2.0 or nc-by-nc-4.0 or mit
+        # TODO eventually ONLY accept apache-2.0
+        if license not in ["apache-2.0", "cc-by-nc-4.0", "mit"]:
+            bt.logging.debug(f"OFFLINE: Skipping model {i+1} of {len(unique_miner_hf_model_names)} due to license: {license}")
+            for miner_uid in hf_model_name_to_miner_uids[hf_model_name]:
+                self.offline_scores[self.competition_version][miner_uid] = 0.0
+            wandb_data['event_name'] = "Skipping Model Due to License"
+            wandb_data['miner_uids'] = hf_model_name_to_miner_uids[hf_model_name]
+            self.log_event(wandb_data)
+            wandb_data.pop('miner_uids')
+            continue
+
+        # confirm model size is less than 10B params (want 8B or less models)
+        if total_size > 10000000000:
+            bt.logging.debug(f"OFFLINE: Skipping model {i+1} of {len(unique_miner_hf_model_names)} due to size: {total_size}")
+            for miner_uid in hf_model_name_to_miner_uids[hf_model_name]:
+                self.offline_scores[self.competition_version][miner_uid] = 0.0
+            wandb_data['event_name'] = "Skipping Model Due to Size"
+            wandb_data['miner_uids'] = hf_model_name_to_miner_uids[hf_model_name]
+            self.log_event(wandb_data)
+            wandb_data.pop('miner_uids')
+            continue
+
+        bt.logging.debug(f"OFFLINE: Starting server for model {i+1} of {len(unique_miner_hf_model_names)}")
+        wandb_data['event_name'] = "HF Model Eval Server Starting"
+        self.log_event(wandb_data)
+
+        # see if we have a snapshot already in the cache
+        latest_snapshot = None
+
+        try:
+            # Start the server for the model
+            try:
+                cache_dir = os.path.expanduser(self.config.validator_hf_cache_dir)
+                snapshot_dir = f"{cache_dir}/models--{hf_model_name.replace('/', '--')}/snapshots/"
+                
+                # Get all snapshot directories
+                snapshots = [os.path.join(snapshot_dir, d) for d in os.listdir(snapshot_dir) if os.path.isdir(os.path.join(snapshot_dir, d))]
+                
+                # Sort snapshots by creation time (os.path.getctime) or modification time (os.path.getmtime)
+                latest_snapshot = max(snapshots, key=os.path.getctime)
+                # TODO if the latest snapshot is older than a week, delete it and download a new one
+
+            except Exception as e:
+                bt.logging.debug(f"OFFLINE: Error getting latest snapshot")
+                latest_snapshot = None
+            
+            # # either load an existing snapshot or download the model
+            # if os.path.exists(snapshot_dir) and latest_snapshot:    
+            #     model_path = latest_snapshot
+            # else:
+            #     # need to download from hugging face
+            model_path = hf_model_name
+
+            server_process = await asyncio.to_thread(execute_shell_command,
+                f"""
+                {os.getcwd()}/.venvsglang/bin/python -m sglang.launch_server \
+                --model-path {model_path} \
+                --port {self.config.validator_hf_server_port} \ 
+                --host 0.0.0.0 \
+                --mem-fraction-static {self.config.validator_hf_server_mem_fraction_static} \
+                --context-length 25000
+                """, 
+                hf_model_name
+            )
+
+            bt.logging.debug(f"OFFLINE: Started server for model {i+1} of {len(unique_miner_hf_model_names)}, waiting for it to start on port {self.config.validator_hf_server_port} (could take several minutes)")
+            try:
+                await asyncio.wait_for(
+                    asyncio.to_thread(wait_for_server, f"http://localhost:{self.config.validator_hf_server_port}", server_process), 
+                    timeout=60*15 # wait up to 15 minutes
+                )
+                bt.logging.debug(f"OFFLINE: Server for model {i+1} of {len(unique_miner_hf_model_names)} started")
+                wandb_data['event_name'] = "HF Model Eval Server Started"
+                self.log_event(wandb_data)
+            except asyncio.TimeoutError as e:
+                # likely a validator error
+                bt.logging.error(f"OFFLINE: Timeout waiting for server for model {i+1} of {len(unique_miner_hf_model_names)} to start, skipping")
+                wandb_data['event_name'] = "Timeout Waiting for HF Model Eval Server"
+                wandb_data['miner_uids'] = hf_model_name_to_miner_uids[hf_model_name]
+                self.log_event(wandb_data)
+                wandb_data.pop('miner_uids')
+                wandb_data.pop('num_hf_model')
+                # can't score this model, so skipping it for now, the miner will be tried again if this runs again
+                continue
+            except Exception as e:
+                bt.logging.error(f"OFFLINE: Error waiting for server: {e}, skipping")
+                wandb_data['event_name'] = "Error Waiting for HF Model Eval Server"
+                wandb_data['error'] = f"{e}"
+                wandb_data['miner_uids'] = hf_model_name_to_miner_uids[hf_model_name]
+                self.log_event(wandb_data)
+                wandb_data.pop('error')
+                wandb_data.pop('num_hf_model')
+                wandb_data.pop('miner_uids')
+                continue
+
+        except Exception as e:
+            # likely a validator error
+            bt.logging.error(f"OFFLINE: Error starting sglang server for model: {i+1} of {len(unique_miner_hf_model_names)}: {e}")
+            wandb_data['event_name'] = "Error Starting HF Model Eval Server"
+            wandb_data['error'] = f"{e}"
+            wandb_data['miner_uids'] = hf_model_name_to_miner_uids[hf_model_name]
+            self.log_event(wandb_data)
+            wandb_data.pop('error')
+            wandb_data.pop('num_hf_model')
+            wandb_data.pop('miner_uids')
+            # can't score this model, so skipping it for now, the miner will be tried again if this runs again
+            # could be an issue with model size
+            continue
+
+        # get LLM responses
+        bt.logging.debug(f"OFFLINE: Getting LLM responses for model {i+1} of {len(unique_miner_hf_model_names)}")
+        wandb_data['event_name'] = "Getting LLM Responses"
+        self.log_event(wandb_data)
+
+        # at most 5 LLM calls concurrently
+        sem = asyncio.Semaphore(5) 
+
+        async def call_llm_with_semaphore(task):
+            async with sem:
+                return await asyncio.to_thread(
+                    llm, self, task.synapse.messages, task.synapse.tools, hf_model_name, hugging_face=True
+                )
+
+        llm_responses_and_finishes = await asyncio.gather(
+            *[call_llm_with_semaphore(task) for task in tasks]
+        )
+        try:    
+            llm_responses = [r[0] for r in llm_responses_and_finishes]
+            llm_finishes = [r[1] for r in llm_responses_and_finishes]
+        except Exception as e:
+            bt.logging.error(f"OFFLINE: Error getting LLM responses: {e}, have to skip this model")
+            continue
+
+        # TODO actually use the finishes to provide more detail to the miners in wandb
+
+        bt.logging.debug(f"OFFLINE: Got {len(llm_responses)} LLM responses for model: {i+1} of {len(unique_miner_hf_model_names)}")
+        wandb_data['event_name'] = "Got LLM Responses"
+        self.log_event(wandb_data)
+
+        # terminate the server after getting all the responses
+        bt.logging.debug(f"OFFLINE: Terminating server for model: {i+1} of {len(unique_miner_hf_model_names)}")
+        wandb_data['event_name'] = "HF Model Eval Server Terminating"
+        self.log_event(wandb_data)
+        await asyncio.to_thread(terminate_process, server_process)
+        bt.logging.debug(f"OFFLINE: Terminated server for model: {i+1} of {len(unique_miner_hf_model_names)}")
+        wandb_data['event_name'] = "HF Model Eval Server Terminated"
+        self.log_event(wandb_data)
+
+        these_miner_uids = hf_model_name_to_miner_uids[hf_model_name]
+        responses = []
+        for j, llm_response in enumerate(llm_responses):
+            task = tasks[j]
+            response = task.synapse.model_copy()
+            response.response = llm_response.strip()
+            response.dendrite.process_time = 5.0 # TODO may be useful to test performance of the model itself
+            response.dendrite.status_code = 200 
+            response.axon.status_code = 200
+            response.competition_version = self.competition_version
+            responses.append(response)
+
+        # evaluate, track score and add to wandb
+        # TODO need to see if this SCORE is higher than the all-time top score
+        # TODO if so, update the all-time top score and model name and reward TOP miners
+        # TODO if not, then temporal decay of scores
+        bt.logging.debug(f"OFFLINE: Processing rewards for model: {i+1} of {len(unique_miner_hf_model_names)}, for miners: {these_miner_uids}")
+        wandb_data['event_name'] = "Processing Rewards"
+        self.log_event(wandb_data)
+
+        # TODO This is blocking the main loop
+        # blocking due to await, attempting to remove await and create a task and move on
+
+        #await process_rewards_update_scores_for_many_tasks_and_many_miners(self, tasks=tasks, responses=responses, miner_uids=these_miner_uids, wandb_data=wandb_data)
+        asyncio.create_task(process_rewards_update_scores_for_many_tasks_and_many_miners(self, tasks=tasks, responses=responses, miner_uids=these_miner_uids, wandb_data=wandb_data))
+                            
+
+        # remove newly downloaded files from HF cache if were not already in cache
+        if not latest_snapshot:
+            bt.logging.debug(f"OFFLINE: Deleting model from HF cache: {i+1} of {len(unique_miner_hf_model_names)}")
+            wandb_data['event_name'] = "Deleting HF Model from Cache"
+            self.log_event(wandb_data)
+            await asyncio.to_thread(delete_model_from_hf_cache, self, hf_model_name)
+        else:
+            bt.logging.debug(f"OFFLINE: NOT Deleting model from HF cache: {i+1} of {len(unique_miner_hf_model_names)}")
+            wandb_data['event_name'] = "NOT Deleting HF Model from Cache - snapshot found, so no new download to revert"
+            self.log_event(wandb_data)
+
+        # TODO handle temporal decay of scores if no miners outperform all time top score
+        # TODO handle temporal decay of all scores depending on a) if no new TOP score and b) if new TOP score
+        wandb_data['event_name'] = "Finished Processing Rewards"
+        wandb_data['miner_uids'] = these_miner_uids
+        self.log_event(wandb_data)
+        wandb_data.pop('num_hf_model')
+        wandb_data.pop('miner_uids')
+
+    bt.logging.debug(f"OFFLINE: Finished processing offline tasks")
+    self.running_offline_mode = False
+    wandb_data['event_name'] = "Finished Processing Offline Tasks"
+    wandb_data['miner_uids'] = miner_uids
+    self.log_event(wandb_data)
+    wandb_data.pop('miner_uids')

bitagent_subnet-main/bitagent/validator/reward.py

@@ -0,0 +1,266 @@
+# The MIT License (MIT)
+# Copyright © 2023 Yuma Rao
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+import asyncio
+import numpy as np
+import bittensor as bt
+from typing import List, Any
+from rich.console import Console
+from bitagent.tasks.task import Task
+from bitagent.protocol import QueryResult
+from common.base.validator import BaseValidatorNeuron
+
+rich_console = Console()
+
+async def send_results_to_miner(validator, result, miner_axon):
+    # extra transparent details for miners
+
+    # For generated/evaluated tasks, we send the results back to the miner so they know how they did and why
+    # The dendrite client queries the network to send feedback to the miner
+    _ = validator.dendrite.query(
+        # Send the query to selected miner axons in the network.
+        axons=[miner_axon],
+        # Construct a query. 
+        synapse=QueryResult(results=result),
+        # All responses have the deserialize function called on them before returning.
+        # You are encouraged to define your own deserialization function.
+        deserialize=False,
+        timeout=5.0 # quick b/c we are not awaiting a response
+    )
+
+async def evaluate_task(validator, task, response):
+    try:
+        return [task.reward(validator, response)]
+    except Exception as e:
+        bt.logging.warning(f"An exception calling task.reward: {e}")
+
+async def return_results(validator, task, miner_uid, reward, response):
+    # means we got all of the information we need to score the miner and update wandb
+    if len(reward) == 4:
+        score, max_possible_score, task_results, correct_answer = reward
+        # make sure the score is not None
+        if score and max_possible_score:
+            normalized_score = score/max_possible_score
+
+            result = f"""
+[bold]Task: {task.name}[/bold]
+[bold]Messages:[/bold] {task.synapse.messages}
+[bold]Tools:[/bold] {[t.name for t in task.synapse.tools]}
+[bold]Response:[/bold] `{response.response}`
+\n[bold]Results:[/bold]\n
+=====================\n"""+"\n".join(task_results) + f"""
+[bold]Total reward:[/bold] {score}
+[bold]Total possible reward:[/bold] {max_possible_score}
+[bold]Normalized reward:[/bold] {normalized_score}
+---
+Stats with this validator:
+Your Average Score: {validator.scores[miner_uid]}
+Highest Score across all miners: {validator.scores.max()}
+Median Score across all miners: {np.median(validator.scores)}
+Your Offline Model Score for Competition {validator.previous_competition_version}: {validator.offline_scores[validator.previous_competition_version][miner_uid]}
+Your Offline Model Score for Competition {validator.competition_version}: {validator.offline_scores[validator.competition_version][miner_uid]}"""
+# TODO need to add BFCL scores when we do them
+            # send results
+            if task.mode == "online":
+                await send_results_to_miner(validator, result, validator.metagraph.axons[miner_uid])
+            else:
+                # useful if validators want to see progress or results of offline tasks
+                # rich_console.print("this is a non-online task")
+                # rich_console.print(result)
+                pass
+            return task_results
+        return None
+    elif len(reward) == 2: # skip it
+        #bt.logging.debug(f"Skipping results for this task b/c Task API seems to have rebooted: {reward[1]}")
+        #time.sleep(25)
+        return None
+    else:
+        #bt.logging.debug(f"Skipping results for this task b/c not enough information")
+        #time.sleep(25)
+        return None
+
+async def write_to_wandb(validator: BaseValidatorNeuron, task: Task, responses: List[Any], miner_uids: List[int], rewards: List[List[float]], results: List[List[str]]) -> None:
+    # common wandb setup
+    try:
+        messages = task.synapse.messages
+        tools = task.synapse.tools
+        task_name = task.name
+        task_mode = task.mode
+    except Exception as e:
+        bt.logging.error("Could not setup common data - ", e)
+
+    for i in range(len(responses)):
+        response = responses[i]
+        miner_uid = miner_uids[i]
+        score,max_possible_score,_,correct_answer = rewards[i][0]
+        normalized_score = score/max_possible_score
+
+        resp = "None"
+        try:
+            resp = response.response
+            run_model = response.hf_run_model_name
+        except:
+            pass
+
+        try:
+            data = {
+                "task_name": task_name,
+                "task_mode": task_mode,
+                "messages": [{'role': m.role, 'content': m.content} for m in messages],
+                "tools": [{'name': t.name, 'description': t.description, 'arguments': t.arguments} for t in tools],
+                "miners_count": len(miner_uids),
+                "messages_count": len(messages),
+                "tools_count": len(tools),
+                "response": resp,
+                "miner_uid": miner_uids[i],
+                "score": score,
+                "normalized_score": normalized_score,
+                "average_score_for_miner_with_this_validator": validator.scores[miner_uid],
+                "stake": validator.metagraph.S[miner_uid],
+                "trust": validator.metagraph.T[miner_uid],
+                "incentive": validator.metagraph.I[miner_uid],
+                "consensus": validator.metagraph.C[miner_uid],
+                "dividends": validator.metagraph.D[miner_uid],
+                "results": "\n".join(str(item) for item in results[i]) if results[i] else "None",
+                "dendrite_process_time": response.dendrite.process_time,
+                "dendrite_status_code": response.dendrite.status_code,
+                "axon_status_code": response.axon.status_code,
+                "validator_uid": validator.metagraph.hotkeys.index(validator.wallet.hotkey.ss58_address),
+                "val_spec_version": validator.spec_version,
+                "highest_score_for_miners_with_this_validator": validator.scores.max(),
+                "median_score_for_miners_with_this_validator": np.median(validator.scores),
+                "offline_score_for_miner_with_this_validator": validator.offline_scores[validator.competition_version][miner_uid],
+                "highest_offline_score_for_miners_with_this_validator": validator.offline_scores[validator.competition_version].max(),
+                "median_offline_score_for_miners_with_this_validator": np.median(validator.offline_scores[validator.competition_version]),
+                "average_offline_score_for_miners_with_this_validator": np.mean(validator.offline_scores[validator.competition_version]),
+                "prior_highest_offline_score_for_miners_with_this_validator": validator.offline_scores[validator.previous_competition_version].max(),
+                "prior_median_offline_score_for_miners_with_this_validator": np.median(validator.offline_scores[validator.previous_competition_version]),
+                "prior_average_offline_score_for_miners_with_this_validator": np.mean(validator.offline_scores[validator.previous_competition_version]),
+                "competition_version": validator.competition_version,
+                # TODO add BFCL scores
+                #"correct_answer": correct_answer, # TODO best way to send this without lookup attack?
+            }
+
+            try:
+                #if task.mode == "offline":
+                #    bt.logging.debug(f"OFFLINE Logging to WandB")
+                #else:
+                #    bt.logging.debug(f"ONLINE Logging to WandB")
+                validator.log_event(data)
+                #if task.mode == "offline":
+                #    bt.logging.debug(f"OFFLINE Logged to WandB")
+                #else:
+                #    bt.logging.debug(f"ONLINE Logged to WandB")
+            except Exception as e:
+                bt.logging.warning("WandB failed to log, moving on ... exception: {}".format(e))
+
+        except Exception as e:
+            bt.logging.warning("Exception in logging to WandB: {}".format(e))
+
+# all of these miners are scored the same way with the same tasks b/c this is scoring offline models
+async def process_rewards_update_scores_for_many_tasks_and_many_miners(
+    validator: BaseValidatorNeuron, tasks: List[Task], responses: List[Any], 
+    miner_uids: List[int], wandb_data: dict
+) -> None:
+    # Gather rewards in parallel
+    rewards = await asyncio.gather(*[
+        evaluate_task(validator, tasks[i], responses[i]) for i in range(len(responses))
+    ])
+
+    try:
+        scores = []
+        miner_tasks = []  # Collect tasks to execute in parallel for each miner
+        for i, reward in enumerate(rewards):
+            if len(reward[0]) == 4 and reward[0][0] is not None and reward[0][1] is not None:
+                scores.append(reward[0][0] / reward[0][1])
+
+                # Create a coroutine chain for each miner_uid
+                for miner_uid in miner_uids:
+                    async def process_miner_task(task_idx, miner_uid, reward, response):
+                        # Get the result for this miner
+                        result = await return_results(validator, tasks[task_idx], miner_uid, reward[0], response)
+                        # Write the result to wandb
+                        await write_to_wandb(validator, tasks[task_idx], [response], [miner_uid], rewards, result)
+                    
+                    # Append the task for execution
+                    miner_tasks.append(process_miner_task(i, miner_uid, reward, responses[i]))
+            else:
+                # Bad reward, so 0 score
+                scores.append(0.0)
+
+        # Await all miner-specific tasks concurrently
+        await asyncio.gather(*miner_tasks)
+
+    except Exception as e:
+        bt.logging.warning(f"OFFLINE: Error logging reward data: {e}")
+        wandb_data['event_name'] = "Processing Rewards - Error"
+        wandb_data['miner_uids'] = miner_uids
+        wandb_data['error'] = e
+        validator.log_event(wandb_data)
+        wandb_data.pop('error')
+        wandb_data.pop('miner_uids')
+
+    # Compute and log the mean score
+    score = np.mean(scores)
+    wandb_data['event_name'] = "Processing Rewards - Score"
+    wandb_data['score'] = score
+    wandb_data['miner_uids'] = miner_uids
+    validator.log_event(wandb_data)
+    wandb_data.pop('score')
+    wandb_data.pop('miner_uids')
+
+    # Update scores
+    validator.update_offline_scores([score] * len(miner_uids), miner_uids)
+
+    return score
+
+async def process_rewards_update_scores_and_send_feedback(validator: BaseValidatorNeuron, task: Task, responses: List[Any], 
+                miner_uids: List[int]) -> None:
+    """
+    Returns a tensor of rewards for the given query and responses.
+
+    Args:
+    - task (Task): The task sent to the miner.
+    - responses (List[float]): A list of responses from the miner.
+    - miner_uids (List[int]): A list of miner UIDs. The miner at a particular index has a response in responses at the same index.
+    """
+    # run these in parallel but wait for the reuslts b/c we need them downstream
+    rewards = await asyncio.gather(*[evaluate_task(validator, task, response) for response in responses])
+    try:
+        # track which miner uids are scored for updating the scores
+        #temp_miner_uids = [miner_uids[i] for i, reward in enumerate(rewards) if len(reward[0]) == 4 and reward[0][0] is not None and reward[0][1] is not None]
+        scores = []
+        results = []
+        for i, reward in enumerate(rewards):
+            if len(reward[0]) == 4 and reward[0][0] is not None and reward[0][1] is not None:
+                scores.append(reward[0][0]/reward[0][1])
+                results.append(await return_results(validator, task, miner_uids[i], reward[0], responses[i]))
+            else:
+                # bad reward, so 0 score
+                scores.append(0.0)
+                results.append(None)
+
+        await write_to_wandb(validator, task, responses, miner_uids, rewards, results)
+
+    except Exception as e:
+        bt.logging.warning(f"ONLINE: Error logging reward data: {e}")
+
+    # Update the scores based on the rewards. You may want to define your own update_scores function for custom behavior.
+    #miner_uids = temp_miner_uids
+    validator.update_scores(scores, miner_uids, alpha=task.weight)
+
+    return scores

bitagent_subnet-main/common/__init__.py

@@ -0,0 +1,29 @@
+# The MIT License (MIT)
+# Copyright © 2023 Yuma Rao
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# Define the version of the common module.
+__version__ = "1.0.15"
+version_split = __version__.split(".")
+__spec_version__ = (
+    (1000 * int(version_split[0]))
+    + (10 * int(version_split[1]))
+    + (1 * int(version_split[2]))
+)
+
+# Import all submodules.
+from . import base

bitagent_subnet-main/common/base/miner.py

@@ -0,0 +1,266 @@
+# The MIT License (MIT)
+# Copyright © 2023 Yuma Rao
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import time
+import asyncio
+import uvicorn
+import threading
+import traceback
+
+import bittensor as bt
+from bittensor.core.axon import FastAPIThreadedServer # type: ignore
+
+#from collections import Counter
+from common.base.neuron import BaseNeuron
+
+class BaseMinerNeuron(BaseNeuron):
+    """
+    Base class for Bittensor miners.
+    """
+
+    neuron_type: str = "MinerNeuron"
+
+    def __init__(self, config=None):
+        super().__init__(config=config)
+
+        # Warn if allowing incoming requests from anyone.
+        if not self.config.blacklist.force_validator_permit:
+            bt.logging.warning(
+                "You are allowing non-validators to send requests to your miner. This is a security risk."
+            )
+        if self.config.blacklist.allow_non_registered:
+            bt.logging.warning(
+                "You are allowing non-registered entities to send requests to your miner. This is a security risk."
+            )
+
+        # The axon handles request processing, allowing validators to send this miner requests.
+        self.axon = bt.axon(
+            wallet=self.wallet,
+            config=self.config,
+            port=self.config.axon.port,
+            ip=self.config.axon.ip,
+            external_ip=self.config.axon.external_ip,
+            external_port=self.config.axon.external_port,
+            max_workers=self.config.axon.max_workers
+        )
+        fast_config = uvicorn.Config(
+            self.axon.app, host="0.0.0.0", port=self.config.axon.port, log_level="trace", loop="asyncio"
+        )
+        self.axon.fast_server = FastAPIThreadedServer(config=fast_config)
+
+        # Attach determiners which functions are called when servicing a request.
+        bt.logging.info(f"Attaching forward function to miner axon.")
+        # NOTE - only big change made to common miner - RogueTensor
+        for forward_capability in self.forward_capabilities:
+            forward_fn = forward_capability['forward']
+            blacklist_fn = forward_capability['blacklist']
+            priority_fn = forward_capability['priority']
+            self.axon.attach(
+                forward_fn=forward_fn,
+                blacklist_fn=blacklist_fn,
+                priority_fn=priority_fn,
+            )
+
+        bt.logging.info(f"Axon created: {self.axon}")
+
+        # Instantiate runners
+        self.should_exit: bool = False
+        self.is_running: bool = False
+        self.thread: threading.Thread = None
+        self.lock = asyncio.Lock()
+
+    def run(self):
+        """
+        Initiates and manages the main loop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors.
+
+        This function performs the following primary tasks:
+        1. Check for registration on the Bittensor network.
+        2. Starts the miner's axon, making it active on the network.
+        3. Periodically resynchronizes with the chain; updating the metagraph with the latest network state and setting weights.
+
+        The miner continues its operations until `should_exit` is set to True or an external interruption occurs.
+        During each epoch of its operation, the miner waits for new blocks on the Bittensor network, updates its
+        knowledge of the network (metagraph), and sets its weights. This process ensures the miner remains active
+        and up-to-date with the network's latest state.
+
+        Note:
+            - The function leverages the global configurations set during the initialization of the miner.
+            - The miner's axon serves as its interface to the Bittensor network, handling incoming and outgoing requests.
+
+        Raises:
+            KeyboardInterrupt: If the miner is stopped by a manual interruption.
+            Exception: For unforeseen errors during the miner's operation, which are logged for diagnosis.
+        """
+
+        # Check that miner is registered on the network.
+        try:
+            self.sync()
+        except Exception as e:
+            bt.logging.error(f"Could not sync, will try again later. Error: {e}")
+
+        # Serve passes the axon information to the network + netuid we are hosting on.
+        # This will auto-update if the axon port of external ip have changed.
+        bt.logging.info(
+            f"Serving miner axon {self.axon} on network: {self.config.subtensor.chain_endpoint} with netuid: {self.config.netuid}"
+        )
+        try:
+            self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor)
+
+            # Start  starts the miner's axon, making it active on the network.
+            self.axon.start()
+
+            bt.logging.info(f"Miner starting at block: {self.block}")
+        except Exception as e:
+            bt.logging.error(f"Could not start miner, errored: {e}")
+
+        # This loop maintains the miner's operations until intentionally stopped.
+        try:
+            while not self.should_exit:
+                while(self.block - self.last_block_sync < self.config.neuron.epoch_length):
+                    # Wait before checking again.
+                    time.sleep(1)
+
+                    # Check if we should exit.
+                    if self.should_exit:
+                        break
+
+                # Sync metagraph
+                self.sync()
+                self.step += 1
+
+                time.sleep(1)
+
+        # If someone intentionally stops the miner, it'll safely terminate operations.
+        except KeyboardInterrupt:
+            self.axon.stop()
+            bt.logging.success("Miner killed by keyboard interrupt.")
+            exit()
+
+        # In case of unforeseen errors, the miner will log the error and continue operations.
+        except Exception as e:
+            bt.logging.error(traceback.format_exc())
+
+    def run_in_background_thread(self):
+        """
+        Starts the miner's operations in a separate background thread.
+        This is useful for non-blocking operations.
+        """
+        if not self.is_running:
+            bt.logging.debug("Starting miner in background thread.")
+            self.should_exit = False
+            self.thread = threading.Thread(target=self.run, daemon=True)
+            self.thread.start()
+            self.is_running = True
+            bt.logging.debug("Started")
+
+    def stop_run_thread(self):
+        """
+        Stops the miner's operations that are running in the background thread.
+        """
+        if self.is_running:
+            bt.logging.debug("Stopping miner in background thread.")
+            self.should_exit = True
+            self.thread.join(5)
+            self.is_running = False
+            bt.logging.debug("Stopped")
+
+    def __enter__(self):
+        """
+        Starts the miner's operations in a background thread upon entering the context.
+        This method facilitates the use of the miner in a 'with' statement.
+        """
+        self.run_in_background_thread()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """
+        Stops the miner's background operations upon exiting the context.
+        This method facilitates the use of the miner in a 'with' statement.
+
+        Args:
+            exc_type: The type of the exception that caused the context to be exited.
+                      None if the context was exited without an exception.
+            exc_value: The instance of the exception that caused the context to be exited.
+                       None if the context was exited without an exception.
+            traceback: A traceback object encoding the stack trace.
+                       None if the context was exited without an exception.
+        """
+        self.stop_run_thread()
+
+    def resync_metagraph(self):
+        """Resyncs the metagraph and updates the hotkeys and moving averages based on the new metagraph."""
+        bt.logging.info("resync_metagraph()")
+
+        # Sync the metagraph.
+        try:
+            self.metagraph.sync(subtensor=self.subtensor)
+            self.last_block_sync = self.block
+        except Exception as e:
+            bt.logging.error(f"Could not sync with metagraph right now, will try later. Error: {e}")
+
+    # each validator sends their top miner's HF model name to each miner
+    def get_top_miner_HF_model_name(self):
+        # miner can specify a HF model name to run
+        if self.config.hf_model_name_to_run != "none":
+            return self.config.hf_model_name_to_run
+
+        return "Salesforce/xLAM-7b-r"
+
+        ## TODO might consider selecting the TOP model that each validator votes for
+        ## if no specific model name is specified, miner will use the top model name from the validators' votes
+        #if not self.hf_top_model_names or len(list(self.hf_top_model_names.keys())) == 0:
+        #    return self.config.hf_model_name_to_run
+        #else:
+        #    # get the most common model name
+        #    most_common_model_name = Counter(self.hf_top_model_names).most_common(1)[0][0]
+        #    return most_common_model_name
+
+
+    # we might do this in the future, not doing for now
+    ## validator sends the miner the top model name to run from HF
+    ## store it
+    #def save_top_model_from_validator(self, top_hf_model_name, validator_uid):
+    #    # save off the top model from this validator
+    #    bt.logging.debug(f"Saving top HF model name from validator {validator_uid} state - {self.config.neuron.full_path}/miner_state.npz.")
+    #
+    #    self.hf_top_model_names[validator_uid] = top_hf_model_name
+    #
+    #    # Save the state of the miner to file.
+    #    np.savez(
+    #        self.config.neuron.full_path + "/miner_state.npz",
+    #        hf_top_model_names=self.hf_top_model_names,
+    #    )
+    #    
+    ## load that top model and run with it
+    #def load_state(self):
+    #    """Loads the state of the miner from a file."""
+    #    bt.logging.info("Loading miner state.")
+    #    if os.path.exists(self.config.neuron.full_path + "/miner_state.npz"):
+    #        state = np.load(self.config.neuron.full_path + "/miner_state.npz", allow_pickle=True)
+    #    else:
+    #        np.savez(
+    #            self.config.neuron.full_path + "/miner_state.npz",
+    #            hf_top_model_names={},
+    #        )
+    #        state = np.load(self.config.neuron.full_path + "/miner_state.npz", allow_pickle=True)
+    #        
+    #    if 'hf_top_model_names' in state:
+    #        loaded_hf_top_model_names = state["hf_top_model_names"]
+    #        self.hf_top_model_names = loaded_hf_top_model_names
+    #    else:
+    #        self.hf_top_model_names = {}

bitagent_subnet-main/common/base/neuron.py

@@ -0,0 +1,187 @@
+# The MIT License (MIT)
+# Copyright © 2023 Yuma Rao
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import copy
+import bittensor as bt
+from abc import ABC, abstractmethod
+
+# Sync calls set weights and also resyncs the metagraph.
+from common.utils.config import check_config, add_args, config
+from common.utils.misc import ttl_get_block
+from common import __spec_version__ as spec_version
+
+
+class BaseNeuron(ABC):
+    """
+    Base class for Bittensor miners. This class is abstract and should be inherited by a subclass. It contains the core logic for all neurons; validators and miners.
+
+    In addition to creating a wallet, subtensor, and metagraph, this class also handles the synchronization of the network state via a basic checkpointing mechanism based on epoch length.
+    """
+
+    neuron_type: str = "BaseNeuron"
+
+    @classmethod
+    def check_config(cls, config: "bt.Config"):
+        check_config(cls, config)
+
+    @classmethod
+    def add_args(cls, parser):
+        add_args(cls, parser)
+
+    @classmethod
+    def config(cls):
+        return config(cls)
+
+    subtensor: "bt.subtensor"
+    wallet: "bt.wallet"
+    metagraph: "bt.metagraph" # type: ignore
+    spec_version: int = spec_version
+
+    @property
+    def block(self):
+        return ttl_get_block(self)
+
+    def __init__(self, config=None):
+        base_config = copy.deepcopy(config or BaseNeuron.config())
+        self.config = self.config()
+        self.config.merge(base_config)
+        self.check_config(self.config)
+
+        # Set up logging with the provided configuration and directory.
+        bt.logging(config=self.config, logging_dir=self.config.full_path)
+
+        # If a gpu is required, set the device to cuda:N (e.g. cuda:0)
+        self.device = self.config.neuron.device
+
+        # Log the configuration for reference.
+        bt.logging.info(self.config)
+
+        # Build Bittensor objects
+        # These are core Bittensor classes to interact with the network.
+        bt.logging.info("Setting up bittensor objects.")
+
+        # The wallet holds the cryptographic key pairs for the miner.
+        self.wallet = bt.wallet(config=self.config)
+        bt.logging.info(f"Wallet: {self.wallet}")
+
+        # The subtensor is our connection to the Bittensor blockchain.
+        try:
+            while True:
+                try:
+                    self.subtensor = bt.subtensor(config=self.config)
+                    bt.logging.info(f"Subtensor: {self.subtensor}")
+
+                    # The metagraph holds the state of the network, letting us know about other validators and miners.
+                    self.metagraph = self.subtensor.metagraph(self.config.netuid)
+                    bt.logging.info(f"Metagraph: {self.metagraph}")
+                    break
+                except Exception as e:
+                    bt.logging.error(f"Error trying to connect to subtensor: {e}")
+
+            # Check if the miner is registered on the Bittensor network before proceeding further.
+            self.check_registered()
+
+            # Each miner gets a unique identity (UID) in the network for differentiation.
+            self.uid = self.metagraph.hotkeys.index(
+                self.wallet.hotkey.ss58_address
+            )
+            bt.logging.info(
+                f"Running neuron on subnet: {self.config.netuid} with uid {self.uid} using network: {self.subtensor.chain_endpoint}"
+            )
+            self.last_block_sync = self.block
+            self.step = 0
+        except Exception as e:
+            bt.logging.error(f"Error trying to connect to subtensor: {e}")
+
+    @abstractmethod
+    async def forward(self, synapse: bt.Synapse) -> bt.Synapse:
+        ...
+
+    @abstractmethod
+    def run(self):
+        ...
+
+    def sync(self, save_state=True):
+        """
+        Wrapper for synchronizing the state of the network for the given miner or validator.
+        """
+        try:
+            # Ensure miner or validator hotkey is still registered on the network.
+            self.check_registered()
+
+            if self.should_sync_metagraph():
+                self.resync_metagraph()
+
+            if self.should_set_weights():
+                self.set_weights()
+
+            # Always save state unless during reinitiation.
+            if save_state:
+                self.save_state()
+        except Exception as e:
+            # Reconnect to subtensor if there is an error.
+            self.subtensor = bt.subtensor(config=self.config)
+            bt.logging.error(f"Error trying to sync, reconnected subtensor and skipping this round: {e}")
+
+    def check_registered(self):
+        # --- Check for registration.
+        if not self.subtensor.is_hotkey_registered(
+            netuid=self.config.netuid,
+            hotkey_ss58=self.wallet.hotkey.ss58_address,
+        ):
+            bt.logging.error(
+                f"Wallet: {self.wallet} is not registered on netuid {self.config.netuid}."
+                f" Please register the hotkey using `btcli subnets register` before trying again"
+            )
+            exit()
+
+    def should_sync_metagraph(self):
+        """
+        Check if enough epoch blocks have elapsed since the last checkpoint to sync.
+        """
+        bt.logging.debug(f"Checking if ready to resync the metagraph at block {self.block}, last sync at {self.last_block_sync} and given epoch size of {self.config.neuron.epoch_length}: ", self.block - self.last_block_sync > self.config.neuron.epoch_length)
+        return self.block - self.last_block_sync > self.config.neuron.epoch_length
+
+    def should_set_weights(self) -> bool:
+        # Don't set weights for miners
+        if self.config.neuron.type == "miner":
+            return False
+        
+        # Don't set weights on initialization.
+        if self.step == 0:
+            return False
+
+        # Check if enough epoch blocks have elapsed since the last epoch.
+        if self.config.neuron.disable_set_weights:
+            return False
+
+        # Define appropriate logic for when set weights.
+        return (
+            (self.block - self.metagraph.last_update[self.uid])
+            > self.config.neuron.epoch_length 
+            and self.neuron_type != "MinerNeuron" 
+        ) # don't set weights if you're a miner
+
+    def save_state(self):
+        bt.logging.warning(
+            "save_state() not implemented for this neuron. You can implement this function to save model checkpoints or other useful data."
+        )
+
+    def load_state(self):
+        bt.logging.warning(
+            "load_state() not implemented for this neuron. You can implement this function to load model checkpoints or other useful data."
+        )

bitagent_subnet-main/common/base/validator.py

@@ -0,0 +1,576 @@
+# The MIT License (MIT)
+# Copyright © 2023 Yuma Rao
+# Copyright © 2023 RogueTensor
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import os
+import copy
+import asyncio
+import threading
+import numpy as np
+import bittensor as bt
+from datetime import datetime, timezone
+from scoring_utils import score_spreading
+from common.utils.uids import get_alive_uids
+from bitagent.validator.constants import DEPLOYED_DATE, COMPETITION_LENGTH_DAYS, TESTNET_COMPETITION_LENGTH_DAYS, COMPETITION_PREFIX, COMPETITION_PREVIOUS_PREFIX
+from common.utils.weight_utils import (
+    process_weights_for_netuid,
+    convert_weights_and_uids_for_emit,
+)
+from typing import List
+from traceback import print_exception
+
+from common.base.neuron import BaseNeuron
+from common.utils.uids import check_uid_availability
+    
+class BaseValidatorNeuron(BaseNeuron):
+    """
+    Base class for Bittensor validators. Your validator should inherit from this class.
+    """
+
+    neuron_type: str = "ValidatorNeuron"
+
+    def __init__(self, config=None):
+        super().__init__(config=config)
+
+        # Save a copy of the hotkeys to local memory.
+        self.hotkeys = copy.deepcopy(self.metagraph.hotkeys)
+
+        # Dendrite lets us send messages to other nodes (axons) in the network.
+        self.dendrite = bt.dendrite(wallet=self.wallet)
+        bt.logging.info(f"Dendrite: {self.dendrite}")
+
+        # Set up initial scoring weights for validation
+        bt.logging.info("Building validation weights.")
+        self.scores = np.zeros(self.metagraph.n, dtype=np.float32)
+        self.offline_scores = {}
+        self.offline_miners_scored = {}
+        self.offline_model_names = {}
+        self.running_offline_mode = False
+        self.offline_status = None
+        self.regrade_version = 1025
+        self.update_competition_numbers()
+        self.max_div = 0.0006
+        self.min_div = 0.0002
+        self.state_file_name = "ft_state.npz"
+
+        # Init sync with the network. Updates the metagraph.
+        if os.path.exists(self.config.neuron.full_path + f"/{self.state_file_name}"):
+            # if we are booting up and have this file, then we'll want to load it
+            # otherwise, if we save state, it will overwrite from the sync
+            self.sync(save_state=False)
+        else:
+            # if no state file then we'll create one on init
+            self.sync()
+        # Serve axon to enable external connections.
+        if not self.config.neuron.axon_off:
+            #self.serve_axon()
+            pass
+        else:
+            bt.logging.warning("axon off, not serving ip to chain.")
+
+        # Create asyncio event loop to manage async tasks.
+        self.loop = asyncio.get_event_loop()
+
+        # Instantiate runners
+        self.should_exit: bool = False
+        self.is_running: bool = False
+        self.thread: threading.Thread = None
+        self.lock = asyncio.Lock()
+
+    def serve_axon(self):
+        """Serve axon to enable external connections."""
+
+        bt.logging.info("serving ip to chain...")
+        try:
+            self.axon = bt.axon(wallet=self.wallet, config=self.config)
+
+            self.axon.attach(
+                forward_fn=self.forward_fn,
+                blacklist_fn=self.blacklist_fn,
+                priority_fn=self.priority_fn,
+            )
+
+            try:
+                self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor)
+                self.axon.start()
+            except Exception as e:
+                bt.logging.error(f"Failed to serve Axon with exception: {e}")
+                pass
+
+        except Exception as e:
+            bt.logging.error(
+                f"Failed to create Axon initialize with exception: {e}"
+            )
+            pass
+
+    async def concurrent_forward(self):
+        coroutines = [
+            self.forward()
+            for _ in range(self.config.neuron.num_concurrent_forwards)
+        ]
+        await asyncio.gather(*coroutines,return_exceptions=True)
+
+    def run(self):
+        """
+        Initiates and manages the main loop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors.
+
+        This function performs the following primary tasks:
+        1. Check for registration on the Bittensor network.
+        2. Continuously forwards queries to the miners on the network, rewarding their responses and updating the scores accordingly.
+        3. Periodically resynchronizes with the chain; updating the metagraph with the latest network state and setting weights.
+
+        The essence of the validator's operations is in the forward function, which is called every step. The forward function is responsible for querying the network and scoring the responses.
+
+        Note:
+            - The function leverages the global configurations set during the initialization of the miner.
+            - The miner's axon serves as its interface to the Bittensor network, handling incoming and outgoing requests.
+
+        Raises:
+            KeyboardInterrupt: If the miner is stopped by a manual interruption.
+            Exception: For unforeseen errors during the miner's operation, which are logged for diagnosis.
+        """
+
+        # Check that validator is registered on the network.
+        self.sync()
+        bt.logging.info(
+            f"Running validator on network: {self.config.subtensor.chain_endpoint} with netuid: {self.config.netuid}"
+        )
+
+        bt.logging.info(f"Validator starting at block: {self.block}")
+        # This loop maintains the validator's operations until intentionally stopped.
+        try:
+            while True:
+                try:
+                    bt.logging.info(f"step({self.step}) block({self.block})")
+                except Exception as e:
+                    bt.logging.error(f"Error logging step and block, likely socket issue, will update next round: {e}")
+                    #if "Broken pipe" in str(e):
+                    #    print("======= Exiting due to a broken pipe ========")
+                    #    self.axon.stop()
+                    #    self.should_exit = True
+                    #    exit()
+
+                # Run multiple forwards concurrently.
+                self.loop.run_until_complete(self.concurrent_forward())
+
+                # Check if we should exit.
+                if self.should_exit:
+                    break
+
+                # Sync metagraph and potentially set weights.
+                try:
+                    self.sync()
+                except Exception as e:
+                    bt.logging.error(f"Error syncing metagraph during run loop: {e}")
+
+                self.step += 1
+        except Exception as e:
+            bt.logging.error(f"Unexpected error during run: {e}")
+
+        # If someone intentionally stops the validator, it'll safely terminate operations.
+        except KeyboardInterrupt:
+            self.axon.stop()
+            bt.logging.success("Validator killed by keyboard interrupt.")
+            exit()
+
+        # In case of unforeseen errors, the validator will log the error and continue operations.
+        except Exception as err:
+            bt.logging.error("Error during validation", str(err))
+            bt.logging.debug(
+                print_exception(type(err), err, err.__traceback__)
+            )
+
+    def run_in_background_thread(self):
+        """
+        Starts the validator's operations in a background thread upon entering the context.
+        This method facilitates the use of the validator in a 'with' statement.
+        """
+        if not self.is_running:
+            bt.logging.debug("Starting validator in background thread.")
+            self.should_exit = False
+            self.thread = threading.Thread(target=self.run, daemon=True)
+            self.thread.start()
+            self.is_running = True
+            bt.logging.debug("Started")
+
+    def stop_run_thread(self):
+        """
+        Stops the validator's operations that are running in the background thread.
+        """
+        if self.is_running:
+            bt.logging.debug("Stopping validator in background thread.")
+            self.should_exit = True
+            self.thread.join(5)
+            self.is_running = False
+            bt.logging.debug("Stopped")
+
+    def __enter__(self):
+        self.run_in_background_thread()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """
+        Stops the validator's background operations upon exiting the context.
+        This method facilitates the use of the validator in a 'with' statement.
+
+        Args:
+            exc_type: The type of the exception that caused the context to be exited.
+                      None if the context was exited without an exception.
+            exc_value: The instance of the exception that caused the context to be exited.
+                       None if the context was exited without an exception.
+            traceback: A traceback object encoding the stack trace.
+                       None if the context was exited without an exception.
+        """
+        if self.is_running:
+            bt.logging.debug("Stopping validator in background thread.")
+            self.should_exit = True
+            self.thread.join(5)
+            self.is_running = False
+            bt.logging.debug("Stopped")
+
+    def set_weights(self):
+        """
+        Sets the validator weights to the metagraph hotkeys based on the scores it has received from the miners. The weights determine the trust and incentive level the validator assigns to miner nodes on the network.
+        """
+        bt.logging.debug(f"set_weights()")
+        if self.config.subtensor.network == "test":
+            return # Don't set weights on testnet.
+
+        self.divisions = int(np.floor(self.block / 1000))
+        current_odds = (0.2 * self.scores) + (0.8 * self.offline_scores[self.previous_competition_version])
+
+        # Check if self.scores contains any NaN values and log a warning if it does.
+        if np.isnan(self.scores).any():
+            bt.logging.warning(
+                f"Scores contain NaN values. This may be due to a lack of responses from miners, or a bug in your reward functions."
+            )
+        # correct validator scores to be 0 
+        for uid, hotkey in enumerate(self.hotkeys):
+            if not check_uid_availability(self.metagraph, uid, self.config.neuron.vpermit_tao_limit): 
+                # if validator, set validators scores to 0
+                self.scores[uid] = 0 
+                self.offline_scores[self.previous_competition_version][uid] = 0
+                self.offline_scores[self.competition_version][uid] = 0
+                self.offline_miners_scored[self.competition_version][self.regrade_version].append(uid)
+                self.offline_model_names[self.competition_version][uid] = ""
+
+        # always fit scores to weighted curve
+        weighted_scores = score_spreading(current_odds,self.divisions,self.min_div,self.max_div)
+
+        #bt.logging.info(f"weighted_scores: {weighted_scores}")
+
+        # Calculate the average reward for each uid across non-zero values.
+        # Replace any NaN values with 0.
+        norm = np.linalg.norm(weighted_scores, ord=1, axis=0, keepdims=True)
+        if np.any(norm == 0) or np.isnan(norm).any():
+            norm = np.ones_like(norm)
+        raw_weights = weighted_scores/norm
+
+        # bt.logging.debug("raw_weights: ")
+        # bt.logging.debug(raw_weights)
+        # bt.logging.debug("raw_weight_uids: ")
+        # bt.logging.debug(self.metagraph.uids)
+
+        # Process the raw weights to final_weights via subtensor limitations.
+        (
+            processed_weight_uids,
+            processed_weights,
+        ) = process_weights_for_netuid(
+            uids=self.metagraph.uids,
+            weights=raw_weights,
+            netuid=self.config.netuid,
+            subtensor=self.subtensor,
+            metagraph=self.metagraph,
+        )
+        # bt.logging.debug("processed_weights: ")
+        # bt.logging.debug(processed_weights)
+        # bt.logging.debug("processed_weight_uids: ")
+        # bt.logging.debug(processed_weight_uids)
+
+        # Convert to uint16 weights and uids.
+        (
+            uint_uids,
+            uint_weights,
+        ) = convert_weights_and_uids_for_emit(
+            uids=processed_weight_uids, weights=processed_weights
+        )
+
+        # Set the weights on chain via our subtensor connection.
+
+        result, msg = self.subtensor.set_weights(
+            wallet=self.wallet,
+            netuid=self.config.netuid,
+            uids=uint_uids,
+            weights=uint_weights,
+            wait_for_finalization=False,
+            wait_for_inclusion=False,
+            version_key=self.spec_version,
+        )
+        if result is True:
+            bt.logging.info(f"set_weights on chain for version: {self.spec_version} successfully!")
+        else:
+            bt.logging.error(f"set_weights failed: {msg}")
+
+    def get_weighted_scores(self):
+        # scores are largely based on PREVIOUS competition scores
+        scaled_scores = ((0.2 * self.scores) + (0.8 * self.offline_scores[self.previous_competition_version])) * 5
+        exp_scores = np.exp(scaled_scores)
+        return exp_scores / np.sum(exp_scores)    
+
+    def resync_metagraph(self):
+        """Resyncs the metagraph and updates the hotkeys and moving averages based on the new metagraph."""
+        bt.logging.info("resync_metagraph()")
+
+        # Copies state of metagraph before syncing.
+        previous_metagraph = copy.deepcopy(self.metagraph)
+
+        # Sync the metagraph.
+        try:
+            self.metagraph.sync(subtensor=self.subtensor)
+            self.last_block_sync = self.block
+
+            # Check if the metagraph axon info has changed.
+            if previous_metagraph.axons == self.metagraph.axons:
+                bt.logging.debug("Metagraph axons are the same, skipping resync")
+                return
+
+            bt.logging.info("Metagraph updated, resyncing hotkeys, dendrite pool and moving averages")  
+            # Normalize all hotkeys that have been replaced, and zero out all hotkeys that are no longer available
+            for uid, hotkey in enumerate(self.hotkeys):
+                if hotkey != self.metagraph.hotkeys[uid]:
+                    bt.logging.debug(f"RESYNC: hotkey changed for uid: {uid}")
+                    self.scores[uid] = np.median(self.scores)
+                    self.offline_scores[self.previous_competition_version][uid] = 0
+                    self.offline_scores[self.competition_version][uid] = 0
+                    if uid in self.offline_miners_scored[self.competition_version][self.regrade_version]:
+                        self.offline_miners_scored[self.competition_version][self.regrade_version].remove(uid)
+                    self.offline_model_names[self.competition_version][uid] = ""
+                    self.offline_model_names[self.previous_competition_version][uid] = ""
+
+            # Check to see if the metagraph has changed size.
+            # If so, we need to add new hotkeys and moving averages.
+            if len(self.hotkeys) < len(self.metagraph.hotkeys):
+                # Update the size of the moving average scores.
+                new_moving_average = np.zeros((self.metagraph.n))
+                min_len = min(len(self.hotkeys), len(self.scores))
+                new_moving_average[:min_len] = self.scores[:min_len]
+                self.scores = new_moving_average
+
+                # previous offline scores
+                new_moving_average = np.zeros((self.metagraph.n))
+                min_len = min(len(self.hotkeys), len(self.offline_scores[self.previous_competition_version]))
+                new_moving_average[:min_len] = self.offline_scores[self.previous_competition_version][:min_len]
+                self.offline_scores[self.previous_competition_version] = new_moving_average
+
+                # current offline scores
+                new_moving_average = np.zeros((self.metagraph.n))
+                min_len = min(len(self.hotkeys), len(self.offline_scores[self.competition_version]))
+                new_moving_average[:min_len] = self.offline_scores[self.competition_version][:min_len]
+                self.offline_scores[self.competition_version] = new_moving_average
+
+            # Update the hotkeys.
+            self.hotkeys = copy.deepcopy(self.metagraph.hotkeys)
+        except Exception as e:
+            bt.logging.error(f"Could not resync with metagraph right now, will try later. Error: {e}")
+
+    def update_offline_scores(self, rewards: np.ndarray, uids: List[int]):
+        """Performs exponential moving average on the scores based on the rewards received from the miners."""
+        if np.isnan(rewards).any():
+            #bt.logging.debug(f"NaN values detected in rewards: {rewards}")
+            # Replace any NaN values in rewards with 0.
+            rewards = np.nan_to_num(rewards, nan=0)
+
+        if isinstance(uids, np.ndarray):
+            uids_array = uids.copy()
+        else:
+            uids_array = np.array(uids)
+
+        scattered_rewards: np.ndarray = self.offline_scores[self.competition_version].copy()
+        scattered_rewards[uids_array] = rewards
+
+        bt.logging.debug(f"OFFLINE Scattered rewards: {rewards}")
+
+        self.offline_scores[self.competition_version]: np.ndarray = scattered_rewards # type: ignore
+        self.offline_miners_scored[self.competition_version][self.regrade_version].extend([int(x) for x in uids_array])
+        bt.logging.debug(f"Updated moving avg OFFLINE scores for Competition {self.competition_version}: {self.offline_scores[self.competition_version]}")
+        self.save_state()
+
+    def update_scores(self, rewards: np.ndarray, uids: List[int], alpha=None):
+        """Performs exponential moving average on the scores based on the rewards received from the miners."""
+        if np.isnan(rewards).any():
+            #bt.logging.debug(f"NaN values detected in rewards: {rewards}")
+            # Replace any NaN values in rewards with 0.
+            rewards = np.nan_to_num(rewards, nan=0)
+
+        if isinstance(uids, np.ndarray):
+            uids_array = uids.copy()
+        else:
+            uids_array = np.array(uids)
+
+        scattered_rewards: np.ndarray = self.scores.copy()
+        scattered_rewards[uids_array] = rewards
+
+        bt.logging.debug(f"ONLINE Scattered rewards: {rewards}")
+
+        # Update scores with rewards produced by this step.
+        # shape: [ metagraph.n ]
+        if not alpha:
+            alpha: float = self.config.neuron.moving_average_alpha
+        self.scores: np.ndarray = alpha * scattered_rewards + ( 1 - alpha) * self.scores
+        bt.logging.debug(f"Updated moving avg ONLINE scores: {self.scores}")
+    
+    def save_state(self):
+        """Saves the state of the validator to a file."""
+        bt.logging.debug(f"Saving validator state - {self.state_file_name}.")
+
+        # Save the state of the validator to file.
+        try:
+            np.savez(
+                self.config.neuron.full_path + f"/{self.state_file_name}",
+                step=self.step,
+                scores=self.scores,
+                offline_scores=self.offline_scores,
+                offline_miners_scored=np.array(list(self.offline_miners_scored.items()), dtype=object),
+                offline_model_names=self.offline_model_names,
+                hotkeys=self.hotkeys,
+                allow_pickle=True,
+            )
+        except Exception as e:
+            bt.logging.error(f"OFFLINE: Error saving validator state: {e}")
+        
+    def load_state(self):
+        """Loads the state of the validator from a file."""
+        bt.logging.info("Loading validator state.")
+        state = np.load(self.config.neuron.full_path + f"/{self.state_file_name}",allow_pickle=True)
+        bt.logging.debug(f"OFFLINE: LOADING STATE: {state}")
+
+        self.step = state["step"]
+        if 'hotkeys' in state:
+            self.hotkeys = state["hotkeys"]
+
+        if 'scores' in state:
+            loaded_scores = state["scores"]
+            self.scores[:len(loaded_scores)] = loaded_scores
+
+        if 'offline_scores' in state:
+            loaded_offline_scores = state["offline_scores"]
+            if isinstance(loaded_offline_scores, dict):
+                self.offline_scores = loaded_offline_scores
+            elif isinstance(loaded_offline_scores, np.ndarray):
+                self.offline_scores = loaded_offline_scores.item()
+            else:
+                bt.logging.error(f"OFFLINE: loaded_offline_scores is not a dict or array, type: {type(loaded_offline_scores)}")
+
+            if self.offline_scores.get(self.previous_competition_version) is None:
+                self.offline_scores[self.previous_competition_version] = np.zeros(self.metagraph.n, dtype=np.float32)
+            #for uid in self.metagraph.uids:
+            #    if uid not in self.offline_scores[self.previous_competition_version]:
+            #        self.offline_scores[self.previous_competition_version][uid] = 0
+        if 'offline_miners_scored' in state:
+            loaded_offline_miners_scored = state["offline_miners_scored"]
+            self.offline_miners_scored = dict(loaded_offline_miners_scored)
+
+        if 'offline_model_names' in state:
+            loaded_offline_model_names = state["offline_model_names"]
+            if isinstance(loaded_offline_model_names, dict):
+                self.offline_model_names = loaded_offline_model_names
+            elif isinstance(loaded_offline_model_names, np.ndarray):
+                self.offline_model_names = loaded_offline_model_names.item()
+            else:
+                bt.logging.error(f"OFFLINE: loaded_offline_model_names is not a dict or array, type: {type(loaded_offline_model_names)}")
+
+    def update_competition_numbers(self):
+        try:
+            # get competition details
+            competition_start_date = datetime.strptime(DEPLOYED_DATE, "%Y-%m-%d").replace(tzinfo=timezone.utc)
+            delta = datetime.now(timezone.utc) - competition_start_date  
+            number_of_days_since_start = delta.days + (delta.seconds / (24*3600))
+            number_of_competitions_since_start = int(number_of_days_since_start / COMPETITION_LENGTH_DAYS)
+            if self.config.subtensor.network == "test":
+                bt.logging.debug(f"OFFLINE TESTNET: using {TESTNET_COMPETITION_LENGTH_DAYS} days per competition")
+                number_of_competitions_since_start = int(number_of_days_since_start / TESTNET_COMPETITION_LENGTH_DAYS)
+
+            #bt.logging.debug(f"OFFLINE: number_of_competitions_since_start: {number_of_competitions_since_start}")
+
+            if number_of_competitions_since_start < 1:
+                # we have not completed any competitions with this prefix, so the previous competition number is the last one we completed with the old prefix
+                largest_previous_competition_number = 0
+                # search through all the previous competition numbers to find the largest (most recent) one
+                for k,_ in self.offline_scores.items():
+                    if k.startswith(f"{COMPETITION_PREVIOUS_PREFIX}-"):
+                        if int(k.split("-")[1]) > largest_previous_competition_number:
+                            largest_previous_competition_number = int(k.split("-")[1])
+                self.previous_competition_version = f"{COMPETITION_PREVIOUS_PREFIX}-{largest_previous_competition_number}"
+            else:
+                # we have completed at least one competition with this prefix, so the previous competition number is the last one we completed
+                self.previous_competition_version = f"{COMPETITION_PREFIX}-{int(number_of_competitions_since_start-1)}"
+
+            if self.offline_scores.get(self.previous_competition_version) is None:
+                self.offline_scores[self.previous_competition_version] = np.zeros(self.metagraph.n, dtype=np.float32)
+
+            self.competition_version = f"{COMPETITION_PREFIX}-{int(number_of_competitions_since_start)}"
+
+            if self.offline_scores.get(self.competition_version) is None:
+                self.offline_scores[self.competition_version] = np.zeros(self.metagraph.n, dtype=np.float32)
+
+            # SETUP OFFLINE MINERS SCORED
+            if self.offline_miners_scored.get(self.competition_version) is None:
+                self.offline_miners_scored[self.competition_version] = {}
+
+            if not isinstance(self.offline_miners_scored[self.competition_version], dict):
+                self.offline_miners_scored[self.competition_version] = {}
+
+            if self.offline_miners_scored[self.competition_version].get(self.regrade_version) is None:
+                self.offline_miners_scored[self.competition_version][self.regrade_version] = []
+
+            # SETUP OFFLINE MODEL NAMES
+            if self.offline_model_names.get(self.competition_version) is None:
+                self.offline_model_names[self.competition_version] = {}
+
+            self.miners_left_to_score = []
+
+            # if an offline_score is 0 (we should try again), we need to add the miner to the list of miners left to score
+            # so clear out the offline_miners_scored for this competition, for those miners
+            for uid in self.offline_miners_scored[self.competition_version][self.regrade_version]:
+                if self.offline_scores[self.competition_version][uid] <= 0.01: # little wiggle room
+                    #bt.logging.debug(f"OFFLINE: removing miner {uid} from offline_miners_scored for competition {self.competition_version} because score is less than 0.01")
+                    self.offline_miners_scored[self.competition_version][self.regrade_version].remove(uid)
+
+            # add all miners that are alive and not already scored to the list of miners left to score
+            for uid in get_alive_uids(self):
+                if uid not in [int(x) for x in self.offline_miners_scored[self.competition_version][self.regrade_version]]:
+                    self.miners_left_to_score.append(int(uid))
+
+            # if a regrade has been set for the comp, then reset the scores for the miners
+            #bt.logging.debug(f"OFFLINE: regrade version: {self.regrade_version}")
+            #bt.logging.debug(f"OFFLINE: regrade check - offline miners scored: {self.offline_miners_scored[self.competition_version][self.regrade_version]}")
+            #bt.logging.debug(f"OFFLINE: regrade check - offline scores: {self.offline_scores[self.competition_version]}")
+            for uid,score in enumerate(self.offline_scores[self.competition_version]):
+                #bt.logging.debug(f"OFFLINE: regrade check for uid: {uid}")
+                if score > 0.0 and uid not in [int(x) for x in self.offline_miners_scored[self.competition_version][self.regrade_version]]:
+                    #bt.logging.debug(f"OFFLINE: resetting miner {uid}'s score for competition {self.competition_version} for regrade")
+                    self.offline_scores[self.competition_version][uid] = 0.0
+                #bt.logging.debug(f"OFFLINE: regrade check for uid done: {uid}")
+
+            # if number of keys in offline_scores is greater than 5, we need to delete the oldest one
+            # if len(self.offline_scores.keys()) > 6:
+            #     oldest_key = list(self.offline_scores.keys())[0]
+            #     del self.offline_scores[oldest_key]
+            #     del self.offline_miners_scored[oldest_key]
+            #     del self.offline_model_names[oldest_key]
+        except Exception as e:
+            bt.logging.error(f"Error updating competition numbers: {e}")

bitagent_subnet-main/common/utils/__init__.py

@@ -0,0 +1,4 @@
+from . import config
+from . import misc
+from . import uids
+from . import shell

bitagent_subnet-main/common/utils/config.py

@@ -0,0 +1,284 @@
+# The MIT License (MIT)
+# Copyright © 2023 Yuma Rao
+# Copyright © 2023 Opentensor Foundation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import os
+import argparse
+import subprocess
+import bittensor as bt
+
+def is_cuda_available():
+    try:
+        output = subprocess.check_output(["nvidia-smi", "-L"], stderr=subprocess.STDOUT)
+        if "NVIDIA" in output.decode("utf-8"):
+            return "cuda"
+    except Exception:
+        pass
+    try:
+        output = subprocess.check_output(["nvcc", "--version"]).decode("utf-8")
+        if "release" in output:
+            return "cuda"
+    except Exception:
+        pass
+    return "cpu"
+
+def check_config(cls, config: "bt.Config"):
+    r"""Checks/validates the config namespace object."""
+    bt.logging.check_config(config)
+
+    full_path = os.path.expanduser(
+        "{}/{}/{}/netuid{}/{}".format(
+            config.logging.logging_dir,  # TODO: change from ~/.bittensor/miners to ~/.bittensor/neurons
+            config.wallet.name,
+            config.wallet.hotkey,
+            config.netuid,
+            config.neuron.name,
+        )
+    )
+    print("full path:", full_path)
+    config.neuron.full_path = os.path.expanduser(full_path)
+    if not os.path.exists(config.neuron.full_path):
+        os.makedirs(config.neuron.full_path, exist_ok=True)
+
+    #if not config.neuron.dont_save_events:
+    #    # Add custom event logger for the events.
+    #    logger.level("EVENTS", no=38, icon="📝")
+    #    logger.add(
+    #        os.path.join(config.neuron.full_path, "events.log"),
+    #        rotation=config.neuron.events_retention_size,
+    #        serialize=True,
+    #        enqueue=True,
+    #        backtrace=False,
+    #        diagnose=False,
+    #        level="EVENTS",
+    #        format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}",
+    #    )
+
+
+def add_args(cls, parser):
+    """
+    Adds relevant arguments to the parser for operation.
+    """
+    # Netuid Arg: The netuid of the subnet to connect to.
+    parser.add_argument("--netuid", type=int, help="Subnet netuid", default=1)
+    neuron_type = (
+        "validator" if "miner" not in cls.__name__.lower() else "miner"
+    )
+
+    parser.add_argument(
+        "--openai-api-key",
+        type=str,
+        default="EMPTY",
+        help="the OpenAI API key defaults to EMPTY"
+    )
+    parser.add_argument(
+        "--openai-api-base",
+        type=str,
+        default="http://localhost:8000/v1",
+        help="the OpenAI API base url - defaults to a local LLM server (like VLLM)",
+    )
+    parser.add_argument(
+        "--neuron.name",
+        type=str,
+        help="Trials for this neuron go in neuron.root / (wallet_cold - wallet_hot) / neuron.name. ",
+        default=neuron_type,
+    )
+
+    parser.add_argument(
+        "--neuron.visible_devices",
+        type=str,
+        help="Comma separated list of visible cuda devices.",
+        default="",
+    )
+
+    parser.add_argument(
+        "--neuron.device",
+        type=str,
+        help="Device to run on.",
+        default=is_cuda_available(),
+    )
+
+    parser.add_argument(
+        "--neuron.epoch_length",
+        type=int,
+        help="The default epoch length (how often we set weights, measured in 12 second blocks).",
+        default=100,
+    )
+
+    parser.add_argument(
+        "--neuron.events_retention_size",
+        type=str,
+        help="Events retention size.",
+        default="2 GB",
+    )
+
+    parser.add_argument(
+        "--neuron.dont_save_events",
+        action="store_true",
+        help="If set, we dont save events to a log file.",
+        default=False,
+    )
+    
+    parser.add_argument(
+        "--log_level",
+        type=str,
+        choices=["trace", "debug", "info"],  # Add more levels if needed
+        help="Logging level to use",
+        default="info"
+    )
+     
+    if neuron_type == "validator":
+
+        parser.add_argument(
+            "--validator-hf-cache-dir",
+            type=str,
+            default="~/.cache/huggingface/hub",
+            help="the directory where the HF models are stored on your system - this is where we delete the models from after we're done serving them",
+        )
+        parser.add_argument(
+            "--validator-hf-server-port",
+            type=int,
+            default=8028, 
+            help="the port of the docker container to run the offline HF model check",
+        )
+        parser.add_argument(
+            "--validator-hf-server-mem-fraction-static",
+            type=float,
+            default=0.40,
+            help="the fraction of the GPU memory to use for the HF server",
+        )
+
+        parser.add_argument(
+            "--validator-model-name",
+            type=str,
+            default="thesven/Mistral-7B-Instruct-v0.3-GPTQ",
+            help="the OpenAI model name defaults to thesven/Mistral-7B-Instruct-v0.3-GPTQ, the model that the validator uses to rewrite user queries",
+        )
+
+        parser.add_argument(
+            "--neuron.num_concurrent_forwards",
+            type=int,
+            help="The number of concurrent forwards running at any time.",
+            default=1,
+        )
+
+        parser.add_argument(
+            "--neuron.sample_size",
+            type=int,
+            help="The number of miners to query in a single step.",
+            default=10
+        )
+
+        parser.add_argument(
+            "--neuron.disable_set_weights",
+            action="store_true",
+            help="Disables setting weights.",
+            default=False,
+        )
+
+        parser.add_argument(
+            "--neuron.moving_average_alpha",
+            type=float,
+            help="Moving average alpha parameter, how much to add of the new observation.",
+            default=0.05,
+        )
+
+        parser.add_argument(
+            "--wandb.on",
+            type=bool,
+            default=True,
+            help="Enable wandb logging.",
+        )
+
+        parser.add_argument(
+            "--neuron.axon_off",
+            "--axon_off",
+            action="store_true",
+            # Note: the validator needs to serve an Axon with their IP or they may
+            #   be blacklisted by the firewall of serving peers on the network.
+            help="Set this flag to not attempt to serve an Axon.",
+            default=False,
+        )
+
+        parser.add_argument(
+            "--neuron.vpermit_tao_limit",
+            type=int,
+            help="The maximum number of TAO allowed to query a validator with a vpermit.",
+            default=4096,
+        )
+
+    else:
+        # grab the command line arguments to find the netuid and set the default accordingly
+        # for mainnet (SN20), we'll set to blacklist any validator without a validator permit
+        # for testnet (SN76), or any other netuid besides 20, we'll allow validators without a permit
+        # this is b/c our testnet validator does not have enough stake to have a "permit"
+        args, _ = parser.parse_known_args()
+        parser.add_argument(
+            "--blacklist.force_validator_permit",
+            action="store_true",
+            help="If set, we will force incoming requests to have a permit.",
+            default=(args.netuid==20),
+        )
+
+        parser.add_argument(
+            "--blacklist.allow_non_registered",
+            action="store_true",
+            help="If set, miners will accept queries from non registered entities. (Dangerous!)",
+            default=False,
+        )
+
+        parser.add_argument(
+            "--hf-model-name-to-run",
+            type=str,
+            default="Salesforce/xLAM-7b-r",
+            help="the OpenAI model name defaults to Salesforce/xLAM-7b-r"
+        )
+
+        parser.add_argument(
+            "--miner",
+            type=str,
+            default="default",
+            help="Miner to load. Default choices are 'default' and 'mock'.  Pass your custom miner name as appropriate."
+        )
+
+        parser.add_argument(
+            "--miner-hf-model-name-to-submit",
+            type=str,
+            default="Salesforce/xLAM-7b-r",
+            help="the HF model name that you've uploaded to the HF hub to be evaluated, will be returned when validator asks for your model to evaluate."
+        )
+
+def config(cls):
+    """
+    Returns the configuration object specific to this miner or validator after adding relevant arguments.
+    """
+    parser = argparse.ArgumentParser()
+    bt.wallet.add_args(parser)
+    bt.subtensor.add_args(parser)
+    bt.logging.add_args(parser)
+    bt.axon.add_args(parser)
+    cls.add_args(parser)
+    args = parser.parse_args()
+
+    # Conditional logging based on the argument
+    logging_level = args.log_level
+    if logging_level == "trace":
+        bt.trace()
+    elif logging_level == "debug":
+        bt.debug()
+
+    return bt.config(parser)

bitagent_subnet-main/common/utils/misc.py

@@ -0,0 +1,112 @@
+# The MIT License (MIT)
+# Copyright © 2023 Yuma Rao
+# Copyright © 2023 Opentensor Foundation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the “Software”), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+# the Software.
+
+# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import time
+import math
+import hashlib as rpccheckhealth
+from math import floor
+from typing import Callable, Any
+from functools import lru_cache, update_wrapper
+
+
+# LRU Cache with TTL
+def ttl_cache(maxsize: int = 128, typed: bool = False, ttl: int = -1):
+    """
+    Decorator that creates a cache of the most recently used function calls with a time-to-live (TTL) feature.
+    The cache evicts the least recently used entries if the cache exceeds the `maxsize` or if an entry has
+    been in the cache longer than the `ttl` period.
+
+    Args:
+        maxsize (int): Maximum size of the cache. Once the cache grows to this size, subsequent entries
+                       replace the least recently used ones. Defaults to 128.
+        typed (bool): If set to True, arguments of different types will be cached separately. For example,
+                      f(3) and f(3.0) will be treated as distinct calls with distinct results. Defaults to False.
+        ttl (int): The time-to-live for each cache entry, measured in seconds. If set to a non-positive value,
+                   the TTL is set to a very large number, effectively making the cache entries permanent. Defaults to -1.
+
+    Returns:
+        Callable: A decorator that can be applied to functions to cache their return values.
+
+    The decorator is useful for caching results of functions that are expensive to compute and are called
+    with the same arguments frequently within short periods of time. The TTL feature helps in ensuring
+    that the cached values are not stale.
+
+    Example:
+        @ttl_cache(ttl=10)
+        def get_data(param):
+            # Expensive data retrieval operation
+            return data
+    """
+    if ttl <= 0:
+        ttl = 65536
+    hash_gen = _ttl_hash_gen(ttl)
+
+    def wrapper(func: Callable) -> Callable:
+        @lru_cache(maxsize, typed)
+        def ttl_func(ttl_hash, *args, **kwargs):
+            return func(*args, **kwargs)
+
+        def wrapped(*args, **kwargs) -> Any:
+            th = next(hash_gen)
+            return ttl_func(th, *args, **kwargs)
+
+        return update_wrapper(wrapped, func)
+
+    return wrapper
+
+
+def _ttl_hash_gen(seconds: int):
+    """
+    Internal generator function used by the `ttl_cache` decorator to generate a new hash value at regular
+    time intervals specified by `seconds`.
+
+    Args:
+        seconds (int): The number of seconds after which a new hash value will be generated.
+
+    Yields:
+        int: A hash value that represents the current time interval.
+
+    This generator is used to create time-based hash values that enable the `ttl_cache` to determine
+    whether cached entries are still valid or if they have expired and should be recalculated.
+    """
+    start_time = time.time()
+    while True:
+        yield floor((time.time() - start_time) / seconds)
+
+
+# 12 seconds updating block.
+@ttl_cache(maxsize=1, ttl=12)
+def ttl_get_block(self) -> int:
+    """
+    Retrieves the current block number from the blockchain. This method is cached with a time-to-live (TTL)
+    of 12 seconds, meaning that it will only refresh the block number from the blockchain at most every 12 seconds,
+    reducing the number of calls to the underlying blockchain interface.
+
+    Returns:
+        int: The current block number on the blockchain.
+
+    This method is useful for applications that need to access the current block number frequently and can
+    tolerate a delay of up to 12 seconds for the latest information. By using a cache with TTL, the method
+    efficiently reduces the workload on the blockchain interface.
+
+    Example:
+        current_block = ttl_get_block(self)
+
+    Note: self here is the miner or validator instance
+    """
+    return self.subtensor.get_current_block()

bitagent_subnet-main/common/utils/shell.py

@@ -0,0 +1,47 @@
+import shlex
+import subprocess
+import bittensor as bt
+from threading import Thread
+
+def execute_shell_command(command: str, model_name: str) -> subprocess.Popen:
+    """
+    Execute a shell command and stream the output to the caller in real-time.
+
+    Args:
+        command: Shell command as a string (can include \\ line continuations)
+    Returns:
+        subprocess.Popen: The process handle for further interaction.
+    """
+    # Replace \ newline with space and split using shlex
+    command = command.replace("\\\n", " ").replace("\\", " ")
+    parts = shlex.split(command)  # Handles quoted strings correct
+
+    try:
+        # Run the process
+        process = subprocess.Popen(
+            parts, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+
+        def stream_output(stream, stream_name):
+            for line in iter(stream.readline, ''):
+                line = line.rstrip('\n')
+                if stream_name == "STDERR":
+                    # log everything except for token generation metrics
+                    if "#new-token" not in line and "Decode batch." not in line:
+                        redacted_line = line.replace(model_name, "[REDACTED]")
+                        bt.logging.debug(f"{stream_name}: {redacted_line}")
+
+                # Uncomment this if you want STDOUT logging as well:
+                # else:
+                #     print(f"{stream_name}: {line}")
+
+            stream.close()
+
+        # Stream both stdout and stderr
+        Thread(target=stream_output, args=(process.stdout, "STDOUT")).start()
+        Thread(target=stream_output, args=(process.stderr, "STDERR")).start()
+
+        return process
+    except Exception as e:
+        print(f"Error executing command: {command}. Exception: {e}")
+        raise

bitagent_subnet-main/common/utils/uids.py

@@ -0,0 +1,113 @@
+import random
+import numpy as np
+import bittensor as bt
+from typing import List
+from bitagent.protocol import IsAlive
+from cachetools import cached, TTLCache
+
+def check_uid_availability(
+    metagraph: "bt.metagraph.Metagraph", uid: int, vpermit_tao_limit: int # type: ignore
+) -> bool:
+    """Check if uid is available. The UID should be available if it is serving and has less than vpermit_tao_limit stake
+    Args:
+        metagraph (:obj: bt.metagraph.Metagraph): Metagraph object
+        uid (int): uid to be checked
+        vpermit_tao_limit (int): Validator permit tao limit
+    Returns:
+        bool: True if uid is available, False otherwise
+    """
+    # Filter non serving axons.
+    #if not metagraph.axons[uid].is_serving:
+    #    return False
+    # don't hit sn owner
+    if uid == 0:
+        return False
+    # Filter validator permit > 1024 stake.
+    if metagraph.validator_permit[uid]:
+        if metagraph.S[uid] > vpermit_tao_limit:
+            return False
+    # any miner receiving incentive should be queried
+    if metagraph.I[uid] > 0:
+        return True
+    # Available otherwise.
+    return True
+
+# Create a cache with a maximum size of 256 items and a TTL of 1 hour (3600 seconds)
+cache = TTLCache(maxsize=256, ttl=3600)
+
+@cached(cache)
+def get_alive_uids(self):
+    start = 0
+    finish = start + 10
+    results = []
+    # query 10 at a time
+    while start < len(self.metagraph.axons):
+        result = self.dendrite.query(
+            axons=self.metagraph.axons[start:finish], synapse=IsAlive(), deserialize=False, timeout=5.0
+        )
+        results.extend(result)
+        start = finish
+        finish = start + 10
+        if finish > len(self.metagraph.axons):
+            finish = len(self.metagraph.axons)  
+    alive_uids = [uid for uid, response in zip(range(self.metagraph.n.item()), results) if response.response and response.dendrite.status_code == 200]
+
+    # if not alive for querying, they won't get tasks for an hour, set their score to -0.5
+    for uid in self.metagraph.uids:
+        if uid not in alive_uids:
+            self.offline_scores[self.competition_version][uid] = -0.5
+            self.scores[uid] = -0.5
+    #bt.logging.debug(f"Found {len(alive_uids)} alive UIDs, caching for 1 hour")
+    return alive_uids
+
+def get_random_uids(
+    self, k: int, exclude: List[int] = None
+) -> np.ndarray:
+    """Returns k available random uids from the metagraph.
+    Args:
+        k (int): Number of uids to return.
+        exclude (List[int]): List of uids to exclude from the random sampling.
+    Returns:
+        uids (np.ndarray): Randomly sampled available uids.
+    Notes:
+        If `k` is larger than the number of available `uids`, set `k` to the number of available `uids`.
+    """
+    candidate_uids = []
+    avail_uids = []
+
+    for uid in get_alive_uids(self):
+        uid_is_available = check_uid_availability(
+            self.metagraph, uid, self.config.neuron.vpermit_tao_limit
+        )
+        uid_is_not_excluded = exclude is None or uid not in exclude
+
+        if uid_is_available:
+            avail_uids.append(uid)
+            if uid_is_not_excluded:
+                candidate_uids.append(uid)
+
+    # Check if candidate_uids contain enough for querying, if not grab all avaliable uids
+    available_uids = candidate_uids
+    while True:
+        try:
+            if len(candidate_uids) < k:
+                available_uids += random.sample(
+                    [uid for uid in avail_uids if uid not in candidate_uids],
+                    k - len(candidate_uids),
+                )
+            uids = random.sample(available_uids, k)
+            return uids
+        except Exception as e:
+            #bt.logging.debug(f"Reduced sample size from {k} to {k-1} and trying again.")
+            k -= 1
+
+def get_uid_rank(self, uid: int) -> int:
+    """Returns the rank of the uid in the metagraph.
+    Args:
+        uid (int): uid to get the rank of.
+    Returns:
+        rank (int): Rank of the uid in the metagraph.
+    """
+    # Get the rank of the uid in the metagraph.
+    rank = (-self.metagraph.I).argsort().tolist().index(uid)
+    return rank

bitagent_subnet-main/common/utils/weight_utils.py

@@ -0,0 +1,216 @@
+import numpy as np
+from typing import Tuple, List, Union, Any
+import bittensor
+from numpy import ndarray, dtype, floating, complexfloating
+
+U32_MAX = 4294967295
+U16_MAX = 65535
+
+
+def normalize_max_weight(
+        x: np.ndarray, limit: float = 0.1
+) -> np.ndarray:
+    r"""Normalizes the numpy array x so that sum(x) = 1 and the max value is not greater than the limit.
+    Args:
+        x (:obj:`np.ndarray`):
+            Array to be max_value normalized.
+        limit: float:
+            Max value after normalization.
+    Returns:
+        y (:obj:`np.ndarray`):
+            Normalized x array.
+    """
+    epsilon = 1e-7  # For numerical stability after normalization
+
+    weights = x.copy()
+    values = np.sort(weights)
+
+    if x.sum() == 0 or len(x) * limit <= 1:
+        return np.ones_like(x) / x.size
+    else:
+        estimation = values / values.sum()
+
+        if estimation.max() <= limit:
+            return weights / weights.sum()
+
+        # Find the cumulative sum and sorted array
+        cumsum = np.cumsum(estimation, 0)
+
+        # Determine the index of cutoff
+        estimation_sum = np.array(
+            [(len(values) - i - 1) * estimation[i] for i in range(len(values))]
+        )
+        n_values = (estimation / (estimation_sum + cumsum + epsilon) < limit).sum()
+
+        # Determine the cutoff based on the index
+        cutoff_scale = (limit * cumsum[n_values - 1] - epsilon) / (
+                1 - (limit * (len(estimation) - n_values))
+        )
+        cutoff = cutoff_scale * values.sum()
+
+        # Applying the cutoff
+        weights[weights > cutoff] = cutoff
+
+        y = weights / weights.sum()
+
+        return y
+
+
+def convert_weights_and_uids_for_emit(
+        uids: np.ndarray, weights: np.ndarray
+) -> Tuple[List[int], List[int]]:
+    r"""Converts weights into integer u32 representation that sum to MAX_INT_WEIGHT.
+    Args:
+        uids (:obj:`np.ndarray,`):
+            Array of uids as destinations for passed weights.
+        weights (:obj:`np.ndarray,`):
+            Array of weights.
+    Returns:
+        weight_uids (List[int]):
+            Uids as a list.
+        weight_vals (List[int]):
+            Weights as a list.
+    """
+    # Checks.
+    uids = np.asarray(uids)
+    weights = np.asarray(weights)
+
+    # Get non-zero weights and corresponding uids
+    non_zero_weights = weights[weights > 0]
+    non_zero_weight_uids = uids[weights > 0]
+
+    # Debugging information
+    bittensor.logging.debug(f"weights: {weights}")
+    bittensor.logging.debug(f"non_zero_weights: {non_zero_weights}")
+    bittensor.logging.debug(f"uids: {uids}")
+    bittensor.logging.debug(f"non_zero_weight_uids: {non_zero_weight_uids}")
+
+    if np.min(weights) < 0:
+        raise ValueError(
+            "Passed weight is negative cannot exist on chain {}".format(weights)
+        )
+    if np.min(uids) < 0:
+        raise ValueError("Passed uid is negative cannot exist on chain {}".format(uids))
+    if len(uids) != len(weights):
+        raise ValueError(
+            "Passed weights and uids must have the same length, got {} and {}".format(
+                len(uids), len(weights)
+            )
+        )
+    if np.sum(weights) == 0:
+        bittensor.logging.debug("nothing to set on chain")
+        return [], []  # Nothing to set on chain.
+    else:
+        max_weight = float(np.max(weights))
+        weights = [
+            float(value) / max_weight for value in weights
+        ]  # max-upscale values (max_weight = 1).
+        bittensor.logging.debug(f"setting on chain max: {max_weight} and weights: {weights}")
+
+    weight_vals = []
+    weight_uids = []
+    for i, (weight_i, uid_i) in enumerate(list(zip(weights, uids))):
+        uint16_val = round(
+            float(weight_i) * int(U16_MAX)
+        )  # convert to int representation.
+
+        # Filter zeros
+        if uint16_val != 0:  # Filter zeros
+            weight_vals.append(uint16_val)
+            weight_uids.append(uid_i)
+    bittensor.logging.debug(f"final params: {weight_uids} : {weight_vals}")
+    return weight_uids, weight_vals
+
+
+def process_weights_for_netuid(
+        uids,
+        weights: np.ndarray,
+        netuid: int,
+        subtensor: "bittensor.subtensor",
+        metagraph: "bittensor.metagraph" = None,
+        exclude_quantile: int = 0,
+) -> Union[tuple[ndarray[Any, dtype[Any]], Union[
+    Union[ndarray[Any, dtype[floating[Any]]], ndarray[Any, dtype[complexfloating[Any, Any]]]], Any]], tuple[
+    ndarray[Any, dtype[Any]], ndarray], tuple[Any, ndarray]]:
+    bittensor.logging.debug("process_weights_for_netuid()")
+    bittensor.logging.debug("weights: ")
+    bittensor.logging.debug(weights)
+    bittensor.logging.debug("netuid: ")
+    bittensor.logging.debug(netuid)
+    bittensor.logging.debug("subtensor: ")
+    bittensor.logging.debug(subtensor)
+    bittensor.logging.debug("metagraph: ")
+    bittensor.logging.debug(metagraph)
+
+    # Get latest metagraph from chain if metagraph is None.
+    if metagraph is None:
+        metagraph = subtensor.metagraph(netuid)
+
+    # Cast weights to floats.
+    if not isinstance(weights, np.ndarray) or weights.dtype != np.float32:
+        weights = weights.astype(np.float32)
+
+    # Network configuration parameters from an subtensor.
+    # These parameters determine the range of acceptable weights for each neuron.
+    quantile = exclude_quantile / U16_MAX
+    min_allowed_weights = subtensor.min_allowed_weights(netuid=netuid)
+    max_weight_limit = subtensor.max_weight_limit(netuid=netuid)
+    bittensor.logging.debug("quantile", quantile)
+    bittensor.logging.debug("min_allowed_weights", min_allowed_weights)
+    bittensor.logging.debug("max_weight_limit", max_weight_limit)
+
+    # Find all non zero weights.
+    non_zero_weight_idx = np.argwhere(weights > 0).squeeze()
+    non_zero_weight_uids = uids[non_zero_weight_idx]
+    non_zero_weights = weights[non_zero_weight_idx]
+    if non_zero_weights.size == 0 or metagraph.n < min_allowed_weights:
+        bittensor.logging.warning("No non-zero weights returning all ones.")
+        final_weights = np.ones(metagraph.n) / metagraph.n
+        bittensor.logging.debug("final_weights", final_weights)
+        return np.arange(len(final_weights)), final_weights
+
+    elif non_zero_weights.size < min_allowed_weights:
+        bittensor.logging.warning(
+            "No non-zero weights less then min allowed weight, returning all ones."
+        )
+        weights = (
+                np.ones(metagraph.n) * 1e-5
+        )  # creating minimum even non-zero weights
+        weights[non_zero_weight_idx] += non_zero_weights
+        bittensor.logging.debug("final_weights", weights)
+        normalized_weights = normalize_max_weight(
+            x=weights, limit=max_weight_limit
+        )
+        return np.arange(len(normalized_weights)), normalized_weights
+
+    bittensor.logging.debug("non_zero_weights: ")
+    bittensor.logging.debug(non_zero_weights)
+
+    # Compute the exclude quantile and find the weights in the lowest quantile
+    max_exclude = max(0, len(non_zero_weights) - min_allowed_weights) / len(
+        non_zero_weights
+    )
+    exclude_quantile = min([quantile, max_exclude])
+    lowest_quantile = np.quantile(non_zero_weights, exclude_quantile)
+    bittensor.logging.debug("max_exclude", max_exclude)
+    bittensor.logging.debug("exclude_quantile: ")
+    bittensor.logging.debug(exclude_quantile)
+    bittensor.logging.debug("lowest_quantile: ")
+    bittensor.logging.debug(lowest_quantile)
+
+    # Exclude all weights below the allowed quantile.
+    non_zero_weight_uids = non_zero_weight_uids[lowest_quantile <= non_zero_weights]
+    non_zero_weights = non_zero_weights[lowest_quantile <= non_zero_weights]
+    bittensor.logging.debug("non_zero_weight_uids: ")
+    bittensor.logging.debug(non_zero_weight_uids)
+    bittensor.logging.debug("non_zero_weights: ")
+    bittensor.logging.debug(non_zero_weights)
+
+    # Normalize weights and return.
+    normalized_weights = normalize_max_weight(
+        x=non_zero_weights, limit=max_weight_limit
+    )
+    bittensor.logging.debug("final_weights: ")
+    bittensor.logging.debug(normalized_weights)
+
+    return non_zero_weight_uids, normalized_weights

bitagent_subnet-main/contrib/CODE_REVIEW_DOCS.md

@@ -0,0 +1,72 @@
+# Code Review
+### Conceptual Review
+
+A review can be a conceptual review, where the reviewer leaves a comment
+ * `Concept (N)ACK`, meaning "I do (not) agree with the general goal of this pull
+   request",
+ * `Approach (N)ACK`, meaning `Concept ACK`, but "I do (not) agree with the
+   approach of this change".
+
+A `NACK` needs to include a rationale why the change is not worthwhile.
+NACKs without accompanying reasoning may be disregarded.
+After conceptual agreement on the change, code review can be provided. A review
+begins with `ACK BRANCH_COMMIT`, where `BRANCH_COMMIT` is the top of the PR
+branch, followed by a description of how the reviewer did the review. The
+following language is used within pull request comments:
+
+  - "I have tested the code", involving change-specific manual testing in
+    addition to running the unit, functional, or fuzz tests, and in case it is
+    not obvious how the manual testing was done, it should be described;
+  - "I have not tested the code, but I have reviewed it and it looks
+    OK, I agree it can be merged";
+  - A "nit" refers to a trivial, often non-blocking issue.
+
+### Code Review
+Project maintainers reserve the right to weigh the opinions of peer reviewers
+using common sense judgement and may also weigh based on merit. Reviewers that
+have demonstrated a deeper commitment and understanding of the project over time
+or who have clear domain expertise may naturally have more weight, as one would
+expect in all walks of life.
+
+Where a patch set affects consensus-critical code, the bar will be much
+higher in terms of discussion and peer review requirements, keeping in mind that
+mistakes could be very costly to the wider community. This includes refactoring
+of consensus-critical code.
+
+Where a patch set proposes to change the Bittensor consensus, it must have been
+discussed extensively on the discord server and other channels, be accompanied by a widely
+discussed BIP and have a generally widely perceived technical consensus of being
+a worthwhile change based on the judgement of the maintainers.
+
+### Finding Reviewers
+
+As most reviewers are themselves developers with their own projects, the review
+process can be quite lengthy, and some amount of patience is required. If you find
+that you've been waiting for a pull request to be given attention for several
+months, there may be a number of reasons for this, some of which you can do something
+about:
+
+  - It may be because of a feature freeze due to an upcoming release. During this time,
+    only bug fixes are taken into consideration. If your pull request is a new feature,
+    it will not be prioritized until after the release. Wait for the release.
+  - It may be because the changes you are suggesting do not appeal to people. Rather than
+    nits and critique, which require effort and means they care enough to spend time on your
+    contribution, thundering silence is a good sign of widespread (mild) dislike of a given change
+    (because people don't assume *others* won't actually like the proposal). Don't take
+    that personally, though! Instead, take another critical look at what you are suggesting
+    and see if it: changes too much, is too broad, doesn't adhere to the
+    [developer notes](DEVELOPMENT_WORKFLOW.md), is dangerous or insecure, is messily written, etc.
+    Identify and address any of the issues you find. Then ask e.g. on IRC if someone could give
+    their opinion on the concept itself.
+  - It may be because your code is too complex for all but a few people, and those people
+    may not have realized your pull request even exists. A great way to find people who
+    are qualified and care about the code you are touching is the
+    [Git Blame feature](https://docs.github.com/en/github/managing-files-in-a-repository/managing-files-on-github/tracking-changes-in-a-file). Simply
+    look up who last modified the code you are changing and see if you can find
+    them and give them a nudge. Don't be incessant about the nudging, though.
+  - Finally, if all else fails, ask on IRC or elsewhere for someone to give your pull request
+    a look. If you think you've been waiting for an unreasonably long time (say,
+    more than a month) for no particular reason (a few lines changed, etc.),
+    this is totally fine. Try to return the favor when someone else is asking
+    for feedback on their code, and the universe balances out.
+  - Remember that the best thing you can do while waiting is give review to others!

bitagent_subnet-main/contrib/CONTRIBUTING.md

@@ -0,0 +1,213 @@
+# Contributing to Bittensor Subnet Development
+
+The following is a set of guidelines for contributing to the Bittensor ecosystem. These are **HIGHLY RECOMMENDED** guidelines, but not hard-and-fast rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
+
+## Table Of Contents
+1. [How Can I Contribute?](#how-can-i-contribute)
+   1. [Communication Channels](#communication-channels)
+   1. [Code Contribution General Guideline](#code-contribution-general-guidelines)
+   1. [Pull Request Philosophy](#pull-request-philosophy)
+   1. [Pull Request Process](#pull-request-process)
+   1. [Addressing Feedback](#addressing-feedback)
+   1. [Squashing Commits](#squashing-commits)
+   1. [Refactoring](#refactoring)
+   1. [Peer Review](#peer-review)
+ 1. [Suggesting Features](#suggesting-enhancements-and-features)
+
+
+## How Can I Contribute?
+TODO(developer): Define your desired contribution procedure.
+
+## Communication Channels
+TODO(developer): Place your communication channels here
+
+> Please follow the Bittensor Subnet [style guide](./STYLE.md) regardless of your contribution type. 
+
+Here is a high-level summary:
+- Code consistency is crucial; adhere to established programming language conventions.
+- Use `black` to format your Python code; it ensures readability and consistency.
+- Write concise Git commit messages; summarize changes in ~50 characters.
+- Follow these six commit rules:
+  - Atomic Commits: Focus on one task or fix per commit.
+  - Subject and Body Separation: Use a blank line to separate the subject from the body.
+  - Subject Line Length: Keep it under 50 characters for readability.
+  - Imperative Mood: Write subject line as if giving a command or instruction.
+  - Body Text Width: Wrap text manually at 72 characters.
+  - Body Content: Explain what changed and why, not how.
+- Make use of your commit messages to simplify project understanding and maintenance.
+
+> For clear examples of each of the commit rules, see the style guide's [rules](./STYLE.md#the-six-rules-of-a-great-commit) section.
+
+### Code Contribution General Guidelines
+
+> Review the Bittensor Subnet [style guide](./STYLE.md) and [development workflow](./DEVELOPMENT_WORKFLOW.md) before contributing. 
+
+
+#### Pull Request Philosophy
+
+Patchsets and enhancements should always be focused. A pull request could add a feature, fix a bug, or refactor code, but it should not contain a mixture of these. Please also avoid 'super' pull requests which attempt to do too much, are overly large, or overly complex as this makes review difficult. 
+
+Specifically, pull requests must adhere to the following criteria:
+- Contain fewer than 50 files. PRs with more than 50 files will be closed.
+- If a PR introduces a new feature, it *must* include corresponding tests.
+- Other PRs (bug fixes, refactoring, etc.) should ideally also have tests, as they provide proof of concept and prevent regression.
+- Categorize your PR properly by using GitHub labels. This aids in the review process by informing reviewers about the type of change at a glance.
+- Make sure your code includes adequate comments. These should explain why certain decisions were made and how your changes work.
+- If your changes are extensive, consider breaking your PR into smaller, related PRs. This makes your contributions easier to understand and review.
+- Be active in the discussion about your PR. Respond promptly to comments and questions to help reviewers understand your changes and speed up the acceptance process.
+
+Generally, all pull requests must:
+
+  - Have a clear use case, fix a demonstrable bug or serve the greater good of the project (e.g. refactoring for modularisation).
+  - Be well peer-reviewed.
+  - Follow code style guidelines.
+  - Not break the existing test suite.
+  - Where bugs are fixed, where possible, there should be unit tests demonstrating the bug and also proving the fix.
+  - Change relevant comments and documentation when behaviour of code changes.
+
+#### Pull Request Process
+
+Please follow these steps to have your contribution considered by the maintainers:
+
+*Before* creating the PR:
+1. Read the [development workflow](./DEVELOPMENT_WORKFLOW.md) defined for this repository to understand our workflow.
+2. Ensure your PR meets the criteria stated in the 'Pull Request Philosophy' section.
+3. Include relevant tests for any fixed bugs or new features as stated in the [testing guide](./TESTING.md).
+4. Ensure your commit messages are clear and concise. Include the issue number if applicable.
+5. If you have multiple commits, rebase them into a single commit using `git rebase -i`.
+6. Explain what your changes do and why you think they should be merged in the PR description consistent with the [style guide](./STYLE.md).
+
+*After* creating the PR:
+1. Verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing after you submit your pull request. 
+2. Label your PR using GitHub's labeling feature. The labels help categorize the PR and streamline the review process.
+3. Document your code with comments that provide a clear understanding of your changes. Explain any non-obvious parts of your code or design decisions you've made.
+4. If your PR has extensive changes, consider splitting it into smaller, related PRs. This reduces the cognitive load on the reviewers and speeds up the review process.
+
+Please be responsive and participate in the discussion on your PR! This aids in clarifying any confusion or concerns and leads to quicker resolution and merging of your PR.
+
+> Note: If your changes are not ready for merge but you want feedback, create a draft pull request.
+
+Following these criteria will aid in quicker review and potential merging of your PR.
+While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted.
+
+When you are ready to submit your changes, create a pull request:
+
+> **Always** follow the [style guide](./STYLE.md) and [development workflow](./DEVELOPMENT_WORKFLOW.md) before submitting pull requests.
+
+After you submit a pull request, it will be reviewed by the maintainers. They may ask you to make changes. Please respond to any comments and push your changes as a new commit.
+
+> Note: Be sure to merge the latest from "upstream" before making a pull request:
+
+```bash
+git remote add upstream https://github.com/opentensor/bittensor.git # TODO(developer): replace with your repo URL
+git fetch upstream
+git merge upstream/<your-branch-name>
+git push origin <your-branch-name>
+```
+
+#### Addressing Feedback
+
+After submitting your pull request, expect comments and reviews from other contributors. You can add more commits to your pull request by committing them locally and pushing to your fork.
+
+You are expected to reply to any review comments before your pull request is merged. You may update the code or reject the feedback if you do not agree with it, but you should express so in a reply. If there is outstanding feedback and you are not actively working on it, your pull request may be closed.
+
+#### Squashing Commits
+
+If your pull request contains fixup commits (commits that change the same line of code repeatedly) or too fine-grained commits, you may be asked to [squash](https://git-scm.com/docs/git-rebase#_interactive_mode) your commits before it will be reviewed. The basic squashing workflow is shown below.
+
+    git checkout your_branch_name
+    git rebase -i HEAD~n
+    # n is normally the number of commits in the pull request.
+    # Set commits (except the one in the first line) from 'pick' to 'squash', save and quit.
+    # On the next screen, edit/refine commit messages.
+    # Save and quit.
+    git push -f # (force push to GitHub)
+
+Please update the resulting commit message, if needed. It should read as a coherent message. In most cases, this means not just listing the interim commits.
+
+If your change contains a merge commit, the above workflow may not work and you will need to remove the merge commit first. See the next section for details on how to rebase.
+
+Please refrain from creating several pull requests for the same change. Use the pull request that is already open (or was created earlier) to amend changes. This preserves the discussion and review that happened earlier for the respective change set.
+
+The length of time required for peer review is unpredictable and will vary from pull request to pull request.
+
+#### Refactoring
+
+Refactoring is a necessary part of any software project's evolution. The following guidelines cover refactoring pull requests for the project.
+
+There are three categories of refactoring: code-only moves, code style fixes, and code refactoring. In general, refactoring pull requests should not mix these three kinds of activities in order to make refactoring pull requests easy to review and uncontroversial. In all cases, refactoring PRs must not change the behaviour of code within the pull request (bugs must be preserved as is).
+
+Project maintainers aim for a quick turnaround on refactoring pull requests, so where possible keep them short, uncomplex and easy to verify.
+
+Pull requests that refactor the code should not be made by new contributors. It requires a certain level of experience to know where the code belongs to and to understand the full ramification (including rebase effort of open pull requests). Trivial pull requests or pull requests that refactor the code with no clear benefits may be immediately closed by the maintainers to reduce unnecessary workload on reviewing.
+
+#### Peer Review
+
+Anyone may participate in peer review which is expressed by comments in the pull request. Typically reviewers will review the code for obvious errors, as well as test out the patch set and opine on the technical merits of the patch. Project maintainers take into account the peer review when determining if there is consensus to merge a pull request (remember that discussions may have taken place elsewhere, not just on GitHub). The following language is used within pull-request comments:
+
+- ACK means "I have tested the code and I agree it should be merged";
+- NACK means "I disagree this should be merged", and must be accompanied by sound technical justification. NACKs without accompanying reasoning may be disregarded;
+- utACK means "I have not tested the code, but I have reviewed it and it looks OK, I agree it can be merged";
+- Concept ACK means "I agree in the general principle of this pull request";
+- Nit refers to trivial, often non-blocking issues.
+
+Reviewers should include the commit(s) they have reviewed in their comments. This can be done by copying the commit SHA1 hash.
+
+A pull request that changes consensus-critical code is considerably more involved than a pull request that adds a feature to the wallet, for example. Such patches must be reviewed and thoroughly tested by several reviewers who are knowledgeable about the changed subsystems. Where new features are proposed, it is helpful for reviewers to try out the patch set on a test network and indicate that they have done so in their review. Project maintainers will take this into consideration when merging changes.
+
+For a more detailed description of the review process, see the [Code Review Guidelines](CODE_REVIEW_DOCS.md).
+
+> **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new issue and include a link to the original issue in the body of your new one.
+
+#### How Do I Submit A (Good) Bug Report?
+
+Please track bugs as GitHub issues.
+
+Explain the problem and include additional details to help maintainers reproduce the problem:
+
+* **Use a clear and descriptive title** for the issue to identify the problem.
+* **Describe the exact steps which reproduce the problem** in as many details as possible. For example, start by explaining how you started the application, e.g. which command exactly you used in the terminal, or how you started Bittensor otherwise. When listing steps, **don't just say what you did, but explain how you did it**. For example, if you ran with a set of custom configs, explain if you used a config file or command line arguments. 
+* **Provide specific examples to demonstrate the steps**. Include links to files or GitHub projects, or copy/pasteable snippets, which you use in those examples. If you're providing snippets in the issue, use [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
+* **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior.
+* **Explain which behavior you expected to see instead and why.**
+* **Include screenshots and animated GIFs** which show you following the described steps and clearly demonstrate the problem. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux.
+* **If you're reporting that Bittensor crashed**, include a crash report with a stack trace from the operating system. On macOS, the crash report will be available in `Console.app` under "Diagnostic and usage information" > "User diagnostic reports". Include the crash report in the issue in a [code block](https://help.github.com/articles/markdown-basics/#multiple-lines), a [file attachment](https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/), or put it in a [gist](https://gist.github.com/) and provide link to that gist.
+* **If the problem is related to performance or memory**, include a CPU profile capture with your report, if you're using a GPU then include a GPU profile capture as well. Look into the [PyTorch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) to look at memory usage of your model.
+* **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened and share more information using the guidelines below.
+
+Provide more context by answering these questions:
+
+* **Did the problem start happening recently** (e.g. after updating to a new version) or was this always a problem?
+* If the problem started happening recently, **can you reproduce the problem in an older version of Bittensor?** 
+* **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which conditions it normally happens.
+
+Include details about your configuration and environment:
+
+* **Which version of Bittensor Subnet are you using?**
+* **What commit hash are you on?** You can get the exact commit hash by checking `git log` and pasting the full commit hash.
+* **What's the name and version of the OS you're using**?
+* **Are you running Bittensor Subnet in a virtual machine?** If so, which VM software are you using and which operating systems and versions are used for the host and the guest?
+* **Are you running Bittensor Subnet in a dockerized container?** If so, have you made sure that your docker container contains your latest changes and is up to date with Master branch?
+
+### Suggesting Enhancements and Features
+
+This section guides you through submitting an enhancement suggestion, including completely new features and minor improvements to existing functionality. Following these guidelines helps maintainers and the community understand your suggestion :pencil: and find related suggestions :mag_right:.
+
+When you are creating an enhancement suggestion, please [include as many details as possible](#how-do-i-submit-a-good-enhancement-suggestion). Fill in [the template](https://bit.ly/atom-behavior-pr), including the steps that you imagine you would take if the feature you're requesting existed.
+
+#### Before Submitting An Enhancement Suggestion
+
+* **Check the [debugging guide](./DEBUGGING.md).** for tips — you might discover that the enhancement is already available. Most importantly, check if you're using the latest version of the project first.
+
+#### How Submit A (Good) Feature Suggestion
+
+* **Use a clear and descriptive title** for the issue to identify the problem.
+* **Provide a step-by-step description of the suggested enhancement** in as many details as possible.
+* **Provide specific examples to demonstrate the steps**. Include copy/pasteable snippets which you use in those examples, as [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
+* **Describe the current behavior** and **explain which behavior you expected to see instead** and why.
+* **Include screenshots and animated GIFs** which help you demonstrate the steps or point out the part of the project which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux.
+* **Explain why this enhancement would be useful** to most users.
+* **List some other text editors or applications where this enhancement exists.**
+* **Specify the name and version of the OS you're using.**
+
+Thank you for considering contributing to Bittensor! Any help is greatly appreciated along this journey to incentivize open and permissionless intelligence.

bitagent_subnet-main/contrib/DEVELOPMENT_WORKFLOW.md

@@ -0,0 +1,165 @@
+# Bittensor Subnet Development Workflow
+
+This is a highly advisable workflow to follow to keep your subtensor project organized and foster ease of contribution.
+
+## Table of contents
+
+- [Bittensor Subnet Development Workflow](#bittensor-subnet-development-workflow)
+  - [Main Branches](#main-branches)
+  - [Development Model](#development-model)
+      - [Feature Branches](#feature-branches)
+      - [Release Branches](#release-branches)
+      - [Hotfix Branches](#hotfix-branches)
+  - [Git Operations](#git-operations)
+      - [Creating a Feature Branch](#creating-a-feature-branch)
+      - [Merging Feature Branch into Staging](#merging-feature-branch-into-staging)
+      - [Creating a Release Branch](#creating-a-release-branch)
+      - [Finishing a Release Branch](#finishing-a-release-branch)
+      - [Creating a Hotfix Branch](#creating-a-hotfix-branch)
+      - [Finishing a Hotfix Branch](#finishing-a-hotfix-branch)
+  - [Continuous Integration (CI) and Continuous Deployment (CD)](#continuous-integration-ci-and-continuous-deployment-cd)
+  - [Versioning and Release Notes](#versioning-and-release-notes)
+  - [Pending Tasks](#pending-tasks)
+
+## Main Branches
+
+Bittensor's codebase consists of two main branches: **main** and **staging**.
+
+**main**
+- This is Bittensor's live production branch, which should only be updated by the core development team. This branch is protected, so refrain from pushing or merging into it unless authorized.
+
+**staging**
+- This branch is continuously updated and is where you propose and merge changes. It's essentially Bittensor's active development branch.
+
+## Development Model
+
+### Feature Branches
+
+- Branch off from: `staging`
+- Merge back into: `staging`
+- Naming convention: `feature/<ticket>/<descriptive-sentence>`
+
+Feature branches are used to develop new features for upcoming or future releases. They exist as long as the feature is in development, but will eventually be merged into `staging` or discarded. Always delete your feature branch after merging to avoid unnecessary clutter.
+
+### Release Branches
+
+- Branch off from: `staging`
+- Merge back into: `staging` and then `main`
+- Naming convention: `release/<version>/<descriptive-message>/<creator's-name>`
+
+Release branches support the preparation of a new production release, allowing for minor bug fixes and preparation of metadata (version number, configuration, etc). All new features should be merged into `staging` and wait for the next big release.
+
+### Hotfix Branches
+
+General workflow:
+
+- Branch off from: `main` or `staging`
+- Merge back into: `staging` then `main`
+- Naming convention: `hotfix/<version>/<descriptive-message>/<creator's-name>` 
+
+Hotfix branches are meant for quick fixes in the production environment. When a critical bug in a production version must be resolved immediately, a hotfix branch is created.
+
+## Git Operations
+
+#### Create a feature branch
+
+1. Branch from the **staging** branch.
+    1. Command: `git checkout -b feature/my-feature staging`
+
+> Rebase frequently with the updated staging branch so you do not face big conflicts before submitting your pull request. Remember, syncing your changes with other developers could also help you avoid big conflicts.
+
+#### Merge feature branch into staging
+
+In other words, integrate your changes into a branch that will be tested and prepared for release.
+
+1. Switch branch to staging: `git checkout staging`
+2. Merging feature branch into staging: `git merge --no-ff feature/my-feature`
+3. Pushing changes to staging: `git push origin staging`
+4. Delete feature branch: `git branch -d feature/my-feature` (alternatively, this can be navigated on the GitHub web UI)
+
+This operation is done by Github when merging a PR.
+
+So, what you have to keep in mind is:
+- Open the PR against the `staging` branch.
+- After merging a PR you should delete your feature branch. This will be strictly enforced.
+
+#### Creating a release branch
+
+1. Create branch from staging: `git checkout -b release/3.4.0/descriptive-message/creator's_name staging`
+2. Updating version with major or minor: `./scripts/update_version.sh major|minor`
+3. Commit file changes with new version: `git commit -a -m "Updated version to 3.4.0"`
+
+
+#### Finishing a Release Branch
+
+This involves releasing stable code and generating a new version for bittensor.
+
+1. Switch branch to main: `git checkout main`
+2. Merge release branch into main: `git merge --no-ff release/3.4.0/optional-descriptive-message`
+3. Tag changeset: `git tag -a v3.4.0 -m "Releasing v3.4.0: some comment about it"`
+4. Push changes to main: `git push origin main`
+5. Push tags to origin: `git push origin --tags`
+
+To keep the changes made in the __release__ branch, we need to merge those back into `staging`:
+
+- Switch branch to staging: `git checkout staging`.
+- Merging release branch into staging: `git merge --no-ff release/3.4.0/optional-descriptive-message`
+
+This step may well lead to a merge conflict (probably even, since we have changed the version number). If so, fix it and commit.
+
+
+#### Creating a hotfix branch
+1. Create branch from main: `git checkout -b hotfix/3.3.4/descriptive-message/creator's-name main`
+2. Update patch version: `./scripts/update_version.sh patch`
+3. Commit file changes with new version: `git commit -a -m "Updated version to 3.3.4"`
+4. Fix the bug and commit the fix: `git commit -m "Fixed critical production issue X"`
+
+#### Finishing a Hotfix Branch
+
+Finishing a hotfix branch involves merging the bugfix into both `main` and `staging`.
+
+1. Switch branch to main: `git checkout main`
+2. Merge hotfix into main: `git merge --no-ff hotfix/3.3.4/optional-descriptive-message`
+3. Tag new version: `git tag -a v3.3.4 -m "Releasing v3.3.4: descriptive comment about the hotfix"`
+4. Push changes to main: `git push origin main`
+5. Push tags to origin: `git push origin --tags`
+6. Switch branch to staging: `git checkout staging`
+7. Merge hotfix into staging: `git merge --no-ff hotfix/3.3.4/descriptive-message/creator's-name`
+8. Push changes to origin/staging: `git push origin staging`
+9. Delete hotfix branch: `git branch -d hotfix/3.3.4/optional-descriptive-message`
+
+The one exception to the rule here is that, **when a release branch currently exists, the hotfix changes need to be merged into that release branch, instead of** `staging`. Back-merging the bugfix into the __release__ branch will eventually result in the bugfix being merged into `develop` too, when the release branch is finished. (If work in develop immediately requires this bugfix and cannot wait for the release branch to be finished, you may safely merge the bugfix into develop now already as well.)
+
+Finally, we remove the temporary branch:
+
+- `git branch -d hotfix/3.3.4/optional-descriptive-message`
+## Continuous Integration (CI) and Continuous Deployment (CD)
+
+Continuous Integration (CI) is a software development practice where members of a team integrate their work frequently. Each integration is verified by an automated build and test process to detect integration errors as quickly as possible. 
+
+Continuous Deployment (CD) is a software engineering approach in which software functionalities are delivered frequently through automated deployments.
+
+- **CircleCI job**: Create jobs in CircleCI to automate the merging of staging into main and release version (needed to release code) and building and testing Bittensor (needed to merge PRs).
+
+> It is highly recommended to set up your own circleci pipeline with your subnet
+
+## Versioning and Release Notes
+
+Semantic versioning helps keep track of the different versions of the software. When code is merged into main, generate a new version. 
+
+Release notes provide documentation for each version released to the users, highlighting the new features, improvements, and bug fixes. When merged into main, generate GitHub release and release notes.
+
+## Pending Tasks
+
+Follow these steps when you are contributing to the bittensor subnet:
+
+- Determine if main and staging are different
+- Determine what is in staging that is not merged yet
+    - Document not released developments
+    - When merged into staging, generate information about what's merged into staging but not released.
+    - When merged into main, generate GitHub release and release notes.
+- CircleCI jobs 
+    - Merge staging into main and release version (needed to release code)
+    - Build and Test Bittensor (needed to merge PRs)
+
+This document can be improved as the Bittensor project continues to develop and change.

bitagent_subnet-main/contrib/STYLE.md

@@ -0,0 +1,348 @@
+# Style Guide
+
+A project’s long-term success rests (among other things) on its maintainability, and a maintainer has few tools more powerful than his or her project’s log. It’s worth taking the time to learn how to care for one properly. What may be a hassle at first soon becomes habit, and eventually a source of pride and productivity for all involved.
+
+Most programming languages have well-established conventions as to what constitutes idiomatic style, i.e. naming, formatting and so on. There are variations on these conventions, of course, but most developers agree that picking one and sticking to it is far better than the chaos that ensues when everybody does their own thing.
+
+# Table of Contents
+1. [Code Style](#code-style)
+2. [Naming Conventions](#naming-conventions)
+3. [Git Commit Style](#git-commit-style)
+4. [The Six Rules of a Great Commit](#the-six-rules-of-a-great-commit)
+   - [1. Atomic Commits](#1-atomic-commits)
+   - [2. Separate Subject from Body with a Blank Line](#2-separate-subject-from-body-with-a-blank-line)
+   - [3. Limit the Subject Line to 50 Characters](#3-limit-the-subject-line-to-50-characters)
+   - [4. Use the Imperative Mood in the Subject Line](#4-use-the-imperative-mood-in-the-subject-line)
+   - [5. Wrap the Body at 72 Characters](#5-wrap-the-body-at-72-characters)
+   - [6. Use the Body to Explain What and Why vs. How](#6-use-the-body-to-explain-what-and-why-vs-how)
+5. [Tools Worth Mentioning](#tools-worth-mentioning)
+   - [Using `--fixup`](#using---fixup)
+   - [Interactive Rebase](#interactive-rebase)
+6. [Pull Request and Squashing Commits Caveats](#pull-request-and-squashing-commits-caveats)
+
+
+### Code style
+
+#### General Style
+Python's official style guide is PEP 8, which provides conventions for writing code for the main Python distribution. Here are some key points:
+
+- `Indentation:` Use 4 spaces per indentation level.
+
+- `Line Length:` Limit all lines to a maximum of 79 characters.
+
+- `Blank Lines:` Surround top-level function and class definitions with two blank lines. Method definitions inside a class are surrounded by a single blank line.
+
+- `Imports:` Imports should usually be on separate lines and should be grouped in the following order:
+
+    - Standard library imports.
+    - Related third party imports.
+    - Local application/library specific imports.
+- `Whitespace:` Avoid extraneous whitespace in the following situations:
+
+    - Immediately inside parentheses, brackets or braces.
+    - Immediately before a comma, semicolon, or colon.
+    - Immediately before the open parenthesis that starts the argument list of a function call.
+- `Comments:` Comments should be complete sentences and should be used to clarify code and are not a substitute for poorly written code.
+
+#### For Python
+
+- `List Comprehensions:` Use list comprehensions for concise and readable creation of lists.
+
+- `Generators:` Use generators when dealing with large amounts of data to save memory.
+
+- `Context Managers:` Use context managers (with statement) for resource management.
+
+- `String Formatting:` Use f-strings for formatting strings in Python 3.6 and above.
+
+- `Error Handling:` Use exceptions for error handling whenever possible.
+
+#### More details
+
+Use `black` to format your python code before commiting for consistency across such a large pool of contributors. Black's code [style](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#code-style) ensures consistent and opinionated code formatting. It automatically formats your Python code according to the Black style guide, enhancing code readability and maintainability.
+
+Key Features of Black:
+
+    Consistency: Black enforces a single, consistent coding style across your project, eliminating style debates and allowing developers to focus on code logic.
+
+    Readability: By applying a standard formatting style, Black improves code readability, making it easier to understand and collaborate on projects.
+
+    Automation: Black automates the code formatting process, saving time and effort. It eliminates the need for manual formatting and reduces the likelihood of inconsistencies.
+
+### Naming Conventions
+
+- `Classes:` Class names should normally use the CapWords Convention.
+- `Functions and Variables:` Function names should be lowercase, with words separated by underscores as necessary to improve readability. Variable names follow the same convention as function names.
+
+- `Constants:` Constants are usually defined on a module level and written in all capital letters with underscores separating words.
+
+- `Non-public Methods and Instance Variables:` Use a single leading underscore (_). This is a weak "internal use" indicator.
+
+- `Strongly "private" methods and variables:` Use a double leading underscore (__). This triggers name mangling in Python.
+
+
+### Git commit style
+
+Here’s a model Git commit message when contributing:
+```
+Summarize changes in around 50 characters or less
+
+More detailed explanatory text, if necessary. Wrap it to about 72
+characters or so. In some contexts, the first line is treated as the
+subject of the commit and the rest of the text as the body. The
+blank line separating the summary from the body is critical (unless
+you omit the body entirely); various tools like `log`, `shortlog`
+and `rebase` can get confused if you run the two together.
+
+Explain the problem that this commit is solving. Focus on why you
+are making this change as opposed to how (the code explains that).
+Are there side effects or other unintuitive consequences of this
+change? Here's the place to explain them.
+
+Further paragraphs come after blank lines.
+
+ - Bullet points are okay, too
+
+ - Typically a hyphen or asterisk is used for the bullet, preceded
+   by a single space, with blank lines in between, but conventions
+   vary here
+
+If you use an issue tracker, put references to them at the bottom,
+like this:
+
+Resolves: #123
+See also: #456, #789
+```
+
+
+## The six rules of a great commit.
+
+#### 1. Atomic Commits
+An “atomic” change revolves around one task or one fix.
+
+Atomic Approach
+ - Commit each fix or task as a separate change
+ - Only commit when a block of work is complete
+ - Commit each layout change separately
+ - Joint commit for layout file, code behind file, and additional resources
+
+Benefits
+
+- Easy to roll back without affecting other changes
+- Easy to make other changes on the fly
+- Easy to merge features to other branches
+
+#### Avoid trivial commit messages
+
+Commit messages like "fix", "fix2", or "fix3" don't provide any context or clear understanding of what changes the commit introduces. Here are some examples of good vs. bad commit messages:
+
+**Bad Commit Message:** 
+
+    $ git commit -m "fix"
+
+**Good Commit Message:**
+
+    $ git commit -m "Fix typo in README file"
+
+> **Caveat**: When working with new features, an atomic commit will often consist of multiple files, since a layout file, code behind file, and additional resources may have been added/modified. You don’t want to commit all of these separately, because if you had to roll back the application to a state before the feature was added, it would involve multiple commit entries, and that can get confusing
+
+#### 2. Separate subject from body with a blank line
+
+Not every commit requires both a subject and a body. Sometimes a single line is fine, especially when the change is so simple that no further context is necessary. 
+
+For example:
+
+    Fix typo in introduction to user guide
+
+Nothing more need be said; if the reader wonders what the typo was, she can simply take a look at the change itself, i.e. use     git show or git diff or git log -p.
+
+If you’re committing something like this at the command line, it’s easy to use the -m option to git commit:
+
+    $ git commit -m"Fix typo in introduction to user guide"
+
+However, when a commit merits a bit of explanation and context, you need to write a body. For example:
+
+    Derezz the master control program
+
+    MCP turned out to be evil and had become intent on world domination.
+    This commit throws Tron's disc into MCP (causing its deresolution)
+    and turns it back into a chess game.
+
+Commit messages with bodies are not so easy to write with the -m option. You’re better off writing the message in a proper text editor. [See Pro Git](https://git-scm.com/book/en/v2/Customizing-Git-Git-Configuration).
+
+In any case, the separation of subject from body pays off when browsing the log. Here’s the full log entry:
+
+    $ git log
+    commit 42e769bdf4894310333942ffc5a15151222a87be
+    Author: Kevin Flynn <kevin@flynnsarcade.com>
+    Date:   Fri Jan 01 00:00:00 1982 -0200
+    
+     Derezz the master control program
+    
+     MCP turned out to be evil and had become intent on world domination.
+     This commit throws Tron's disc into MCP (causing its deresolution)
+     and turns it back into a chess game.
+
+
+#### 3. Limit the subject line to 50 characters
+50 characters is not a hard limit, just a rule of thumb. Keeping subject lines at this length ensures that they are readable, and forces the author to think for a moment about the most concise way to explain what’s going on.
+
+GitHub’s UI is fully aware of these conventions. It will warn you if you go past the 50 character limit. Git will truncate any subject line longer than 72 characters with an ellipsis, thus keeping it to 50 is best practice.
+
+#### 4. Use the imperative mood in the subject line
+Imperative mood just means “spoken or written as if giving a command or instruction”. A few examples:
+
+    Clean your room
+    Close the door
+    Take out the trash
+
+Each of the seven rules you’re reading about right now are written in the imperative (“Wrap the body at 72 characters”, etc.).
+
+The imperative can sound a little rude; that’s why we don’t often use it. But it’s perfect for Git commit subject lines. One reason for this is that Git itself uses the imperative whenever it creates a commit on your behalf.
+
+For example, the default message created when using git merge reads:
+
+    Merge branch 'myfeature'
+
+And when using git revert:
+
+    Revert "Add the thing with the stuff"
+
+    This reverts commit cc87791524aedd593cff5a74532befe7ab69ce9d.
+
+Or when clicking the “Merge” button on a GitHub pull request:
+
+    Merge pull request #123 from someuser/somebranch
+
+So when you write your commit messages in the imperative, you’re following Git’s own built-in conventions. For example:
+
+    Refactor subsystem X for readability
+    Update getting started documentation
+    Remove deprecated methods
+    Release version 1.0.0
+
+Writing this way can be a little awkward at first. We’re more used to speaking in the indicative mood, which is all about reporting facts. That’s why commit messages often end up reading like this:
+
+    Fixed bug with Y
+    Changing behavior of X
+
+And sometimes commit messages get written as a description of their contents:
+
+    More fixes for broken stuff
+    Sweet new API methods
+
+To remove any confusion, here’s a simple rule to get it right every time.
+
+**A properly formed Git commit subject line should always be able to complete the following sentence:**
+
+    If applied, this commit will <your subject line here>
+
+For example:
+
+    If applied, this commit will refactor subsystem X for readability
+    If applied, this commit will update getting started documentation
+    If applied, this commit will remove deprecated methods
+    If applied, this commit will release version 1.0.0
+    If applied, this commit will merge pull request #123 from user/branch
+
+#### 5. Wrap the body at 72 characters
+Git never wraps text automatically. When you write the body of a commit message, you must mind its right margin, and wrap text manually.
+
+The recommendation is to do this at 72 characters, so that Git has plenty of room to indent text while still keeping everything under 80 characters overall.
+
+A good text editor can help here. It’s easy to configure Vim, for example, to wrap text at 72 characters when you’re writing a Git commit.
+
+#### 6. Use the body to explain what and why vs. how
+This [commit](https://github.com/bitcoin/bitcoin/commit/eb0b56b19017ab5c16c745e6da39c53126924ed6) from Bitcoin Core is a great example of explaining what changed and why:
+
+```
+commit eb0b56b19017ab5c16c745e6da39c53126924ed6
+Author: Pieter Wuille <pieter.wuille@gmail.com>
+Date:   Fri Aug 1 22:57:55 2014 +0200
+
+   Simplify serialize.h's exception handling
+
+   Remove the 'state' and 'exceptmask' from serialize.h's stream
+   implementations, as well as related methods.
+
+   As exceptmask always included 'failbit', and setstate was always
+   called with bits = failbit, all it did was immediately raise an
+   exception. Get rid of those variables, and replace the setstate
+   with direct exception throwing (which also removes some dead
+   code).
+
+   As a result, good() is never reached after a failure (there are
+   only 2 calls, one of which is in tests), and can just be replaced
+   by !eof().
+
+   fail(), clear(n) and exceptions() are just never called. Delete
+   them.
+```
+
+Take a look at the [full diff](https://github.com/bitcoin/bitcoin/commit/eb0b56b19017ab5c16c745e6da39c53126924ed6) and just think how much time the author is saving fellow and future committers by taking the time to provide this context here and now. If he didn’t, it would probably be lost forever.
+
+In most cases, you can leave out details about how a change has been made. Code is generally self-explanatory in this regard (and if the code is so complex that it needs to be explained in prose, that’s what source comments are for). Just focus on making clear the reasons why you made the change in the first place—the way things worked before the change (and what was wrong with that), the way they work now, and why you decided to solve it the way you did.
+
+The future maintainer that thanks you may be yourself!
+
+
+
+#### Tools worth mentioning
+
+##### Using `--fixup`
+
+If you've made a commit and then realize you've missed something or made a minor mistake, you can use the `--fixup` option. 
+
+For example, suppose you've made a commit with a hash `9fceb02`. Later, you realize you've left a debug statement in your code. Instead of making a new commit titled "remove debug statement" or "fix", you can do the following:
+
+    $ git commit --fixup 9fceb02
+
+This will create a new commit to fix the issue, with a message like "fixup! The original commit message".
+
+##### Interactive Rebase
+
+Interactive rebase, or `rebase -i`, can be used to squash these fixup commits into the original commits they're fixing, which cleans up your commit history. You can use the `autosquash` option to automatically squash any commits marked as "fixup" into their target commits.
+
+For example:
+
+    $ git rebase -i --autosquash HEAD~5
+
+This command starts an interactive rebase for the last 5 commits (`HEAD~5`). Any commits marked as "fixup" will be automatically moved to squash with their target commits.
+
+The benefit of using `--fixup` and interactive rebase is that it keeps your commit history clean and readable. It groups fixes with the commits they are related to, rather than having a separate "fix" commit that might not make sense to other developers (or even to you) in the future.
+
+
+---
+
+#### Pull Request and Squashing Commits Caveats
+
+While atomic commits are great for development and for understanding the changes within the branch, the commit history can get messy when merging to the main branch. To keep a cleaner and more understandable commit history in our main branch, we encourage squashing all the commits of a PR into one when merging.
+
+This single commit should provide an overview of the changes that the PR introduced. It should follow the guidelines for atomic commits (an atomic commit is complete, self-contained, and understandable) but on the scale of the entire feature, task, or fix that the PR addresses. This approach combines the benefits of atomic commits during development with a clean commit history in our main branch.
+
+Here is how you can squash commits:
+
+```bash
+git rebase -i HEAD~n
+```
+
+where `n` is the number of commits to squash. After running the command, replace `pick` with `squash` for the commits you want to squash into the previous commit. This will combine the commits and allow you to write a new commit message.
+
+In this context, an atomic commit message could look like:
+
+```
+Add feature X
+
+This commit introduces feature X which does A, B, and C. It adds 
+new files for layout, updates the code behind the file, and introduces
+new resources. This change is important because it allows users to 
+perform task Y more efficiently. 
+
+It includes:
+- Creation of new layout file
+- Updates in the code-behind file
+- Addition of new resources
+
+Resolves: #123
+```
+
+In your PRs, remember to detail what the PR is introducing or fixing. This will be helpful for reviewers to understand the context and the reason behind the changes.