models/Megatron/docs/source/api-guide/datasets.rst

datasets package
================

.. mdinclude :: ../../../megatron/core/datasets/readme.md

Submodules
----------

datasets.blended\_megatron\_dataset\_config module
---------------------------------------------------

.. automodule:: core.datasets.blended_megatron_dataset_config
   :members:
   :undoc-members:
   :show-inheritance:

datasets.blended\_megatron\_dataset\_builder module
---------------------------------------------------

.. automodule:: core.datasets.blended_megatron_dataset_builder
   :members:
   :undoc-members:
   :show-inheritance:

datasets.megatron\_tokenizer module
-----------------------------------

.. automodule:: core.datasets.megatron_tokenizer
   :members:
   :undoc-members:
   :show-inheritance:

datasets.indexed\_dataset module
--------------------------------

.. automodule:: core.datasets.indexed_dataset
   :members:
   :undoc-members:
   :show-inheritance:

datasets.megatron\_dataset module
---------------------------------

.. automodule:: core.datasets.megatron_dataset
   :members:
   :undoc-members:
   :show-inheritance:

datasets.gpt\_dataset module
----------------------------

.. automodule:: core.datasets.gpt_dataset
   :members:
   :undoc-members:
   :show-inheritance:

datasets.masked\_dataset module
-------------------------------

.. automodule:: core.datasets.masked_dataset
   :members:
   :undoc-members:
   :show-inheritance:

datasets.bert\_dataset module
-----------------------------

.. automodule:: core.datasets.bert_dataset
   :members:
   :undoc-members:
   :show-inheritance:

datasets.t5\_dataset module
---------------------------

.. automodule:: core.datasets.t5_dataset
   :members:
   :undoc-members:
   :show-inheritance:

datasets.blended\_dataset module
----------------------------------

.. automodule:: core.datasets.blended_dataset
   :members:
   :undoc-members:
   :show-inheritance:

datasets.utils module
---------------------

.. automodule:: core.datasets.utils
   :members:
   :undoc-members:
   :show-inheritance:

Module contents
---------------

.. automodule:: core.datasets
   :members:
   :undoc-members:
   :show-inheritance: