# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.

"""T5 Style dataset."""

import collections

import numpy as np
import torch

from megatron import get_tokenizer
from megatron.data.dataset_utils import (
    create_masked_lm_predictions,
    get_samples_mapping
)

class T5Dataset(torch.utils.data.Dataset):

    def __init__(self, name, indexed_dataset, data_prefix,
                 num_epochs, max_num_samples, masked_lm_prob,
                 max_seq_length, max_seq_length_dec,
                 short_seq_prob, seed):

        # Params to store.
        self.name = name
        self.seed = seed
        self.masked_lm_prob = masked_lm_prob
        self.max_seq_length = max_seq_length
        self.max_seq_length_dec = max_seq_length_dec

        # Dataset.
        self.indexed_dataset = indexed_dataset

        # Build the samples mapping.
        self.samples_mapping = get_samples_mapping(self.indexed_dataset,
                                                   data_prefix,
                                                   num_epochs,
                                                   max_num_samples,
                                                   self.max_seq_length - 2, # account for added tokens
                                                   short_seq_prob,
                                                   self.seed,
                                                   self.name,
                                                   False)

        # Vocab stuff.
        tokenizer = get_tokenizer()
        self.vocab_id_list = list(tokenizer.inv_vocab.keys())
        self.vocab_id_to_token_dict = tokenizer.inv_vocab
        self.cls_id = tokenizer.cls
        self.sep_id = tokenizer.sep
        self.mask_id = tokenizer.mask
        self.pad_id = tokenizer.pad
        self.bos_id = tokenizer.bos_token_id
        self.eos_id = tokenizer.eos_token_id
        self.sentinel_tokens = tokenizer.additional_special_tokens_ids
        assert len(self.sentinel_tokens) > 0, "Provide the argument --vocab-extra-ids 100 to the script"

    def __len__(self):
        return self.samples_mapping.shape[0]

    def __getitem__(self, idx):

        start_index, end_index, seq_length = self.samples_mapping[idx]
        sample = []
        for index in range(start_index, end_index):
            sample.append(self.indexed_dataset[index])
        # Note that this rng state should be numpy and not python since
        # python randint is inclusive whereas the numpy one is exclusive.
        np_rng = np.random.RandomState(seed=(self.seed + idx))
        return build_training_sample(sample, seq_length,
                                     self.max_seq_length,  # needed for padding
                                     self.max_seq_length_dec,
                                     self.vocab_id_list,
                                     self.vocab_id_to_token_dict,
                                     self.cls_id, self.sep_id,
                                     self.mask_id, self.pad_id,
                                     self.masked_lm_prob, np_rng,
                                     self.bos_id, self.eos_id,
                                     self.sentinel_tokens)

def build_training_sample(sample, target_seq_length,
                          max_seq_length, max_seq_length_dec,
                          vocab_id_list, vocab_id_to_token_dict,
                          cls_id, sep_id, mask_id, pad_id,
                          masked_lm_prob, np_rng, bos_id=None,
                          eos_id=None, sentinel_tokens=None):
    """Build training sample.

    Arguments:
        sample: A list of sentences in which each sentence is a list token ids.
        target_seq_length: Desired sequence length.
        max_seq_length: Maximum length of the sequence. All values are padded to
            this length.
        vocab_id_list: List of vocabulary ids. Used to pick a random id.
        vocab_id_to_token_dict: A dictionary from vocab ids to text tokens.
        cls_id: Start of example id.
        sep_id: Separator id.
        mask_id: Mask token id.


        start_index, end_index, seq_length = self.samples_mapping[idx]
        sample = []
        for index in range(start_index, end_index):
            sample.append(self.indexed_dataset[index])
        # Note that this rng state should be numpy and not python since
        # python randint is inclusive whereas the numpy one is exclusive.
        np_rng = np.random.RandomState(seed=(self.seed + idx))
        return build_training_sample(sample, seq_length,
                                     self.max_seq_length,  # needed for padding
                                     self.max_seq_length_dec,
                                     self.vocab_id_list,
                                     self.vocab_id_to_token_dict,
                                     self.cls_id, self.sep_id,
                                     self.mask_id, self.pad_id,
                                     self.masked_lm_prob, np_rng,
                                     self.bos_id, self.eos_id,
                                     self.sentinel_tokens)

def build_training_sample(sample, target_seq_length,
                          max_seq_length, max_seq_length_dec,
                          vocab_id_list, vocab_id_to_token_dict,
                          cls_id, sep_id, mask_id, pad_id,
                          masked_lm_prob, np_rng, bos_id=None,
                          eos_id=None, sentinel_tokens=None):
    """Build training sample.

    Arguments:
        sample: A list of sentences in which each sentence is a list token ids.
        target_seq_length: Desired sequence length.
        max_seq_length: Maximum length of the sequence. All values are padded to
            this length.
        vocab_id_list: List of vocabulary ids. Used to pick a random id.
        vocab_id_to_token_dict: A dictionary from vocab ids to text tokens.
        cls_id: Start of example id.
        sep_id: Separator id.
        mask_id: Mask token id.
        pad_id: Padding token id.
        masked_lm_prob: Probability to mask tokens.
        np_rng: Random number genenrator. Note that this rng state should be
              numpy and not python since python randint is inclusive for
              the opper bound whereas the numpy one is exclusive.
        bos_id: start of decoder example id
        eos_id: end of generation id
        sentinel_tokens: unique value to be substituted for every replaced span
    """

    assert target_seq_length <= max_seq_length

    # flatten sentences into one list
    tokens = [token for sentence in sample for token in sentence]

    # Truncate to `target_sequence_length`.
    max_num_tokens = target_seq_length
    truncated = len(tokens) > max_num_tokens
    tokens = tokens[:max_num_tokens]

    # Masking.
    max_predictions_per_seq = masked_lm_prob * max_num_tokens
    (tokens, masked_positions, masked_labels, _, masked_spans) = create_masked_lm_predictions(
        tokens, vocab_id_list, vocab_id_to_token_dict, masked_lm_prob,
        cls_id, sep_id, mask_id, max_predictions_per_seq, np_rng,
        max_ngrams=10, geometric_dist=True, masking_style="t5")

    # Padding.
    tokens_enc, tokens_dec_in, labels, enc_mask, \
    dec_mask, enc_dec_mask, loss_mask \
        = pad_and_convert_to_numpy(tokens, masked_positions,
                                   masked_labels, pad_id, max_seq_length,
                                   max_seq_length_dec, masked_spans,
                                   bos_id, eos_id, sentinel_tokens)

    train_sample = {
        'text_enc': tokens_enc,
        'text_dec': tokens_dec_in,
        'labels': labels,
        'loss_mask': loss_mask,
        'truncated': int(truncated),
        'enc_mask': enc_mask,
        'dec_mask': dec_mask,
        'enc_dec_mask': enc_dec_mask,
    }
    return train_sample

def pad_and_convert_to_numpy(tokens, masked_positions,
                             masked_labels, pad_id,
                             max_seq_length, max_seq_length_dec,
                             masked_spans=None, bos_id=None,
                             eos_id=None, sentinel_tokens=None):
    """Pad sequences and convert them to numpy."""

    sentinel_tokens = collections.deque(sentinel_tokens)
    t5_input = []
    (t5_decoder_in, t5_decoder_out) = ([bos_id], [])
    (start_index, end_index) = (0, None)
    for span in masked_spans:
        flag = sentinel_tokens.popleft()

        # Append the same tokens in decoder input and output
        t5_decoder_in.append(flag)
        t5_decoder_in.extend(span.label)
        t5_decoder_out.append(flag)


    tokens_enc, tokens_dec_in, labels, enc_mask, \
    dec_mask, enc_dec_mask, loss_mask \
        = pad_and_convert_to_numpy(tokens, masked_positions,
                                   masked_labels, pad_id, max_seq_length,
                                   max_seq_length_dec, masked_spans,
                                   bos_id, eos_id, sentinel_tokens)

    train_sample = {
        'text_enc': tokens_enc,
        'text_dec': tokens_dec_in,
        'labels': labels,
        'loss_mask': loss_mask,
        'truncated': int(truncated),
        'enc_mask': enc_mask,
        'dec_mask': dec_mask,
        'enc_dec_mask': enc_dec_mask,
    }
    return train_sample

def pad_and_convert_to_numpy(tokens, masked_positions,
                             masked_labels, pad_id,
                             max_seq_length, max_seq_length_dec,
                             masked_spans=None, bos_id=None,
                             eos_id=None, sentinel_tokens=None):
    """Pad sequences and convert them to numpy."""

    sentinel_tokens = collections.deque(sentinel_tokens)
    t5_input = []
    (t5_decoder_in, t5_decoder_out) = ([bos_id], [])
    (start_index, end_index) = (0, None)
    for span in masked_spans:
        flag = sentinel_tokens.popleft()

        # Append the same tokens in decoder input and output
        t5_decoder_in.append(flag)
        t5_decoder_in.extend(span.label)
        t5_decoder_out.append(flag)
        t5_decoder_out.extend(span.label)

        end_index = span.index[0]
        t5_input.extend(tokens[start_index: end_index])
        t5_input.append(flag)

        # the next start index is the token after the last span token
        start_index = span.index[-1] + 1

    # Add <eos> token to the t5_decoder_out
    t5_decoder_out.append(eos_id)

    # Add the remaining tokens to the t5 input
    t5_input.extend(tokens[start_index:])

    # assert (len(t5_input) - len(masked_spans)) + \
    #        (len(t5_decoder_in) - (len(masked_spans) + 1)) == len(tokens)

    # Some checks.

    # Encoder-side padding mask.
    num_tokens = len(t5_input)
    padding_length = max_seq_length - num_tokens
    assert padding_length >= 0
    assert len(masked_positions) == len(masked_labels)

    # Tokens..
    filler = [pad_id] * padding_length
    tokens_enc = np.array(t5_input + filler, dtype=np.int64)

    # Decoder-side padding mask.
    num_tokens_dec = len(t5_decoder_in)
    padding_length_dec = max_seq_length_dec - num_tokens_dec
    assert padding_length_dec >= 0
    filler_dec = [pad_id] * padding_length_dec
    tokens_dec_in = np.array(t5_decoder_in + filler_dec, dtype=np.int64)

    # Create attention masks
    enc_mask = make_attention_mask(tokens_enc, tokens_enc)
    enc_dec_mask = make_attention_mask(tokens_dec_in, tokens_enc)
    dec_mask = make_attention_mask(tokens_dec_in, tokens_dec_in)
    dec_mask = dec_mask * make_history_mask(tokens_dec_in)

    # Labels mask.
    labels = t5_decoder_out + ([-1] * padding_length_dec)
    labels = np.array(labels, dtype=np.int64)

    # Loss mask
    loss_mask = ([1] * num_tokens_dec) + ([0] * padding_length_dec)
    loss_mask = np.array(loss_mask, dtype=np.int64)

    return tokens_enc, tokens_dec_in, labels, enc_mask, \
           dec_mask, enc_dec_mask, loss_mask

def make_attention_mask(source_block, target_block):
    """
    Returns a 2-dimensional (2-D) attention mask
    :param source_block: 1-D array
    :param target_block: 1-D array
    """
    mask = (target_block[None, :] >= 1) * (source_block[:, None] >= 1)
    mask = mask.astype(np.int64)
    # (source_length, target_length)
    return mask

def make_attention_mask_3d(source_block, target_block):
    """
    Returns a 3-dimensional (3-D) attention mask
    :param source_block: 1-D array
    :param target_block: 1-D array
    """


    tokens_dec_in = np.array(t5_decoder_in + filler_dec, dtype=np.int64)

    # Create attention masks
    enc_mask = make_attention_mask(tokens_enc, tokens_enc)
    enc_dec_mask = make_attention_mask(tokens_dec_in, tokens_enc)
    dec_mask = make_attention_mask(tokens_dec_in, tokens_dec_in)
    dec_mask = dec_mask * make_history_mask(tokens_dec_in)

    # Labels mask.
    labels = t5_decoder_out + ([-1] * padding_length_dec)
    labels = np.array(labels, dtype=np.int64)

    # Loss mask
    loss_mask = ([1] * num_tokens_dec) + ([0] * padding_length_dec)
    loss_mask = np.array(loss_mask, dtype=np.int64)

    return tokens_enc, tokens_dec_in, labels, enc_mask, \
           dec_mask, enc_dec_mask, loss_mask

def make_attention_mask(source_block, target_block):
    """
    Returns a 2-dimensional (2-D) attention mask
    :param source_block: 1-D array
    :param target_block: 1-D array
    """
    mask = (target_block[None, :] >= 1) * (source_block[:, None] >= 1)
    mask = mask.astype(np.int64)
    # (source_length, target_length)
    return mask

def make_attention_mask_3d(source_block, target_block):
    """
    Returns a 3-dimensional (3-D) attention mask
    :param source_block: 1-D array
    :param target_block: 1-D array
    """
    mask = (target_block[:, None, :] >= 1) * (source_block[:, :, None] >= 1)
    # (batch, source_length, target_length)
    # mask = mask.astype(np.int64)
    return mask

def make_history_mask(block):
    length = block.shape[0]
    arange = np.arange(length)
    history_mask = (arange[None, ] <= arange[:, None])
    history_mask = history_mask.astype(np.int64)
    return history_mask


    # Loss mask
    loss_mask = ([1] * num_tokens_dec) + ([0] * padding_length_dec)
    loss_mask = np.array(loss_mask, dtype=np.int64)

    return tokens_enc, tokens_dec_in, labels, enc_mask, \
           dec_mask, enc_dec_mask, loss_mask

def make_attention_mask(source_block, target_block):
    """
    Returns a 2-dimensional (2-D) attention mask
    :param source_block: 1-D array
    :param target_block: 1-D array
    """
    mask = (target_block[None, :] >= 1) * (source_block[:, None] >= 1)
    mask = mask.astype(np.int64)
    # (source_length, target_length)
    return mask

def make_attention_mask_3d(source_block, target_block):
    """
    Returns a 3-dimensional (3-D) attention mask
    :param source_block: 1-D array
    :param target_block: 1-D array
    """
    mask = (target_block[:, None, :] >= 1) * (source_block[:, :, None] >= 1)
    # (batch, source_length, target_length)
    # mask = mask.astype(np.int64)
    return mask

def make_history_mask(block):
    length = block.shape[0]
    arange = np.arange(length)
    history_mask = (arange[None, ] <= arange[:, None])
    history_mask = history_mask.astype(np.int64)
    return history_mask

def make_history_mask_3d(block):
    batch, length = block.shape
    arange = torch.arange(length, device=block.device)
    history_mask = (arange[None, ] <= arange[:, None])[None, ]
    history_mask = history_mask.expand(batch, length, length)
    return history_mask


    :param target_block: 1-D array
    """
    mask = (target_block[None, :] >= 1) * (source_block[:, None] >= 1)
    mask = mask.astype(np.int64)
    # (source_length, target_length)
    return mask

def make_attention_mask_3d(source_block, target_block):
    """
    Returns a 3-dimensional (3-D) attention mask
    :param source_block: 1-D array
    :param target_block: 1-D array
    """
    mask = (target_block[:, None, :] >= 1) * (source_block[:, :, None] >= 1)
    # (batch, source_length, target_length)
    # mask = mask.astype(np.int64)
    return mask

def make_history_mask(block):
    length = block.shape[0]
    arange = np.arange(length)
    history_mask = (arange[None, ] <= arange[:, None])
    history_mask = history_mask.astype(np.int64)
    return history_mask

def make_history_mask_3d(block):
    batch, length = block.shape
    arange = torch.arange(length, device=block.device)
    history_mask = (arange[None, ] <= arange[:, None])[None, ]
    history_mask = history_mask.expand(batch, length, length)
    return history_mask


def make_attention_mask_3d(source_block, target_block):
    """
    Returns a 3-dimensional (3-D) attention mask
    :param source_block: 1-D array
    :param target_block: 1-D array
    """
    mask = (target_block[:, None, :] >= 1) * (source_block[:, :, None] >= 1)
    # (batch, source_length, target_length)
    # mask = mask.astype(np.int64)
    return mask

def make_history_mask(block):
    length = block.shape[0]
    arange = np.arange(length)
    history_mask = (arange[None, ] <= arange[:, None])
    history_mask = history_mask.astype(np.int64)
    return history_mask

def make_history_mask_3d(block):
    batch, length = block.shape
    arange = torch.arange(length, device=block.device)
    history_mask = (arange[None, ] <= arange[:, None])[None, ]
    history_mask = history_mask.expand(batch, length, length)
    return history_mask


# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.

"""T5 Style dataset."""

import collections

import numpy as np
import torch

from megatron import get_tokenizer
from megatron.data.dataset_utils import (
    create_masked_lm_predictions,
    get_samples_mapping
)

class T5Dataset(torch.utils.data.Dataset):

    def __init__(self, name, indexed_dataset, data_prefix,
                 num_epochs, max_num_samples, masked_lm_prob,
                 max_seq_length, max_seq_length_dec,
                 short_seq_prob, seed):

        # Params to store.
        self.name = name
        self.seed = seed
        self.masked_lm_prob = masked_lm_prob
        self.max_seq_length = max_seq_length
        self.max_seq_length_dec = max_seq_length_dec

        # Dataset.
        self.indexed_dataset = indexed_dataset

        # Build the samples mapping.
        self.samples_mapping = get_samples_mapping(self.indexed_dataset,
                                                   data_prefix,
                                                   num_epochs,
                                                   max_num_samples,
                                                   self.max_seq_length - 2, # account for added tokens
                                                   short_seq_prob,
                                                   self.seed,
                                                   self.name,
                                                   False)

        # Vocab stuff.
        tokenizer = get_tokenizer()
        self.vocab_id_list = list(tokenizer.inv_vocab.keys())
        self.vocab_id_to_token_dict = tokenizer.inv_vocab
        self.cls_id = tokenizer.cls
        self.sep_id = tokenizer.sep
        self.mask_id = tokenizer.mask
        self.pad_id = tokenizer.pad
        self.bos_id = tokenizer.bos_token_id
        self.eos_id = tokenizer.eos_token_id
        self.sentinel_tokens = tokenizer.additional_special_tokens_ids
        assert len(self.sentinel_tokens) > 0, "Provide the argument --vocab-extra-ids 100 to the script"

    def __len__(self):
        return self.samples_mapping.shape[0]

    def __getitem__(self, idx):

        start_index, end_index, seq_length = self.samples_mapping[idx]
        sample = []
        for index in range(start_index, end_index):
            sample.append(self.indexed_dataset[index])
        # Note that this rng state should be numpy and not python since
        # python randint is inclusive whereas the numpy one is exclusive.
        np_rng = np.random.RandomState(seed=(self.seed + idx))
        return build_training_sample(sample, seq_length,
                                     self.max_seq_length,  # needed for padding
                                     self.max_seq_length_dec,
                                     self.vocab_id_list,
                                     self.vocab_id_to_token_dict,
                                     self.cls_id, self.sep_id,
                                     self.mask_id, self.pad_id,


                                                   max_num_samples,
                                                   self.max_seq_length - 2, # account for added tokens
                                                   short_seq_prob,
                                                   self.seed,
                                                   self.name,
                                                   False)

        # Vocab stuff.
        tokenizer = get_tokenizer()
        self.vocab_id_list = list(tokenizer.inv_vocab.keys())
        self.vocab_id_to_token_dict = tokenizer.inv_vocab
        self.cls_id = tokenizer.cls
        self.sep_id = tokenizer.sep
        self.mask_id = tokenizer.mask
        self.pad_id = tokenizer.pad
        self.bos_id = tokenizer.bos_token_id
        self.eos_id = tokenizer.eos_token_id
        self.sentinel_tokens = tokenizer.additional_special_tokens_ids
        assert len(self.sentinel_tokens) > 0, "Provide the argument --vocab-extra-ids 100 to the script"

    def __len__(self):
        return self.samples_mapping.shape[0]

    def __getitem__(self, idx):

        start_index, end_index, seq_length = self.samples_mapping[idx]
        sample = []
        for index in range(start_index, end_index):
            sample.append(self.indexed_dataset[index])
        # Note that this rng state should be numpy and not python since
        # python randint is inclusive whereas the numpy one is exclusive.
        np_rng = np.random.RandomState(seed=(self.seed + idx))
        return build_training_sample(sample, seq_length,
                                     self.max_seq_length,  # needed for padding
                                     self.max_seq_length_dec,
                                     self.vocab_id_list,
                                     self.vocab_id_to_token_dict,
                                     self.cls_id, self.sep_id,
                                     self.mask_id, self.pad_id,
                                     self.masked_lm_prob, np_rng,
                                     self.bos_id, self.eos_id,
                                     self.sentinel_tokens)


                                                   self.seed,
                                                   self.name,
                                                   False)

        # Vocab stuff.
        tokenizer = get_tokenizer()
        self.vocab_id_list = list(tokenizer.inv_vocab.keys())
        self.vocab_id_to_token_dict = tokenizer.inv_vocab
        self.cls_id = tokenizer.cls
        self.sep_id = tokenizer.sep
        self.mask_id = tokenizer.mask
        self.pad_id = tokenizer.pad
        self.bos_id = tokenizer.bos_token_id
        self.eos_id = tokenizer.eos_token_id
        self.sentinel_tokens = tokenizer.additional_special_tokens_ids
        assert len(self.sentinel_tokens) > 0, "Provide the argument --vocab-extra-ids 100 to the script"

    def __len__(self):
        return self.samples_mapping.shape[0]

    def __getitem__(self, idx):

        start_index, end_index, seq_length = self.samples_mapping[idx]
        sample = []
        for index in range(start_index, end_index):
            sample.append(self.indexed_dataset[index])
        # Note that this rng state should be numpy and not python since
        # python randint is inclusive whereas the numpy one is exclusive.
        np_rng = np.random.RandomState(seed=(self.seed + idx))
        return build_training_sample(sample, seq_length,
                                     self.max_seq_length,  # needed for padding
                                     self.max_seq_length_dec,
                                     self.vocab_id_list,
                                     self.vocab_id_to_token_dict,
                                     self.cls_id, self.sep_id,
                                     self.mask_id, self.pad_id,
                                     self.masked_lm_prob, np_rng,
                                     self.bos_id, self.eos_id,
                                     self.sentinel_tokens)

def build_training_sample(sample, target_seq_length,
                          max_seq_length, max_seq_length_dec,
                          vocab_id_list, vocab_id_to_token_dict,
                          cls_id, sep_id, mask_id, pad_id,
                          masked_lm_prob, np_rng, bos_id=None,
                          eos_id=None, sentinel_tokens=None):
    """Build training sample.

    Arguments:
        sample: A list of sentences in which each sentence is a list token ids.
        target_seq_length: Desired sequence length.
        max_seq_length: Maximum length of the sequence. All values are padded to
            this length.
        vocab_id_list: List of vocabulary ids. Used to pick a random id.
        vocab_id_to_token_dict: A dictionary from vocab ids to text tokens.
        cls_id: Start of example id.
        sep_id: Separator id.
        mask_id: Mask token id.


# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# Essentially re-written in entirety

import os
import shutil
import struct
from enum import Enum
from functools import lru_cache
from itertools import accumulate
from types import TracebackType
from typing import List, Optional, Tuple, Type, Union

import numpy as np
import torch

from megatron import print_rank_0

_INDEX_HEADER = b"MMIDIDX\x00\x00"

class DType(Enum):
    uint8 = 1
    int8 = 2
    int16 = 3
    int32 = 4
    int64 = 5
    float64 = 6
    float32 = 7
    uint16 = 8

    @classmethod
    def code_from_dtype(cls, value: Type[np.number]) -> int:
        return cls[value.__name__].value

    @classmethod
    def dtype_from_code(cls, value: int) -> Type[np.number]:
        return getattr(np, cls(value).name)

    @staticmethod
    def size(key: Union[int, Type[np.number]]) -> int:
        if isinstance(key, int):
            return DType.dtype_from_code(key)().itemsize
        elif np.number in key.__mro__:
            return key().itemsize
        else:
            raise ValueError

    @staticmethod
    def optimal_dtype(cardinality: int) -> Type[np.number]:
        if cardinality is not None and cardinality < 65500:
            return np.uint16
        else:
            return np.int32

class _IndexWriter(object):
    """
    Object class to write the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, dtype: Type[np.number]) -> None:
        self.path = path
        self.dtype = dtype

    def __enter__(self) -> "_IndexWriter":
        self.idx_path = open(self.path, "wb")
        # fixed, vestigial practice
        self.idx_path.write(_INDEX_HEADER)
        # fixed, vestigial practice
        self.idx_path.write(struct.pack("<Q", 1))
        # the numeric code for the dtype
        self.idx_path.write(struct.pack("<B", DType.code_from_dtype(self.dtype)))
        return self

    def __exit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> Optional[bool]:
        self.idx_path.close()

    def write(
        self,
        sequence_lengths: List[int],
        sequence_modes: Optional[List[int]],
        document_indices: List[int],
    ) -> None:
        sequence_pointers = self._sequence_pointers(sequence_lengths)

        # the number of sequences in the dataset
        sequence_count = len(sequence_lengths)
        self.idx_path.write(struct.pack("<Q", sequence_count))

        # the number of documents in the dataset
        document_count = len(document_indices)
        self.idx_path.write(struct.pack("<Q", document_count))

        # the number of tokens per sequence
        sequence_lengths = np.array(sequence_lengths, dtype=np.int32)
        self.idx_path.write(sequence_lengths.tobytes(order="C"))
        del sequence_lengths

        # the byte offsets for all sequences
        sequence_pointers = np.array(sequence_pointers, dtype=np.int64)
        self.idx_path.write(sequence_pointers.tobytes(order="C"))
        del sequence_pointers

        # the sequence indices marking the end of each document
        document_indices = np.array(document_indices, dtype=np.int64)
        self.idx_path.write(document_indices.tobytes(order="C"))

        # the mode per sequence
        if sequence_modes is not None:
            sequence_modes = np.array(sequence_modes, dtype=np.int32)
            self._file.write(sequence_modes.tobytes(order='C'))
            del sequence_modes

    def _sequence_pointers(self, sequence_lengths: List[int]) -> List[int]:
        itemsize = DType.size(self.dtype)
        curr_ptr = 0
        list_ptr = []
        for length in sequence_lengths:
            list_ptr.append(curr_ptr)
            curr_ptr += length * itemsize
        return list_ptr

class _IndexReader(object):
    """
    Object class to read the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, multimodal: bool) -> None:
        with open(path, "rb") as stream:
            header = stream.read(9)
            assert header == _INDEX_HEADER, f"bad header, cannot read: {path}"

            version = struct.unpack("<Q", stream.read(8))[0]
            assert version == 1, f"bad version, cannot read: {path}"

            code = struct.unpack("<B", stream.read(1))[0]
            self._dtype = DType.dtype_from_code(code)
            self._dtype_size = DType.size(self._dtype)

            self._sequence_count = struct.unpack("<Q", stream.read(8))[0]
            self._document_count = struct.unpack("<Q", stream.read(8))[0]

            offset = stream.tell()

        self._multimodal = multimodal

        self._bin_buffer_mmap = np.memmap(path, mode="r", order="C")
        self._bin_buffer = memoryview(self._bin_buffer_mmap)

        print_rank_0("    reading sequence lengths...")
        self._sequence_lengths = np.frombuffer(
            self._bin_buffer, dtype=np.int32, count=self._sequence_count, offset=offset
        )

        print_rank_0("    reading sequence pointers...")
        self._sequence_pointers = np.frombuffer(
            self._bin_buffer,
            dtype=np.int64,
            count=self._sequence_count,
            offset=offset + self._sequence_lengths.nbytes,
        )

        print_rank_0("    reading document indices...")
        self._document_indices = np.frombuffer(
            self._bin_buffer,
            dtype=np.int64,
            count=self._document_count,
            offset=offset + self._sequence_lengths.nbytes + self._sequence_pointers.nbytes,
        )

        self._sequence_modes = None
        if self._multimodal:
            print_rank_0("    reading sequence modes...")
            self._sequence_modes = np.frombuffer(
                self._bin_buffer,
                dtype=np.int8,
                count=self._len,
                offset=offset
                + self._sequence_lengths.nbytes
                + self._sequence_pointers.nbytes
                + self._document_indices.nbytes,
            )

    def __del__(self) -> None:
        self._bin_buffer_mmap._mmap.close()
        del self._bin_buffer_mmap

    def __len__(self) -> int:
        return self._sequence_count

    @lru_cache(maxsize=8)
    def __getitem__(self, i: int) -> Tuple[np.int32, np.int64, Optional[np.int8]]:
        return (
            self._sequence_pointers[i],
            self._sequence_lengths[i],
            self._sequence_modes[i] if self._multimodal else None,
        )

    @property
    def dtype(self) -> Type[np.number]:
        return self._dtype

    @property
    def sizes(self) -> np.ndarray:
        return self._sequence_lengths

    @property
    def doc_idx(self) -> np.ndarray:
        return self._document_indices

    @property
    def modes(self) -> np.ndarray:
        return self._sequence_modes

class MMapIndexedDataset(torch.utils.data.Dataset):
    def __init__(self, path: str, skip_warmup: bool = False, multimodal: bool = False) -> None:
        super().__init__()

        self._path = None
        self._index = None
        self._bin_buffer = None
        self._multimodal = multimodal

        self._do_init(path, skip_warmup, multimodal)

    def __getstate__(self) -> str:
        return self._path

    def __setstate__(self, path: str) -> None:
        self._do_init(path, skip_warmup=True, multimodal=False)

    def __del__(self) -> None:
        self._bin_buffer_mmap._mmap.close()
        del self._bin_buffer_mmap
        del self._index

    def __len__(self) -> int:
        return len(self._index)

    def __getitem__(self, idx: Union[int, np.integer, slice]) -> np.ndarray:
        if isinstance(idx, (int, np.integer)):
            sequence_pointer, sequence_length, sequence_mode = self._index[idx]
            sequence = np.frombuffer(
                self._bin_buffer,
                dtype=self._index.dtype,
                count=sequence_length,
                offset=sequence_pointer,
            )
            return (sequence, sequence_mode) if sequence_mode is not None else sequence
        elif isinstance(idx, slice):
            start, stop, step = idx.indices(len(self))
            if step != 1:
                raise ValueError("Slices into indexed_dataset must be contiguous")
            sequence_lengths = self._index._sequence_lengths[idx]
            sequence_modes = self._index._sequence_modes[idx] if self._multimodal else None
            sequence_offsets = list(accumulate(sequence_lengths))
            sequences = np.split(
                np.frombuffer(
                    self._bin_buffer,
                    dtype=self._index.dtype,
                    count=sum(sequence_lengths),
                    offset=self._index._sequence_pointers[start],
                ),
                sequence_offsets[:-1],
            )
            return (sequences, sequence_modes) if sequence_modes is not None else sequences
        else:
            raise TypeError("Unexpected type received for idx: {}".format(type(idx)))

    def _do_init(self, path: str, skip_warmup: bool, multimodal: bool) -> None:
        self._path = path

        if not skip_warmup:
            print_rank_0("    warming up index mmap file...")
            self.warmup_mmap_file(get_idx_path(self._path))

        self._index = _IndexReader(get_idx_path(self._path), multimodal)

        if not skip_warmup:
            print_rank_0("    warming up data mmap file...")
            self.warmup_mmap_file(get_bin_path(self._path))

        print_rank_0("    creating np buffer of mmap...")
        self._bin_buffer_mmap = np.memmap(get_bin_path(self._path), mode="r", order="C")

        print_rank_0("    creating memory view of np buffer...")
        self._bin_buffer = memoryview(self._bin_buffer_mmap)

    def get(self, idx: int, offset: int = 0, length: Optional[int] = None) -> np.ndarray:
        """Retrieves a single item from the dataset with the option to only
        return a portion of the item.

        get(idx) is the same as [idx] but get() does not support slicing.
        """
        sequence_pointer, sequence_length, sequence_mode = self._index[idx]
        if length is None:
            length = sequence_length - offset
        sequence_pointer += offset * DType.size(self._index.dtype)
        sequence = np.frombuffer(
            self._bin_buffer, dtype=self._index.dtype, count=length, offset=sequence_pointer
        )
        return (sequence, sequence_mode) if sequence_mode is not None else sequence

    @property
    def sizes(self) -> np.ndarray:
        return self._index.sizes

    @property
    def doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def get_doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def set_doc_idx(self, doc_idx: np.ndarray) -> None:
        self._index._document_indices = doc_idx

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:
        return os.path.exists(get_idx_path(path_prefix)) and os.path.exists(
            get_bin_path(path_prefix)
        )

    @staticmethod
    def warmup_mmap_file(path: str) -> None:
        with open(path, "rb") as stream:
            while stream.read(100 * 1024 * 1024):
                pass

class MMapIndexedDatasetBuilder(object):
    def __init__(
        self, bin_path: str, dtype: Type[np.number] = np.int32, multimodal: bool = False
    ) -> None:
        self._data_file = open(bin_path, "wb")
        self._dtype = dtype
        self._multimodal = multimodal

        self._sequence_lengths = []
        self._document_indices = [0]
        self._sequence_modes = [] if self._multimodal else None

    def add_item(self, tensor: torch.Tensor, mode: int = 0) -> None:
        np_array = np.array(tensor.numpy(), dtype=self._dtype)
        self._data_file.write(np_array.tobytes(order="C"))
        self._sequence_lengths.append(np_array.size)
        if self._multimodal:
            self._sequence_modes.append(mode)

    def add_doc(
        self, tensor: torch.Tensor, lengths: List[int], modes: Optional[List[int]] = None
    ) -> None:
        np_array = np.array(tensor, dtype=self._dtype)
        self._data_file.write(np_array.tobytes(order="C"))
        self._sequence_lengths.extend(lengths)
        self._document_indices.append(len(self._sequence_lengths))
        if self._multimodal:
            self._sequence_modes.extend(modes if modes is not None else [0] * lengths)

    def end_document(self) -> None:
        self._document_indices.append(len(self._sequence_lengths))

    def merge_file_(self, path_prefix: str) -> None:
        # Concatenate index
        index = _IndexReader(get_idx_path(path_prefix), multimodal=self._multimodal)
        assert index.dtype == self._dtype

        offset = len(self._sequence_lengths)
        self._sequence_lengths.extend(index.sizes)
        self._document_indices.extend((offset + index.doc_idx)[1:])

        if self._multimodal:
            self._sequence_modes.extend(index._sequence_modes)

        # Concatenate data
        with open(get_bin_path(path_prefix), "rb") as f:
            shutil.copyfileobj(f, self._data_file)

    def finalize(self, idx_path: str) -> None:
        self._data_file.close()
        with _IndexWriter(idx_path, self._dtype) as writer:
            writer.write(self._sequence_lengths, self._sequence_modes, self._document_indices)

def get_idx_path(path_prefix: str) -> str:
    return path_prefix + ".idx"

def get_bin_path(path_prefix: str) -> str:
    return path_prefix + ".bin"


DType

# Essentially re-written in entirety

import os
import shutil
import struct
from enum import Enum
from functools import lru_cache
from itertools import accumulate
from types import TracebackType
from typing import List, Optional, Tuple, Type, Union

import numpy as np
import torch

from megatron import print_rank_0

_INDEX_HEADER = b"MMIDIDX\x00\x00"

class DType(Enum):
    uint8 = 1
    int8 = 2
    int16 = 3
    int32 = 4
    int64 = 5
    float64 = 6
    float32 = 7
    uint16 = 8

    @classmethod
    def code_from_dtype(cls, value: Type[np.number]) -> int:
        return cls[value.__name__].value

    @classmethod
    def dtype_from_code(cls, value: int) -> Type[np.number]:
        return getattr(np, cls(value).name)

    @staticmethod
    def size(key: Union[int, Type[np.number]]) -> int:
        if isinstance(key, int):
            return DType.dtype_from_code(key)().itemsize
        elif np.number in key.__mro__:
            return key().itemsize
        else:
            raise ValueError

    @staticmethod
    def optimal_dtype(cardinality: int) -> Type[np.number]:
        if cardinality is not None and cardinality < 65500:
            return np.uint16
        else:
            return np.int32

class _IndexWriter(object):
    """
    Object class to write the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, dtype: Type[np.number]) -> None:
        self.path = path
        self.dtype = dtype

    def __enter__(self) -> "_IndexWriter":
        self.idx_path = open(self.path, "wb")
        # fixed, vestigial practice
        self.idx_path.write(_INDEX_HEADER)
        # fixed, vestigial practice
        self.idx_path.write(struct.pack("<Q", 1))
        # the numeric code for the dtype
        self.idx_path.write(struct.pack("<B", DType.code_from_dtype(self.dtype)))
        return self


    def dtype_from_code(cls, value: int) -> Type[np.number]:
        return getattr(np, cls(value).name)

    @staticmethod
    def size(key: Union[int, Type[np.number]]) -> int:
        if isinstance(key, int):
            return DType.dtype_from_code(key)().itemsize
        elif np.number in key.__mro__:
            return key().itemsize
        else:
            raise ValueError

    @staticmethod
    def optimal_dtype(cardinality: int) -> Type[np.number]:
        if cardinality is not None and cardinality < 65500:
            return np.uint16
        else:
            return np.int32

class _IndexWriter(object):
    """
    Object class to write the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, dtype: Type[np.number]) -> None:
        self.path = path
        self.dtype = dtype

    def __enter__(self) -> "_IndexWriter":
        self.idx_path = open(self.path, "wb")
        # fixed, vestigial practice
        self.idx_path.write(_INDEX_HEADER)
        # fixed, vestigial practice
        self.idx_path.write(struct.pack("<Q", 1))
        # the numeric code for the dtype
        self.idx_path.write(struct.pack("<B", DType.code_from_dtype(self.dtype)))
        return self

    def __exit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> Optional[bool]:
        self.idx_path.close()

    def write(
        self,
        sequence_lengths: List[int],
        sequence_modes: Optional[List[int]],
        document_indices: List[int],
    ) -> None:
        sequence_pointers = self._sequence_pointers(sequence_lengths)

        # the number of sequences in the dataset
        sequence_count = len(sequence_lengths)
        self.idx_path.write(struct.pack("<Q", sequence_count))

        # the number of documents in the dataset
        document_count = len(document_indices)
        self.idx_path.write(struct.pack("<Q", document_count))

        # the number of tokens per sequence
        sequence_lengths = np.array(sequence_lengths, dtype=np.int32)
        self.idx_path.write(sequence_lengths.tobytes(order="C"))
        del sequence_lengths

        # the byte offsets for all sequences
        sequence_pointers = np.array(sequence_pointers, dtype=np.int64)
        self.idx_path.write(sequence_pointers.tobytes(order="C"))
        del sequence_pointers

        # the sequence indices marking the end of each document
        document_indices = np.array(document_indices, dtype=np.int64)
        self.idx_path.write(document_indices.tobytes(order="C"))

        # the mode per sequence
        if sequence_modes is not None:
            sequence_modes = np.array(sequence_modes, dtype=np.int32)
            self._file.write(sequence_modes.tobytes(order='C'))
            del sequence_modes

    def _sequence_pointers(self, sequence_lengths: List[int]) -> List[int]:
        itemsize = DType.size(self.dtype)
        curr_ptr = 0
        list_ptr = []
        for length in sequence_lengths:
            list_ptr.append(curr_ptr)
            curr_ptr += length * itemsize
        return list_ptr

class _IndexReader(object):
    """
    Object class to read the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, multimodal: bool) -> None:
        with open(path, "rb") as stream:
            header = stream.read(9)
            assert header == _INDEX_HEADER, f"bad header, cannot read: {path}"

            version = struct.unpack("<Q", stream.read(8))[0]
            assert version == 1, f"bad version, cannot read: {path}"

            code = struct.unpack("<B", stream.read(1))[0]
            self._dtype = DType.dtype_from_code(code)
            self._dtype_size = DType.size(self._dtype)

            self._sequence_count = struct.unpack("<Q", stream.read(8))[0]


        # the sequence indices marking the end of each document
        document_indices = np.array(document_indices, dtype=np.int64)
        self.idx_path.write(document_indices.tobytes(order="C"))

        # the mode per sequence
        if sequence_modes is not None:
            sequence_modes = np.array(sequence_modes, dtype=np.int32)
            self._file.write(sequence_modes.tobytes(order='C'))
            del sequence_modes

    def _sequence_pointers(self, sequence_lengths: List[int]) -> List[int]:
        itemsize = DType.size(self.dtype)
        curr_ptr = 0
        list_ptr = []
        for length in sequence_lengths:
            list_ptr.append(curr_ptr)
            curr_ptr += length * itemsize
        return list_ptr

class _IndexReader(object):
    """
    Object class to read the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, multimodal: bool) -> None:
        with open(path, "rb") as stream:
            header = stream.read(9)
            assert header == _INDEX_HEADER, f"bad header, cannot read: {path}"

            version = struct.unpack("<Q", stream.read(8))[0]
            assert version == 1, f"bad version, cannot read: {path}"

            code = struct.unpack("<B", stream.read(1))[0]
            self._dtype = DType.dtype_from_code(code)
            self._dtype_size = DType.size(self._dtype)

            self._sequence_count = struct.unpack("<Q", stream.read(8))[0]
            self._document_count = struct.unpack("<Q", stream.read(8))[0]

            offset = stream.tell()

        self._multimodal = multimodal

        self._bin_buffer_mmap = np.memmap(path, mode="r", order="C")
        self._bin_buffer = memoryview(self._bin_buffer_mmap)

        print_rank_0("    reading sequence lengths...")
        self._sequence_lengths = np.frombuffer(
            self._bin_buffer, dtype=np.int32, count=self._sequence_count, offset=offset
        )

        print_rank_0("    reading sequence pointers...")
        self._sequence_pointers = np.frombuffer(
            self._bin_buffer,
            dtype=np.int64,
            count=self._sequence_count,
            offset=offset + self._sequence_lengths.nbytes,
        )

        print_rank_0("    reading document indices...")
        self._document_indices = np.frombuffer(
            self._bin_buffer,
            dtype=np.int64,
            count=self._document_count,
            offset=offset + self._sequence_lengths.nbytes + self._sequence_pointers.nbytes,
        )

        self._sequence_modes = None
        if self._multimodal:
            print_rank_0("    reading sequence modes...")
            self._sequence_modes = np.frombuffer(
                self._bin_buffer,
                dtype=np.int8,
                count=self._len,
                offset=offset
                + self._sequence_lengths.nbytes
                + self._sequence_pointers.nbytes
                + self._document_indices.nbytes,
            )

    def __del__(self) -> None:
        self._bin_buffer_mmap._mmap.close()
        del self._bin_buffer_mmap

    def __len__(self) -> int:
        return self._sequence_count

    @lru_cache(maxsize=8)
    def __getitem__(self, i: int) -> Tuple[np.int32, np.int64, Optional[np.int8]]:
        return (
            self._sequence_pointers[i],
            self._sequence_lengths[i],
            self._sequence_modes[i] if self._multimodal else None,
        )

    @property
    def dtype(self) -> Type[np.number]:
        return self._dtype

    @property
    def sizes(self) -> np.ndarray:
        return self._sequence_lengths

    @property
    def doc_idx(self) -> np.ndarray:
        return self._document_indices

    @property
    def modes(self) -> np.ndarray:
        return self._sequence_modes

class MMapIndexedDataset(torch.utils.data.Dataset):
    def __init__(self, path: str, skip_warmup: bool = False, multimodal: bool = False) -> None:
        super().__init__()

        self._path = None
        self._index = None
        self._bin_buffer = None
        self._multimodal = multimodal

        self._do_init(path, skip_warmup, multimodal)

    def __getstate__(self) -> str:
        return self._path

    def __setstate__(self, path: str) -> None:
        self._do_init(path, skip_warmup=True, multimodal=False)

    def __del__(self) -> None:


MMapIndexedDataset

            self._sequence_modes[i] if self._multimodal else None,
        )

    @property
    def dtype(self) -> Type[np.number]:
        return self._dtype

    @property
    def sizes(self) -> np.ndarray:
        return self._sequence_lengths

    @property
    def doc_idx(self) -> np.ndarray:
        return self._document_indices

    @property
    def modes(self) -> np.ndarray:
        return self._sequence_modes

class MMapIndexedDataset(torch.utils.data.Dataset):
    def __init__(self, path: str, skip_warmup: bool = False, multimodal: bool = False) -> None:
        super().__init__()

        self._path = None
        self._index = None
        self._bin_buffer = None
        self._multimodal = multimodal

        self._do_init(path, skip_warmup, multimodal)

    def __getstate__(self) -> str:
        return self._path

    def __setstate__(self, path: str) -> None:
        self._do_init(path, skip_warmup=True, multimodal=False)

    def __del__(self) -> None:
        self._bin_buffer_mmap._mmap.close()
        del self._bin_buffer_mmap
        del self._index

    def __len__(self) -> int:
        return len(self._index)

    def __getitem__(self, idx: Union[int, np.integer, slice]) -> np.ndarray:
        if isinstance(idx, (int, np.integer)):
            sequence_pointer, sequence_length, sequence_mode = self._index[idx]
            sequence = np.frombuffer(
                self._bin_buffer,
                dtype=self._index.dtype,
                count=sequence_length,
                offset=sequence_pointer,
            )
            return (sequence, sequence_mode) if sequence_mode is not None else sequence
        elif isinstance(idx, slice):
            start, stop, step = idx.indices(len(self))
            if step != 1:
                raise ValueError("Slices into indexed_dataset must be contiguous")
            sequence_lengths = self._index._sequence_lengths[idx]
            sequence_modes = self._index._sequence_modes[idx] if self._multimodal else None
            sequence_offsets = list(accumulate(sequence_lengths))
            sequences = np.split(
                np.frombuffer(
                    self._bin_buffer,
                    dtype=self._index.dtype,
                    count=sum(sequence_lengths),
                    offset=self._index._sequence_pointers[start],
                ),
                sequence_offsets[:-1],
            )
            return (sequences, sequence_modes) if sequence_modes is not None else sequences
        else:
            raise TypeError("Unexpected type received for idx: {}".format(type(idx)))

    def _do_init(self, path: str, skip_warmup: bool, multimodal: bool) -> None:
        self._path = path

        if not skip_warmup:
            print_rank_0("    warming up index mmap file...")
            self.warmup_mmap_file(get_idx_path(self._path))

        self._index = _IndexReader(get_idx_path(self._path), multimodal)

        if not skip_warmup:
            print_rank_0("    warming up data mmap file...")
            self.warmup_mmap_file(get_bin_path(self._path))

        print_rank_0("    creating np buffer of mmap...")
        self._bin_buffer_mmap = np.memmap(get_bin_path(self._path), mode="r", order="C")

        print_rank_0("    creating memory view of np buffer...")
        self._bin_buffer = memoryview(self._bin_buffer_mmap)

    def get(self, idx: int, offset: int = 0, length: Optional[int] = None) -> np.ndarray:
        """Retrieves a single item from the dataset with the option to only
        return a portion of the item.

        get(idx) is the same as [idx] but get() does not support slicing.
        """
        sequence_pointer, sequence_length, sequence_mode = self._index[idx]
        if length is None:
            length = sequence_length - offset
        sequence_pointer += offset * DType.size(self._index.dtype)
        sequence = np.frombuffer(
            self._bin_buffer, dtype=self._index.dtype, count=length, offset=sequence_pointer
        )
        return (sequence, sequence_mode) if sequence_mode is not None else sequence

    @property
    def sizes(self) -> np.ndarray:
        return self._index.sizes

    @property
    def doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def get_doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def set_doc_idx(self, doc_idx: np.ndarray) -> None:
        self._index._document_indices = doc_idx

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:
        return os.path.exists(get_idx_path(path_prefix)) and os.path.exists(
            get_bin_path(path_prefix)
        )

    @staticmethod
    def warmup_mmap_file(path: str) -> None:
        with open(path, "rb") as stream:
            while stream.read(100 * 1024 * 1024):
                pass

class MMapIndexedDatasetBuilder(object):
    def __init__(
        self, bin_path: str, dtype: Type[np.number] = np.int32, multimodal: bool = False
    ) -> None:
        self._data_file = open(bin_path, "wb")
        self._dtype = dtype
        self._multimodal = multimodal

        self._sequence_lengths = []
        self._document_indices = [0]
        self._sequence_modes = [] if self._multimodal else None

    def add_item(self, tensor: torch.Tensor, mode: int = 0) -> None:
        np_array = np.array(tensor.numpy(), dtype=self._dtype)
        self._data_file.write(np_array.tobytes(order="C"))
        self._sequence_lengths.append(np_array.size)
        if self._multimodal:
            self._sequence_modes.append(mode)


MMapIndexedDatasetBuilder

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:
        return os.path.exists(get_idx_path(path_prefix)) and os.path.exists(
            get_bin_path(path_prefix)
        )

    @staticmethod
    def warmup_mmap_file(path: str) -> None:
        with open(path, "rb") as stream:
            while stream.read(100 * 1024 * 1024):
                pass

class MMapIndexedDatasetBuilder(object):
    def __init__(
        self, bin_path: str, dtype: Type[np.number] = np.int32, multimodal: bool = False
    ) -> None:
        self._data_file = open(bin_path, "wb")
        self._dtype = dtype
        self._multimodal = multimodal

        self._sequence_lengths = []
        self._document_indices = [0]
        self._sequence_modes = [] if self._multimodal else None

    def add_item(self, tensor: torch.Tensor, mode: int = 0) -> None:
        np_array = np.array(tensor.numpy(), dtype=self._dtype)
        self._data_file.write(np_array.tobytes(order="C"))
        self._sequence_lengths.append(np_array.size)
        if self._multimodal:
            self._sequence_modes.append(mode)

    def add_doc(
        self, tensor: torch.Tensor, lengths: List[int], modes: Optional[List[int]] = None
    ) -> None:
        np_array = np.array(tensor, dtype=self._dtype)
        self._data_file.write(np_array.tobytes(order="C"))
        self._sequence_lengths.extend(lengths)
        self._document_indices.append(len(self._sequence_lengths))
        if self._multimodal:
            self._sequence_modes.extend(modes if modes is not None else [0] * lengths)

    def end_document(self) -> None:
        self._document_indices.append(len(self._sequence_lengths))

    def merge_file_(self, path_prefix: str) -> None:
        # Concatenate index
        index = _IndexReader(get_idx_path(path_prefix), multimodal=self._multimodal)
        assert index.dtype == self._dtype

        offset = len(self._sequence_lengths)
        self._sequence_lengths.extend(index.sizes)
        self._document_indices.extend((offset + index.doc_idx)[1:])

        if self._multimodal:
            self._sequence_modes.extend(index._sequence_modes)

        # Concatenate data
        with open(get_bin_path(path_prefix), "rb") as f:
            shutil.copyfileobj(f, self._data_file)

    def finalize(self, idx_path: str) -> None:
        self._data_file.close()
        with _IndexWriter(idx_path, self._dtype) as writer:
            writer.write(self._sequence_lengths, self._sequence_modes, self._document_indices)

def get_idx_path(path_prefix: str) -> str:
    return path_prefix + ".idx"

def get_bin_path(path_prefix: str) -> str:
    return path_prefix + ".bin"


        index = _IndexReader(get_idx_path(path_prefix), multimodal=self._multimodal)
        assert index.dtype == self._dtype

        offset = len(self._sequence_lengths)
        self._sequence_lengths.extend(index.sizes)
        self._document_indices.extend((offset + index.doc_idx)[1:])

        if self._multimodal:
            self._sequence_modes.extend(index._sequence_modes)

        # Concatenate data
        with open(get_bin_path(path_prefix), "rb") as f:
            shutil.copyfileobj(f, self._data_file)

    def finalize(self, idx_path: str) -> None:
        self._data_file.close()
        with _IndexWriter(idx_path, self._dtype) as writer:
            writer.write(self._sequence_lengths, self._sequence_modes, self._document_indices)

def get_idx_path(path_prefix: str) -> str:
    return path_prefix + ".idx"

def get_bin_path(path_prefix: str) -> str:
    return path_prefix + ".bin"


        self._sequence_lengths.extend(index.sizes)
        self._document_indices.extend((offset + index.doc_idx)[1:])

        if self._multimodal:
            self._sequence_modes.extend(index._sequence_modes)

        # Concatenate data
        with open(get_bin_path(path_prefix), "rb") as f:
            shutil.copyfileobj(f, self._data_file)

    def finalize(self, idx_path: str) -> None:
        self._data_file.close()
        with _IndexWriter(idx_path, self._dtype) as writer:
            writer.write(self._sequence_lengths, self._sequence_modes, self._document_indices)

def get_idx_path(path_prefix: str) -> str:
    return path_prefix + ".idx"

def get_bin_path(path_prefix: str) -> str:
    return path_prefix + ".bin"


import numpy as np
import torch

from megatron import print_rank_0

_INDEX_HEADER = b"MMIDIDX\x00\x00"

class DType(Enum):
    uint8 = 1
    int8 = 2
    int16 = 3
    int32 = 4
    int64 = 5
    float64 = 6
    float32 = 7
    uint16 = 8

    @classmethod
    def code_from_dtype(cls, value: Type[np.number]) -> int:
        return cls[value.__name__].value

    @classmethod
    def dtype_from_code(cls, value: int) -> Type[np.number]:
        return getattr(np, cls(value).name)

    @staticmethod
    def size(key: Union[int, Type[np.number]]) -> int:
        if isinstance(key, int):
            return DType.dtype_from_code(key)().itemsize
        elif np.number in key.__mro__:
            return key().itemsize
        else:
            raise ValueError

    @staticmethod
    def optimal_dtype(cardinality: int) -> Type[np.number]:
        if cardinality is not None and cardinality < 65500:
            return np.uint16
        else:
            return np.int32


from megatron import print_rank_0

_INDEX_HEADER = b"MMIDIDX\x00\x00"

class DType(Enum):
    uint8 = 1
    int8 = 2
    int16 = 3
    int32 = 4
    int64 = 5
    float64 = 6
    float32 = 7
    uint16 = 8

    @classmethod
    def code_from_dtype(cls, value: Type[np.number]) -> int:
        return cls[value.__name__].value

    @classmethod
    def dtype_from_code(cls, value: int) -> Type[np.number]:
        return getattr(np, cls(value).name)

    @staticmethod
    def size(key: Union[int, Type[np.number]]) -> int:
        if isinstance(key, int):
            return DType.dtype_from_code(key)().itemsize
        elif np.number in key.__mro__:
            return key().itemsize
        else:
            raise ValueError

    @staticmethod
    def optimal_dtype(cardinality: int) -> Type[np.number]:
        if cardinality is not None and cardinality < 65500:
            return np.uint16
        else:
            return np.int32

class _IndexWriter(object):
    """


size

class DType(Enum):
    uint8 = 1
    int8 = 2
    int16 = 3
    int32 = 4
    int64 = 5
    float64 = 6
    float32 = 7
    uint16 = 8

    @classmethod
    def code_from_dtype(cls, value: Type[np.number]) -> int:
        return cls[value.__name__].value

    @classmethod
    def dtype_from_code(cls, value: int) -> Type[np.number]:
        return getattr(np, cls(value).name)

    @staticmethod
    def size(key: Union[int, Type[np.number]]) -> int:
        if isinstance(key, int):
            return DType.dtype_from_code(key)().itemsize
        elif np.number in key.__mro__:
            return key().itemsize
        else:
            raise ValueError

    @staticmethod
    def optimal_dtype(cardinality: int) -> Type[np.number]:
        if cardinality is not None and cardinality < 65500:
            return np.uint16
        else:
            return np.int32

class _IndexWriter(object):
    """
    Object class to write the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, dtype: Type[np.number]) -> None:
        self.path = path
        self.dtype = dtype

    def __enter__(self) -> "_IndexWriter":
        self.idx_path = open(self.path, "wb")


    uint16 = 8

    @classmethod
    def code_from_dtype(cls, value: Type[np.number]) -> int:
        return cls[value.__name__].value

    @classmethod
    def dtype_from_code(cls, value: int) -> Type[np.number]:
        return getattr(np, cls(value).name)

    @staticmethod
    def size(key: Union[int, Type[np.number]]) -> int:
        if isinstance(key, int):
            return DType.dtype_from_code(key)().itemsize
        elif np.number in key.__mro__:
            return key().itemsize
        else:
            raise ValueError

    @staticmethod
    def optimal_dtype(cardinality: int) -> Type[np.number]:
        if cardinality is not None and cardinality < 65500:
            return np.uint16
        else:
            return np.int32

class _IndexWriter(object):
    """
    Object class to write the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, dtype: Type[np.number]) -> None:
        self.path = path
        self.dtype = dtype

    def __enter__(self) -> "_IndexWriter":
        self.idx_path = open(self.path, "wb")
        # fixed, vestigial practice
        self.idx_path.write(_INDEX_HEADER)
        # fixed, vestigial practice
        self.idx_path.write(struct.pack("<Q", 1))
        # the numeric code for the dtype
        self.idx_path.write(struct.pack("<B", DType.code_from_dtype(self.dtype)))
        return self


        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:
        return os.path.exists(get_idx_path(path_prefix)) and os.path.exists(
            get_bin_path(path_prefix)
        )

    @staticmethod
    def warmup_mmap_file(path: str) -> None:
        with open(path, "rb") as stream:
            while stream.read(100 * 1024 * 1024):
                pass

class MMapIndexedDatasetBuilder(object):
    def __init__(
        self, bin_path: str, dtype: Type[np.number] = np.int32, multimodal: bool = False
    ) -> None:
        self._data_file = open(bin_path, "wb")
        self._dtype = dtype
        self._multimodal = multimodal

        self._sequence_lengths = []
        self._document_indices = [0]
        self._sequence_modes = [] if self._multimodal else None

    def add_item(self, tensor: torch.Tensor, mode: int = 0) -> None:
        np_array = np.array(tensor.numpy(), dtype=self._dtype)
        self._data_file.write(np_array.tobytes(order="C"))
        self._sequence_lengths.append(np_array.size)
        if self._multimodal:
            self._sequence_modes.append(mode)

    def add_doc(
        self, tensor: torch.Tensor, lengths: List[int], modes: Optional[List[int]] = None
    ) -> None:
        np_array = np.array(tensor, dtype=self._dtype)
        self._data_file.write(np_array.tobytes(order="C"))
        self._sequence_lengths.extend(lengths)
        self._document_indices.append(len(self._sequence_lengths))
        if self._multimodal:
            self._sequence_modes.extend(modes if modes is not None else [0] * lengths)

    def end_document(self) -> None:
        self._document_indices.append(len(self._sequence_lengths))


        else:
            raise ValueError

    @staticmethod
    def optimal_dtype(cardinality: int) -> Type[np.number]:
        if cardinality is not None and cardinality < 65500:
            return np.uint16
        else:
            return np.int32

class _IndexWriter(object):
    """
    Object class to write the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, dtype: Type[np.number]) -> None:
        self.path = path
        self.dtype = dtype

    def __enter__(self) -> "_IndexWriter":
        self.idx_path = open(self.path, "wb")
        # fixed, vestigial practice
        self.idx_path.write(_INDEX_HEADER)
        # fixed, vestigial practice
        self.idx_path.write(struct.pack("<Q", 1))
        # the numeric code for the dtype
        self.idx_path.write(struct.pack("<B", DType.code_from_dtype(self.dtype)))
        return self

    def __exit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> Optional[bool]:
        self.idx_path.close()

    def write(
        self,
        sequence_lengths: List[int],
        sequence_modes: Optional[List[int]],
        document_indices: List[int],
    ) -> None:
        sequence_pointers = self._sequence_pointers(sequence_lengths)

        # the number of sequences in the dataset
        sequence_count = len(sequence_lengths)
        self.idx_path.write(struct.pack("<Q", sequence_count))


class _IndexWriter(object):
    """
    Object class to write the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, dtype: Type[np.number]) -> None:
        self.path = path
        self.dtype = dtype

    def __enter__(self) -> "_IndexWriter":
        self.idx_path = open(self.path, "wb")
        # fixed, vestigial practice
        self.idx_path.write(_INDEX_HEADER)
        # fixed, vestigial practice
        self.idx_path.write(struct.pack("<Q", 1))
        # the numeric code for the dtype
        self.idx_path.write(struct.pack("<B", DType.code_from_dtype(self.dtype)))
        return self

    def __exit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> Optional[bool]:
        self.idx_path.close()

    def write(
        self,
        sequence_lengths: List[int],
        sequence_modes: Optional[List[int]],
        document_indices: List[int],
    ) -> None:
        sequence_pointers = self._sequence_pointers(sequence_lengths)

        # the number of sequences in the dataset
        sequence_count = len(sequence_lengths)
        self.idx_path.write(struct.pack("<Q", sequence_count))

        # the number of documents in the dataset
        document_count = len(document_indices)
        self.idx_path.write(struct.pack("<Q", document_count))

        # the number of tokens per sequence
        sequence_lengths = np.array(sequence_lengths, dtype=np.int32)
        self.idx_path.write(sequence_lengths.tobytes(order="C"))


write

        self.dtype = dtype

    def __enter__(self) -> "_IndexWriter":
        self.idx_path = open(self.path, "wb")
        # fixed, vestigial practice
        self.idx_path.write(_INDEX_HEADER)
        # fixed, vestigial practice
        self.idx_path.write(struct.pack("<Q", 1))
        # the numeric code for the dtype
        self.idx_path.write(struct.pack("<B", DType.code_from_dtype(self.dtype)))
        return self

    def __exit__(
        self,
        exc_type: Optional[Type[BaseException]],
        exc_val: Optional[BaseException],
        exc_tb: Optional[TracebackType],
    ) -> Optional[bool]:
        self.idx_path.close()

    def write(
        self,
        sequence_lengths: List[int],
        sequence_modes: Optional[List[int]],
        document_indices: List[int],
    ) -> None:
        sequence_pointers = self._sequence_pointers(sequence_lengths)

        # the number of sequences in the dataset
        sequence_count = len(sequence_lengths)
        self.idx_path.write(struct.pack("<Q", sequence_count))

        # the number of documents in the dataset
        document_count = len(document_indices)
        self.idx_path.write(struct.pack("<Q", document_count))

        # the number of tokens per sequence
        sequence_lengths = np.array(sequence_lengths, dtype=np.int32)
        self.idx_path.write(sequence_lengths.tobytes(order="C"))
        del sequence_lengths

        # the byte offsets for all sequences
        sequence_pointers = np.array(sequence_pointers, dtype=np.int64)
        self.idx_path.write(sequence_pointers.tobytes(order="C"))
        del sequence_pointers

        # the sequence indices marking the end of each document
        document_indices = np.array(document_indices, dtype=np.int64)
        self.idx_path.write(document_indices.tobytes(order="C"))

        # the mode per sequence
        if sequence_modes is not None:
            sequence_modes = np.array(sequence_modes, dtype=np.int32)
            self._file.write(sequence_modes.tobytes(order='C'))
            del sequence_modes

    def _sequence_pointers(self, sequence_lengths: List[int]) -> List[int]:
        itemsize = DType.size(self.dtype)
        curr_ptr = 0
        list_ptr = []
        for length in sequence_lengths:
            list_ptr.append(curr_ptr)
            curr_ptr += length * itemsize
        return list_ptr

class _IndexReader(object):
    """
    Object class to read the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, multimodal: bool) -> None:
        with open(path, "rb") as stream:
            header = stream.read(9)
            assert header == _INDEX_HEADER, f"bad header, cannot read: {path}"


        # the number of tokens per sequence
        sequence_lengths = np.array(sequence_lengths, dtype=np.int32)
        self.idx_path.write(sequence_lengths.tobytes(order="C"))
        del sequence_lengths

        # the byte offsets for all sequences
        sequence_pointers = np.array(sequence_pointers, dtype=np.int64)
        self.idx_path.write(sequence_pointers.tobytes(order="C"))
        del sequence_pointers

        # the sequence indices marking the end of each document
        document_indices = np.array(document_indices, dtype=np.int64)
        self.idx_path.write(document_indices.tobytes(order="C"))

        # the mode per sequence
        if sequence_modes is not None:
            sequence_modes = np.array(sequence_modes, dtype=np.int32)
            self._file.write(sequence_modes.tobytes(order='C'))
            del sequence_modes

    def _sequence_pointers(self, sequence_lengths: List[int]) -> List[int]:
        itemsize = DType.size(self.dtype)
        curr_ptr = 0
        list_ptr = []
        for length in sequence_lengths:
            list_ptr.append(curr_ptr)
            curr_ptr += length * itemsize
        return list_ptr

class _IndexReader(object):
    """
    Object class to read the index file i.e. <data-path>.idx
    """

    def __init__(self, path: str, multimodal: bool) -> None:
        with open(path, "rb") as stream:
            header = stream.read(9)
            assert header == _INDEX_HEADER, f"bad header, cannot read: {path}"

            version = struct.unpack("<Q", stream.read(8))[0]
            assert version == 1, f"bad version, cannot read: {path}"

            code = struct.unpack("<B", stream.read(1))[0]
            self._dtype = DType.dtype_from_code(code)
            self._dtype_size = DType.size(self._dtype)

            self._sequence_count = struct.unpack("<Q", stream.read(8))[0]


        return self._sequence_modes

class MMapIndexedDataset(torch.utils.data.Dataset):
    def __init__(self, path: str, skip_warmup: bool = False, multimodal: bool = False) -> None:
        super().__init__()

        self._path = None
        self._index = None
        self._bin_buffer = None
        self._multimodal = multimodal

        self._do_init(path, skip_warmup, multimodal)

    def __getstate__(self) -> str:
        return self._path

    def __setstate__(self, path: str) -> None:
        self._do_init(path, skip_warmup=True, multimodal=False)

    def __del__(self) -> None:
        self._bin_buffer_mmap._mmap.close()
        del self._bin_buffer_mmap
        del self._index

    def __len__(self) -> int:
        return len(self._index)

    def __getitem__(self, idx: Union[int, np.integer, slice]) -> np.ndarray:
        if isinstance(idx, (int, np.integer)):
            sequence_pointer, sequence_length, sequence_mode = self._index[idx]
            sequence = np.frombuffer(
                self._bin_buffer,
                dtype=self._index.dtype,
                count=sequence_length,
                offset=sequence_pointer,
            )
            return (sequence, sequence_mode) if sequence_mode is not None else sequence
        elif isinstance(idx, slice):
            start, stop, step = idx.indices(len(self))
            if step != 1:
                raise ValueError("Slices into indexed_dataset must be contiguous")
            sequence_lengths = self._index._sequence_lengths[idx]
            sequence_modes = self._index._sequence_modes[idx] if self._multimodal else None


        super().__init__()

        self._path = None
        self._index = None
        self._bin_buffer = None
        self._multimodal = multimodal

        self._do_init(path, skip_warmup, multimodal)

    def __getstate__(self) -> str:
        return self._path

    def __setstate__(self, path: str) -> None:
        self._do_init(path, skip_warmup=True, multimodal=False)

    def __del__(self) -> None:
        self._bin_buffer_mmap._mmap.close()
        del self._bin_buffer_mmap
        del self._index

    def __len__(self) -> int:
        return len(self._index)

    def __getitem__(self, idx: Union[int, np.integer, slice]) -> np.ndarray:
        if isinstance(idx, (int, np.integer)):
            sequence_pointer, sequence_length, sequence_mode = self._index[idx]
            sequence = np.frombuffer(
                self._bin_buffer,
                dtype=self._index.dtype,
                count=sequence_length,
                offset=sequence_pointer,
            )
            return (sequence, sequence_mode) if sequence_mode is not None else sequence
        elif isinstance(idx, slice):
            start, stop, step = idx.indices(len(self))
            if step != 1:
                raise ValueError("Slices into indexed_dataset must be contiguous")
            sequence_lengths = self._index._sequence_lengths[idx]
            sequence_modes = self._index._sequence_modes[idx] if self._multimodal else None
            sequence_offsets = list(accumulate(sequence_lengths))
            sequences = np.split(
                np.frombuffer(


        self._index = None
        self._bin_buffer = None
        self._multimodal = multimodal

        self._do_init(path, skip_warmup, multimodal)

    def __getstate__(self) -> str:
        return self._path

    def __setstate__(self, path: str) -> None:
        self._do_init(path, skip_warmup=True, multimodal=False)

    def __del__(self) -> None:
        self._bin_buffer_mmap._mmap.close()
        del self._bin_buffer_mmap
        del self._index

    def __len__(self) -> int:
        return len(self._index)

    def __getitem__(self, idx: Union[int, np.integer, slice]) -> np.ndarray:
        if isinstance(idx, (int, np.integer)):
            sequence_pointer, sequence_length, sequence_mode = self._index[idx]
            sequence = np.frombuffer(
                self._bin_buffer,
                dtype=self._index.dtype,
                count=sequence_length,
                offset=sequence_pointer,
            )
            return (sequence, sequence_mode) if sequence_mode is not None else sequence
        elif isinstance(idx, slice):
            start, stop, step = idx.indices(len(self))
            if step != 1:
                raise ValueError("Slices into indexed_dataset must be contiguous")
            sequence_lengths = self._index._sequence_lengths[idx]
            sequence_modes = self._index._sequence_modes[idx] if self._multimodal else None
            sequence_offsets = list(accumulate(sequence_lengths))
            sequences = np.split(
                np.frombuffer(
                    self._bin_buffer,
                    dtype=self._index.dtype,
                    count=sum(sequence_lengths),
                    offset=self._index._sequence_pointers[start],
                ),
                sequence_offsets[:-1],
            )
            return (sequences, sequence_modes) if sequence_modes is not None else sequences
        else:
            raise TypeError("Unexpected type received for idx: {}".format(type(idx)))

    def _do_init(self, path: str, skip_warmup: bool, multimodal: bool) -> None:
        self._path = path

        if not skip_warmup:
            print_rank_0("    warming up index mmap file...")
            self.warmup_mmap_file(get_idx_path(self._path))

        self._index = _IndexReader(get_idx_path(self._path), multimodal)

        if not skip_warmup:
            print_rank_0("    warming up data mmap file...")
            self.warmup_mmap_file(get_bin_path(self._path))

        print_rank_0("    creating np buffer of mmap...")
        self._bin_buffer_mmap = np.memmap(get_bin_path(self._path), mode="r", order="C")

        print_rank_0("    creating memory view of np buffer...")
        self._bin_buffer = memoryview(self._bin_buffer_mmap)


dtype

                + self._sequence_pointers.nbytes
                + self._document_indices.nbytes,
            )

    def __del__(self) -> None:
        self._bin_buffer_mmap._mmap.close()
        del self._bin_buffer_mmap

    def __len__(self) -> int:
        return self._sequence_count

    @lru_cache(maxsize=8)
    def __getitem__(self, i: int) -> Tuple[np.int32, np.int64, Optional[np.int8]]:
        return (
            self._sequence_pointers[i],
            self._sequence_lengths[i],
            self._sequence_modes[i] if self._multimodal else None,
        )

    @property
    def dtype(self) -> Type[np.number]:
        return self._dtype

    @property
    def sizes(self) -> np.ndarray:
        return self._sequence_lengths

    @property
    def doc_idx(self) -> np.ndarray:
        return self._document_indices

    @property
    def modes(self) -> np.ndarray:
        return self._sequence_modes

class MMapIndexedDataset(torch.utils.data.Dataset):
    def __init__(self, path: str, skip_warmup: bool = False, multimodal: bool = False) -> None:
        super().__init__()

        self._path = None
        self._index = None


sizes

        print_rank_0("    creating memory view of np buffer...")
        self._bin_buffer = memoryview(self._bin_buffer_mmap)

    def get(self, idx: int, offset: int = 0, length: Optional[int] = None) -> np.ndarray:
        """Retrieves a single item from the dataset with the option to only
        return a portion of the item.

        get(idx) is the same as [idx] but get() does not support slicing.
        """
        sequence_pointer, sequence_length, sequence_mode = self._index[idx]
        if length is None:
            length = sequence_length - offset
        sequence_pointer += offset * DType.size(self._index.dtype)
        sequence = np.frombuffer(
            self._bin_buffer, dtype=self._index.dtype, count=length, offset=sequence_pointer
        )
        return (sequence, sequence_mode) if sequence_mode is not None else sequence

    @property
    def sizes(self) -> np.ndarray:
        return self._index.sizes

    @property
    def doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def get_doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def set_doc_idx(self, doc_idx: np.ndarray) -> None:
        self._index._document_indices = doc_idx

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:


    def get(self, idx: int, offset: int = 0, length: Optional[int] = None) -> np.ndarray:
        """Retrieves a single item from the dataset with the option to only
        return a portion of the item.

        get(idx) is the same as [idx] but get() does not support slicing.
        """
        sequence_pointer, sequence_length, sequence_mode = self._index[idx]
        if length is None:
            length = sequence_length - offset
        sequence_pointer += offset * DType.size(self._index.dtype)
        sequence = np.frombuffer(
            self._bin_buffer, dtype=self._index.dtype, count=length, offset=sequence_pointer
        )
        return (sequence, sequence_mode) if sequence_mode is not None else sequence

    @property
    def sizes(self) -> np.ndarray:
        return self._index.sizes

    @property
    def doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def get_doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def set_doc_idx(self, doc_idx: np.ndarray) -> None:
        self._index._document_indices = doc_idx

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:
        return os.path.exists(get_idx_path(path_prefix)) and os.path.exists(
            get_bin_path(path_prefix)
        )


modes

        sequence_pointer += offset * DType.size(self._index.dtype)
        sequence = np.frombuffer(
            self._bin_buffer, dtype=self._index.dtype, count=length, offset=sequence_pointer
        )
        return (sequence, sequence_mode) if sequence_mode is not None else sequence

    @property
    def sizes(self) -> np.ndarray:
        return self._index.sizes

    @property
    def doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def get_doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def set_doc_idx(self, doc_idx: np.ndarray) -> None:
        self._index._document_indices = doc_idx

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:
        return os.path.exists(get_idx_path(path_prefix)) and os.path.exists(
            get_bin_path(path_prefix)
        )

    @staticmethod
    def warmup_mmap_file(path: str) -> None:
        with open(path, "rb") as stream:
            while stream.read(100 * 1024 * 1024):
                pass

class MMapIndexedDatasetBuilder(object):
    def __init__(


    @property
    def doc_idx(self) -> np.ndarray:
        return self._document_indices

    @property
    def modes(self) -> np.ndarray:
        return self._sequence_modes

class MMapIndexedDataset(torch.utils.data.Dataset):
    def __init__(self, path: str, skip_warmup: bool = False, multimodal: bool = False) -> None:
        super().__init__()

        self._path = None
        self._index = None
        self._bin_buffer = None
        self._multimodal = multimodal

        self._do_init(path, skip_warmup, multimodal)

    def __getstate__(self) -> str:
        return self._path

    def __setstate__(self, path: str) -> None:
        self._do_init(path, skip_warmup=True, multimodal=False)

    def __del__(self) -> None:
        self._bin_buffer_mmap._mmap.close()
        del self._bin_buffer_mmap
        del self._index

    def __len__(self) -> int:
        return len(self._index)

    def __getitem__(self, idx: Union[int, np.integer, slice]) -> np.ndarray:
        if isinstance(idx, (int, np.integer)):
            sequence_pointer, sequence_length, sequence_mode = self._index[idx]
            sequence = np.frombuffer(
                self._bin_buffer,
                dtype=self._index.dtype,
                count=sequence_length,
                offset=sequence_pointer,


    @property
    def modes(self) -> np.ndarray:
        return self._sequence_modes

class MMapIndexedDataset(torch.utils.data.Dataset):
    def __init__(self, path: str, skip_warmup: bool = False, multimodal: bool = False) -> None:
        super().__init__()

        self._path = None
        self._index = None
        self._bin_buffer = None
        self._multimodal = multimodal

        self._do_init(path, skip_warmup, multimodal)

    def __getstate__(self) -> str:
        return self._path

    def __setstate__(self, path: str) -> None:
        self._do_init(path, skip_warmup=True, multimodal=False)

    def __del__(self) -> None:
        self._bin_buffer_mmap._mmap.close()
        del self._bin_buffer_mmap
        del self._index

    def __len__(self) -> int:
        return len(self._index)

    def __getitem__(self, idx: Union[int, np.integer, slice]) -> np.ndarray:
        if isinstance(idx, (int, np.integer)):
            sequence_pointer, sequence_length, sequence_mode = self._index[idx]
            sequence = np.frombuffer(
                self._bin_buffer,
                dtype=self._index.dtype,
                count=sequence_length,
                offset=sequence_pointer,
            )
            return (sequence, sequence_mode) if sequence_mode is not None else sequence
        elif isinstance(idx, slice):


        elif isinstance(idx, slice):
            start, stop, step = idx.indices(len(self))
            if step != 1:
                raise ValueError("Slices into indexed_dataset must be contiguous")
            sequence_lengths = self._index._sequence_lengths[idx]
            sequence_modes = self._index._sequence_modes[idx] if self._multimodal else None
            sequence_offsets = list(accumulate(sequence_lengths))
            sequences = np.split(
                np.frombuffer(
                    self._bin_buffer,
                    dtype=self._index.dtype,
                    count=sum(sequence_lengths),
                    offset=self._index._sequence_pointers[start],
                ),
                sequence_offsets[:-1],
            )
            return (sequences, sequence_modes) if sequence_modes is not None else sequences
        else:
            raise TypeError("Unexpected type received for idx: {}".format(type(idx)))

    def _do_init(self, path: str, skip_warmup: bool, multimodal: bool) -> None:
        self._path = path

        if not skip_warmup:
            print_rank_0("    warming up index mmap file...")
            self.warmup_mmap_file(get_idx_path(self._path))

        self._index = _IndexReader(get_idx_path(self._path), multimodal)

        if not skip_warmup:
            print_rank_0("    warming up data mmap file...")
            self.warmup_mmap_file(get_bin_path(self._path))

        print_rank_0("    creating np buffer of mmap...")
        self._bin_buffer_mmap = np.memmap(get_bin_path(self._path), mode="r", order="C")

        print_rank_0("    creating memory view of np buffer...")
        self._bin_buffer = memoryview(self._bin_buffer_mmap)

    def get(self, idx: int, offset: int = 0, length: Optional[int] = None) -> np.ndarray:
        """Retrieves a single item from the dataset with the option to only
        return a portion of the item.

        get(idx) is the same as [idx] but get() does not support slicing.
        """
        sequence_pointer, sequence_length, sequence_mode = self._index[idx]
        if length is None:
            length = sequence_length - offset
        sequence_pointer += offset * DType.size(self._index.dtype)
        sequence = np.frombuffer(
            self._bin_buffer, dtype=self._index.dtype, count=length, offset=sequence_pointer
        )
        return (sequence, sequence_mode) if sequence_mode is not None else sequence

    @property
    def sizes(self) -> np.ndarray:
        return self._index.sizes


get

    def _do_init(self, path: str, skip_warmup: bool, multimodal: bool) -> None:
        self._path = path

        if not skip_warmup:
            print_rank_0("    warming up index mmap file...")
            self.warmup_mmap_file(get_idx_path(self._path))

        self._index = _IndexReader(get_idx_path(self._path), multimodal)

        if not skip_warmup:
            print_rank_0("    warming up data mmap file...")
            self.warmup_mmap_file(get_bin_path(self._path))

        print_rank_0("    creating np buffer of mmap...")
        self._bin_buffer_mmap = np.memmap(get_bin_path(self._path), mode="r", order="C")

        print_rank_0("    creating memory view of np buffer...")
        self._bin_buffer = memoryview(self._bin_buffer_mmap)

    def get(self, idx: int, offset: int = 0, length: Optional[int] = None) -> np.ndarray:
        """Retrieves a single item from the dataset with the option to only
        return a portion of the item.

        get(idx) is the same as [idx] but get() does not support slicing.
        """
        sequence_pointer, sequence_length, sequence_mode = self._index[idx]
        if length is None:
            length = sequence_length - offset
        sequence_pointer += offset * DType.size(self._index.dtype)
        sequence = np.frombuffer(
            self._bin_buffer, dtype=self._index.dtype, count=length, offset=sequence_pointer
        )
        return (sequence, sequence_mode) if sequence_mode is not None else sequence

    @property
    def sizes(self) -> np.ndarray:
        return self._index.sizes

    @property
    def doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def get_doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def set_doc_idx(self, doc_idx: np.ndarray) -> None:
        self._index._document_indices = doc_idx

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:


        get(idx) is the same as [idx] but get() does not support slicing.
        """
        sequence_pointer, sequence_length, sequence_mode = self._index[idx]
        if length is None:
            length = sequence_length - offset
        sequence_pointer += offset * DType.size(self._index.dtype)
        sequence = np.frombuffer(
            self._bin_buffer, dtype=self._index.dtype, count=length, offset=sequence_pointer
        )
        return (sequence, sequence_mode) if sequence_mode is not None else sequence

    @property
    def sizes(self) -> np.ndarray:
        return self._index.sizes

    @property
    def doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def get_doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def set_doc_idx(self, doc_idx: np.ndarray) -> None:
        self._index._document_indices = doc_idx

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:
        return os.path.exists(get_idx_path(path_prefix)) and os.path.exists(
            get_bin_path(path_prefix)
        )

    @staticmethod
    def warmup_mmap_file(path: str) -> None:
        with open(path, "rb") as stream:


        sequence_pointer, sequence_length, sequence_mode = self._index[idx]
        if length is None:
            length = sequence_length - offset
        sequence_pointer += offset * DType.size(self._index.dtype)
        sequence = np.frombuffer(
            self._bin_buffer, dtype=self._index.dtype, count=length, offset=sequence_pointer
        )
        return (sequence, sequence_mode) if sequence_mode is not None else sequence

    @property
    def sizes(self) -> np.ndarray:
        return self._index.sizes

    @property
    def doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def get_doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def set_doc_idx(self, doc_idx: np.ndarray) -> None:
        self._index._document_indices = doc_idx

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:
        return os.path.exists(get_idx_path(path_prefix)) and os.path.exists(
            get_bin_path(path_prefix)
        )

    @staticmethod
    def warmup_mmap_file(path: str) -> None:
        with open(path, "rb") as stream:
            while stream.read(100 * 1024 * 1024):
                pass


        return (sequence, sequence_mode) if sequence_mode is not None else sequence

    @property
    def sizes(self) -> np.ndarray:
        return self._index.sizes

    @property
    def doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def get_doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def set_doc_idx(self, doc_idx: np.ndarray) -> None:
        self._index._document_indices = doc_idx

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:
        return os.path.exists(get_idx_path(path_prefix)) and os.path.exists(
            get_bin_path(path_prefix)
        )

    @staticmethod
    def warmup_mmap_file(path: str) -> None:
        with open(path, "rb") as stream:
            while stream.read(100 * 1024 * 1024):
                pass

class MMapIndexedDatasetBuilder(object):
    def __init__(
        self, bin_path: str, dtype: Type[np.number] = np.int32, multimodal: bool = False
    ) -> None:
        self._data_file = open(bin_path, "wb")
        self._dtype = dtype


exists

        return self._index.sizes

    @property
    def doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def get_doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def set_doc_idx(self, doc_idx: np.ndarray) -> None:
        self._index._document_indices = doc_idx

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:
        return os.path.exists(get_idx_path(path_prefix)) and os.path.exists(
            get_bin_path(path_prefix)
        )

    @staticmethod
    def warmup_mmap_file(path: str) -> None:
        with open(path, "rb") as stream:
            while stream.read(100 * 1024 * 1024):
                pass

class MMapIndexedDatasetBuilder(object):
    def __init__(
        self, bin_path: str, dtype: Type[np.number] = np.int32, multimodal: bool = False
    ) -> None:
        self._data_file = open(bin_path, "wb")
        self._dtype = dtype
        self._multimodal = multimodal

        self._sequence_lengths = []
        self._document_indices = [0]
        self._sequence_modes = [] if self._multimodal else None


    def get_doc_idx(self) -> np.ndarray:
        return self._index._document_indices

    def set_doc_idx(self, doc_idx: np.ndarray) -> None:
        self._index._document_indices = doc_idx

    def modes(self) -> np.ndarray:
        return self._index.modes

    @property
    def supports_prefetch(self) -> bool:
        return False

    @staticmethod
    def exists(path_prefix: str) -> bool:
        return os.path.exists(get_idx_path(path_prefix)) and os.path.exists(
            get_bin_path(path_prefix)
        )

    @staticmethod
    def warmup_mmap_file(path: str) -> None:
        with open(path, "rb") as stream:
            while stream.read(100 * 1024 * 1024):
                pass

class MMapIndexedDatasetBuilder(object):
    def __init__(
        self, bin_path: str, dtype: Type[np.number] = np.int32, multimodal: bool = False
    ) -> None:
        self._data_file = open(bin_path, "wb")
        self._dtype = dtype
        self._multimodal = multimodal

        self._sequence_lengths = []
        self._document_indices = [0]
        self._sequence_modes = [] if self._multimodal else None

    def add_item(self, tensor: torch.Tensor, mode: int = 0) -> None:
        np_array = np.array(tensor.numpy(), dtype=self._dtype)
        self._data_file.write(np_array.tobytes(order="C"))
        self._sequence_lengths.append(np_array.size)
        if self._multimodal:
            self._sequence_modes.append(mode)