# Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # import logging import re from dataclasses import asdict, dataclass, fields from enum import Enum from pathlib import Path from typing import Any, Dict, List, Optional, Sequence, Tuple, TypeVar # XXX: these should be kept for eval of filters expressions import pyarrow as pa import pyarrow.compute as pc import pyarrow.parquet as pq from fairseq2.assets import default_asset_store from omegaconf import MISSING logger = logging.getLogger(__name__) class ParquetBatchFormat(Enum): pyarrow = 0 pandas = 1 torch = 2 class ColumnsNames(Enum): source_column = "_source_column" source_text_column = "_source_text_column" target_column = "_target_column" target_text_column = "_target_text_column" dataset_name = "_dataset_name" @dataclass class SonarTextColumn: text_value: Optional[str] = None """ Raw text expression that will be used as constant colum after being sententized and sonarized. """ text_column: Optional[str] = None sonar_column: Optional[str] = None """ Note `text_column` and `sonar_column` should be aligned (so `sonar_column` should be sonar encoded `text_column`). If `sonar_column` is None and `text_column` is provided, we set `sonar_column = f"{text_column}_sonar_emb"` as default processing value! """ @dataclass class ParquetDatasetLimitOptions: fraction_of_files: Optional[float] = None nb_files: Optional[int] = None nb_fragments: Optional[int] = None nb_rows: Optional[int] = None @dataclass(frozen=True) class SonarDecoderConfig: tokenizer: str = "text_sonar_basic_decoder" """ SONAR tokenizer """ decoder: str = "text_sonar_basic_decoder" """ SONAR decoder""" lang: str = "eng_Latn" """ Target language """ max_tokens_in_sentence: int = 256 """Maximum number of tokens generated in the text""" temperature: float = 1.0 """The decoding logit temperature, where values greater than 1.0 produce more uniform logits; values less than 1.0 produce sharper logits.""" @dataclass(frozen=True) class SonarEncoderConfig: tokenizer: str = "text_sonar_basic_encoder" """ SONAR tokenizer """ encoder: str = "text_sonar_basic_encoder" """ SONAR decoder""" lang: str = "eng_Latn" """ Target language """ @dataclass class DatasetConfig: """ Generic dataset config """ columns: Optional[List[str]] = None """The list of columns to load. Columns such as `source_column`, ..., will be added automatically. """ source_text_column: Optional[str] = None """ Column to load as source raw text""" target_text_column: Optional[str] = None """ Column to load as target raw text for paired data""" source_prefix_text: Optional[str] = None """ Text to prepend to the content of the source_column""" source_suffix_text: Optional[str] = None """ Text to append to the content of the target_column""" target_prefix_text: Optional[str] = None """ Text to prepend to the content of the source_column""" target_suffix_text: Optional[str] = None """ Text to append to the content of the target_column""" source_sequences: Optional[List[SonarTextColumn]] = None """ Designed to make on-the-fly prompts from existing columns that are more complex than prefix and suffix. Each element of source_sequences is a SonarTextColumn, which can be either: - constant raw text (with the text_value argument) - text column (with the text_column argument) - sonar column (with the sonar_column argument) Note that text_value cannot co-exist with text_column or sonar_column, and sonar column cannot be specified without a text column. Further behaviour for parquet datasets: - If text_value is specified, this will be split to sentences and sonarized - If only text_column is specified, a new column named "_sonar_emb" will be added as sonar_column. - If both (text_column, sonar_column) is specified, All SonarTextColumn elements from source_sequences will be concatenated together to produce new source_column and source_text_column (same for target), which will have names as defined in ColumnsNames. Using source_sequences is NOT compatible with using source_column or source_text_column, as well as quality filtering. """ target_sequences: Optional[List[SonarTextColumn]] = None """Designed to make on-the-fly prompts / instructions for target column, see `source_sequences` for more details""" silent_freeze: bool = False """If set to true, the config value can only be set once, i.e. it will not be able to update after the being set is instantiated. This is helpful to avoid side-effect in setting some configs after being specified by the user application (Hydra, CLI)""" def __post_init__(self): if self.source_sequences is not None: if self.source_text_column is not None: logger.warning( f"Both `source_sequence` and `source_text_column` is specified. " f"Ignore `source_text_column` and use default value `{ColumnsNames.source_text_column.value}`.\n" f"(`source_sequences` = {self.source_sequences}, \n" f"`source_text_column` = {self.source_text_column} )" ) self.source_text_column = ColumnsNames.source_text_column.value if self.target_sequences is not None: if self.target_text_column is not None: logger.warning( f"Both `target_sequences` and `target_text_column` is specified. " f"Ignore `target_text_column` and use default value `{ColumnsNames.target_text_column.value}`.\n" f"(`target_sequences` = {self.target_sequences}, \n" f"`target_text_column` = {self.target_text_column} )" ) self.target_text_column = ColumnsNames.target_text_column.value for col in (self.source_sequences or []) + (self.target_sequences or []): if col.text_value is not None: assert col.text_column is None and col.sonar_column is None else: assert col.text_column is not None self._has_initialized_: bool = True def __setattr__(self, name: str, value: Any) -> None: if not getattr(self, "_has_initialized_", False): return super().__setattr__(name, value) if name == "silent_freeze": raise ValueError( "Direct change of silent_freeze outside __init__ is forbidden" ) if self.silent_freeze and getattr(self, name) not in ("", None, MISSING): logger.debug( f"Ignore change of {name} since silent_freeze is set and value is not empty ({getattr(self, name)})" ) return super().__setattr__(name, value) def override_attr(self, name: str, value: Any) -> None: try: self._has_initialized_ = False super().__setattr__(name, value) finally: self._has_initialized_ = True def freeze(self) -> None: """Turn the `silent_freeze` flag on""" try: self._has_initialized_ = False self.silent_freeze = True finally: self._has_initialized_ = True @dataclass class JSONDatasetConfig(DatasetConfig): """Config for datasets stored in JsonL format.""" file_path: str = str() """ Path to the directory containing the Jsonl dataset. Each task will replace this wil a real Json files TODO: Add support for remote JsonL file (e.g. with "s3://...") """ prompt_template: Optional[str] = None """ A jinja-format string to apply for each item in the dataset to transform into a string. Useful for example when compiling a dynamic instruction / prompt for training or evaluation. Note that when this is specified, it will take precedence over the "affix" option, i.e. the columns `source_prefix_text`, `source_suffix_text`,... will be ignored. """ def __setattr__(self, name: str, value: Any) -> None: if not getattr(self, "_has_initialized_", False): return super().__setattr__(name, value) if name == "silent_freeze": raise ValueError("Direct change of silent_freeze is forbidden") if self.silent_freeze: if getattr(self, name) not in ("", None, MISSING): logger.debug( f"Ignore change of {name} in silent frozen mode when value is not empty ({getattr(self, name)})" ) return # Ensure we cannot set the default `prompt_template` value when the user specifies # source_sequences or source_text_column explicitly for hi_prior_col, lo_prior_col, lo_prior_value in [ ("source_sequences", "source_text_column", self.source_text_column), ("target_sequences", "target_text_column", self.target_text_column), ("prompt_template", "source_sequences", self.source_sequences), ("prompt_template", "source_prefix_text", self.source_prefix_text), ("prompt_template", "source_suffix_text", self.source_suffix_text), ]: if name == hi_prior_col and lo_prior_value not in ("", None, MISSING): logger.warning( f"Updating value of {hi_prior_col} will cause conflicts with the user-defined " f"value in {lo_prior_col}. The update will be ignored.\n" ) return super().__setattr__(name, value) @dataclass class ParquetDatasetConfig(DatasetConfig): """ Config for datasets stored in Parquet format. XXX: this config should not hold non-trival default values. We want this to make datacards info and hydra config merge easier. All None value should be filled up in downstream `build_parquet_iterator_pipeline`. """ name: Optional[str] = None """When name is provided, it will use preregistered cards to populate all attributes. name convention is the following - {card_name}={split}:{weight} Example: - wiki - wiki:0.2 # no split - wiki=dev # default weight=1 - wiki=dev:0.2 Cards attributes will be overwritten by user defined ParquetDatasetConfig in `create_dataset_config_from_cards`. """ parquet_path: str = str() """The path to parquet dataset file. if `parquet_path` is remote (like stats with "s3://..."), the filesystem will be automatically detected and `filesystem_expr` should remain None """ weight: float = 1.0 """ Indicates relative weight of dataset that can be used for sampling from different datasets. """ limit: Optional[ParquetDatasetLimitOptions] = None """ Contains different options that allows to load only a part of the provided dataset. It will **always** take some number of **first** fragments according to the order in which they appear in the dataset and this logic will not be depedent on suffling/seed. When several limits are provided, each of them will be applied (resulting in the strongest limit). """ source_column: Optional[str] = None """ Column to load as source embeddings""" target_column: Optional[str] = None """ Column to load as target embeddings for paired data""" source_quality_column: Optional[str] = None source_quality_range: Optional[Any] = None partition_filters: Optional[str] = None """ Filters that should be applied only on partition columns for fast partition prunning. This filters should not be duplicated in `filters` (below) which are used on materialized data. To know the partition columns on dataset : ```python >>> pq.ParquetDataset(parquet_path).partitioning.schema.names ``` Note that for if `parquet_path` references a single file -> the result above will NOT be correct (returns all columns). Note that for a single file case, there should no partition_filters since there're no partitions !! """ filters: Optional[str] = None """See https://arrow.apache.org/docs/python/generated/pyarrow.dataset.Expression.html#pyarrow.dataset.Expression Some examples : >>> import pyarrow.compute as pc >>> import pyarrow as pa >>> filters = (pc.field("data_split") == pc.scalar("train")) & (pc.field("duration") > 7) >>> filters = pa.compute.greater(pa.compute.utf8_length(ds.field("lang1_text")), 4) >>> filters = pa.compute.less_equal(pa.compute.list_value_length(pa.dataset.field("audio_wav")), 16_000 * 30) Note that all fields used here should be among existing columns in the dataset schema. For hydra compatibility, we need to pass this filters as an str expression that'll be passed to `eval(...)` """ filesystem_expr: Optional[str] = None """ DEPRECATED : not used any more and will be remove soon """ filesystem: Optional[Any] = None """ DEPRECATED: not used any more and will be remove soon """ split_to_row_groups: Optional[bool] = None """If ``True``, uses Parquet row groups instead of simple partitions which are generally smaller. Highly recommended for non-partitioned parquet files.""" nb_parallel_fragments: Optional[int] = None """ This parameter can be dataset specific: For dataset with large number of sentences per document (sample), it's enough to set `nb_parallel_fragments=2 or 3`. For datasets, with smaller number of sentences (~10) and small row_group_size (~200-600), `nb_parallel_fragments` could be increase to 10 - 20. The number of Parquet fragments allowed to be read in parallel. Higher values will result in higher speeds, better randomization, and higher memory footprint. If partition size is rather small compared to the batch size, we recommend to increase ``nb_parallel_fragments``. Leaving ``nb_parallel_fragments`` to None will trigger auto-detection based on dataset metadata. """ sharding_in_memory: bool = False """ This option should be activated for sharding small datasets whose total number of row groups is small that makes sharding per row group impossible. """ def __post_init__(self): super().__post_init__() if self.source_sequences is not None: if self.source_column is not None: logger.warning( f"Both `source_sequences` and `source_column` is specified. " f"Ignore `source_column` and use default value `{ColumnsNames.source_column.value}`.\n" f"(`source_sequences` = {self.source_sequences}, \n" f"`source_column` = {self.source_column} )" ) assert self.source_quality_range is None self.source_column = ColumnsNames.source_column.value if self.target_sequences is not None: if self.target_column is not None: logger.warning( f"Both `target_sequences` and `target_column` is specified. " f"Ignore `target_column` and use default value `{ColumnsNames.target_column.value}`.\n" f"(`target_sequences` = {self.target_sequences}, \n" f"`target_column` = {self.target_column} )" ) self.target_column = ColumnsNames.target_column.value for col in (self.source_sequences or []) + (self.target_sequences or []): if col.sonar_column is None and col.text_value is None: assert col.text_column, f"Invalid SonarTextColumn: {col}" col.sonar_column = col.text_column + "_sonar_emb" if self.source_quality_range is None: self.source_quality_column = None DatasetConfigT = TypeVar("DatasetConfigT", bound=DatasetConfig, contravariant=True) @dataclass class DataLoadingConfig: multiple_dataset_chaining: str = "sample" """ This option allows to chain several datasets together. The chaining can be done in two ways: - `sample` : each dataset will be sampled with the provided weight - `concat` : datasets will be concatenated together (no weights taken into account) - `round_robin`: datasets will be sampled in a round robin fashion (no weights taken into account) """ batch_size: Optional[int] = None """The output batch size.""" order_by_length: bool = True """ Whether to create the batches with homogeneous tokens length for more efficient padding. """ max_tokens: Optional[int] = None """Used with the ``order_by_length`` option to control the total number of padded tokens in each batch. Typically, this option is preferred over ``batch_size`` to reduce the memory footprint. """ len_to_wrap_long_seq: Optional[int] = None """ Wrapping a source sequences to the length of `len_to_wrap_long_seq`. For instance, for a `len_to_wrap_long_seq=2` batch = { "source": [["v1", "v2", "v3", "v4", "v5"], ["u1", "u2", "u3"], ["w1"]], } will be transormed to 1. if packing is False : batch = { "source": [['v1', 'v2'], ['v3', 'v4'], ['v5'], ["u1", "u2"], ["u3"], ["w1"]] } 1. if packing is True : batch = { "source": [['v1', 'v2'], ['v3', 'v4'], ['v5', 'u1'], ["u2", "u3"], ["w1"]] } Note: currently only allowed to be used with no "target" provided (unsupervised style) ! """ packing: bool = False """ If True, all sequential documents (seqs of sentences) will be concated into one big document before applying wrapping. This will result in all samples (except maybe one) having exactly `len_to_wrap_long_seq` length ! """ wrap_before_affixing: bool = False """ If True, we will wrap the sequences before adding the source prefix/suffix. Recommended when pre-training with packed data i.e len_to_wrap_long_seq not None and packing=True """ max_sentence_len_in_doc: Optional[int] = None """ Remove samples (documents) whose `source_text_column` contains at least one sentence of len > `max_sentence_len_in_doc`. This operations is done after long sequences wrapping (if applicable). Typically values: 100 - 300 """ min_sentence_len_in_doc: Optional[int] = None """ Remove samples (documents) `source_text_column` contains at least one sentence of len < `min_sentence_len_in_doc`. This operations is done after long sequences wrapping (if applicable). Typically values: 5 - 15 """ max_sentence_len_in_target_doc: Optional[int] = None """ same filtering option as above but for `target_text_column` """ min_sentence_len_in_target_doc: Optional[int] = None """ same filtering option as above but for `target_text_column` """ min_length_of_sequences: Optional[int] = 1 """ Remove samples (documents) whose `source_text_column` are scrictly shorter than `min_length_of_sequences`. This operations is done after long sequences wrapping (if applicable). One can use here the same value as for sequences wrapping in order to produce all sequences with the same length. """ min_length_of_sequences_after_batching: Optional[int] = 1 """ Remove source sequences shorter than `min_length_of_sequences_after_batching` This filtering is applied after batching and potentially affixing and wrapping. """ min_length_of_target_sequences: Optional[int] = 1 """ Same as above applied for `target_text_column` """ min_length_of_target_sequences_after_batching: Optional[int] = 1 """ Same as above applied for `target_text_column` """ output_format: ParquetBatchFormat = ParquetBatchFormat.torch """The format to use for output batches.""" shuffle: bool = True """If ``True``, shuffles the dataset samples during the iteration. If ``False`` and ``order_by_length`` is ``None``, the batch samples will be produced in natural Parquet dataset reading order.""" drop_null: bool = True """If ``True``, drops rows containing any null value.""" seed: int = 123 """The RNG seed value for deterministic behavior.""" nb_epochs: int = 100 """ Number of passes over the data before iterations stop """ min_batch_size: int = 1 """Drops batches whose length is less than ``min_batch_size``""" nb_prefetch: float = 3.0 """The number of producer groups (of size `nb_parallel_fragments`) to prefetch.""" num_parallel_calls: float = 1.5 """The number of parallel calls in map operations.""" use_threads: bool = False """Whether pyarrow should use its internal threads to read the Parquet file. Since we rely on the external parallelism, this param is tuned off by default.""" ignore_checkpointed_pipeline: bool = False """Whether to ignore the saved datapipeline state or load it when resuming. Temporary fix for issues re-loading saved checkpoints""" even_sharding: bool = False """ This option should be activated ONLY for validataion on small datasets to guarantee the perfect data sharding accross the workers. Note that in current impmentation, activating `even_sharding` requires `sharding_in_memory=True` which will lead to big overhead for big dataset. Note also that some fraction of the data may be dropped due to even sharding. For big validation datasets, prefer using large `nb_epoch` + limiting `max_validation_iterations` instead of using `even_sharding` ! For training use case, it should left to False and combined with large number of epochs. For evaluation use case, it also should be False since we dont care about the batch syncronization across different workers. """ max_iteration_steps: Optional[int] = None """ If not None, it will be used to limit the number of batches produced per each dataset """ @dataclass class ValidationDataLoadingConfig(DataLoadingConfig): """ This class allows to have some hardcoded parameters for data loading of validation datasets """ multiple_dataset_chaining: str = "concat" nb_epochs: int = 1 min_batch_size: int = 1 # we want to keep all samples shuffle: bool = False # we dont need the randomness here batch_size: Optional[int] = None max_tokens: Optional[int] = None """ Leaving both `max_tokens` and `batch_size` to None will trigger auto-detection based on dataset metadata and distributed training world size. to make more or less even distribution of samples across workers. Typically, if worker_batch_size = total_batch_size // world_size <= 40, we will use batch_size=worker_batch_size, otherwise we will use max_tokens=min(total_tokens_number // world_size, 3000). See dataloading:SingleParquetDatasetDataloader::set_validation_params for more details. """ @dataclass class EvaluationDataLoadingConfig(DataLoadingConfig): """ This class allows to have some hardcoded parameters for data loading of evaluation datasets. In partitcular, even in distributed setup evaluation should not require workers syncronization. Therefore, we set `even_sharding` = False to get the all data samples ! """ multiple_dataset_chaining: str = "concat" nb_epochs: int = 1 # only ONE full pass over the full data ! min_batch_size: int = 1 # we want to keep all samples shuffle: bool = False # we dont need the randomness here batch_size: Optional[int] = 10 max_tokens: Optional[int] = None # this should be ok for most of models even_sharding: bool = False # we dont want to lose any sample ! sharding_in_memory: bool = True # activate sharding by rank and world size rank: int = 0 world_size: int = 1 max_samples: Optional[int] = None # fmt: skip """evaluate only the first n samples (for debugging)""" def setup_fairseq2_extensions() -> None: # path where all datacards should be located ! cards_dir = Path(__file__).parent.parent.joinpath("datacards") if cards_dir.exists(): default_asset_store.add_file_metadata_provider(cards_dir) setup_fairseq2_extensions() def get_cluster() -> Optional[str]: """Returns the cluster name of the current environment. User can implement their own logic to load datasets living in different locations/clusters """ return "s3" def _resolve_parquet_path(options: Dict[str, str]) -> Optional[str]: cluster_name = get_cluster() or "s3" parquet_path = options.get(cluster_name) if parquet_path is None: # best effort - taking first element parquet_path = next(iter(options.values())) return parquet_path def _resolve_filters( split: Optional[str], card_filter: Optional[str], user_filter: Optional[str], card_partition_filters: Optional[str], user_partition_filters: Optional[str], ) -> Tuple[Optional[pc.Expression], Optional[pc.Expression]]: custom_filters = user_filter or card_filter partition_filters = user_partition_filters or card_partition_filters if custom_filters is not None: custom_filters = pq.filters_to_expression(eval(custom_filters)) if partition_filters is not None: partition_filters = pq.filters_to_expression(eval(partition_filters)) if split: split_filter = pc.equal(pc.field("split"), split) if partition_filters is None: partition_filters = split_filter else: partition_filters = pa.compute.if_else( split_filter, partition_filters, False ) return custom_filters, partition_filters def _default_resolver(a, b): res = a if bool(a) and a is not MISSING else b return res def get_parquet_config_from_name( name: str, config: Optional[ParquetDatasetConfig] = None ) -> ParquetDatasetConfig: """ name convention is the following - {card_name}={split}:{weight} """ # parsing name pattern = r"^(?P[a-zA-Z0-9_]+)=?(?P[a-zA-Z0-9_]*)?:?(?P\d+(?:\.\d+)?)?$" match_ = re.match(pattern, name) assert match_ is not None, f"name parsing failed: {name}" card_name = match_.group("card_name") split = match_.group("split") weight = match_.group("weight") if weight: weight = float(weight) logger.info( f"Parsing {name} : card_name={card_name}, split={split}, weight={weight}" ) reload_config = default_asset_store.retrieve_card(card_name) cards_metadata: Dict[str, Any] = {**reload_config._metadata} if config is None: config = ParquetDatasetConfig(name=card_name, parquet_path="") assert config is not None if isinstance(config, ParquetDatasetConfig): config_dict = asdict(config) else: config_dict = config # type: ignore metadata = {} # resolve parquet_path according to the cluster for field in fields(ParquetDatasetConfig): field_name = field.name metadata[field_name] = _default_resolver( config_dict.get(field_name), cards_metadata.get(field_name) ) if isinstance(metadata["source_sequences"], list): metadata["source_sequences"] = [ SonarTextColumn(**item) for item in metadata["source_sequences"] ] if isinstance(metadata["target_sequences"], list): metadata["target_sequences"] = [ SonarTextColumn(**item) for item in metadata["target_sequences"] ] metadata["parquet_path"] = _default_resolver( config_dict.get("parquet_path"), _resolve_parquet_path(cards_metadata["parquet_path"]), ) metadata["filters"], metadata["partition_filters"] = _resolve_filters( split, card_filter=cards_metadata.get("filters"), user_filter=config_dict.get("filters"), card_partition_filters=cards_metadata.get("partition_filters"), user_partition_filters=config_dict.get("partition_filters"), ) if weight: # priority from parsed name metadata["weight"] = weight metadata["name"] = name # to patch nested hydra case ! if metadata["limit"] is not None and isinstance(metadata["limit"], dict): metadata["limit"] = ParquetDatasetLimitOptions(**metadata["limit"]) return ParquetDatasetConfig(**metadata) def create_dataset_config_from_cards( config: DatasetConfig, ) -> DatasetConfig: if getattr(config, "name", None) is None: return config output_config = get_parquet_config_from_name(config.name, config) # type: ignore return output_config def get_renaming_mappers(configs: Sequence[DatasetConfig]) -> List[dict]: used_columns = [x for x in ColumnsNames.__members__ if x != "dataset_name"] pre_mapping = { att: [getattr(cc, att) for cc in configs if hasattr(cc, att)] for att in used_columns } mappers: List[dict] = [{} for _ in configs] for att, val in pre_mapping.items(): if all(x is None for x in val): continue for i, name in enumerate(val): if name is None: raise ValueError( f"All datasets should provide {att} param, but got {configs[i]}" ) mappers[i][name] = getattr(ColumnsNames, att).value return mappers