| from argparse import ArgumentParser |
| from typing import Optional |
|
|
| from datasets.commands import BaseDatasetsCLICommand |
| from datasets.hub import convert_to_parquet |
|
|
|
|
| def _command_factory(args): |
| return ConvertToParquetCommand( |
| args.dataset_id, |
| args.token, |
| args.revision, |
| args.trust_remote_code, |
| ) |
|
|
|
|
| class ConvertToParquetCommand(BaseDatasetsCLICommand): |
| @staticmethod |
| def register_subcommand(parser): |
| parser: ArgumentParser = parser.add_parser("convert_to_parquet", help="Convert dataset to Parquet") |
| parser.add_argument( |
| "dataset_id", help="source dataset ID, e.g. USERNAME/DATASET_NAME or ORGANIZATION/DATASET_NAME" |
| ) |
| parser.add_argument("--token", help="access token to the Hugging Face Hub (defaults to logged-in user's one)") |
| parser.add_argument("--revision", help="source revision") |
| parser.add_argument( |
| "--trust_remote_code", action="store_true", help="whether to trust the code execution of the load script" |
| ) |
| parser.set_defaults(func=_command_factory) |
|
|
| def __init__( |
| self, |
| dataset_id: str, |
| token: Optional[str], |
| revision: Optional[str], |
| trust_remote_code: bool, |
| ): |
| self._dataset_id = dataset_id |
| self._token = token |
| self._revision = revision |
| self._trust_remote_code = trust_remote_code |
|
|
| def run(self) -> None: |
| _ = convert_to_parquet( |
| self._dataset_id, revision=self._revision, token=self._token, trust_remote_code=self._trust_remote_code |
| ) |
|
|
