def get_args(input_args=None):
    parser = argparse.ArgumentParser()
    group = parser.add_argument_group(title="input data")
    group.add_argument(
        "--input",
        type=str,
        required=True,
        help="Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated "
        "list",
    )
    group.add_argument(
        "--jsonl-keys",
        nargs="+",
        default=["text"],
        help="space separate listed of keys to extract from jsonl. Defa",
    )
    group.add_argument(
        "--num-docs",
        default=None,
        help="Optional: Number of documents in the input data (if known) for an accurate progress bar.",
        type=int,
    )
    group = parser.add_argument_group(title="tokenizer")
    group.add_argument(
        "--tokenizer-type",
        type=str,
        required=True,
        choices=[
            "HFGPT2Tokenizer",
            "HFTokenizer",
            "GPT2BPETokenizer",
            "CharLevelTokenizer",
            "TiktokenTokenizer",
            "SPMTokenizer",
        ],
        help="What type of tokenizer to use.",
    )
    group.add_argument(
        "--vocab-file", type=str, default=None, help="Path to the vocab file"
    )
    group.add_argument(
        "--merge-file",
        type=str,
        default=None,
        help="Path to the BPE merge file (if necessary).",
    )
    group.add_argument(
        "--append-eod",
        action="store_true",
        help="Append an <eod> token to the end of a document.",
    )
    group.add_argument("--ftfy", action="store_true", help="Use ftfy to clean text")
    group = parser.add_argument_group(title="output data")
    group.add_argument(
        "--output-prefix",
        type=str,
        required=True,
        help="Path to binary output file without suffix",
    )
    group.add_argument(
        "--dataset-impl",
        type=str,
        default="mmap",
        choices=["lazy", "cached", "mmap"],
        help="Dataset implementation to use. Default: mmap",
    )

    group = parser.add_argument_group(title="runtime")
    group.add_argument(
        "--workers", type=int, default=1, help="Number of worker processes to launch"
    )
    group.add_argument(
        "--log-interval",
        type=int,
        default=100,
        help="Interval between progress updates",
    )
    args = parser.parse_args(input_args)
    args.keep_empty = False

    # some default/dummy values for the tokenizer
    args.rank = 0
    args.make_vocab_size_divisible_by = 128
    args.model_parallel_size = 1

    return args

get_args(input_args=['--input', './tests/data/enwik8_first100.txt', '--output-prefix', './tests/data/enwik8_first100', '--vocab', 'gpt2', '--tokenizer-type', 'HFGPT2Tokenizer', '--merge-file', './data/gpt2-merges.txt', '--append-eod'])

ArgumentParser(prog='__main__.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='input data', _group_actions=[]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='input data', _group_actions=[_StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None)]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='input data', _group_actions=[_StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None)]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='input data', _group_actions=[_StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None)]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[_StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None)]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[_StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None)]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[_StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None)]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[_StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None)]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='tokenizer', _group_actions=[_StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None)]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='output data', _group_actions=[]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), '--output-prefix': _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='output data', _group_actions=[_StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None)]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), '--output-prefix': _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), '--dataset-impl': _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='output data', _group_actions=[_StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None)]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), '--output-prefix': _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), '--dataset-impl': _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='runtime', _group_actions=[]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None), _StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), '--output-prefix': _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), '--dataset-impl': _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None), '--workers': _StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='runtime', _group_actions=[_StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None)]}

{description=None, argument_default=None, prefix_chars='-', conflict_handler='error', _registries={'action': {None: <class 'argparse._StoreAction'>, 'store': <class 'argparse._StoreAction'>, 'store_const': <class 'argparse._StoreConstAction'>, 'store_true': <class 'argparse._StoreTrueAction'>, 'store_false': <class 'argparse._StoreFalseAction'>, 'append': <class 'argparse._AppendAction'>, 'append_const': <class 'argparse._AppendConstAction'>, 'count': <class 'argparse._CountAction'>, 'help': <class 'argparse._HelpAction'>, 'version': <class 'argparse._VersionAction'>, 'parsers': <class 'argparse._SubParsersAction'>, 'extend': <class 'argparse._ExtendAction'>}, 'type': {None: <function ArgumentParser.__init__.<locals>.identity at 0x7f1a9c171d30>}}, _actions=[_HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None), _StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None), _StoreAction(option_strings=['--log-interval'], dest='log_interval', nargs=None, const=None, default=100, type=<class 'int'>, choices=None, required=False, help='Interval between progress updates', metavar=None)], _option_string_actions={'-h': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--help': _HelpAction(option_strings=['-h', '--help'], dest='help', nargs=0, const=None, default='==SUPPRESS==', type=None, choices=None, required=False, help='show this help message and exit', metavar=None), '--input': _StoreAction(option_strings=['--input'], dest='input', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to input jsonl files or lmd archive(s) - if using multiple archives, put them in a comma separated list', metavar=None), '--jsonl-keys': _StoreAction(option_strings=['--jsonl-keys'], dest='jsonl_keys', nargs='+', const=None, default=['text'], type=None, choices=None, required=False, help='space separate listed of keys to extract from jsonl. Defa', metavar=None), '--num-docs': _StoreAction(option_strings=['--num-docs'], dest='num_docs', nargs=None, const=None, default=None, type=<class 'int'>, choices=None, required=False, help='Optional: Number of documents in the input data (if known) for an accurate progress bar.', metavar=None), '--tokenizer-type': _StoreAction(option_strings=['--tokenizer-type'], dest='tokenizer_type', nargs=None, const=None, default=None, type=<class 'str'>, choices=['HFGPT2Tokenizer', 'HFTokenizer', 'GPT2BPETokenizer', 'CharLevelTokenizer', 'TiktokenTokenizer', 'SPMTokenizer'], required=True, help='What type of tokenizer to use.', metavar=None), '--vocab-file': _StoreAction(option_strings=['--vocab-file'], dest='vocab_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the vocab file', metavar=None), '--merge-file': _StoreAction(option_strings=['--merge-file'], dest='merge_file', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=False, help='Path to the BPE merge file (if necessary).', metavar=None), '--append-eod': _StoreTrueAction(option_strings=['--append-eod'], dest='append_eod', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Append an <eod> token to the end of a document.', metavar=None), '--ftfy': _StoreTrueAction(option_strings=['--ftfy'], dest='ftfy', nargs=0, const=True, default=False, type=None, choices=None, required=False, help='Use ftfy to clean text', metavar=None), '--output-prefix': _StoreAction(option_strings=['--output-prefix'], dest='output_prefix', nargs=None, const=None, default=None, type=<class 'str'>, choices=None, required=True, help='Path to binary output file without suffix', metavar=None), '--dataset-impl': _StoreAction(option_strings=['--dataset-impl'], dest='dataset_impl', nargs=None, const=None, default='mmap', type=<class 'str'>, choices=['lazy', 'cached', 'mmap'], required=False, help='Dataset implementation to use. Default: mmap', metavar=None), '--workers': _StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None), '--log-interval': _StoreAction(option_strings=['--log-interval'], dest='log_interval', nargs=None, const=None, default=100, type=<class 'int'>, choices=None, required=False, help='Interval between progress updates', metavar=None)}, _action_groups=[], _mutually_exclusive_groups=[], _defaults={}, _negative_number_matcher=re.compile('^-\\d+$|^-\\d*\\.\\d+$'), _has_negative_number_optionals=[], title='runtime', _group_actions=[_StoreAction(option_strings=['--workers'], dest='workers', nargs=None, const=None, default=1, type=<class 'int'>, choices=None, required=False, help='Number of worker processes to launch', metavar=None), _StoreAction(option_strings=['--log-interval'], dest='log_interval', nargs=None, const=None, default=100, type=<class 'int'>, choices=None, required=False, help='Interval between progress updates', metavar=None)]}

Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100)

Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False)

Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False, rank=0)

Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False, rank=0, make_vocab_size_divisible_by=128)

Namespace(input='./tests/data/enwik8_first100.txt', jsonl_keys=['text'], num_docs=None, tokenizer_type='HFGPT2Tokenizer', vocab_file='gpt2', merge_file='./data/gpt2-merges.txt', append_eod=True, ftfy=False, output_prefix='./tests/data/enwik8_first100', dataset_impl='mmap', workers=1, log_interval=100, keep_empty=False, rank=0, make_vocab_size_divisible_by=128, model_parallel_size=1)


def gen_search_gzh_url(keyword, page=1):
        """拼接搜索 公众号 URL

        Parameters
        ----------
        keyword : str or unicode
            搜索文字
        page : int, optional
            页数 the default is 1

        Returns
        -------
        str
            search_gzh_url
        """
        assert isinstance(page, int) and page > 0

        qs_dict = OrderedDict()
        qs_dict['type'] = _search_type_gzh
        qs_dict['page'] = page
        qs_dict['ie'] = 'utf8'
        qs_dict['query'] = keyword

        return 'http://weixin.sogou.com/weixin?{}'.format(urlencode(qs_dict))

gen_search_gzh_url(keyword='高考', page=1)


def check_just_one_not_none(*value_refname):
    nones=0
    for a_tuple in value_refname:
        if a_tuple[0] is not None:
            nones += 1
    if nones != 1:
        raise ValueError(error_message_for_non_compatible_references([a_tuple[1] for a_tuple in value_refname]))

check_just_one_not_none(value_refname=((None, 'graph_file_input'), (None, 'graph_list_of_files_input'), ('\n@prefix : <http://example.org/> .\n@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .\n\n:Marie a :person ;\n\t:likes :Justine ;\n\t:likes :Antoine ;\n\t:likes :football ;\n\t:name "Marie" ;\n\t:age "30"^^xsd:int .\n\n:Justine a :person ;\n    :likes :Marie ;\n    :likes :basketball ;\n    :name "Justine" .\n\n:Antoine a :person ;\n\t:likes :Marie ;\n\t:likes :Justine ;\n\t:name "Antoine" ;\n\t:age "32"^^xsd:int .\n', 'raw_graph'), (None, 'url_input'), (None, 'list_of_url_input'), (None, 'url_endpoint')))


def check_one_or_zero_not_none(*value_refname):
    nones=0
    for a_tuple in value_refname:
        if a_tuple[0] is not None:
            nones += 1
    if nones > 1:
        raise ValueError(error_message_for_non_compatible_references(
            list_of_ref_names=[a_tuple[1] for a_tuple in value_refname],
            one_mandatory=False))

check_one_or_zero_not_none(value_refname=((None, 'namespaces_dict'), (None, 'namespaces_dict_file')))


def cd(args: list) -> None:
    """
        Change the current working directory of the device.

        While this method does not actually change any directories,
        it simply updates the value in the file_manager_state property
        that keeps record of the current directory.

        Before changing directories though, some checks are performed
        on the device to at least ensure that the destination directory
        exists.

        :param args:
        :return:
    """

    if len(args) <= 0:
        click.secho('Usage: cd <destination directory>', bold=True)
        return

    path = args[0]
    current_dir = pwd()

    # nothing to do
    if path == '.':
        return

    # moving one directory back
    if path == '..':

        split_path = os.path.split(current_dir)

        # nothing to do if we are already at root
        if len(split_path) == 1:
            return

        new_path = ''.join(split_path[:-1])
        click.secho(new_path, fg='green', bold=True)

        file_manager_state.cwd = new_path

        return

    # if we got an absolute path, check if the path
    # actually exists, and then cd to it if we can
    if os.path.isabs(path):

        # assume the path does not exist by default
        does_exist = False

        # check for existence based on the runtime
        if device_state.platform == Ios:
            does_exist = _path_exists_ios(path)

        if device_state.platform == Android:
            does_exist = _path_exists_android(path)

        # if we checked with the device that the path exists
        # and it did, update the state manager, otherwise
        # show an error that the path may be invalid
        if does_exist:
            click.secho(path, fg='green', bold=True)

            file_manager_state.cwd = path
            return

        else:
            click.secho('Invalid path: `{0}`'.format(path), fg='red')

    # directory is not absolute, tack it on at the end and
    # see if its legit.
    else:

        proposed_path = device_state.platform.path_separator.join([current_dir, path])

        # assume the proposed_path does not exist by default
        does_exist = False

        # check for existence based on the runtime
        if device_state.platform == Ios:
            does_exist = _path_exists_ios(proposed_path)

        if device_state.platform == Android:
            does_exist = _path_exists_android(proposed_path)

        # if we checked with the device that the path exists
        # and it did, update the state manager, otherwise
        # show an error that the path may be invalid
        if does_exist:
            click.secho(proposed_path, fg='green', bold=True)

            file_manager_state.cwd = proposed_path
            return

        else:
            click.secho('Invalid path: `{0}`'.format(proposed_path), fg='red')

cd(args=['/foo/bar/baz'])


def dump_all(args: list) -> None:
    """
        Dump memory from the currently injected process.
        Loosely based on:
            https://github.com/Nightbringer21/fridump

        :param args:
        :return:
    """

    if len(clean_argument_flags(args)) <= 0:
        click.secho('Usage: memory dump all <local destination>', bold=True)
        return

    # the destination file to write the dump to
    destination = args[0]

    # Check for file override
    if os.path.exists(destination):
        click.secho('Destination file {dest} already exists'.format(dest=destination), fg='yellow', bold=True)
        if not click.confirm('Continue, appending to the file?'):
            return

    # access type used when enumerating ranges
    access = 'rw-'

    api = state_connection.get_api()
    ranges = api.memory_list_ranges(access)

    total_size = sum([x['size'] for x in ranges])
    click.secho('Will dump {0} {1} images, totalling {2}'.format(
        len(ranges), access, sizeof_fmt(total_size)), fg='green', dim=True)

    with click.progressbar(ranges) as bar:
        for image in bar:
            dump = bytearray()
            bar.label = 'Dumping {0} from base: {1}'.format(sizeof_fmt(image['size']), hex(int(image['base'], 16)))

            # catch and exception thrown while dumping.
            # this could for a few reasons like if the protection
            # changes or the range is reallocated
            try:
                # grab the (size) bytes starting at the (base_address) in chunks of BLOCK_SIZE
                chunks = _get_chunks(int(image['base'], 16), int(image['size']), BLOCK_SIZE)
                for chunk in chunks:
                    dump.extend(bytearray(api.memory_dump(chunk[0], chunk[1])))

            except Exception as e:
                continue

            # append the results to the destination file
            with open(destination, 'ab') as f:
                f.write(dump)

    click.secho('Memory dumped to file: {0}'.format(destination), fg='green')

dump_all(args=['/foo'])

{fill_char='#', empty_char='-', bar_template='%(label)s  [%(bar)s]  %(info)s', info_sep='  ', show_eta=True, show_percent=None, show_pos=False, item_show_func=None, label='', file=<_io.StringIO object at 0x7f5e07a3a3a0>, color=None, update_min_steps=1, _completed_intervals=0, width=36, autowidth=False, iter=<list_iterator object at 0x7f5e079fda90>, length=1, pos=0, avg=[], start=1712231192.5557132, last_eta=1712231192.5557132, eta_known=False, finished=False, max_width=None, entered=True, current_item=None, is_hidden=True, _last_line=''}

{fill_char='#', empty_char='-', bar_template='%(label)s  [%(bar)s]  %(info)s', info_sep='  ', show_eta=True, show_percent=None, show_pos=False, item_show_func=None, label='Dumping 100.0 B from base: 0x7fff90800000', file=<_io.StringIO object at 0x7f5e07a3a3a0>, color=None, update_min_steps=1, _completed_intervals=0, width=36, autowidth=False, iter=<list_iterator object at 0x7f5e079fda90>, length=1, pos=0, avg=[], start=1712231192.5557132, last_eta=1712231192.5557132, eta_known=False, finished=False, max_width=None, entered=True, current_item=None, is_hidden=True, _last_line=''}


def num_parameters(module: nn.Module, requires_grad: Optional[bool] = None) -> int:
    total = 0
    for p in module.parameters():
        if requires_grad is None or p.requires_grad == requires_grad:
            if hasattr(p, "quant_state"):
                # bitsandbytes 4bit layer support
                total += math.prod(p.quant_state[1])
            else:
                total += p.numel()
    return total

num_parameters(module=GPT(  (lm_head): Linear(in_features=8, out_features=8, bias=False)  (transformer): ModuleDict(    (wte): Embedding(8, 8)    (h): ModuleList(      (0-1): 2 x Block(        (norm_1): LayerNorm((8,), eps=1e-05, elementwise_affine=True)        (attn): CausalSelfAttention(          (attn): Linear(in_features=8, out_features=24, bias=True)          (proj): Linear(in_features=8, out_features=8, bias=True)          (adapter_wte): Embedding(10, 8)        )        (norm_2): LayerNorm((8,), eps=1e-05, elementwise_affine=True)        (mlp): GptNeoxMLP(          (fc): Linear(in_features=8, out_features=32, bias=True)          (proj): Linear(in_features=32, out_features=8, bias=True)        )      )    )    (ln_f): LayerNorm((8,), eps=1e-05, elementwise_affine=True)  )), requires_grad=True)

Parameter containing:tensor([1., 1., 1., 1., 1., 1., 1., 1.], device='cuda:0')

Parameter containing:tensor([0., 0., 0., 0., 0., 0., 0., 0.], device='cuda:0')

Parameter containing:tensor([[[[0.],          [0.],          [0.],          [0.]]]], device='cuda:0', requires_grad=True)


def layer_template(layer_name: str, idx: int) -> Tuple[str, int]:
    split = layer_name.split(".")
    number = int(split[idx])
    split[idx] = "{}"
    from_name = ".".join(split)
    return from_name, number

layer_template(layer_name='model.layers.0.self_attn.q_proj.weight', idx=2)

['model', 'layers', '0', 'self_attn', 'q_proj', 'weight']

['model', 'layers', '{}', 'self_attn', 'q_proj', 'weight']


def rowcol_to_a1(row, col):
    """Translates a row and column cell address to A1 notation.

    :param row: The row of the cell to be converted.
        Rows start at index 1.
    :type row: int, str

    :param col: The column of the cell to be converted.
        Columns start at index 1.
    :type row: int, str

    :returns: a string containing the cell's coordinates in A1 notation.

    Example:

    >>> rowcol_to_a1(1, 1)
    A1

    """
    row = int(row)
    col = int(col)

    if row < 1 or col < 1:
        raise IncorrectCellLabel("({}, {})".format(row, col))

    div = col
    column_label = ""

    while div:
        (div, mod) = divmod(div, 26)
        if mod == 0:
            mod = 26
            div -= 1
        column_label = chr(mod + MAGIC_NUMBER) + column_label

    label = "{}{}".format(column_label, row)

    return label

rowcol_to_a1(row=4, col=4)


def a1_to_rowcol(label):
    """Translates a cell's address in A1 notation to a tuple of integers.

    :param str label: A cell label in A1 notation, e.g. 'B1'.
        Letter case is ignored.
    :returns: a tuple containing `row` and `column` numbers. Both indexed
              from 1 (one).
    :rtype: tuple

    Example:

    >>> a1_to_rowcol('A1')
    (1, 1)

    """
    m = CELL_ADDR_RE.match(label)
    if m:
        column_label = m.group(1).upper()
        row = int(m.group(2))

        col = 0
        for i, c in enumerate(reversed(column_label)):
            col += (ord(c) - MAGIC_NUMBER) * (26**i)
    else:
        raise IncorrectCellLabel(label)

    return (row, col)

a1_to_rowcol(label='B1')

<re.Match object; span=(0, 2), match='B1'>


def _a1_to_rowcol_unbounded(label):
    """Translates a cell's address in A1 notation to a tuple of integers.

    Same as `a1_to_rowcol()` but allows for missing row or column part
    (e.g. "A" for the first column)

    :returns: a tuple containing `row` and `column` numbers. Both indexed
        from 1 (one).
    :rtype: tuple

    Example:

    >>> _a1_to_rowcol_unbounded('A1')
    (1, 1)

    >>> _a1_to_rowcol_unbounded('A')
    (inf, 1)

    >>> _a1_to_rowcol_unbounded('1')
    (1, inf)

    >>> _a1_to_rowcol_unbounded('ABC123')
    (123, 731)

    >>> _a1_to_rowcol_unbounded('ABC')
    (inf, 731)

    >>> _a1_to_rowcol_unbounded('123')
    (123, inf)

    >>> _a1_to_rowcol_unbounded('1A')
    Traceback (most recent call last):
        ...
    gspread.exceptions.IncorrectCellLabel: 1A

    >>> _a1_to_rowcol_unbounded('')
    (inf, inf)

    """
    m = A1_ADDR_ROW_COL_RE.match(label)
    if m:
        column_label, row = m.groups()

        if column_label:
            col = 0
            for i, c in enumerate(reversed(column_label.upper())):
                col += (ord(c) - MAGIC_NUMBER) * (26**i)
        else:
            col = inf

        if row:
            row = int(row)
        else:
            row = inf
    else:
        raise IncorrectCellLabel(label)

    return (row, col)

_a1_to_rowcol_unbounded(label='A1')

<re.Match object; span=(0, 2), match='A1'>