Upload folder using huggingface_hub

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 v11-33e-2001/weights/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
+v11-33e-2001/context/qwen2-annulus-v5f/tokenizer.json filter=lfs diff=lfs merge=lfs -text

v11-33e-2001/context/io.json

@@ -0,0 +1 @@
+{"root": {"type": "ref", "key": "trainer_context_1"}, "objects": {"tuple_1": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "tensor_model_parallel_size"], ["Index(index=1)", "pipeline_model_parallel_comm_backend"], ["Index(index=2)", "pipeline_model_parallel_size"], ["Index(index=3)", "virtual_pipeline_model_parallel_size"], ["Index(index=4)", "sequence_parallel"], ["Index(index=5)", "context_parallel_size"], ["Index(index=6)", "expert_model_parallel_size"], ["Index(index=7)", "expert_tensor_parallel_size"], ["Index(index=8)", "moe_extended_tp"], ["Index(index=9)", "params_dtype"], ["Index(index=10)", "autocast_dtype"], ["Index(index=11)", "use_te_rng_tracker"], ["Index(index=12)", "pipeline_dtype"], ["Index(index=13)", "microbatch_group_size_per_vp_stage"], ["Index(index=14)", "num_layers_in_first_pipeline_stage"], ["Index(index=15)", "num_layers_in_last_pipeline_stage"], ["Index(index=16)", "pipeline_model_parallel_layout"], ["Index(index=17)", "account_for_embedding_in_pipeline_split"], ["Index(index=18)", "account_for_loss_in_pipeline_split"], ["Index(index=19)", "num_layers_at_start_in_bf16"], ["Index(index=20)", "num_layers_at_end_in_bf16"]], "metadata": null}, "dict_1": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_2": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "annulus.model.base_config", "name": "AnnulusGPTConfig"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_1"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_1"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_2"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "annulus_gptconfig_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='tensor_model_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.model.config.tensor_model_parallel_size"]}], ["Attr(name='pipeline_model_parallel_comm_backend')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.pipeline_model_parallel_comm_backend"]}], ["Attr(name='pipeline_model_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.model.config.pipeline_model_parallel_size"]}], ["Attr(name='virtual_pipeline_model_parallel_size')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.virtual_pipeline_model_parallel_size"]}], ["Attr(name='sequence_parallel')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.sequence_parallel"]}], ["Attr(name='context_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.model.config.context_parallel_size"]}], ["Attr(name='expert_model_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.model.config.expert_model_parallel_size"]}], ["Attr(name='expert_tensor_parallel_size')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.expert_tensor_parallel_size"]}], ["Attr(name='moe_extended_tp')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.moe_extended_tp"]}], ["Attr(name='params_dtype')", {"type": "pyref", "module": "torch", "name": "float32", "paths": ["<root>.model.config.params_dtype", "<root>.model.optim.config.params_dtype", "<root>.model.optim.config.main_grads_dtype", "<root>.model.optim.config.main_params_dtype", "<root>.model.optim.config.exp_avg_dtype", "<root>.model.optim.config.exp_avg_sq_dtype"]}], ["Attr(name='autocast_dtype')", {"type": "pyref", "module": "torch", "name": "bfloat16", "paths": ["<root>.model.config.autocast_dtype", "<root>.model.config.pipeline_dtype"]}], ["Attr(name='use_te_rng_tracker')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.use_te_rng_tracker"]}], ["Attr(name='pipeline_dtype')", {"type": "pyref", "module": "torch", "name": "bfloat16", "paths": ["<root>.model.config.autocast_dtype", "<root>.model.config.pipeline_dtype"]}], ["Attr(name='microbatch_group_size_per_vp_stage')", {"type": "leaf", "value": 1, "paths": ["<root>.model.config.microbatch_group_size_per_vp_stage"]}], ["Attr(name='num_layers_in_first_pipeline_stage')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.num_layers_in_first_pipeline_stage"]}], ["Attr(name='num_layers_in_last_pipeline_stage')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.num_layers_in_last_pipeline_stage"]}], ["Attr(name='pipeline_model_parallel_layout')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.pipeline_model_parallel_layout"]}], ["Attr(name='account_for_embedding_in_pipeline_split')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.account_for_embedding_in_pipeline_split"]}], ["Attr(name='account_for_loss_in_pipeline_split')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.account_for_loss_in_pipeline_split"]}], ["Attr(name='num_layers_at_start_in_bf16')", {"type": "leaf", "value": 0, "paths": ["<root>.model.config.num_layers_at_start_in_bf16"]}], ["Attr(name='num_layers_at_end_in_bf16')", {"type": "leaf", "value": 0, "paths": ["<root>.model.config.num_layers_at_end_in_bf16"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_1"}, "paths": ["<root>.model.config"]}, "tuple_2": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "lr"], ["Index(index=1)", "min_lr"], ["Index(index=2)", "decoupled_lr"], ["Index(index=3)", "decoupled_min_lr"], ["Index(index=4)", "weight_decay"], ["Index(index=5)", "apply_wd_to_qk_layernorm"], ["Index(index=6)", "fp8_recipe"], ["Index(index=7)", "fp16"], ["Index(index=8)", "bf16"], ["Index(index=9)", "reuse_grad_buf_for_mxfp8_param_ag"], ["Index(index=10)", "params_dtype"], ["Index(index=11)", "use_precision_aware_optimizer"], ["Index(index=12)", "store_param_remainders"], ["Index(index=13)", "main_grads_dtype"], ["Index(index=14)", "main_params_dtype"], ["Index(index=15)", "exp_avg_dtype"], ["Index(index=16)", "exp_avg_sq_dtype"], ["Index(index=17)", "optimizer"], ["Index(index=18)", "loss_scale"], ["Index(index=19)", "initial_loss_scale"], ["Index(index=20)", "min_loss_scale"], ["Index(index=21)", "loss_scale_window"], ["Index(index=22)", "hysteresis"], ["Index(index=23)", "adam_beta1"], ["Index(index=24)", "adam_beta2"], ["Index(index=25)", "adam_eps"], ["Index(index=26)", "decoupled_weight_decay"], ["Index(index=27)", "sgd_momentum"], ["Index(index=28)", "muon_momentum"], ["Index(index=29)", "muon_split_qkv"], ["Index(index=30)", "muon_use_nesterov"], ["Index(index=31)", "muon_scale_mode"], ["Index(index=32)", "muon_fp32_matmul_prec"], ["Index(index=33)", "muon_num_ns_steps"], ["Index(index=34)", "muon_tp_mode"], ["Index(index=35)", "muon_extra_scale_factor"], ["Index(index=36)", "use_distributed_optimizer"], ["Index(index=37)", "overlap_param_gather"], ["Index(index=38)", "overlap_param_gather_with_optimizer_step"], ["Index(index=39)", "optimizer_cpu_offload"], ["Index(index=40)", "optimizer_offload_fraction"], ["Index(index=41)", "use_torch_optimizer_for_cpu_offload"], ["Index(index=42)", "overlap_cpu_optimizer_d2h_h2d"], ["Index(index=43)", "pin_cpu_grads"], ["Index(index=44)", "pin_cpu_params"], ["Index(index=45)", "clip_grad"], ["Index(index=46)", "log_num_zeros_in_grad"], ["Index(index=47)", "barrier_with_L1_time"], ["Index(index=48)", "timers"], ["Index(index=49)", "config_logger_dir"]], "metadata": null}, "dict_3": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_4": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_2": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "megatron.core.optimizer.optimizer_config", "name": "OptimizerConfig"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_2"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_3"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_4"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "optimizer_config_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='lr')", {"type": "leaf", "value": 0.0003, "paths": ["<root>.model.optim.config.lr"]}], ["Attr(name='min_lr')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.min_lr"]}], ["Attr(name='decoupled_lr')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.decoupled_lr"]}], ["Attr(name='decoupled_min_lr')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.decoupled_min_lr"]}], ["Attr(name='weight_decay')", {"type": "leaf", "value": 0.1, "paths": ["<root>.model.optim.config.weight_decay"]}], ["Attr(name='apply_wd_to_qk_layernorm')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.apply_wd_to_qk_layernorm"]}], ["Attr(name='fp8_recipe')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.fp8_recipe"]}], ["Attr(name='fp16')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.fp16"]}], ["Attr(name='bf16')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.bf16"]}], ["Attr(name='reuse_grad_buf_for_mxfp8_param_ag')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.reuse_grad_buf_for_mxfp8_param_ag"]}], ["Attr(name='params_dtype')", {"type": "pyref", "module": "torch", "name": "float32", "paths": ["<root>.model.config.params_dtype", "<root>.model.optim.config.params_dtype", "<root>.model.optim.config.main_grads_dtype", "<root>.model.optim.config.main_params_dtype", "<root>.model.optim.config.exp_avg_dtype", "<root>.model.optim.config.exp_avg_sq_dtype"]}], ["Attr(name='use_precision_aware_optimizer')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.use_precision_aware_optimizer"]}], ["Attr(name='store_param_remainders')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.store_param_remainders"]}], ["Attr(name='main_grads_dtype')", {"type": "pyref", "module": "torch", "name": "float32", "paths": ["<root>.model.config.params_dtype", "<root>.model.optim.config.params_dtype", "<root>.model.optim.config.main_grads_dtype", "<root>.model.optim.config.main_params_dtype", "<root>.model.optim.config.exp_avg_dtype", "<root>.model.optim.config.exp_avg_sq_dtype"]}], ["Attr(name='main_params_dtype')", {"type": "pyref", "module": "torch", "name": "float32", "paths": ["<root>.model.config.params_dtype", "<root>.model.optim.config.params_dtype", "<root>.model.optim.config.main_grads_dtype", "<root>.model.optim.config.main_params_dtype", "<root>.model.optim.config.exp_avg_dtype", "<root>.model.optim.config.exp_avg_sq_dtype"]}], ["Attr(name='exp_avg_dtype')", {"type": "pyref", "module": "torch", "name": "float32", "paths": ["<root>.model.config.params_dtype", "<root>.model.optim.config.params_dtype", "<root>.model.optim.config.main_grads_dtype", "<root>.model.optim.config.main_params_dtype", "<root>.model.optim.config.exp_avg_dtype", "<root>.model.optim.config.exp_avg_sq_dtype"]}], ["Attr(name='exp_avg_sq_dtype')", {"type": "pyref", "module": "torch", "name": "float32", "paths": ["<root>.model.config.params_dtype", "<root>.model.optim.config.params_dtype", "<root>.model.optim.config.main_grads_dtype", "<root>.model.optim.config.main_params_dtype", "<root>.model.optim.config.exp_avg_dtype", "<root>.model.optim.config.exp_avg_sq_dtype"]}], ["Attr(name='optimizer')", {"type": "leaf", "value": "adam", "paths": ["<root>.model.optim.config.optimizer"]}], ["Attr(name='loss_scale')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.loss_scale"]}], ["Attr(name='initial_loss_scale')", {"type": "leaf", "value": 4294967296, "paths": ["<root>.model.optim.config.initial_loss_scale"]}], ["Attr(name='min_loss_scale')", {"type": "leaf", "value": 1.0, "paths": ["<root>.model.optim.config.min_loss_scale"]}], ["Attr(name='loss_scale_window')", {"type": "leaf", "value": 1000, "paths": ["<root>.model.optim.config.loss_scale_window"]}], ["Attr(name='hysteresis')", {"type": "leaf", "value": 2, "paths": ["<root>.model.optim.config.hysteresis"]}], ["Attr(name='adam_beta1')", {"type": "leaf", "value": 0.9, "paths": ["<root>.model.optim.config.adam_beta1"]}], ["Attr(name='adam_beta2')", {"type": "leaf", "value": 0.98, "paths": ["<root>.model.optim.config.adam_beta2"]}], ["Attr(name='adam_eps')", {"type": "leaf", "value": 1e-08, "paths": ["<root>.model.optim.config.adam_eps"]}], ["Attr(name='decoupled_weight_decay')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.decoupled_weight_decay"]}], ["Attr(name='sgd_momentum')", {"type": "leaf", "value": 0.9, "paths": ["<root>.model.optim.config.sgd_momentum"]}], ["Attr(name='muon_momentum')", {"type": "leaf", "value": 0.95, "paths": ["<root>.model.optim.config.muon_momentum"]}], ["Attr(name='muon_split_qkv')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.muon_split_qkv"]}], ["Attr(name='muon_use_nesterov')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.muon_use_nesterov"]}], ["Attr(name='muon_scale_mode')", {"type": "leaf", "value": "spectral", "paths": ["<root>.model.optim.config.muon_scale_mode"]}], ["Attr(name='muon_fp32_matmul_prec')", {"type": "leaf", "value": "medium", "paths": ["<root>.model.optim.config.muon_fp32_matmul_prec"]}], ["Attr(name='muon_num_ns_steps')", {"type": "leaf", "value": 5, "paths": ["<root>.model.optim.config.muon_num_ns_steps"]}], ["Attr(name='muon_tp_mode')", {"type": "leaf", "value": "blockwise", "paths": ["<root>.model.optim.config.muon_tp_mode"]}], ["Attr(name='muon_extra_scale_factor')", {"type": "leaf", "value": 1.0, "paths": ["<root>.model.optim.config.muon_extra_scale_factor"]}], ["Attr(name='use_distributed_optimizer')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.use_distributed_optimizer"]}], ["Attr(name='overlap_param_gather')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.overlap_param_gather"]}], ["Attr(name='overlap_param_gather_with_optimizer_step')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.overlap_param_gather_with_optimizer_step"]}], ["Attr(name='optimizer_cpu_offload')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.optimizer_cpu_offload"]}], ["Attr(name='optimizer_offload_fraction')", {"type": "leaf", "value": 0.0, "paths": ["<root>.model.optim.config.optimizer_offload_fraction"]}], ["Attr(name='use_torch_optimizer_for_cpu_offload')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.use_torch_optimizer_for_cpu_offload"]}], ["Attr(name='overlap_cpu_optimizer_d2h_h2d')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.overlap_cpu_optimizer_d2h_h2d"]}], ["Attr(name='pin_cpu_grads')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.pin_cpu_grads"]}], ["Attr(name='pin_cpu_params')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.pin_cpu_params"]}], ["Attr(name='clip_grad')", {"type": "leaf", "value": 1.0, "paths": ["<root>.model.optim.config.clip_grad"]}], ["Attr(name='log_num_zeros_in_grad')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.log_num_zeros_in_grad"]}], ["Attr(name='barrier_with_L1_time')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.barrier_with_L1_time"]}], ["Attr(name='timers')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.timers"]}], ["Attr(name='config_logger_dir')", {"type": "leaf", "value": "", "paths": ["<root>.model.optim.config.config_logger_dir"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_2"}, "paths": ["<root>.model.optim.config"]}, "tuple_3": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "max_steps"], ["Index(index=1)", "warmup_steps"], ["Index(index=2)", "constant_steps"], ["Index(index=3)", "min_lr"]], "metadata": null}, "dict_5": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_6": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_3": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.lightning.pytorch.optim.lr_scheduler", "name": "CosineAnnealingScheduler"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_3"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_5"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_6"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "cosine_annealing_scheduler_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='max_steps')", {"type": "leaf", "value": 50000, "paths": ["<root>.model.optim.lr_scheduler.max_steps"]}], ["Attr(name='warmup_steps')", {"type": "leaf", "value": 2500, "paths": ["<root>.model.optim.lr_scheduler.warmup_steps"]}], ["Attr(name='constant_steps')", {"type": "leaf", "value": 0, "paths": ["<root>.model.optim.lr_scheduler.constant_steps"]}], ["Attr(name='min_lr')", {"type": "leaf", "value": 3e-06, "paths": ["<root>.model.optim.lr_scheduler.min_lr"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_3"}, "paths": ["<root>.model.optim.lr_scheduler"]}, "tuple_4": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "config"], ["Index(index=1)", "lr_scheduler"]], "metadata": null}, "dict_7": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_8": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_4": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.lightning.pytorch.optim.megatron", "name": "MegatronOptimizerModule"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_4"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_7"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_8"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "megatron_optimizer_module_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='config')", {"type": "ref", "key": "optimizer_config_1"}], ["Attr(name='lr_scheduler')", {"type": "ref", "key": "cosine_annealing_scheduler_1"}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_4"}, "paths": ["<root>.model.optim"]}, "tuple_5": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "pretrained_model_name"]], "metadata": null}, "dict_9": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_10": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_5": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.collections.common.tokenizers.huggingface.auto_tokenizer", "name": "AutoTokenizer"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_5"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_9"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_10"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "auto_tokenizer_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='pretrained_model_name')", {"type": "leaf", "value": "qwen2-annulus-v5f", "paths": ["<root>.model.tokenizer.pretrained_model_name"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_5"}, "paths": ["<root>.model.tokenizer"]}, "tuple_6": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "config"], ["Index(index=1)", "optim"], ["Index(index=2)", "tokenizer"]], "metadata": null}, "dict_11": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_12": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_6": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.collections.llm.gpt.model.base", "name": "GPTModel"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_6"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_11"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_12"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "gptmodel_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='config')", {"type": "ref", "key": "annulus_gptconfig_1"}], ["Attr(name='optim')", {"type": "ref", "key": "megatron_optimizer_module_1"}], ["Attr(name='tokenizer')", {"type": "ref", "key": "auto_tokenizer_1"}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_6"}, "paths": ["<root>.model"]}, "tuple_7": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "tensor_model_parallel_size"], ["Index(index=1)", "pipeline_model_parallel_size"], ["Index(index=2)", "sequence_parallel"], ["Index(index=3)", "expert_model_parallel_size"], ["Index(index=4)", "ckpt_async_save"]], "metadata": null}, "dict_13": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_14": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_7": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.lightning.pytorch.strategies.megatron_strategy", "name": "MegatronStrategy"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_7"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_13"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_14"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "megatron_strategy_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='tensor_model_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.trainer.strategy.tensor_model_parallel_size"]}], ["Attr(name='pipeline_model_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.trainer.strategy.pipeline_model_parallel_size"]}], ["Attr(name='sequence_parallel')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.sequence_parallel"]}], ["Attr(name='expert_model_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.trainer.strategy.expert_model_parallel_size"]}], ["Attr(name='ckpt_async_save')", {"type": "leaf", "value": true, "paths": ["<root>.trainer.strategy.ckpt_async_save"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_7"}, "paths": ["<root>.trainer.strategy"]}, "_wiring_audit_1": {"type": {"type": "pyref", "module": "__main__", "name": "_WiringAudit"}, "items": [["IdentityElement()", {"type": "leaf", "value": "4f633fed-0394-40b3-8474-720e689c8640", "paths": ["<root>.trainer.callbacks[0]"]}]], "metadata": null, "paths": ["<root>.trainer.callbacks[0]"]}, "_na_nskip_callback_1": {"type": {"type": "pyref", "module": "__main__", "name": "_NaNSkipCallback"}, "items": [["IdentityElement()", {"type": "leaf", "value": "a3566248-22ed-4b80-a512-49ac46e89b0a", "paths": ["<root>.trainer.callbacks[1]"]}]], "metadata": null, "paths": ["<root>.trainer.callbacks[1]"]}, "_lrprobe_1": {"type": {"type": "pyref", "module": "__main__", "name": "_LRProbe"}, "items": [["IdentityElement()", {"type": "leaf", "value": "1c98447a-ebad-43b9-87f3-82a023f46218", "paths": ["<root>.trainer.callbacks[2]"]}]], "metadata": null, "paths": ["<root>.trainer.callbacks[2]"]}, "list_1": {"type": {"type": "pyref", "module": "builtins", "name": "list"}, "items": [["Index(index=0)", {"type": "ref", "key": "_wiring_audit_1"}], ["Index(index=1)", {"type": "ref", "key": "_na_nskip_callback_1"}], ["Index(index=2)", {"type": "ref", "key": "_lrprobe_1"}]], "metadata": null, "paths": ["<root>.trainer.callbacks"]}, "megatron_mixed_precision_1": {"type": {"type": "pyref", "module": "nemo.lightning.pytorch.plugins.mixed_precision", "name": "MegatronMixedPrecision"}, "items": [["IdentityElement()", {"type": "leaf", "value": "92df67d0-0d99-48e3-8f27-3499b165c65b", "paths": ["<root>.trainer.plugins"]}]], "metadata": null, "paths": ["<root>.trainer.plugins"]}, "tuple_8": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "accelerator"], ["Index(index=1)", "strategy"], ["Index(index=2)", "devices"], ["Index(index=3)", "num_nodes"], ["Index(index=4)", "callbacks"], ["Index(index=5)", "max_steps"], ["Index(index=6)", "limit_val_batches"], ["Index(index=7)", "log_every_n_steps"], ["Index(index=8)", "plugins"]], "metadata": null}, "dict_15": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_16": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_8": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.lightning.pytorch.trainer", "name": "Trainer"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_8"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_15"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_16"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "trainer_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='accelerator')", {"type": "leaf", "value": "gpu", "paths": ["<root>.trainer.accelerator"]}], ["Attr(name='strategy')", {"type": "ref", "key": "megatron_strategy_1"}], ["Attr(name='devices')", {"type": "leaf", "value": 4, "paths": ["<root>.trainer.devices"]}], ["Attr(name='num_nodes')", {"type": "leaf", "value": 1, "paths": ["<root>.trainer.num_nodes"]}], ["Attr(name='callbacks')", {"type": "ref", "key": "list_1"}], ["Attr(name='max_steps')", {"type": "leaf", "value": 50000, "paths": ["<root>.trainer.max_steps"]}], ["Attr(name='limit_val_batches')", {"type": "leaf", "value": 0, "paths": ["<root>.trainer.limit_val_batches"]}], ["Attr(name='log_every_n_steps')", {"type": "leaf", "value": 1, "paths": ["<root>.trainer.log_every_n_steps"]}], ["Attr(name='plugins')", {"type": "ref", "key": "megatron_mixed_precision_1"}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_8"}, "paths": ["<root>.trainer"]}, "dict_17": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}, "paths": ["<root>.extra"]}, "tuple_9": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "model"], ["Index(index=1)", "trainer"], ["Index(index=2)", "extra"]], "metadata": null}, "dict_18": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_19": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_9": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.lightning.io.pl", "name": "TrainerContext"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_9"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_18"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_19"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "trainer_context_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='model')", {"type": "ref", "key": "gptmodel_1"}], ["Attr(name='trainer')", {"type": "ref", "key": "trainer_1"}], ["Attr(name='extra')", {"type": "ref", "key": "dict_17"}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_9"}, "paths": ["<root>"]}}, "refcounts": {"tuple_1": 1, "dict_1": 1, "dict_2": 1, "buildable_traverser_metadata_1": 1, "annulus_gptconfig_1": 1, "tuple_2": 1, "dict_3": 1, "dict_4": 1, "buildable_traverser_metadata_2": 1, "optimizer_config_1": 1, "tuple_3": 1, "dict_5": 1, "dict_6": 1, "buildable_traverser_metadata_3": 1, "cosine_annealing_scheduler_1": 1, "tuple_4": 1, "dict_7": 1, "dict_8": 1, "buildable_traverser_metadata_4": 1, "megatron_optimizer_module_1": 1, "tuple_5": 1, "dict_9": 1, "dict_10": 1, "buildable_traverser_metadata_5": 1, "auto_tokenizer_1": 1, "tuple_6": 1, "dict_11": 1, "dict_12": 1, "buildable_traverser_metadata_6": 1, "gptmodel_1": 1, "tuple_7": 1, "dict_13": 1, "dict_14": 1, "buildable_traverser_metadata_7": 1, "megatron_strategy_1": 1, "_wiring_audit_1": 1, "_na_nskip_callback_1": 1, "_lrprobe_1": 1, "list_1": 1, "megatron_mixed_precision_1": 1, "tuple_8": 1, "dict_15": 1, "dict_16": 1, "buildable_traverser_metadata_8": 1, "trainer_1": 1, "dict_17": 1, "tuple_9": 1, "dict_18": 1, "dict_19": 1, "buildable_traverser_metadata_9": 1, "trainer_context_1": 1}, "version": "0.0.1"}

v11-33e-2001/context/model.yaml

@@ -0,0 +1,373 @@
+_target_: nemo.collections.llm.gpt.model.base.GPTModel
+config:
+  _cpu_offloading_context: null
+  _target_: annulus.model.base_config.AnnulusGPTConfig
+  account_for_embedding_in_pipeline_split: false
+  account_for_loss_in_pipeline_split: false
+  activation_func:
+    _call_: false
+    _target_: torch.nn.functional.silu
+  activation_func_clamp_value: null
+  activation_func_fp8_input_store: false
+  add_bias_linear: false
+  add_qkv_bias: false
+  apply_query_key_layer_scaling: false
+  apply_residual_connection_post_layernorm: false
+  apply_rope_fusion: false
+  async_tensor_model_parallel_allreduce: true
+  attention_backend:
+    _call_: true
+    _target_: megatron.core.transformer.enums.AttnBackend
+  attention_dropout: 0.1
+  attention_output_gate: false
+  attention_softmax_in_fp32: false
+  autocast_dtype:
+    _call_: false
+    _target_: torch.bfloat16
+  barrier_with_L1_time: true
+  batch_invariant_mode: false
+  batch_p2p_comm: true
+  batch_p2p_sync: true
+  bf16: true
+  bias_activation_fusion: false
+  bias_dropout_fusion: false
+  calculate_per_token_loss: false
+  clone_scatter_output_in_embedding: true
+  config_logger_dir: ''
+  context_parallel_size: 1
+  cp_comm_type: null
+  cpu_offloading: false
+  cpu_offloading_activations: true
+  cpu_offloading_double_buffering: false
+  cpu_offloading_num_layers: 0
+  cpu_offloading_weights: false
+  cross_entropy_fusion_impl: native
+  cross_entropy_loss_fusion: true
+  cuda_graph_impl: none
+  cuda_graph_retain_backward_graph: false
+  cuda_graph_scope: full
+  cuda_graph_use_single_mempool: false
+  cuda_graph_warmup_steps: 3
+  data_step_fn:
+    _call_: false
+    _target_: nemo.collections.llm.gpt.model.base.gpt_data_step
+  deallocate_pipeline_outputs: true
+  defer_embedding_wgrad_compute: false
+  delay_wgrad_compute: false
+  deterministic_mode: false
+  disable_bf16_reduced_precision_matmul: false
+  disable_parameter_transpose_cache: false
+  distribute_saved_activations: false
+  dsa_indexer_head_dim: null
+  dsa_indexer_loss_coeff: null
+  dsa_indexer_n_heads: null
+  dsa_indexer_topk: null
+  dsa_indexer_use_sparse_loss: false
+  embedding_init_method: null
+  embedding_init_method_std: null
+  enable_autocast: false
+  enable_cuda_graph: false
+  ep_overlap_early_attn_memory_release: false
+  experimental_attention_variant: null
+  expert_model_parallel_size: 1
+  expert_tensor_parallel_size: null
+  external_cuda_graph: false
+  ffn_hidden_size: 320
+  finalize_model_grads_func: null
+  fine_grained_activation_offloading: false
+  first_last_layers_bf16: false
+  flash_decode: false
+  forward_step_fn:
+    _call_: false
+    _target_: nemo.collections.llm.gpt.model.base.gpt_forward_step
+  fp16: false
+  fp16_lm_cross_entropy: false
+  fp32_residual_connection: false
+  fp4: null
+  fp4_param: false
+  fp4_quantizer_factory: null
+  fp4_recipe: nvfp4
+  fp8: null
+  fp8_amax_compute_algo: most_recent
+  fp8_amax_history_len: 1
+  fp8_dot_product_attention: false
+  fp8_interval: 1
+  fp8_margin: 0
+  fp8_multi_head_attention: false
+  fp8_param: false
+  fp8_quantizer_factory: null
+  fp8_recipe: delayed
+  fp8_wgrad: true
+  fused_single_qkv_rope: false
+  gated_linear_unit: true
+  generation_config: null
+  glu_linear_offset: 0.0
+  grad_scale_func: null
+  grad_sync_func: null
+  gradient_accumulation_fusion: true
+  hetereogenous_dist_checkpoint: false
+  heterogeneous_block_specs: false
+  hidden_dropout: 0.1
+  hidden_size: 1024
+  hierarchical_context_parallel_sizes: null
+  hybrid_context_parallel: false
+  inference_fuse_tp_communication: false
+  inference_rng_tracker: false
+  inference_sampling_seed: 42
+  init_method: null
+  init_method_std: 0.02
+  init_model_with_meta_device: false
+  is_hybrid_model: false
+  kitchen_attention_backend: sdpa
+  kv_channels: 128
+  layernorm_epsilon: 1.0e-05
+  layernorm_zero_centered_gamma: false
+  linear_attention_freq: null
+  linear_conv_kernel_dim: 4
+  linear_key_head_dim: 128
+  linear_num_key_heads: 16
+  linear_num_value_heads: 32
+  linear_value_head_dim: 128
+  log_max_attention_logit: false
+  make_vocab_size_divisible_by: 128
+  mamba_head_dim: 64
+  mamba_num_groups: 8
+  mamba_num_heads: null
+  mamba_state_dim: 128
+  masked_softmax_fusion: true
+  max_seqlen_per_dp_cp_rank: null
+  memory_efficient_layer_norm: false
+  microbatch_group_size_per_vp_stage: 1
+  min_offloaded_tensor_size: 1048576
+  mlp_chunks_for_prefill: 1
+  moe_apply_probs_on_input: false
+  moe_aux_loss_coeff: 0.0
+  moe_deepep_num_sms: 20
+  moe_enable_deepep: false
+  moe_enable_routing_replay: false
+  moe_expert_capacity_factor: null
+  moe_extended_tp: false
+  moe_ffn_hidden_size: 320
+  moe_flex_dispatcher_backend: deepep
+  moe_grouped_gemm: true
+  moe_hybridep_num_sms: 16
+  moe_input_jitter_eps: null
+  moe_latent_size: null
+  moe_layer_freq: 1
+  moe_layer_recompute: false
+  moe_pad_expert_input_to_capacity: false
+  moe_per_layer_logging: false
+  moe_permute_fusion: false
+  moe_router_bias_update_rate: 0.001
+  moe_router_dtype: null
+  moe_router_enable_expert_bias: true
+  moe_router_force_load_balancing: false
+  moe_router_fusion: false
+  moe_router_group_topk: null
+  moe_router_load_balancing_type: none
+  moe_router_num_groups: null
+  moe_router_padding_for_fp8: false
+  moe_router_padding_for_quantization: false
+  moe_router_pre_softmax: false
+  moe_router_score_function: sigmoid
+  moe_router_topk: 8
+  moe_router_topk_limited_devices: null
+  moe_router_topk_scaling_factor: null
+  moe_shared_expert_gate: false
+  moe_shared_expert_intermediate_size: 2048
+  moe_shared_expert_overlap: false
+  moe_token_dispatcher_type: alltoall
+  moe_token_drop_policy: probs
+  moe_token_dropping: false
+  moe_use_legacy_grouped_gemm: false
+  moe_z_loss_coeff: 0.001
+  mrope_section: null
+  mtp_loss_scaling_factor: 0.1
+  mtp_num_layers: null
+  mtp_standalone: false
+  multi_latent_attention: false
+  no_rope_freq: null
+  no_sync_func: null
+  normalization: RMSNorm
+  num_attention_heads: 8
+  num_layers: 24
+  num_layers_at_end_in_bf16: 0
+  num_layers_at_start_in_bf16: 0
+  num_layers_in_first_pipeline_stage: null
+  num_layers_in_last_pipeline_stage: null
+  num_microbatches_with_partial_activation_checkpoints: null
+  num_moe_experts: 33
+  num_query_groups: 4
+  offload_modules: []
+  output_layer_init_method: null
+  overlap_moe_expert_parallel_comm: false
+  overlap_p2p_comm: false
+  overlap_p2p_comm_warmup_flush: false
+  parallel_output: true
+  param_sync_func: null
+  params_dtype:
+    _call_: false
+    _target_: torch.float32
+  perform_initialization: true
+  persist_layer_norm: false
+  pipeline_dtype:
+    _call_: false
+    _target_: torch.bfloat16
+  pipeline_model_parallel_comm_backend: null
+  pipeline_model_parallel_layout: null
+  pipeline_model_parallel_size: 1
+  position_embedding_type: rope
+  qk_clip: false
+  qk_clip_alpha: 0.5
+  qk_clip_threshold: 100
+  qk_l2_norm: false
+  qk_layernorm: false
+  quant_recipe: null
+  recompute_granularity: null
+  recompute_method: null
+  recompute_modules: null
+  recompute_num_layers: null
+  rotary_base: 500000.0
+  rotary_interleaved: false
+  rotary_percent: 1.0
+  scatter_embedding_sequence_parallel: true
+  seq_len_interpolation_factor: null
+  seq_length: 8192
+  sequence_parallel: false
+  share_embeddings_and_output_weights: true
+  softmax_scale: null
+  softmax_type: vanilla
+  symmetric_ar_type: null
+  tensor_model_parallel_size: 1
+  test_mode: false
+  timers: null
+  tp_comm_atomic_ag: false
+  tp_comm_atomic_rs: false
+  tp_comm_bootstrap_backend: nccl
+  tp_comm_bulk_dgrad: true
+  tp_comm_bulk_wgrad: true
+  tp_comm_overlap: false
+  tp_comm_overlap_ag: true
+  tp_comm_overlap_cfg: null
+  tp_comm_overlap_disable_fc1: false
+  tp_comm_overlap_disable_qkv: false
+  tp_comm_overlap_rs: true
+  tp_comm_overlap_rs_dgrad: false
+  tp_comm_split_ag: true
+  tp_comm_split_rs: true
+  tp_only_amax_red: false
+  transformer_impl: transformer_engine
+  transformer_layer_spec:
+    _call_: false
+    _target_: nemo.collections.llm.gpt.model.base.default_layer_spec
+  use_cpu_initialization: false
+  use_fused_weighted_squared_relu: false
+  use_inference_optimized_layers: false
+  use_kitchen: false
+  use_kitchen_attention: false
+  use_mamba_mem_eff_path: true
+  use_ring_exchange_p2p: false
+  use_te_activation_func: false
+  use_te_rng_tracker: false
+  use_transformer_engine_full_layer_spec: false
+  variable_seq_lengths: false
+  virtual_pipeline_model_parallel_size: null
+  vocab_size: null
+  wgrad_deferral_limit: 0
+  window_attn_skip_freq: null
+  window_size: null
+model_context_managers: []
+model_transform: null
+optim:
+  _target_: nemo.lightning.pytorch.optim.megatron.MegatronOptimizerModule
+  config:
+    _target_: megatron.core.optimizer.optimizer_config.OptimizerConfig
+    adam_beta1: 0.9
+    adam_beta2: 0.98
+    adam_eps: 1.0e-08
+    apply_wd_to_qk_layernorm: false
+    barrier_with_L1_time: false
+    bf16: true
+    clip_grad: 1.0
+    config_logger_dir: ''
+    decoupled_lr: null
+    decoupled_min_lr: null
+    decoupled_weight_decay: true
+    exp_avg_dtype:
+      _call_: false
+      _target_: torch.float32
+    exp_avg_sq_dtype:
+      _call_: false
+      _target_: torch.float32
+    fp16: false
+    fp8_recipe: null
+    hysteresis: 2
+    initial_loss_scale: 4294967296
+    log_num_zeros_in_grad: false
+    loss_scale: null
+    loss_scale_window: 1000
+    lr: 0.0003
+    main_grads_dtype:
+      _call_: false
+      _target_: torch.float32
+    main_params_dtype:
+      _call_: false
+      _target_: torch.float32
+    min_loss_scale: 1.0
+    min_lr: null
+    muon_extra_scale_factor: 1.0
+    muon_fp32_matmul_prec: medium
+    muon_momentum: 0.95
+    muon_num_ns_steps: 5
+    muon_scale_mode: spectral
+    muon_split_qkv: true
+    muon_tp_mode: blockwise
+    muon_use_nesterov: false
+    optimizer: adam
+    optimizer_cpu_offload: false
+    optimizer_offload_fraction: 0.0
+    overlap_cpu_optimizer_d2h_h2d: false
+    overlap_param_gather: false
+    overlap_param_gather_with_optimizer_step: false
+    params_dtype:
+      _call_: false
+      _target_: torch.float32
+    pin_cpu_grads: true
+    pin_cpu_params: true
+    reuse_grad_buf_for_mxfp8_param_ag: false
+    sgd_momentum: 0.9
+    store_param_remainders: true
+    timers: null
+    use_distributed_optimizer: false
+    use_precision_aware_optimizer: false
+    use_torch_optimizer_for_cpu_offload: false
+    weight_decay: 0.1
+  lr_mult: 1.0
+  lr_scheduler:
+    _target_: nemo.lightning.pytorch.optim.lr_scheduler.CosineAnnealingScheduler
+    constant_steps: 0
+    frequency: 1
+    interval: step
+    max_steps: 50000
+    min_lr: 3.0e-06
+    monitor: val_loss
+    warmup_steps: 2500
+  no_weight_decay_cond: null
+  scale_lr_cond: null
+tokenizer:
+  _target_: nemo.collections.common.tokenizers.huggingface.auto_tokenizer.AutoTokenizer
+  additional_special_tokens: []
+  bos_token: null
+  chat_template: null
+  cls_token: null
+  eos_token: null
+  include_special_tokens: false
+  mask_token: null
+  merges_file: null
+  pad_token: null
+  pretrained_model_name: qwen2-annulus-v5f
+  sep_token: null
+  trust_remote_code: false
+  unk_token: null
+  use_fast: true
+  vocab_file: null

v11-33e-2001/context/qwen2-annulus-v5f/added_tokens.json

@@ -0,0 +1,236 @@
+{
+  "<answer>": 151875,
+  "<beyond_horizon>": 151872,
+  "<date?>": 151874,
+  "<far_future>": 151873,
+  "<future>": 151876,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "[Y1700s]": 151647,
+  "[Y1710s]": 151648,
+  "[Y1720s]": 151649,
+  "[Y1730s]": 151650,
+  "[Y1740s]": 151651,
+  "[Y1750s]": 151652,
+  "[Y1760s]": 151653,
+  "[Y1770s]": 151654,
+  "[Y1780s]": 151655,
+  "[Y1790s]": 151656,
+  "[Y1800s]": 151657,
+  "[Y1810s]": 151658,
+  "[Y1820s]": 151659,
+  "[Y1830]": 151660,
+  "[Y1831]": 151661,
+  "[Y1832]": 151662,
+  "[Y1833]": 151663,
+  "[Y1834]": 151664,
+  "[Y1835]": 151665,
+  "[Y1836]": 151666,
+  "[Y1837]": 151667,
+  "[Y1838]": 151668,
+  "[Y1839]": 151669,
+  "[Y1840]": 151670,
+  "[Y1841]": 151671,
+  "[Y1842]": 151672,
+  "[Y1843]": 151673,
+  "[Y1844]": 151674,
+  "[Y1845]": 151675,
+  "[Y1846]": 151676,
+  "[Y1847]": 151677,
+  "[Y1848]": 151678,
+  "[Y1849]": 151679,
+  "[Y1850]": 151680,
+  "[Y1851]": 151681,
+  "[Y1852]": 151682,
+  "[Y1853]": 151683,
+  "[Y1854]": 151684,
+  "[Y1855]": 151685,
+  "[Y1856]": 151686,
+  "[Y1857]": 151687,
+  "[Y1858]": 151688,
+  "[Y1859]": 151689,
+  "[Y1860]": 151690,
+  "[Y1861]": 151691,
+  "[Y1862]": 151692,
+  "[Y1863]": 151693,
+  "[Y1864]": 151694,
+  "[Y1865]": 151695,
+  "[Y1866]": 151696,
+  "[Y1867]": 151697,
+  "[Y1868]": 151698,
+  "[Y1869]": 151699,
+  "[Y1870]": 151700,
+  "[Y1871]": 151701,
+  "[Y1872]": 151702,
+  "[Y1873]": 151703,
+  "[Y1874]": 151704,
+  "[Y1875]": 151705,
+  "[Y1876]": 151706,
+  "[Y1877]": 151707,
+  "[Y1878]": 151708,
+  "[Y1879]": 151709,
+  "[Y1880]": 151710,
+  "[Y1881]": 151711,
+  "[Y1882]": 151712,
+  "[Y1883]": 151713,
+  "[Y1884]": 151714,
+  "[Y1885]": 151715,
+  "[Y1886]": 151716,
+  "[Y1887]": 151717,
+  "[Y1888]": 151718,
+  "[Y1889]": 151719,
+  "[Y1890]": 151720,
+  "[Y1891]": 151721,
+  "[Y1892]": 151722,
+  "[Y1893]": 151723,
+  "[Y1894]": 151724,
+  "[Y1895]": 151725,
+  "[Y1896]": 151726,
+  "[Y1897]": 151727,
+  "[Y1898]": 151728,
+  "[Y1899]": 151729,
+  "[Y1900]": 151730,
+  "[Y1901]": 151731,
+  "[Y1902]": 151732,
+  "[Y1903]": 151733,
+  "[Y1904]": 151734,
+  "[Y1905]": 151735,
+  "[Y1906]": 151736,
+  "[Y1907]": 151737,
+  "[Y1908]": 151738,
+  "[Y1909]": 151739,
+  "[Y1910]": 151740,
+  "[Y1911]": 151741,
+  "[Y1912]": 151742,
+  "[Y1913]": 151743,
+  "[Y1914]": 151744,
+  "[Y1915]": 151745,
+  "[Y1916]": 151746,
+  "[Y1917]": 151747,
+  "[Y1918]": 151748,
+  "[Y1919]": 151749,
+  "[Y1920]": 151750,
+  "[Y1921]": 151751,
+  "[Y1922]": 151752,
+  "[Y1923]": 151753,
+  "[Y1924]": 151754,
+  "[Y1925]": 151755,
+  "[Y1926]": 151756,
+  "[Y1927]": 151757,
+  "[Y1928]": 151758,
+  "[Y1929]": 151759,
+  "[Y1930]": 151760,
+  "[Y1931]": 151761,
+  "[Y1932]": 151762,
+  "[Y1933]": 151763,
+  "[Y1934]": 151764,
+  "[Y1935]": 151765,
+  "[Y1936]": 151766,
+  "[Y1937]": 151767,
+  "[Y1938]": 151768,
+  "[Y1939]": 151769,
+  "[Y1940]": 151770,
+  "[Y1941]": 151771,
+  "[Y1942]": 151772,
+  "[Y1943]": 151773,
+  "[Y1944]": 151774,
+  "[Y1945]": 151775,
+  "[Y1946]": 151776,
+  "[Y1947]": 151777,
+  "[Y1948]": 151778,
+  "[Y1949]": 151779,
+  "[Y1950]": 151780,
+  "[Y1951]": 151781,
+  "[Y1952]": 151782,
+  "[Y1953]": 151783,
+  "[Y1954]": 151784,
+  "[Y1955]": 151785,
+  "[Y1956]": 151786,
+  "[Y1957]": 151787,
+  "[Y1958]": 151788,
+  "[Y1959]": 151789,
+  "[Y1960]": 151790,
+  "[Y1961]": 151791,
+  "[Y1962]": 151792,
+  "[Y1963]": 151793,
+  "[Y1964]": 151794,
+  "[Y1965]": 151795,
+  "[Y1966]": 151796,
+  "[Y1967]": 151797,
+  "[Y1968]": 151798,
+  "[Y1969]": 151799,
+  "[Y1970]": 151800,
+  "[Y1971]": 151801,
+  "[Y1972]": 151802,
+  "[Y1973]": 151803,
+  "[Y1974]": 151804,
+  "[Y1975]": 151805,
+  "[Y1976]": 151806,
+  "[Y1977]": 151807,
+  "[Y1978]": 151808,
+  "[Y1979]": 151809,
+  "[Y1980]": 151810,
+  "[Y1981]": 151811,
+  "[Y1982]": 151812,
+  "[Y1983]": 151813,
+  "[Y1984]": 151814,
+  "[Y1985]": 151815,
+  "[Y1986]": 151816,
+  "[Y1987]": 151817,
+  "[Y1988]": 151818,
+  "[Y1989]": 151819,
+  "[Y1990]": 151820,
+  "[Y1991]": 151821,
+  "[Y1992]": 151822,
+  "[Y1993]": 151823,
+  "[Y1994]": 151824,
+  "[Y1995]": 151825,
+  "[Y1996]": 151826,
+  "[Y1997]": 151827,
+  "[Y1998]": 151828,
+  "[Y1999]": 151829,
+  "[Y2000]": 151830,
+  "[Y2001]": 151831,
+  "[Y2002]": 151832,
+  "[Y2003]": 151833,
+  "[Y2004]": 151834,
+  "[Y2005]": 151835,
+  "[Y2006]": 151836,
+  "[Y2007]": 151837,
+  "[Y2008]": 151838,
+  "[Y2009]": 151839,
+  "[Y2010]": 151840,
+  "[Y2011]": 151841,
+  "[Y2012]": 151842,
+  "[Y2013]": 151843,
+  "[Y2014]": 151844,
+  "[Y2015]": 151845,
+  "[Y2016]": 151846,
+  "[Y2017]": 151847,
+  "[Y2018]": 151848,
+  "[Y2019]": 151849,
+  "[Y2020]": 151850,
+  "[Y2021]": 151851,
+  "[Y2022]": 151852,
+  "[Y2023]": 151853,
+  "[Y2024]": 151854,
+  "[Y2025]": 151855,
+  "[Y2026]": 151856,
+  "[Y2027]": 151857,
+  "[Y2028]": 151858,
+  "[Y2029]": 151859,
+  "[Y2030]": 151860,
+  "[Y2031]": 151861,
+  "[Y2032]": 151862,
+  "[Y2033]": 151863,
+  "[Y2034]": 151864,
+  "[Y2035]": 151865,
+  "[Y2036]": 151866,
+  "[Y2037]": 151867,
+  "[Y2038]": 151868,
+  "[Y2039]": 151869,
+  "[Y2040]": 151870,
+  "[Y_UNKNOWN]": 151871,
+  "[Ypre1700]": 151646
+}

v11-33e-2001/context/qwen2-annulus-v5f/special_tokens_map.json

@@ -0,0 +1,25 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<future>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

v11-33e-2001/context/qwen2-annulus-v5f/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de9fa01aedf3f5b78c1630cfa0112bf6668aa99127cfd315eb6b2c7605bd27a1
+size 11460806

v11-33e-2001/context/qwen2-annulus-v5f/tokenizer_config.json

@@ -0,0 +1,1891 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "[Ypre1700]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "[Y1700s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "[Y1710s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "[Y1720s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "[Y1730s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "[Y1740s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "[Y1750s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "[Y1760s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "[Y1770s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "[Y1780s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "[Y1790s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "[Y1800s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151658": {
+      "content": "[Y1810s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151659": {
+      "content": "[Y1820s]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151660": {
+      "content": "[Y1830]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151661": {
+      "content": "[Y1831]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151662": {
+      "content": "[Y1832]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151663": {
+      "content": "[Y1833]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151664": {
+      "content": "[Y1834]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151665": {
+      "content": "[Y1835]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "[Y1836]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "[Y1837]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "[Y1838]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "[Y1839]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "[Y1840]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "[Y1841]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "[Y1842]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "[Y1843]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "[Y1844]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "[Y1845]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151676": {
+      "content": "[Y1846]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151677": {
+      "content": "[Y1847]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151678": {
+      "content": "[Y1848]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151679": {
+      "content": "[Y1849]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151680": {
+      "content": "[Y1850]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151681": {
+      "content": "[Y1851]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151682": {
+      "content": "[Y1852]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151683": {
+      "content": "[Y1853]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151684": {
+      "content": "[Y1854]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151685": {
+      "content": "[Y1855]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151686": {
+      "content": "[Y1856]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151687": {
+      "content": "[Y1857]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151688": {
+      "content": "[Y1858]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151689": {
+      "content": "[Y1859]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151690": {
+      "content": "[Y1860]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151691": {
+      "content": "[Y1861]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151692": {
+      "content": "[Y1862]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151693": {
+      "content": "[Y1863]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151694": {
+      "content": "[Y1864]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151695": {
+      "content": "[Y1865]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151696": {
+      "content": "[Y1866]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151697": {
+      "content": "[Y1867]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151698": {
+      "content": "[Y1868]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151699": {
+      "content": "[Y1869]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151700": {
+      "content": "[Y1870]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151701": {
+      "content": "[Y1871]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151702": {
+      "content": "[Y1872]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151703": {
+      "content": "[Y1873]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151704": {
+      "content": "[Y1874]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151705": {
+      "content": "[Y1875]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151706": {
+      "content": "[Y1876]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151707": {
+      "content": "[Y1877]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151708": {
+      "content": "[Y1878]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151709": {
+      "content": "[Y1879]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151710": {
+      "content": "[Y1880]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151711": {
+      "content": "[Y1881]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151712": {
+      "content": "[Y1882]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151713": {
+      "content": "[Y1883]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151714": {
+      "content": "[Y1884]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151715": {
+      "content": "[Y1885]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151716": {
+      "content": "[Y1886]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151717": {
+      "content": "[Y1887]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151718": {
+      "content": "[Y1888]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151719": {
+      "content": "[Y1889]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151720": {
+      "content": "[Y1890]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151721": {
+      "content": "[Y1891]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151722": {
+      "content": "[Y1892]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151723": {
+      "content": "[Y1893]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151724": {
+      "content": "[Y1894]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151725": {
+      "content": "[Y1895]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151726": {
+      "content": "[Y1896]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151727": {
+      "content": "[Y1897]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151728": {
+      "content": "[Y1898]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151729": {
+      "content": "[Y1899]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151730": {
+      "content": "[Y1900]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151731": {
+      "content": "[Y1901]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151732": {
+      "content": "[Y1902]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151733": {
+      "content": "[Y1903]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151734": {
+      "content": "[Y1904]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151735": {
+      "content": "[Y1905]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151736": {
+      "content": "[Y1906]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151737": {
+      "content": "[Y1907]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151738": {
+      "content": "[Y1908]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151739": {
+      "content": "[Y1909]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151740": {
+      "content": "[Y1910]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151741": {
+      "content": "[Y1911]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151742": {
+      "content": "[Y1912]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151743": {
+      "content": "[Y1913]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151744": {
+      "content": "[Y1914]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151745": {
+      "content": "[Y1915]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151746": {
+      "content": "[Y1916]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151747": {
+      "content": "[Y1917]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151748": {
+      "content": "[Y1918]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151749": {
+      "content": "[Y1919]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151750": {
+      "content": "[Y1920]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151751": {
+      "content": "[Y1921]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151752": {
+      "content": "[Y1922]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151753": {
+      "content": "[Y1923]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151754": {
+      "content": "[Y1924]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151755": {
+      "content": "[Y1925]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151756": {
+      "content": "[Y1926]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151757": {
+      "content": "[Y1927]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151758": {
+      "content": "[Y1928]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151759": {
+      "content": "[Y1929]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151760": {
+      "content": "[Y1930]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151761": {
+      "content": "[Y1931]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151762": {
+      "content": "[Y1932]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151763": {
+      "content": "[Y1933]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151764": {
+      "content": "[Y1934]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151765": {
+      "content": "[Y1935]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151766": {
+      "content": "[Y1936]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151767": {
+      "content": "[Y1937]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151768": {
+      "content": "[Y1938]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151769": {
+      "content": "[Y1939]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151770": {
+      "content": "[Y1940]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151771": {
+      "content": "[Y1941]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151772": {
+      "content": "[Y1942]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151773": {
+      "content": "[Y1943]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151774": {
+      "content": "[Y1944]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151775": {
+      "content": "[Y1945]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151776": {
+      "content": "[Y1946]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151777": {
+      "content": "[Y1947]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151778": {
+      "content": "[Y1948]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151779": {
+      "content": "[Y1949]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151780": {
+      "content": "[Y1950]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151781": {
+      "content": "[Y1951]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151782": {
+      "content": "[Y1952]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151783": {
+      "content": "[Y1953]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151784": {
+      "content": "[Y1954]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151785": {
+      "content": "[Y1955]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151786": {
+      "content": "[Y1956]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151787": {
+      "content": "[Y1957]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151788": {
+      "content": "[Y1958]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151789": {
+      "content": "[Y1959]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151790": {
+      "content": "[Y1960]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151791": {
+      "content": "[Y1961]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151792": {
+      "content": "[Y1962]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151793": {
+      "content": "[Y1963]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151794": {
+      "content": "[Y1964]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151795": {
+      "content": "[Y1965]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151796": {
+      "content": "[Y1966]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151797": {
+      "content": "[Y1967]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151798": {
+      "content": "[Y1968]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151799": {
+      "content": "[Y1969]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151800": {
+      "content": "[Y1970]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151801": {
+      "content": "[Y1971]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151802": {
+      "content": "[Y1972]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151803": {
+      "content": "[Y1973]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151804": {
+      "content": "[Y1974]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151805": {
+      "content": "[Y1975]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151806": {
+      "content": "[Y1976]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151807": {
+      "content": "[Y1977]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151808": {
+      "content": "[Y1978]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151809": {
+      "content": "[Y1979]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151810": {
+      "content": "[Y1980]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151811": {
+      "content": "[Y1981]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151812": {
+      "content": "[Y1982]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151813": {
+      "content": "[Y1983]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151814": {
+      "content": "[Y1984]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151815": {
+      "content": "[Y1985]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151816": {
+      "content": "[Y1986]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151817": {
+      "content": "[Y1987]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151818": {
+      "content": "[Y1988]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151819": {
+      "content": "[Y1989]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151820": {
+      "content": "[Y1990]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151821": {
+      "content": "[Y1991]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151822": {
+      "content": "[Y1992]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151823": {
+      "content": "[Y1993]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151824": {
+      "content": "[Y1994]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151825": {
+      "content": "[Y1995]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151826": {
+      "content": "[Y1996]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151827": {
+      "content": "[Y1997]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151828": {
+      "content": "[Y1998]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151829": {
+      "content": "[Y1999]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151830": {
+      "content": "[Y2000]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151831": {
+      "content": "[Y2001]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151832": {
+      "content": "[Y2002]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151833": {
+      "content": "[Y2003]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151834": {
+      "content": "[Y2004]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151835": {
+      "content": "[Y2005]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151836": {
+      "content": "[Y2006]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151837": {
+      "content": "[Y2007]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151838": {
+      "content": "[Y2008]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151839": {
+      "content": "[Y2009]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151840": {
+      "content": "[Y2010]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151841": {
+      "content": "[Y2011]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151842": {
+      "content": "[Y2012]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151843": {
+      "content": "[Y2013]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151844": {
+      "content": "[Y2014]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151845": {
+      "content": "[Y2015]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151846": {
+      "content": "[Y2016]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151847": {
+      "content": "[Y2017]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151848": {
+      "content": "[Y2018]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151849": {
+      "content": "[Y2019]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151850": {
+      "content": "[Y2020]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151851": {
+      "content": "[Y2021]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151852": {
+      "content": "[Y2022]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151853": {
+      "content": "[Y2023]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151854": {
+      "content": "[Y2024]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151855": {
+      "content": "[Y2025]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151856": {
+      "content": "[Y2026]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151857": {
+      "content": "[Y2027]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151858": {
+      "content": "[Y2028]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151859": {
+      "content": "[Y2029]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151860": {
+      "content": "[Y2030]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151861": {
+      "content": "[Y2031]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151862": {
+      "content": "[Y2032]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151863": {
+      "content": "[Y2033]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151864": {
+      "content": "[Y2034]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151865": {
+      "content": "[Y2035]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151866": {
+      "content": "[Y2036]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151867": {
+      "content": "[Y2037]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151868": {
+      "content": "[Y2038]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151869": {
+      "content": "[Y2039]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151870": {
+      "content": "[Y2040]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151871": {
+      "content": "[Y_UNKNOWN]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151872": {
+      "content": "<beyond_horizon>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151873": {
+      "content": "<far_future>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151874": {
+      "content": "<date?>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151875": {
+      "content": "<answer>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151876": {
+      "content": "<future>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<future>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}