context/io.json

{"root": {"type": "ref", "key": "trainer_context_1"}, "objects": {"tuple_1": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "tensor_model_parallel_size"], ["Index(index=1)", "pipeline_model_parallel_comm_backend"], ["Index(index=2)", "pipeline_model_parallel_size"], ["Index(index=3)", "virtual_pipeline_model_parallel_size"], ["Index(index=4)", "sequence_parallel"], ["Index(index=5)", "context_parallel_size"], ["Index(index=6)", "expert_model_parallel_size"], ["Index(index=7)", "expert_tensor_parallel_size"], ["Index(index=8)", "moe_extended_tp"], ["Index(index=9)", "autocast_dtype"], ["Index(index=10)", "use_te_rng_tracker"], ["Index(index=11)", "tp_comm_overlap"], ["Index(index=12)", "cross_entropy_loss_fusion"], ["Index(index=13)", "pipeline_dtype"], ["Index(index=14)", "microbatch_group_size_per_vp_stage"], ["Index(index=15)", "num_layers_in_first_pipeline_stage"], ["Index(index=16)", "num_layers_in_last_pipeline_stage"], ["Index(index=17)", "pipeline_model_parallel_layout"], ["Index(index=18)", "account_for_embedding_in_pipeline_split"], ["Index(index=19)", "account_for_loss_in_pipeline_split"], ["Index(index=20)", "hidden_dropout"], ["Index(index=21)", "attention_dropout"], ["Index(index=22)", "fp32_residual_connection"], ["Index(index=23)", "calculate_per_token_loss"], ["Index(index=24)", "distribute_saved_activations"], ["Index(index=25)", "fp8_wgrad"], ["Index(index=26)", "num_layers_at_start_in_bf16"], ["Index(index=27)", "num_layers_at_end_in_bf16"], ["Index(index=28)", "seq_length"], ["Index(index=29)", "to_upper"]], "metadata": null}, "dict_1": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_2": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.collections.llm.gpt.model.hyena", "name": "Hyena40bConfig"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_1"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_1"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_2"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "hyena40b_config_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='tensor_model_parallel_size')", {"type": "leaf", "value": 16, "paths": ["<root>.model.config.tensor_model_parallel_size"]}], ["Attr(name='pipeline_model_parallel_comm_backend')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.pipeline_model_parallel_comm_backend"]}], ["Attr(name='pipeline_model_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.model.config.pipeline_model_parallel_size"]}], ["Attr(name='virtual_pipeline_model_parallel_size')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.virtual_pipeline_model_parallel_size"]}], ["Attr(name='sequence_parallel')", {"type": "leaf", "value": true, "paths": ["<root>.model.config.sequence_parallel"]}], ["Attr(name='context_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.model.config.context_parallel_size"]}], ["Attr(name='expert_model_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.model.config.expert_model_parallel_size"]}], ["Attr(name='expert_tensor_parallel_size')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.expert_tensor_parallel_size"]}], ["Attr(name='moe_extended_tp')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.moe_extended_tp"]}], ["Attr(name='autocast_dtype')", {"type": "pyref", "module": "torch", "name": "bfloat16", "paths": ["<root>.model.config.autocast_dtype", "<root>.model.config.pipeline_dtype", "<root>.trainer.strategy.pipeline_dtype"]}], ["Attr(name='use_te_rng_tracker')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.use_te_rng_tracker"]}], ["Attr(name='tp_comm_overlap')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.tp_comm_overlap"]}], ["Attr(name='cross_entropy_loss_fusion')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.cross_entropy_loss_fusion"]}], ["Attr(name='pipeline_dtype')", {"type": "pyref", "module": "torch", "name": "bfloat16", "paths": ["<root>.model.config.autocast_dtype", "<root>.model.config.pipeline_dtype", "<root>.trainer.strategy.pipeline_dtype"]}], ["Attr(name='microbatch_group_size_per_vp_stage')", {"type": "leaf", "value": 1, "paths": ["<root>.model.config.microbatch_group_size_per_vp_stage"]}], ["Attr(name='num_layers_in_first_pipeline_stage')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.num_layers_in_first_pipeline_stage"]}], ["Attr(name='num_layers_in_last_pipeline_stage')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.num_layers_in_last_pipeline_stage"]}], ["Attr(name='pipeline_model_parallel_layout')", {"type": "leaf", "value": null, "paths": ["<root>.model.config.pipeline_model_parallel_layout"]}], ["Attr(name='account_for_embedding_in_pipeline_split')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.account_for_embedding_in_pipeline_split"]}], ["Attr(name='account_for_loss_in_pipeline_split')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.account_for_loss_in_pipeline_split"]}], ["Attr(name='hidden_dropout')", {"type": "leaf", "value": 0.001, "paths": ["<root>.model.config.hidden_dropout"]}], ["Attr(name='attention_dropout')", {"type": "leaf", "value": 0.01, "paths": ["<root>.model.config.attention_dropout"]}], ["Attr(name='fp32_residual_connection')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.fp32_residual_connection"]}], ["Attr(name='calculate_per_token_loss')", {"type": "leaf", "value": true, "paths": ["<root>.model.config.calculate_per_token_loss"]}], ["Attr(name='distribute_saved_activations')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.distribute_saved_activations"]}], ["Attr(name='fp8_wgrad')", {"type": "leaf", "value": false, "paths": ["<root>.model.config.fp8_wgrad"]}], ["Attr(name='num_layers_at_start_in_bf16')", {"type": "leaf", "value": 0, "paths": ["<root>.model.config.num_layers_at_start_in_bf16"]}], ["Attr(name='num_layers_at_end_in_bf16')", {"type": "leaf", "value": 0, "paths": ["<root>.model.config.num_layers_at_end_in_bf16"]}], ["Attr(name='seq_length')", {"type": "leaf", "value": 8192, "paths": ["<root>.model.config.seq_length"]}], ["Attr(name='to_upper')", {"type": "leaf", "value": "normalized_weighted", "paths": ["<root>.model.config.to_upper"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_1"}, "paths": ["<root>.model.config"]}, "tuple_2": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "optimizer"], ["Index(index=1)", "lr"], ["Index(index=2)", "min_lr"], ["Index(index=3)", "decoupled_lr"], ["Index(index=4)", "decoupled_min_lr"], ["Index(index=5)", "weight_decay"], ["Index(index=6)", "fp8_recipe"], ["Index(index=7)", "fp16"], ["Index(index=8)", "bf16"], ["Index(index=9)", "reuse_grad_buf_for_mxfp8_param_ag"], ["Index(index=10)", "params_dtype"], ["Index(index=11)", "use_precision_aware_optimizer"], ["Index(index=12)", "store_param_remainders"], ["Index(index=13)", "main_grads_dtype"], ["Index(index=14)", "main_params_dtype"], ["Index(index=15)", "exp_avg_dtype"], ["Index(index=16)", "exp_avg_sq_dtype"], ["Index(index=17)", "loss_scale"], ["Index(index=18)", "initial_loss_scale"], ["Index(index=19)", "min_loss_scale"], ["Index(index=20)", "loss_scale_window"], ["Index(index=21)", "hysteresis"], ["Index(index=22)", "adam_beta1"], ["Index(index=23)", "adam_beta2"], ["Index(index=24)", "adam_eps"], ["Index(index=25)", "sgd_momentum"], ["Index(index=26)", "use_distributed_optimizer"], ["Index(index=27)", "overlap_param_gather_with_optimizer_step"], ["Index(index=28)", "optimizer_cpu_offload"], ["Index(index=29)", "optimizer_offload_fraction"], ["Index(index=30)", "use_torch_optimizer_for_cpu_offload"], ["Index(index=31)", "overlap_cpu_optimizer_d2h_h2d"], ["Index(index=32)", "pin_cpu_grads"], ["Index(index=33)", "pin_cpu_params"], ["Index(index=34)", "clip_grad"], ["Index(index=35)", "log_num_zeros_in_grad"], ["Index(index=36)", "barrier_with_L1_time"], ["Index(index=37)", "timers"], ["Index(index=38)", "config_logger_dir"]], "metadata": null}, "dict_3": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_4": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_2": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "megatron.core.optimizer.optimizer_config", "name": "OptimizerConfig"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_2"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_3"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_4"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "optimizer_config_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='optimizer')", {"type": "leaf", "value": "adam", "paths": ["<root>.model.optim.config.optimizer"]}], ["Attr(name='lr')", {"type": "leaf", "value": 3e-06, "paths": ["<root>.model.optim.config.lr"]}], ["Attr(name='min_lr')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.min_lr"]}], ["Attr(name='decoupled_lr')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.decoupled_lr"]}], ["Attr(name='decoupled_min_lr')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.decoupled_min_lr"]}], ["Attr(name='weight_decay')", {"type": "leaf", "value": 0.001, "paths": ["<root>.model.optim.config.weight_decay"]}], ["Attr(name='fp8_recipe')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.fp8_recipe"]}], ["Attr(name='fp16')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.fp16"]}], ["Attr(name='bf16')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.bf16"]}], ["Attr(name='reuse_grad_buf_for_mxfp8_param_ag')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.reuse_grad_buf_for_mxfp8_param_ag"]}], ["Attr(name='params_dtype')", {"type": "pyref", "module": "torch", "name": "float32", "paths": ["<root>.model.optim.config.params_dtype", "<root>.model.optim.config.main_grads_dtype", "<root>.model.optim.config.main_params_dtype", "<root>.model.optim.config.exp_avg_dtype", "<root>.model.optim.config.exp_avg_sq_dtype"]}], ["Attr(name='use_precision_aware_optimizer')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.use_precision_aware_optimizer"]}], ["Attr(name='store_param_remainders')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.store_param_remainders"]}], ["Attr(name='main_grads_dtype')", {"type": "pyref", "module": "torch", "name": "float32", "paths": ["<root>.model.optim.config.params_dtype", "<root>.model.optim.config.main_grads_dtype", "<root>.model.optim.config.main_params_dtype", "<root>.model.optim.config.exp_avg_dtype", "<root>.model.optim.config.exp_avg_sq_dtype"]}], ["Attr(name='main_params_dtype')", {"type": "pyref", "module": "torch", "name": "float32", "paths": ["<root>.model.optim.config.params_dtype", "<root>.model.optim.config.main_grads_dtype", "<root>.model.optim.config.main_params_dtype", "<root>.model.optim.config.exp_avg_dtype", "<root>.model.optim.config.exp_avg_sq_dtype"]}], ["Attr(name='exp_avg_dtype')", {"type": "pyref", "module": "torch", "name": "float32", "paths": ["<root>.model.optim.config.params_dtype", "<root>.model.optim.config.main_grads_dtype", "<root>.model.optim.config.main_params_dtype", "<root>.model.optim.config.exp_avg_dtype", "<root>.model.optim.config.exp_avg_sq_dtype"]}], ["Attr(name='exp_avg_sq_dtype')", {"type": "pyref", "module": "torch", "name": "float32", "paths": ["<root>.model.optim.config.params_dtype", "<root>.model.optim.config.main_grads_dtype", "<root>.model.optim.config.main_params_dtype", "<root>.model.optim.config.exp_avg_dtype", "<root>.model.optim.config.exp_avg_sq_dtype"]}], ["Attr(name='loss_scale')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.loss_scale"]}], ["Attr(name='initial_loss_scale')", {"type": "leaf", "value": 4294967296, "paths": ["<root>.model.optim.config.initial_loss_scale"]}], ["Attr(name='min_loss_scale')", {"type": "leaf", "value": 1.0, "paths": ["<root>.model.optim.config.min_loss_scale"]}], ["Attr(name='loss_scale_window')", {"type": "leaf", "value": 1000, "paths": ["<root>.model.optim.config.loss_scale_window"]}], ["Attr(name='hysteresis')", {"type": "leaf", "value": 2, "paths": ["<root>.model.optim.config.hysteresis"]}], ["Attr(name='adam_beta1')", {"type": "leaf", "value": 0.9, "paths": ["<root>.model.optim.config.adam_beta1"]}], ["Attr(name='adam_beta2')", {"type": "leaf", "value": 0.95, "paths": ["<root>.model.optim.config.adam_beta2"]}], ["Attr(name='adam_eps')", {"type": "leaf", "value": 1e-08, "paths": ["<root>.model.optim.config.adam_eps"]}], ["Attr(name='sgd_momentum')", {"type": "leaf", "value": 0.9, "paths": ["<root>.model.optim.config.sgd_momentum"]}], ["Attr(name='use_distributed_optimizer')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.use_distributed_optimizer"]}], ["Attr(name='overlap_param_gather_with_optimizer_step')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.overlap_param_gather_with_optimizer_step"]}], ["Attr(name='optimizer_cpu_offload')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.optimizer_cpu_offload"]}], ["Attr(name='optimizer_offload_fraction')", {"type": "leaf", "value": 0.0, "paths": ["<root>.model.optim.config.optimizer_offload_fraction"]}], ["Attr(name='use_torch_optimizer_for_cpu_offload')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.use_torch_optimizer_for_cpu_offload"]}], ["Attr(name='overlap_cpu_optimizer_d2h_h2d')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.overlap_cpu_optimizer_d2h_h2d"]}], ["Attr(name='pin_cpu_grads')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.pin_cpu_grads"]}], ["Attr(name='pin_cpu_params')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.pin_cpu_params"]}], ["Attr(name='clip_grad')", {"type": "leaf", "value": 1.0, "paths": ["<root>.model.optim.config.clip_grad"]}], ["Attr(name='log_num_zeros_in_grad')", {"type": "leaf", "value": true, "paths": ["<root>.model.optim.config.log_num_zeros_in_grad"]}], ["Attr(name='barrier_with_L1_time')", {"type": "leaf", "value": false, "paths": ["<root>.model.optim.config.barrier_with_L1_time"]}], ["Attr(name='timers')", {"type": "leaf", "value": null, "paths": ["<root>.model.optim.config.timers"]}], ["Attr(name='config_logger_dir')", {"type": "leaf", "value": "", "paths": ["<root>.model.optim.config.config_logger_dir"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_2"}, "paths": ["<root>.model.optim.config"]}, "tuple_3": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "max_steps"], ["Index(index=1)", "warmup_steps"], ["Index(index=2)", "constant_steps"], ["Index(index=3)", "min_lr"]], "metadata": null}, "dict_5": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_6": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_3": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.lightning.pytorch.optim.lr_scheduler", "name": "CosineAnnealingScheduler"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_3"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_5"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_6"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "cosine_annealing_scheduler_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='max_steps')", {"type": "leaf", "value": 200, "paths": ["<root>.model.optim.lr_scheduler.max_steps"]}], ["Attr(name='warmup_steps')", {"type": "leaf", "value": 10, "paths": ["<root>.model.optim.lr_scheduler.warmup_steps"]}], ["Attr(name='constant_steps')", {"type": "leaf", "value": 1800, "paths": ["<root>.model.optim.lr_scheduler.constant_steps"]}], ["Attr(name='min_lr')", {"type": "leaf", "value": 1.5e-06, "paths": ["<root>.model.optim.lr_scheduler.min_lr"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_3"}, "paths": ["<root>.model.optim.lr_scheduler"]}, "tuple_4": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "config"], ["Index(index=1)", "lr_scheduler"], ["Index(index=2)", "no_weight_decay_cond"]], "metadata": null}, "dict_7": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_8": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_4": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.lightning.pytorch.optim.megatron", "name": "MegatronOptimizerModule"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_4"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_7"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_8"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "megatron_optimizer_module_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='config')", {"type": "ref", "key": "optimizer_config_1"}], ["Attr(name='lr_scheduler')", {"type": "ref", "key": "cosine_annealing_scheduler_1"}], ["Attr(name='no_weight_decay_cond')", {"type": "pyref", "module": "bionemo.evo2.utils.config", "name": "hyena_no_weight_decay_cond_with_embeddings", "paths": ["<root>.model.optim.no_weight_decay_cond"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_4"}, "paths": ["<root>.model.optim"]}, "dict_9": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}, "paths": ["<root>.model.tokenizer.special_tokens"]}, "tuple_5": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "special_tokens"]], "metadata": null}, "dict_10": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_11": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_5": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.collections.common.tokenizers.bytelevel_tokenizers", "name": "ByteLevelTokenizer"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_5"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_10"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_11"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "byte_level_tokenizer_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='special_tokens')", {"type": "ref", "key": "dict_9"}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_5"}, "paths": ["<root>.model.tokenizer"]}, "tuple_6": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "config"], ["Index(index=1)", "optim"], ["Index(index=2)", "tokenizer"], ["Index(index=3)", "model_transform"]], "metadata": null}, "dict_12": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_13": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_6": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.collections.llm.gpt.model.hyena", "name": "HyenaModel"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_6"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_12"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_13"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "hyena_model_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='config')", {"type": "ref", "key": "hyena40b_config_1"}], ["Attr(name='optim')", {"type": "ref", "key": "megatron_optimizer_module_1"}], ["Attr(name='tokenizer')", {"type": "ref", "key": "byte_level_tokenizer_1"}], ["Attr(name='model_transform')", {"type": "leaf", "value": null, "paths": ["<root>.model.model_transform"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_6"}, "paths": ["<root>.model"]}, "tuple_7": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "grad_reduce_in_fp32"], ["Index(index=1)", "overlap_grad_reduce"], ["Index(index=2)", "overlap_param_gather"], ["Index(index=3)", "align_param_gather"], ["Index(index=4)", "use_distributed_optimizer"], ["Index(index=5)", "num_distributed_optimizer_instances"], ["Index(index=6)", "check_for_nan_in_grad"], ["Index(index=7)", "check_for_large_grads"], ["Index(index=8)", "bucket_size"], ["Index(index=9)", "pad_buckets_for_high_nccl_busbw"], ["Index(index=10)", "average_in_collective"], ["Index(index=11)", "fp8_param_gather"], ["Index(index=12)", "reuse_grad_buf_for_mxfp8_param_ag"], ["Index(index=13)", "use_custom_fsdp"], ["Index(index=14)", "data_parallel_sharding_strategy"], ["Index(index=15)", "gradient_reduce_div_fusion"], ["Index(index=16)", "suggested_communication_unit_size"], ["Index(index=17)", "preserve_fp32_weights"], ["Index(index=18)", "keep_fp8_transpose_cache_when_using_custom_fsdp"], ["Index(index=19)", "nccl_ub"], ["Index(index=20)", "fsdp_double_buffer"]], "metadata": null}, "dict_14": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_15": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_7": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "megatron.core.distributed.distributed_data_parallel_config", "name": "DistributedDataParallelConfig"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_7"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_14"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_15"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "distributed_data_parallel_config_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='grad_reduce_in_fp32')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.grad_reduce_in_fp32"]}], ["Attr(name='overlap_grad_reduce')", {"type": "leaf", "value": true, "paths": ["<root>.trainer.strategy.ddp.overlap_grad_reduce"]}], ["Attr(name='overlap_param_gather')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.overlap_param_gather"]}], ["Attr(name='align_param_gather')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.align_param_gather"]}], ["Attr(name='use_distributed_optimizer')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.use_distributed_optimizer"]}], ["Attr(name='num_distributed_optimizer_instances')", {"type": "leaf", "value": 1, "paths": ["<root>.trainer.strategy.ddp.num_distributed_optimizer_instances"]}], ["Attr(name='check_for_nan_in_grad')", {"type": "leaf", "value": true, "paths": ["<root>.trainer.strategy.ddp.check_for_nan_in_grad"]}], ["Attr(name='check_for_large_grads')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.check_for_large_grads"]}], ["Attr(name='bucket_size')", {"type": "leaf", "value": null, "paths": ["<root>.trainer.strategy.ddp.bucket_size"]}], ["Attr(name='pad_buckets_for_high_nccl_busbw')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.pad_buckets_for_high_nccl_busbw"]}], ["Attr(name='average_in_collective')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.average_in_collective"]}], ["Attr(name='fp8_param_gather')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.fp8_param_gather"]}], ["Attr(name='reuse_grad_buf_for_mxfp8_param_ag')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.reuse_grad_buf_for_mxfp8_param_ag"]}], ["Attr(name='use_custom_fsdp')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.use_custom_fsdp"]}], ["Attr(name='data_parallel_sharding_strategy')", {"type": "leaf", "value": "no_shard", "paths": ["<root>.trainer.strategy.ddp.data_parallel_sharding_strategy"]}], ["Attr(name='gradient_reduce_div_fusion')", {"type": "leaf", "value": true, "paths": ["<root>.trainer.strategy.ddp.gradient_reduce_div_fusion"]}], ["Attr(name='suggested_communication_unit_size')", {"type": "leaf", "value": null, "paths": ["<root>.trainer.strategy.ddp.suggested_communication_unit_size"]}], ["Attr(name='preserve_fp32_weights')", {"type": "leaf", "value": true, "paths": ["<root>.trainer.strategy.ddp.preserve_fp32_weights"]}], ["Attr(name='keep_fp8_transpose_cache_when_using_custom_fsdp')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.keep_fp8_transpose_cache_when_using_custom_fsdp"]}], ["Attr(name='nccl_ub')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.nccl_ub"]}], ["Attr(name='fsdp_double_buffer')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.strategy.ddp.fsdp_double_buffer"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_7"}, "paths": ["<root>.trainer.strategy.ddp"]}, "tuple_8": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "tensor_model_parallel_size"], ["Index(index=1)", "pipeline_model_parallel_size"], ["Index(index=2)", "context_parallel_size"], ["Index(index=3)", "sequence_parallel"], ["Index(index=4)", "ckpt_load_optimizer"], ["Index(index=5)", "ckpt_save_optimizer"], ["Index(index=6)", "ddp"], ["Index(index=7)", "pipeline_dtype"], ["Index(index=8)", "save_ckpt_format"], ["Index(index=9)", "ckpt_async_save"], ["Index(index=10)", "ckpt_load_strictness"]], "metadata": null}, "dict_16": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_17": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_8": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.lightning.pytorch.strategies.megatron_strategy", "name": "MegatronStrategy"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_8"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_16"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_17"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "megatron_strategy_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='tensor_model_parallel_size')", {"type": "leaf", "value": 16, "paths": ["<root>.trainer.strategy.tensor_model_parallel_size"]}], ["Attr(name='pipeline_model_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.trainer.strategy.pipeline_model_parallel_size"]}], ["Attr(name='context_parallel_size')", {"type": "leaf", "value": 1, "paths": ["<root>.trainer.strategy.context_parallel_size"]}], ["Attr(name='sequence_parallel')", {"type": "leaf", "value": true, "paths": ["<root>.trainer.strategy.sequence_parallel"]}], ["Attr(name='ckpt_load_optimizer')", {"type": "leaf", "value": true, "paths": ["<root>.trainer.strategy.ckpt_load_optimizer"]}], ["Attr(name='ckpt_save_optimizer')", {"type": "leaf", "value": true, "paths": ["<root>.trainer.strategy.ckpt_save_optimizer"]}], ["Attr(name='ddp')", {"type": "ref", "key": "distributed_data_parallel_config_1"}], ["Attr(name='pipeline_dtype')", {"type": "pyref", "module": "torch", "name": "bfloat16", "paths": ["<root>.model.config.autocast_dtype", "<root>.model.config.pipeline_dtype", "<root>.trainer.strategy.pipeline_dtype"]}], ["Attr(name='save_ckpt_format')", {"type": "leaf", "value": "torch_dist", "paths": ["<root>.trainer.strategy.save_ckpt_format"]}], ["Attr(name='ckpt_async_save')", {"type": "leaf", "value": true, "paths": ["<root>.trainer.strategy.ckpt_async_save"]}], ["Attr(name='ckpt_load_strictness')", {"type": "leaf", "value": "log_all", "paths": ["<root>.trainer.strategy.ckpt_load_strictness"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_8"}, "paths": ["<root>.trainer.strategy"]}, "rich_model_summary_1": {"type": {"type": "pyref", "module": "lightning.pytorch.callbacks.rich_model_summary", "name": "RichModelSummary"}, "items": [["IdentityElement()", {"type": "leaf", "value": "ec9b7e65-1fb8-4908-8332-54cc1decba6e", "paths": ["<root>.trainer.callbacks[0]"]}]], "metadata": null, "paths": ["<root>.trainer.callbacks[0]"]}, "learning_rate_monitor_1": {"type": {"type": "pyref", "module": "lightning.pytorch.callbacks.lr_monitor", "name": "LearningRateMonitor"}, "items": [["IdentityElement()", {"type": "leaf", "value": "a4b05bc2-4f51-44d4-8a01-a11055c1dbf7", "paths": ["<root>.trainer.callbacks[1]"]}]], "metadata": null, "paths": ["<root>.trainer.callbacks[1]"]}, "timing_callback_1": {"type": {"type": "pyref", "module": "nemo.utils.exp_manager", "name": "TimingCallback"}, "items": [["IdentityElement()", {"type": "leaf", "value": "214d9bbe-8403-4d57-af90-3eca1e13d619", "paths": ["<root>.trainer.callbacks[2]"]}]], "metadata": null, "paths": ["<root>.trainer.callbacks[2]"]}, "tevcallback_1": {"type": {"type": "pyref", "module": "bionemo.evo2.utils.logging.callbacks", "name": "TEVCallback"}, "items": [["IdentityElement()", {"type": "leaf", "value": "8865f294-4368-4be4-b8b4-0eb2ca437c5e", "paths": ["<root>.trainer.callbacks[3]"]}]], "metadata": null, "paths": ["<root>.trainer.callbacks[3]"]}, "dict_18": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_19": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_9": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.lightning.pytorch.callbacks.preemption", "name": "PreemptionCallback"}], ["Attr(name='argument_names')", {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_18"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_19"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "preemption_callback_1": {"type": {"type": "pyref", "module": "nemo.lightning.pytorch.callbacks.preemption", "name": "PreemptionCallback"}, "items": [], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_9"}, "paths": ["<root>.trainer.callbacks[4]"]}, "model_checkpoint_1": {"type": {"type": "pyref", "module": "nemo.lightning.pytorch.callbacks.model_checkpoint", "name": "ModelCheckpoint"}, "items": [["IdentityElement()", {"type": "leaf", "value": "3b2060da-542b-4d2b-8e04-1775d1e6e5ba", "paths": ["<root>.trainer.callbacks[5]"]}]], "metadata": null, "paths": ["<root>.trainer.callbacks[5]"]}, "list_1": {"type": {"type": "pyref", "module": "builtins", "name": "list"}, "items": [["Index(index=0)", {"type": "ref", "key": "rich_model_summary_1"}], ["Index(index=1)", {"type": "ref", "key": "learning_rate_monitor_1"}], ["Index(index=2)", {"type": "ref", "key": "timing_callback_1"}], ["Index(index=3)", {"type": "ref", "key": "tevcallback_1"}], ["Index(index=4)", {"type": "ref", "key": "preemption_callback_1"}], ["Index(index=5)", {"type": "ref", "key": "model_checkpoint_1"}]], "metadata": null, "paths": ["<root>.trainer.callbacks"]}, "megatron_mixed_precision_1": {"type": {"type": "pyref", "module": "nemo.lightning.pytorch.plugins.mixed_precision", "name": "MegatronMixedPrecision"}, "items": [["IdentityElement()", {"type": "leaf", "value": "fdef73d5-660e-431d-9a73-49b4d561b8dd", "paths": ["<root>.trainer.plugins"]}]], "metadata": null, "paths": ["<root>.trainer.plugins"]}, "tuple_9": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "accelerator"], ["Index(index=1)", "strategy"], ["Index(index=2)", "devices"], ["Index(index=3)", "num_nodes"], ["Index(index=4)", "callbacks"], ["Index(index=5)", "max_steps"], ["Index(index=6)", "limit_val_batches"], ["Index(index=7)", "val_check_interval"], ["Index(index=8)", "num_sanity_val_steps"], ["Index(index=9)", "log_every_n_steps"], ["Index(index=10)", "enable_checkpointing"], ["Index(index=11)", "use_distributed_sampler"], ["Index(index=12)", "plugins"]], "metadata": null}, "dict_20": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_21": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_10": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.lightning.pytorch.trainer", "name": "Trainer"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_9"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_20"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_21"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "trainer_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='accelerator')", {"type": "leaf", "value": "gpu", "paths": ["<root>.trainer.accelerator"]}], ["Attr(name='strategy')", {"type": "ref", "key": "megatron_strategy_1"}], ["Attr(name='devices')", {"type": "leaf", "value": 8, "paths": ["<root>.trainer.devices"]}], ["Attr(name='num_nodes')", {"type": "leaf", "value": 2, "paths": ["<root>.trainer.num_nodes"]}], ["Attr(name='callbacks')", {"type": "ref", "key": "list_1"}], ["Attr(name='max_steps')", {"type": "leaf", "value": 200, "paths": ["<root>.trainer.max_steps"]}], ["Attr(name='limit_val_batches')", {"type": "leaf", "value": 5, "paths": ["<root>.trainer.limit_val_batches"]}], ["Attr(name='val_check_interval')", {"type": "leaf", "value": 10, "paths": ["<root>.trainer.val_check_interval"]}], ["Attr(name='num_sanity_val_steps')", {"type": "leaf", "value": 0, "paths": ["<root>.trainer.num_sanity_val_steps"]}], ["Attr(name='log_every_n_steps')", {"type": "leaf", "value": 1, "paths": ["<root>.trainer.log_every_n_steps"]}], ["Attr(name='enable_checkpointing')", {"type": "leaf", "value": true, "paths": ["<root>.trainer.enable_checkpointing"]}], ["Attr(name='use_distributed_sampler')", {"type": "leaf", "value": false, "paths": ["<root>.trainer.use_distributed_sampler"]}], ["Attr(name='plugins')", {"type": "ref", "key": "megatron_mixed_precision_1"}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_10"}, "paths": ["<root>.trainer"]}, "list_2": {"type": {"type": "pyref", "module": "builtins", "name": "list"}, "items": [["Index(index=0)", {"type": "leaf", "value": 1.0, "paths": ["<root>.extra['datamodule'].paths['train'][0]"]}], ["Index(index=1)", {"type": "leaf", "value": "/workspace/bionemo2/data/preprocessed_data/grcm39_1_4_6_7_8_9_11_12_17_uint8_distinct_byte-level_train", "paths": ["<root>.extra['datamodule'].paths['train'][1]"]}]], "metadata": null, "paths": ["<root>.extra['datamodule'].paths['train']"]}, "list_3": {"type": {"type": "pyref", "module": "builtins", "name": "list"}, "items": [["Index(index=0)", {"type": "leaf", "value": 1.0, "paths": ["<root>.extra['datamodule'].paths['validation'][0]"]}], ["Index(index=1)", {"type": "leaf", "value": "/workspace/bionemo2/data/preprocessed_data/grcm39_1_4_6_7_8_9_11_12_17_uint8_distinct_byte-level_val", "paths": ["<root>.extra['datamodule'].paths['validation'][1]"]}]], "metadata": null, "paths": ["<root>.extra['datamodule'].paths['validation']"]}, "list_4": {"type": {"type": "pyref", "module": "builtins", "name": "list"}, "items": [["Index(index=0)", {"type": "leaf", "value": 1.0, "paths": ["<root>.extra['datamodule'].paths['test'][0]"]}], ["Index(index=1)", {"type": "leaf", "value": "/workspace/bionemo2/data/preprocessed_data/grcm39_1_4_6_7_8_9_11_12_17_uint8_distinct_byte-level_test", "paths": ["<root>.extra['datamodule'].paths['test'][1]"]}]], "metadata": null, "paths": ["<root>.extra['datamodule'].paths['test']"]}, "tuple_10": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "train"], ["Index(index=1)", "validation"], ["Index(index=2)", "test"]], "metadata": null}, "tuple_11": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", {"type": "pyref", "module": "builtins", "name": "list"}], ["Index(index=1)", {"type": "ref", "key": "tuple_10"}]], "metadata": null}, "defaultdict_1": {"type": {"type": "pyref", "module": "collections", "name": "defaultdict"}, "items": [["Key(key='train')", {"type": "ref", "key": "list_2"}], ["Key(key='validation')", {"type": "ref", "key": "list_3"}], ["Key(key='test')", {"type": "ref", "key": "list_4"}]], "metadata": {"type": "ref", "key": "tuple_11"}, "paths": ["<root>.extra['datamodule'].paths"]}, "dict_22": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}, "paths": ["<root>.extra['datamodule'].tokenizer.special_tokens"]}, "tuple_12": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "special_tokens"]], "metadata": null}, "dict_23": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_24": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_11": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.collections.common.tokenizers.bytelevel_tokenizers", "name": "ByteLevelTokenizer"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_12"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_23"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_24"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "byte_level_tokenizer_2": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='special_tokens')", {"type": "ref", "key": "dict_22"}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_11"}, "paths": ["<root>.extra['datamodule'].tokenizer"]}, "tuple_13": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "paths"], ["Index(index=1)", "seq_length"], ["Index(index=2)", "tokenizer"], ["Index(index=3)", "micro_batch_size"], ["Index(index=4)", "global_batch_size"], ["Index(index=5)", "num_workers"], ["Index(index=6)", "eod_mask_loss"], ["Index(index=7)", "seed"], ["Index(index=8)", "dataset_cls"]], "metadata": null}, "dict_25": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_26": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_12": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.collections.llm.gpt.data.pre_training", "name": "PreTrainingDataModule"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_13"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_25"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_26"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "pre_training_data_module_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='paths')", {"type": "ref", "key": "defaultdict_1"}], ["Attr(name='seq_length')", {"type": "leaf", "value": 8192, "paths": ["<root>.extra['datamodule'].seq_length"]}], ["Attr(name='tokenizer')", {"type": "ref", "key": "byte_level_tokenizer_2"}], ["Attr(name='micro_batch_size')", {"type": "leaf", "value": 1, "paths": ["<root>.extra['datamodule'].micro_batch_size"]}], ["Attr(name='global_batch_size')", {"type": "leaf", "value": 8, "paths": ["<root>.extra['datamodule'].global_batch_size"]}], ["Attr(name='num_workers')", {"type": "leaf", "value": 8, "paths": ["<root>.extra['datamodule'].num_workers"]}], ["Attr(name='eod_mask_loss')", {"type": "leaf", "value": true, "paths": ["<root>.extra['datamodule'].eod_mask_loss"]}], ["Attr(name='seed')", {"type": "leaf", "value": 1234, "paths": ["<root>.extra['datamodule'].seed"]}], ["Attr(name='dataset_cls')", {"type": "pyref", "module": "nemo.collections.llm.gpt.data.megatron.hyena.evo2_dataset", "name": "Evo2DatasetPadEodLossMask", "paths": ["<root>.extra['datamodule'].dataset_cls"]}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_12"}, "paths": ["<root>.extra['datamodule']"]}, "tuple_14": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "datamodule"]], "metadata": null}, "dict_27": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [["Key(key='datamodule')", {"type": "ref", "key": "pre_training_data_module_1"}]], "metadata": {"type": "ref", "key": "tuple_14"}, "paths": ["<root>.extra"]}, "tuple_15": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [["Index(index=0)", "model"], ["Index(index=1)", "trainer"], ["Index(index=2)", "extra"]], "metadata": null}, "dict_28": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "dict_29": {"type": {"type": "pyref", "module": "builtins", "name": "dict"}, "items": [], "metadata": {"type": {"type": "pyref", "module": "builtins", "name": "tuple"}, "items": [], "metadata": null}}, "buildable_traverser_metadata_13": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}, "items": [["Attr(name='fn_or_cls')", {"type": "pyref", "module": "nemo.lightning.io.pl", "name": "TrainerContext"}], ["Attr(name='argument_names')", {"type": "ref", "key": "tuple_15"}], ["Attr(name='argument_tags')", {"type": "ref", "key": "dict_28"}], ["Attr(name='argument_history')", {"type": "ref", "key": "dict_29"}]], "metadata": {"type": "pyref", "module": "fiddle._src.config", "name": "BuildableTraverserMetadata"}}, "trainer_context_1": {"type": {"type": "pyref", "module": "fiddle._src.config", "name": "Config"}, "items": [["Attr(name='model')", {"type": "ref", "key": "hyena_model_1"}], ["Attr(name='trainer')", {"type": "ref", "key": "trainer_1"}], ["Attr(name='extra')", {"type": "ref", "key": "dict_27"}]], "metadata": {"type": "ref", "key": "buildable_traverser_metadata_13"}, "paths": ["<root>"]}}, "refcounts": {"tuple_1": 1, "dict_1": 1, "dict_2": 1, "buildable_traverser_metadata_1": 1, "hyena40b_config_1": 1, "tuple_2": 1, "dict_3": 1, "dict_4": 1, "buildable_traverser_metadata_2": 1, "optimizer_config_1": 1, "tuple_3": 1, "dict_5": 1, "dict_6": 1, "buildable_traverser_metadata_3": 1, "cosine_annealing_scheduler_1": 1, "tuple_4": 1, "dict_7": 1, "dict_8": 1, "buildable_traverser_metadata_4": 1, "megatron_optimizer_module_1": 1, "dict_9": 1, "tuple_5": 1, "dict_10": 1, "dict_11": 1, "buildable_traverser_metadata_5": 1, "byte_level_tokenizer_1": 1, "tuple_6": 1, "dict_12": 1, "dict_13": 1, "buildable_traverser_metadata_6": 1, "hyena_model_1": 1, "tuple_7": 1, "dict_14": 1, "dict_15": 1, "buildable_traverser_metadata_7": 1, "distributed_data_parallel_config_1": 1, "tuple_8": 1, "dict_16": 1, "dict_17": 1, "buildable_traverser_metadata_8": 1, "megatron_strategy_1": 1, "rich_model_summary_1": 1, "learning_rate_monitor_1": 1, "timing_callback_1": 1, "tevcallback_1": 1, "dict_18": 1, "dict_19": 1, "buildable_traverser_metadata_9": 1, "preemption_callback_1": 1, "model_checkpoint_1": 1, "list_1": 1, "megatron_mixed_precision_1": 1, "tuple_9": 1, "dict_20": 1, "dict_21": 1, "buildable_traverser_metadata_10": 1, "trainer_1": 1, "list_2": 1, "list_3": 1, "list_4": 1, "tuple_10": 1, "tuple_11": 1, "defaultdict_1": 1, "dict_22": 1, "tuple_12": 1, "dict_23": 1, "dict_24": 1, "buildable_traverser_metadata_11": 1, "byte_level_tokenizer_2": 1, "tuple_13": 1, "dict_25": 1, "dict_26": 1, "buildable_traverser_metadata_12": 1, "pre_training_data_module_1": 1, "tuple_14": 1, "dict_27": 1, "tuple_15": 1, "dict_28": 1, "dict_29": 1, "buildable_traverser_metadata_13": 1, "trainer_context_1": 1}, "version": "0.0.1"}