Training in progress, step 8500

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07328eeab3354e923721dcf5dc81f90b0d35763e3d2da61af29191c7a2e7c269
+oid sha256:9ebf61e4d96852d448117769c201ad05ba342a330472e0c4f7a17e11064f0353
 size 12017472

ray_logs/experiment_state-2025-10-16_06-08-49.json

@@ -1 +1 @@
-{"trial_data": [["{\n  \"stub\": false,\n  \"trainable_name\": \"TorchTrainer\",\n  \"trial_id\": \"95565_00000\",\n  \"storage\": {\n    \"_type\": \"CLOUDPICKLE_FALLBACK\",\n    \"value\": \"800595b1020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c087261795f6c6f6773948c0e747269616c5f6469725f6e616d65948c2e546f726368547261696e65725f39353536355f30303030305f305f323032352d31302d31365f30362d30382d3439948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e74948875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1446696c6553797374656d2e5f66726f6d5f7572699493948c0966696c653a2f2f2f5f94859452948c0f73746f726167655f66735f70617468948c402f726f6f742f6c6561702d66696e6574756e652f6f7574707574732f7366742f6c666d325f3335306d5f6d6172617468695f6f7074696d697a65645f31326570948c0673796e6365729468008c115f46696c6573797374656d53796e6365729493942981947d94286815681b68114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032352d31302d31365f30362d30382d34399475622e\"\n  },\n  \"config\": {\n    \"train_loop_config\": {\n      \"model_name\": \"LFM2-350M\",\n      \"train_config\": {\n        \"training_type\": \"sft\",\n        \"output_dir\": \"/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep\",\n        \"num_train_epochs\": 12,\n        \"per_device_train_batch_size\": 4,\n        \"learning_rate\": 8e-05,\n        \"lr_scheduler_type\": \"cosine\",\n        \"warmup_steps\": 100,\n        \"warmup_ratio\": 0.15,\n        \"logging_steps\": 25,\n        \"save_strategy\": \"steps\",\n        \"eval_strategy\": \"steps\",\n        \"load_best_model_at_end\": true,\n        \"ddp_find_unused_parameters\": false,\n        \"deepspeed\": {\n          \"zero_optimization\": {\n            \"stage\": 2,\n            \"overlap_comm\": true\n          },\n          \"train_batch_size\": \"auto\",\n          \"train_micro_batch_size_per_gpu\": \"auto\",\n          \"gradient_clipping\": \"auto\",\n          \"gradient_accumulation_steps\": \"auto\",\n          \"optimizer\": {\n            \"type\": \"AdamW\",\n            \"params\": {\n              \"lr\": \"auto\",\n              \"betas\": \"auto\",\n              \"eps\": \"auto\",\n              \"weight_decay\": \"auto\"\n            }\n          },\n          \"bf16\": {\n            \"enabled\": \"auto\"\n          },\n          \"activation_checkpointing\": {\n            \"partition_activations\": false,\n            \"cpu_checkpointing\": false,\n            \"contiguous_memory_optimization\": false,\n            \"number_checkpoints\": null,\n            \"synchronize_checkpoint_boundary\": false,\n            \"profile\": false\n          }\n        },\n        \"gradient_accumulation_steps\": 8,\n        \"gradient_checkpointing\": true,\n        \"bf16\": true,\n        \"dataloader_num_workers\": 4,\n        \"logging_first_step\": true,\n        \"eval_steps\": 250,\n        \"save_steps\": 500,\n        \"save_total_limit\": 5,\n        \"metric_for_best_model\": \"eval_loss\",\n        \"greater_is_better\": false,\n        \"weight_decay\": 0.01,\n        \"max_grad_norm\": 1.0,\n        \"push_to_hub\": true,\n        \"hub_model_id\": \"aghatage/lfm2-350m-marathi-optimized\",\n        \"hub_strategy\": \"checkpoint\",\n        \"hub_private_repo\": false\n      },\n      \"peft_config\": {\n        \"_type\": \"CLOUDPICKLE_FALLBACK\",\n        \"value\": \"80059558030000000000008c17706566742e74756e6572732e6c6f72612e636f6e666967948c0a4c6f7261436f6e6669679493942981947d94288c097461736b5f74797065948c15706566742e7574696c732e706566745f7479706573948c085461736b547970659493948c0943415553414c5f4c4d94859452948c09706566745f747970659468068c0850656674547970659493948c044c4f524194859452948c0c6175746f5f6d617070696e67944e8c17626173655f6d6f64656c5f6e616d655f6f725f70617468944e8c087265766973696f6e944e8c0e696e666572656e63655f6d6f646594898c0172944b108c0e7461726765745f6d6f64756c6573948f94288c066b5f70726f6a948c027731948c06715f70726f6a948c027732948c06765f70726f6a948c086f75745f70726f6a948c027733948c07696e5f70726f6a94908c0f6578636c7564655f6d6f64756c6573944e8c0a6c6f72615f616c706861944b208c0c6c6f72615f64726f706f757494473fb999999999999a8c0e66616e5f696e5f66616e5f6f757494898c0462696173948c046e6f6e65948c0a7573655f72736c6f726194898c0f6d6f64756c65735f746f5f73617665944e8c11696e69745f6c6f72615f7765696768747394888c136c61796572735f746f5f7472616e73666f726d944e8c0e6c61796572735f7061747465726e944e8c0c72616e6b5f7061747465726e947d948c0d616c7068615f7061747465726e947d948c0f6d65676174726f6e5f636f6e666967944e8c0d6d65676174726f6e5f636f7265948c0d6d65676174726f6e2e636f7265948c17747261696e61626c655f746f6b656e5f696e6469636573944e8c0c6c6f6674715f636f6e666967947d948c0a6576615f636f6e666967944e8c0c636f7264615f636f6e666967944e8c087573655f646f726194898c0a7573655f71616c6f726194898c1171616c6f72615f67726f75705f73697a65944b108c116c617965725f7265706c69636174696f6e944e8c0e72756e74696d655f636f6e6669679468008c114c6f726152756e74696d65436f6e6669679493942981947d948c15657068656d6572616c5f6770755f6f66666c6f6164948973628c096c6f72615f6269617394898c117461726765745f706172616d6574657273944e8c0f5f637573746f6d5f6d6f64756c6573944e75622e\"\n      },\n      \"dataset\": [\n        {\n          \"_type\": \"CLOUDPICKLE_FALLBACK\",\n          \"value\": \"8005951a080000000000008c1664617461736574732e6172726f775f64617461736574948c07446174617365749493942981947d94288c055f696e666f948c0d64617461736574732e696e666f948c0b44617461736574496e666f9493942981947d94288c0b6465736372697074696f6e948c00948c086369746174696f6e94680c8c08686f6d657061676594680c8c076c6963656e736594680c8c086665617475726573948c1a64617461736574732e66656174757265732e6665617475726573948c0846656174757265739493947d948c086d657373616765739468118c044c6973749493942981947d94288c0766656174757265947d94288c04726f6c659468118c0556616c75659493942981947d94288c056474797065948c06737472696e67948c026964944e8c0770615f74797065948c0b70796172726f772e6c6962948c0e747970655f666f725f616c6961739493948c06737472696e67948594529475628c07636f6e74656e7494681e2981947d942868218c06737472696e679468234e6824682a7562758c066c656e677468944affffffff68234e756273859452948c0e706f73745f70726f636573736564944e8c0f737570657276697365645f6b657973944e8c0c6275696c6465725f6e616d65948c0770617271756574948c0c646174617365745f6e616d65948c1a6d6172617468692d616c706163612d776974682d73797374656d948c0b636f6e6669675f6e616d65948c0764656661756c74948c0776657273696f6e948c1664617461736574732e7574696c732e76657273696f6e948c0756657273696f6e9493942981947d94288c0b76657273696f6e5f737472948c05302e302e3094680b4e8c056d616a6f72944b008c056d696e6f72944b008c057061746368944b0075628c0673706c697473948c0f64617461736574732e73706c697473948c0953706c6974446963749493942981948c05747261696e9468468c0953706c6974496e666f9493942981947d94288c046e616d6594684a8c096e756d5f6279746573944a0faa59068c0c6e756d5f6578616d706c6573944d01bf8c0d73686172645f6c656e67746873944e683668377562737d946836683773628c12646f776e6c6f61645f636865636b73756d73947d948c7c68663a2f2f64617461736574732f61676861746167652f6d6172617468692d616c706163612d776974682d73797374656d40353335353932653162633230373238343431393636636334626134353461643365353461376538612f646174612f747261696e2d30303030302d6f662d30303030312e70617271756574947d942868504ae0391c028c08636865636b73756d944e75738c0d646f776e6c6f61645f73697a65944ae0391c028c14706f73745f70726f63657373696e675f73697a65944e8c0c646174617365745f73697a65944a0faa59068c0d73697a655f696e5f6279746573944aefe3750875628c065f73706c69749468468c0a4e616d656453706c69749493942981947d948c055f6e616d65948c05747261696e9473628c085f696e6465786573947d948c055f64617461948c0e64617461736574732e7461626c65948c114d656d6f72794d61707065645461626c659493942981947d94288c0470617468948caa2f776f726b73706163652f2e63616368652f68756767696e67666163652f64617461736574732f61676861746167655f5f5f6d6172617468692d616c706163612d776974682d73797374656d2f64656661756c742f302e302e302f353335353932653162633230373238343431393636636334626134353461643365353461376538612f6d6172617468692d616c706163612d776974682d73797374656d2d747261696e2e6172726f77948c077265706c617973945d94288c177265706c6163655f736368656d615f6d65746164617461947d948c0b68756767696e6766616365948ca57b22696e666f223a207b226665617475726573223a207b226d65737361676573223a207b2266656174757265223a207b22726f6c65223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d2c2022636f6e74656e74223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d7d2c20225f74797065223a20224c697374227d7d7d7d947385947d94879468707d9468728ca57b22696e666f223a207b226665617475726573223a207b226d65737361676573223a207b2266656174757265223a207b22726f6c65223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d2c2022636f6e74656e74223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d7d2c20225f74797065223a20224c697374227d7d7d7d947385947d9487946575628c085f696e64696365739468692981947d9428686c8ca02f776f726b73706163652f2e63616368652f68756767696e67666163652f64617461736574732f61676861746167655f5f5f6d6172617468692d616c706163612d776974682d73797374656d2f64656661756c742f302e302e302f353335353932653162633230373238343431393636636334626134353461643365353461376538612f63616368652d336361363039326631613730356664322e6172726f7794686e5d9475628c0c5f666f726d61745f74797065944e8c0e5f666f726d61745f6b7761726773947d948c0f5f666f726d61745f636f6c756d6e73944e8c135f6f75747075745f616c6c5f636f6c756d6e7394898c0c5f66696e6765727072696e74948c10336361363039326631613730356664329475622e\"\n        },\n        {\n          \"_type\": \"CLOUDPICKLE_FALLBACK\",\n          \"value\": \"8005951a080000000000008c1664617461736574732e6172726f775f64617461736574948c07446174617365749493942981947d94288c055f696e666f948c0d64617461736574732e696e666f948c0b44617461736574496e666f9493942981947d94288c0b6465736372697074696f6e948c00948c086369746174696f6e94680c8c08686f6d657061676594680c8c076c6963656e736594680c8c086665617475726573948c1a64617461736574732e66656174757265732e6665617475726573948c0846656174757265739493947d948c086d657373616765739468118c044c6973749493942981947d94288c0766656174757265947d94288c04726f6c659468118c0556616c75659493942981947d94288c056474797065948c06737472696e67948c026964944e8c0770615f74797065948c0b70796172726f772e6c6962948c0e747970655f666f725f616c6961739493948c06737472696e67948594529475628c07636f6e74656e7494681e2981947d942868218c06737472696e679468234e6824682a7562758c066c656e677468944affffffff68234e756273859452948c0e706f73745f70726f636573736564944e8c0f737570657276697365645f6b657973944e8c0c6275696c6465725f6e616d65948c0770617271756574948c0c646174617365745f6e616d65948c1a6d6172617468692d616c706163612d776974682d73797374656d948c0b636f6e6669675f6e616d65948c0764656661756c74948c0776657273696f6e948c1664617461736574732e7574696c732e76657273696f6e948c0756657273696f6e9493942981947d94288c0b76657273696f6e5f737472948c05302e302e3094680b4e8c056d616a6f72944b008c056d696e6f72944b008c057061746368944b0075628c0673706c697473948c0f64617461736574732e73706c697473948c0953706c6974446963749493942981948c05747261696e9468468c0953706c6974496e666f9493942981947d94288c046e616d6594684a8c096e756d5f6279746573944a0faa59068c0c6e756d5f6578616d706c6573944d01bf8c0d73686172645f6c656e67746873944e683668377562737d946836683773628c12646f776e6c6f61645f636865636b73756d73947d948c7c68663a2f2f64617461736574732f61676861746167652f6d6172617468692d616c706163612d776974682d73797374656d40353335353932653162633230373238343431393636636334626134353461643365353461376538612f646174612f747261696e2d30303030302d6f662d30303030312e70617271756574947d942868504ae0391c028c08636865636b73756d944e75738c0d646f776e6c6f61645f73697a65944ae0391c028c14706f73745f70726f63657373696e675f73697a65944e8c0c646174617365745f73697a65944a0faa59068c0d73697a655f696e5f6279746573944aefe3750875628c065f73706c69749468468c0a4e616d656453706c69749493942981947d948c055f6e616d65948c05747261696e9473628c085f696e6465786573947d948c055f64617461948c0e64617461736574732e7461626c65948c114d656d6f72794d61707065645461626c659493942981947d94288c0470617468948caa2f776f726b73706163652f2e63616368652f68756767696e67666163652f64617461736574732f61676861746167655f5f5f6d6172617468692d616c706163612d776974682d73797374656d2f64656661756c742f302e302e302f353335353932653162633230373238343431393636636334626134353461643365353461376538612f6d6172617468692d616c706163612d776974682d73797374656d2d747261696e2e6172726f77948c077265706c617973945d94288c177265706c6163655f736368656d615f6d65746164617461947d948c0b68756767696e6766616365948ca57b22696e666f223a207b226665617475726573223a207b226d65737361676573223a207b2266656174757265223a207b22726f6c65223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d2c2022636f6e74656e74223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d7d2c20225f74797065223a20224c697374227d7d7d7d947385947d94879468707d9468728ca57b22696e666f223a207b226665617475726573223a207b226d65737361676573223a207b2266656174757265223a207b22726f6c65223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d2c2022636f6e74656e74223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d7d2c20225f74797065223a20224c697374227d7d7d7d947385947d9487946575628c085f696e64696365739468692981947d9428686c8ca02f776f726b73706163652f2e63616368652f68756767696e67666163652f64617461736574732f61676861746167655f5f5f6d6172617468692d616c706163612d776974682d73797374656d2f64656661756c742f302e302e302f353335353932653162633230373238343431393636636334626134353461643365353461376538612f63616368652d623734326135316238633166363034642e6172726f7794686e5d9475628c0c5f666f726d61745f74797065944e8c0e5f666f726d61745f6b7761726773947d948c0f5f666f726d61745f636f6c756d6e73944e8c135f6f75747075745f616c6c5f636f6c756d6e7394898c0c5f66696e6765727072696e74948c10623734326135316238633166363034649475622e\"\n        }\n      ]\n    }\n  },\n  \"_Trial__unresolved_config\": {\n    \"train_loop_config\": {\n      \"model_name\": \"LFM2-350M\",\n      \"train_config\": {\n        \"training_type\": \"sft\",\n        \"output_dir\": \"/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep\",\n        \"num_train_epochs\": 12,\n        \"per_device_train_batch_size\": 4,\n        \"learning_rate\": 8e-05,\n        \"lr_scheduler_type\": \"cosine\",\n        \"warmup_steps\": 100,\n        \"warmup_ratio\": 0.15,\n        \"logging_steps\": 25,\n        \"save_strategy\": \"steps\",\n        \"eval_strategy\": \"steps\",\n        \"load_best_model_at_end\": true,\n        \"ddp_find_unused_parameters\": false,\n        \"deepspeed\": {\n          \"zero_optimization\": {\n            \"stage\": 2,\n            \"overlap_comm\": true\n          },\n          \"train_batch_size\": \"auto\",\n          \"train_micro_batch_size_per_gpu\": \"auto\",\n          \"gradient_clipping\": \"auto\",\n          \"gradient_accumulation_steps\": \"auto\",\n          \"optimizer\": {\n            \"type\": \"AdamW\",\n            \"params\": {\n              \"lr\": \"auto\",\n              \"betas\": \"auto\",\n              \"eps\": \"auto\",\n              \"weight_decay\": \"auto\"\n            }\n          },\n          \"bf16\": {\n            \"enabled\": \"auto\"\n          },\n          \"activation_checkpointing\": {\n            \"partition_activations\": false,\n            \"cpu_checkpointing\": false,\n            \"contiguous_memory_optimization\": false,\n            \"number_checkpoints\": null,\n            \"synchronize_checkpoint_boundary\": false,\n            \"profile\": false\n          }\n        },\n        \"gradient_accumulation_steps\": 8,\n        \"gradient_checkpointing\": true,\n        \"bf16\": true,\n        \"dataloader_num_workers\": 4,\n        \"logging_first_step\": true,\n        \"eval_steps\": 250,\n        \"save_steps\": 500,\n        \"save_total_limit\": 5,\n        \"metric_for_best_model\": \"eval_loss\",\n        \"greater_is_better\": false,\n        \"weight_decay\": 0.01,\n        \"max_grad_norm\": 1.0,\n        \"push_to_hub\": true,\n        \"hub_model_id\": \"aghatage/lfm2-350m-marathi-optimized\",\n        \"hub_strategy\": \"checkpoint\",\n        \"hub_private_repo\": false\n      },\n      \"peft_config\": [\n        \"__ref_ph\",\n        \"bddb242d\"\n      ],\n      \"dataset\": [\n        [\n          \"__ref_ph\",\n          \"c5244594\"\n        ],\n        [\n          \"__ref_ph\",\n          \"d5c67b24\"\n        ]\n      ]\n    }\n  },\n  \"evaluated_params\": {},\n  \"experiment_tag\": \"0\",\n  \"stopping_criterion\": {},\n  \"_setup_default_resource\": true,\n  \"_default_placement_group_factory\": \"80054e2e\",\n  \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c0343505594473ff00000000000008c0347505594473ff000000000000075618c155f686561645f62756e646c655f69735f656d70747994888c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n  \"log_to_file\": [\n    null,\n    null\n  ],\n  \"max_failures\": 0,\n  \"_default_result_or_future\": null,\n  \"export_formats\": [],\n  \"status\": \"RUNNING\",\n  \"relative_logdir\": \"TorchTrainer_95565_00000_0_2025-10-16_06-08-49\",\n  \"trial_name_creator\": null,\n  \"trial_dirname_creator\": null,\n  \"custom_trial_name\": null,\n  \"custom_dirname\": null,\n  \"restore_path\": null,\n  \"_restore_checkpoint_result\": null,\n  \"_state_json\": null,\n  \"results\": \"80054e2e\",\n  \"extra_arg\": \"80054e2e\",\n  \"_resources\": \"80054e2e\"\n}", "{\n  \"start_time\": 1760594946.4726121,\n  \"num_failures\": 0,\n  \"num_failures_after_restore\": 0,\n  \"error_filename\": null,\n  \"pickled_error_filename\": null,\n  \"last_result\": {},\n  \"last_result_time\": -Infinity,\n  \"metric_analysis\": {},\n  \"_n_steps\": [\n    5,\n    10\n  ],\n  \"metric_n_steps\": {},\n  \"checkpoint_manager\": {\n    \"_type\": \"CLOUDPICKLE_FALLBACK\",\n    \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n  }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 241319, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1760594929.865938, "_session_str": "2025-10-16_06-08-49", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1760594929.865938}}
+{"trial_data": [["{\n  \"stub\": false,\n  \"trainable_name\": \"TorchTrainer\",\n  \"trial_id\": \"95565_00000\",\n  \"storage\": {\n    \"_type\": \"CLOUDPICKLE_FALLBACK\",\n    \"value\": \"800595b1020000000000008c1b7261792e747261696e2e5f696e7465726e616c2e73746f72616765948c0e53746f72616765436f6e746578749493942981947d94288c12637573746f6d5f66735f70726f766964656494898c136578706572696d656e745f6469725f6e616d65948c087261795f6c6f6773948c0e747269616c5f6469725f6e616d65948c2e546f726368547261696e65725f39353536355f30303030305f305f323032352d31302d31365f30362d30382d3439948c1863757272656e745f636865636b706f696e745f696e646578944affffffff8c0b73796e635f636f6e666967948c097261792e747261696e948c0a53796e63436f6e6669679493942981947d94288c0b73796e635f706572696f64944d2c018c0c73796e635f74696d656f7574944d08078c0e73796e635f61727469666163747394898c1c73796e635f6172746966616374735f6f6e5f636865636b706f696e74948875628c1273746f726167655f66696c6573797374656d948c0b70796172726f772e5f6673948c1446696c6553797374656d2e5f66726f6d5f7572699493948c0966696c653a2f2f2f5f94859452948c0f73746f726167655f66735f70617468948c402f726f6f742f6c6561702d66696e6574756e652f6f7574707574732f7366742f6c666d325f3335306d5f6d6172617468695f6f7074696d697a65645f31326570948c0673796e6365729468008c115f46696c6573797374656d53796e6365729493942981947d94286815681b68114d2c0168124d08078c116c6173745f73796e635f75705f74696d659447fff00000000000008c136c6173745f73796e635f646f776e5f74696d659447fff00000000000008c0d5f73796e635f70726f63657373944e8c0c5f63757272656e745f636d64944e75628c0a5f74696d657374616d70948c13323032352d31302d31365f30362d30382d34399475622e\"\n  },\n  \"config\": {\n    \"train_loop_config\": {\n      \"model_name\": \"LFM2-350M\",\n      \"train_config\": {\n        \"training_type\": \"sft\",\n        \"output_dir\": \"/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep\",\n        \"num_train_epochs\": 12,\n        \"per_device_train_batch_size\": 4,\n        \"learning_rate\": 8e-05,\n        \"lr_scheduler_type\": \"cosine\",\n        \"warmup_steps\": 100,\n        \"warmup_ratio\": 0.15,\n        \"logging_steps\": 25,\n        \"save_strategy\": \"steps\",\n        \"eval_strategy\": \"steps\",\n        \"load_best_model_at_end\": true,\n        \"ddp_find_unused_parameters\": false,\n        \"deepspeed\": {\n          \"zero_optimization\": {\n            \"stage\": 2,\n            \"overlap_comm\": true\n          },\n          \"train_batch_size\": \"auto\",\n          \"train_micro_batch_size_per_gpu\": \"auto\",\n          \"gradient_clipping\": \"auto\",\n          \"gradient_accumulation_steps\": \"auto\",\n          \"optimizer\": {\n            \"type\": \"AdamW\",\n            \"params\": {\n              \"lr\": \"auto\",\n              \"betas\": \"auto\",\n              \"eps\": \"auto\",\n              \"weight_decay\": \"auto\"\n            }\n          },\n          \"bf16\": {\n            \"enabled\": \"auto\"\n          },\n          \"activation_checkpointing\": {\n            \"partition_activations\": false,\n            \"cpu_checkpointing\": false,\n            \"contiguous_memory_optimization\": false,\n            \"number_checkpoints\": null,\n            \"synchronize_checkpoint_boundary\": false,\n            \"profile\": false\n          }\n        },\n        \"gradient_accumulation_steps\": 8,\n        \"gradient_checkpointing\": true,\n        \"bf16\": true,\n        \"dataloader_num_workers\": 4,\n        \"logging_first_step\": true,\n        \"eval_steps\": 250,\n        \"save_steps\": 500,\n        \"save_total_limit\": 5,\n        \"metric_for_best_model\": \"eval_loss\",\n        \"greater_is_better\": false,\n        \"weight_decay\": 0.01,\n        \"max_grad_norm\": 1.0,\n        \"push_to_hub\": true,\n        \"hub_model_id\": \"aghatage/lfm2-350m-marathi-optimized\",\n        \"hub_strategy\": \"checkpoint\",\n        \"hub_private_repo\": false\n      },\n      \"peft_config\": {\n        \"_type\": \"CLOUDPICKLE_FALLBACK\",\n        \"value\": \"80059558030000000000008c17706566742e74756e6572732e6c6f72612e636f6e666967948c0a4c6f7261436f6e6669679493942981947d94288c097461736b5f74797065948c15706566742e7574696c732e706566745f7479706573948c085461736b547970659493948c0943415553414c5f4c4d94859452948c09706566745f747970659468068c0850656674547970659493948c044c4f524194859452948c0c6175746f5f6d617070696e67944e8c17626173655f6d6f64656c5f6e616d655f6f725f70617468944e8c087265766973696f6e944e8c0e696e666572656e63655f6d6f646594898c0172944b108c0e7461726765745f6d6f64756c6573948f94288c066b5f70726f6a948c027731948c06715f70726f6a948c027732948c06765f70726f6a948c086f75745f70726f6a948c027733948c07696e5f70726f6a94908c0f6578636c7564655f6d6f64756c6573944e8c0a6c6f72615f616c706861944b208c0c6c6f72615f64726f706f757494473fb999999999999a8c0e66616e5f696e5f66616e5f6f757494898c0462696173948c046e6f6e65948c0a7573655f72736c6f726194898c0f6d6f64756c65735f746f5f73617665944e8c11696e69745f6c6f72615f7765696768747394888c136c61796572735f746f5f7472616e73666f726d944e8c0e6c61796572735f7061747465726e944e8c0c72616e6b5f7061747465726e947d948c0d616c7068615f7061747465726e947d948c0f6d65676174726f6e5f636f6e666967944e8c0d6d65676174726f6e5f636f7265948c0d6d65676174726f6e2e636f7265948c17747261696e61626c655f746f6b656e5f696e6469636573944e8c0c6c6f6674715f636f6e666967947d948c0a6576615f636f6e666967944e8c0c636f7264615f636f6e666967944e8c087573655f646f726194898c0a7573655f71616c6f726194898c1171616c6f72615f67726f75705f73697a65944b108c116c617965725f7265706c69636174696f6e944e8c0e72756e74696d655f636f6e6669679468008c114c6f726152756e74696d65436f6e6669679493942981947d948c15657068656d6572616c5f6770755f6f66666c6f6164948973628c096c6f72615f6269617394898c117461726765745f706172616d6574657273944e8c0f5f637573746f6d5f6d6f64756c6573944e75622e\"\n      },\n      \"dataset\": [\n        {\n          \"_type\": \"CLOUDPICKLE_FALLBACK\",\n          \"value\": \"8005951a080000000000008c1664617461736574732e6172726f775f64617461736574948c07446174617365749493942981947d94288c055f696e666f948c0d64617461736574732e696e666f948c0b44617461736574496e666f9493942981947d94288c0b6465736372697074696f6e948c00948c086369746174696f6e94680c8c08686f6d657061676594680c8c076c6963656e736594680c8c086665617475726573948c1a64617461736574732e66656174757265732e6665617475726573948c0846656174757265739493947d948c086d657373616765739468118c044c6973749493942981947d94288c0766656174757265947d94288c04726f6c659468118c0556616c75659493942981947d94288c056474797065948c06737472696e67948c026964944e8c0770615f74797065948c0b70796172726f772e6c6962948c0e747970655f666f725f616c6961739493948c06737472696e67948594529475628c07636f6e74656e7494681e2981947d942868218c06737472696e679468234e6824682a7562758c066c656e677468944affffffff68234e756273859452948c0e706f73745f70726f636573736564944e8c0f737570657276697365645f6b657973944e8c0c6275696c6465725f6e616d65948c0770617271756574948c0c646174617365745f6e616d65948c1a6d6172617468692d616c706163612d776974682d73797374656d948c0b636f6e6669675f6e616d65948c0764656661756c74948c0776657273696f6e948c1664617461736574732e7574696c732e76657273696f6e948c0756657273696f6e9493942981947d94288c0b76657273696f6e5f737472948c05302e302e3094680b4e8c056d616a6f72944b008c056d696e6f72944b008c057061746368944b0075628c0673706c697473948c0f64617461736574732e73706c697473948c0953706c6974446963749493942981948c05747261696e9468468c0953706c6974496e666f9493942981947d94288c046e616d6594684a8c096e756d5f6279746573944a0faa59068c0c6e756d5f6578616d706c6573944d01bf8c0d73686172645f6c656e67746873944e683668377562737d946836683773628c12646f776e6c6f61645f636865636b73756d73947d948c7c68663a2f2f64617461736574732f61676861746167652f6d6172617468692d616c706163612d776974682d73797374656d40353335353932653162633230373238343431393636636334626134353461643365353461376538612f646174612f747261696e2d30303030302d6f662d30303030312e70617271756574947d942868504ae0391c028c08636865636b73756d944e75738c0d646f776e6c6f61645f73697a65944ae0391c028c14706f73745f70726f63657373696e675f73697a65944e8c0c646174617365745f73697a65944a0faa59068c0d73697a655f696e5f6279746573944aefe3750875628c065f73706c69749468468c0a4e616d656453706c69749493942981947d948c055f6e616d65948c05747261696e9473628c085f696e6465786573947d948c055f64617461948c0e64617461736574732e7461626c65948c114d656d6f72794d61707065645461626c659493942981947d94288c0470617468948caa2f776f726b73706163652f2e63616368652f68756767696e67666163652f64617461736574732f61676861746167655f5f5f6d6172617468692d616c706163612d776974682d73797374656d2f64656661756c742f302e302e302f353335353932653162633230373238343431393636636334626134353461643365353461376538612f6d6172617468692d616c706163612d776974682d73797374656d2d747261696e2e6172726f77948c077265706c617973945d94288c177265706c6163655f736368656d615f6d65746164617461947d948c0b68756767696e6766616365948ca57b22696e666f223a207b226665617475726573223a207b226d65737361676573223a207b2266656174757265223a207b22726f6c65223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d2c2022636f6e74656e74223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d7d2c20225f74797065223a20224c697374227d7d7d7d947385947d94879468707d9468728ca57b22696e666f223a207b226665617475726573223a207b226d65737361676573223a207b2266656174757265223a207b22726f6c65223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d2c2022636f6e74656e74223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d7d2c20225f74797065223a20224c697374227d7d7d7d947385947d9487946575628c085f696e64696365739468692981947d9428686c8ca02f776f726b73706163652f2e63616368652f68756767696e67666163652f64617461736574732f61676861746167655f5f5f6d6172617468692d616c706163612d776974682d73797374656d2f64656661756c742f302e302e302f353335353932653162633230373238343431393636636334626134353461643365353461376538612f63616368652d336361363039326631613730356664322e6172726f7794686e5d9475628c0c5f666f726d61745f74797065944e8c0e5f666f726d61745f6b7761726773947d948c0f5f666f726d61745f636f6c756d6e73944e8c135f6f75747075745f616c6c5f636f6c756d6e7394898c0c5f66696e6765727072696e74948c10336361363039326631613730356664329475622e\"\n        },\n        {\n          \"_type\": \"CLOUDPICKLE_FALLBACK\",\n          \"value\": \"8005951a080000000000008c1664617461736574732e6172726f775f64617461736574948c07446174617365749493942981947d94288c055f696e666f948c0d64617461736574732e696e666f948c0b44617461736574496e666f9493942981947d94288c0b6465736372697074696f6e948c00948c086369746174696f6e94680c8c08686f6d657061676594680c8c076c6963656e736594680c8c086665617475726573948c1a64617461736574732e66656174757265732e6665617475726573948c0846656174757265739493947d948c086d657373616765739468118c044c6973749493942981947d94288c0766656174757265947d94288c04726f6c659468118c0556616c75659493942981947d94288c056474797065948c06737472696e67948c026964944e8c0770615f74797065948c0b70796172726f772e6c6962948c0e747970655f666f725f616c6961739493948c06737472696e67948594529475628c07636f6e74656e7494681e2981947d942868218c06737472696e679468234e6824682a7562758c066c656e677468944affffffff68234e756273859452948c0e706f73745f70726f636573736564944e8c0f737570657276697365645f6b657973944e8c0c6275696c6465725f6e616d65948c0770617271756574948c0c646174617365745f6e616d65948c1a6d6172617468692d616c706163612d776974682d73797374656d948c0b636f6e6669675f6e616d65948c0764656661756c74948c0776657273696f6e948c1664617461736574732e7574696c732e76657273696f6e948c0756657273696f6e9493942981947d94288c0b76657273696f6e5f737472948c05302e302e3094680b4e8c056d616a6f72944b008c056d696e6f72944b008c057061746368944b0075628c0673706c697473948c0f64617461736574732e73706c697473948c0953706c6974446963749493942981948c05747261696e9468468c0953706c6974496e666f9493942981947d94288c046e616d6594684a8c096e756d5f6279746573944a0faa59068c0c6e756d5f6578616d706c6573944d01bf8c0d73686172645f6c656e67746873944e683668377562737d946836683773628c12646f776e6c6f61645f636865636b73756d73947d948c7c68663a2f2f64617461736574732f61676861746167652f6d6172617468692d616c706163612d776974682d73797374656d40353335353932653162633230373238343431393636636334626134353461643365353461376538612f646174612f747261696e2d30303030302d6f662d30303030312e70617271756574947d942868504ae0391c028c08636865636b73756d944e75738c0d646f776e6c6f61645f73697a65944ae0391c028c14706f73745f70726f63657373696e675f73697a65944e8c0c646174617365745f73697a65944a0faa59068c0d73697a655f696e5f6279746573944aefe3750875628c065f73706c69749468468c0a4e616d656453706c69749493942981947d948c055f6e616d65948c05747261696e9473628c085f696e6465786573947d948c055f64617461948c0e64617461736574732e7461626c65948c114d656d6f72794d61707065645461626c659493942981947d94288c0470617468948caa2f776f726b73706163652f2e63616368652f68756767696e67666163652f64617461736574732f61676861746167655f5f5f6d6172617468692d616c706163612d776974682d73797374656d2f64656661756c742f302e302e302f353335353932653162633230373238343431393636636334626134353461643365353461376538612f6d6172617468692d616c706163612d776974682d73797374656d2d747261696e2e6172726f77948c077265706c617973945d94288c177265706c6163655f736368656d615f6d65746164617461947d948c0b68756767696e6766616365948ca57b22696e666f223a207b226665617475726573223a207b226d65737361676573223a207b2266656174757265223a207b22726f6c65223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d2c2022636f6e74656e74223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d7d2c20225f74797065223a20224c697374227d7d7d7d947385947d94879468707d9468728ca57b22696e666f223a207b226665617475726573223a207b226d65737361676573223a207b2266656174757265223a207b22726f6c65223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d2c2022636f6e74656e74223a207b226474797065223a2022737472696e67222c20225f74797065223a202256616c7565227d7d2c20225f74797065223a20224c697374227d7d7d7d947385947d9487946575628c085f696e64696365739468692981947d9428686c8ca02f776f726b73706163652f2e63616368652f68756767696e67666163652f64617461736574732f61676861746167655f5f5f6d6172617468692d616c706163612d776974682d73797374656d2f64656661756c742f302e302e302f353335353932653162633230373238343431393636636334626134353461643365353461376538612f63616368652d623734326135316238633166363034642e6172726f7794686e5d9475628c0c5f666f726d61745f74797065944e8c0e5f666f726d61745f6b7761726773947d948c0f5f666f726d61745f636f6c756d6e73944e8c135f6f75747075745f616c6c5f636f6c756d6e7394898c0c5f66696e6765727072696e74948c10623734326135316238633166363034649475622e\"\n        }\n      ]\n    }\n  },\n  \"_Trial__unresolved_config\": {\n    \"train_loop_config\": {\n      \"model_name\": \"LFM2-350M\",\n      \"train_config\": {\n        \"training_type\": \"sft\",\n        \"output_dir\": \"/root/leap-finetune/outputs/sft/lfm2_350m_marathi_optimized_12ep\",\n        \"num_train_epochs\": 12,\n        \"per_device_train_batch_size\": 4,\n        \"learning_rate\": 8e-05,\n        \"lr_scheduler_type\": \"cosine\",\n        \"warmup_steps\": 100,\n        \"warmup_ratio\": 0.15,\n        \"logging_steps\": 25,\n        \"save_strategy\": \"steps\",\n        \"eval_strategy\": \"steps\",\n        \"load_best_model_at_end\": true,\n        \"ddp_find_unused_parameters\": false,\n        \"deepspeed\": {\n          \"zero_optimization\": {\n            \"stage\": 2,\n            \"overlap_comm\": true\n          },\n          \"train_batch_size\": \"auto\",\n          \"train_micro_batch_size_per_gpu\": \"auto\",\n          \"gradient_clipping\": \"auto\",\n          \"gradient_accumulation_steps\": \"auto\",\n          \"optimizer\": {\n            \"type\": \"AdamW\",\n            \"params\": {\n              \"lr\": \"auto\",\n              \"betas\": \"auto\",\n              \"eps\": \"auto\",\n              \"weight_decay\": \"auto\"\n            }\n          },\n          \"bf16\": {\n            \"enabled\": \"auto\"\n          },\n          \"activation_checkpointing\": {\n            \"partition_activations\": false,\n            \"cpu_checkpointing\": false,\n            \"contiguous_memory_optimization\": false,\n            \"number_checkpoints\": null,\n            \"synchronize_checkpoint_boundary\": false,\n            \"profile\": false\n          }\n        },\n        \"gradient_accumulation_steps\": 8,\n        \"gradient_checkpointing\": true,\n        \"bf16\": true,\n        \"dataloader_num_workers\": 4,\n        \"logging_first_step\": true,\n        \"eval_steps\": 250,\n        \"save_steps\": 500,\n        \"save_total_limit\": 5,\n        \"metric_for_best_model\": \"eval_loss\",\n        \"greater_is_better\": false,\n        \"weight_decay\": 0.01,\n        \"max_grad_norm\": 1.0,\n        \"push_to_hub\": true,\n        \"hub_model_id\": \"aghatage/lfm2-350m-marathi-optimized\",\n        \"hub_strategy\": \"checkpoint\",\n        \"hub_private_repo\": false\n      },\n      \"peft_config\": [\n        \"__ref_ph\",\n        \"bddb242d\"\n      ],\n      \"dataset\": [\n        [\n          \"__ref_ph\",\n          \"c5244594\"\n        ],\n        [\n          \"__ref_ph\",\n          \"d5c67b24\"\n        ]\n      ]\n    }\n  },\n  \"evaluated_params\": {},\n  \"experiment_tag\": \"0\",\n  \"stopping_criterion\": {},\n  \"_setup_default_resource\": true,\n  \"_default_placement_group_factory\": \"80054e2e\",\n  \"placement_group_factory\": \"800595ba000000000000008c237261792e74756e652e657865637574696f6e2e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c085f62756e646c6573945d947d94288c0343505594473ff00000000000008c0347505594473ff000000000000075618c155f686561645f62756e646c655f69735f656d70747994888c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n  \"log_to_file\": [\n    null,\n    null\n  ],\n  \"max_failures\": 0,\n  \"_default_result_or_future\": null,\n  \"export_formats\": [],\n  \"status\": \"RUNNING\",\n  \"relative_logdir\": \"TorchTrainer_95565_00000_0_2025-10-16_06-08-49\",\n  \"trial_name_creator\": null,\n  \"trial_dirname_creator\": null,\n  \"custom_trial_name\": null,\n  \"custom_dirname\": null,\n  \"restore_path\": null,\n  \"_restore_checkpoint_result\": null,\n  \"_state_json\": null,\n  \"results\": \"80054e2e\",\n  \"extra_arg\": \"80054e2e\",\n  \"_resources\": \"80054e2e\"\n}", "{\n  \"start_time\": 1760594946.4726121,\n  \"num_failures\": 0,\n  \"num_failures_after_restore\": 0,\n  \"error_filename\": null,\n  \"pickled_error_filename\": null,\n  \"last_result\": {},\n  \"last_result_time\": -Infinity,\n  \"metric_analysis\": {},\n  \"_n_steps\": [\n    5,\n    10\n  ],\n  \"metric_n_steps\": {},\n  \"checkpoint_manager\": {\n    \"_type\": \"CLOUDPICKLE_FALLBACK\",\n    \"value\": \"80059584010000000000008c267261792e747261696e2e5f696e7465726e616c2e636865636b706f696e745f6d616e61676572948c125f436865636b706f696e744d616e616765729493942981947d94288c125f636865636b706f696e745f636f6e666967948c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394681275628c135f636865636b706f696e745f726573756c7473945d948c195f6c61746573745f636865636b706f696e745f726573756c74944e75622e\"\n  }\n}"]], "runner_data": {"_earliest_stopping_actor": Infinity, "_actor_cleanup_timeout": 600, "_actor_force_cleanup_timeout": 10, "_reuse_actors": false, "_buffer_length": 1, "_buffer_min_time_s": 0.0, "_buffer_max_time_s": 100.0, "_max_pending_trials": 200, "_metric": null, "_total_time": 0, "_iteration": 256319, "_has_errored": false, "_fail_fast": false, "_print_trial_errors": true, "_cached_trial_decisions": {}, "_queued_trial_decisions": {}, "_should_stop_experiment": false, "_stopper": {"_type": "CLOUDPICKLE_FALLBACK", "value": "8005952c000000000000008c157261792e74756e652e73746f707065722e6e6f6f70948c0b4e6f6f7053746f707065729493942981942e"}, "_start_time": 1760594929.865938, "_session_str": "2025-10-16_06-08-49", "_checkpoint_period": "auto", "_trial_checkpoint_config": {"_type": "CLOUDPICKLE_FALLBACK", "value": "800595f2000000000000008c097261792e747261696e948c10436865636b706f696e74436f6e6669679493942981947d94288c0b6e756d5f746f5f6b656570944e8c1a636865636b706f696e745f73636f72655f617474726962757465944e8c16636865636b706f696e745f73636f72655f6f72646572948c036d6178948c14636865636b706f696e745f6672657175656e6379944b008c11636865636b706f696e745f61745f656e6494898c1a5f636865636b706f696e745f6b6565705f616c6c5f72616e6b73948c0a44455052454341544544948c1f5f636865636b706f696e745f75706c6f61645f66726f6d5f776f726b65727394680c75622e"}, "_resumed": false}, "stats": {"start_time": 1760594929.865938}}

runs/Oct16_06-09-16_5586afa3f988/events.out.tfevents.1760595098.5586afa3f988

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b7985bcb6f5c42bab7ad55a0675694bfd9a8edaba44827f7f3fb73be36846531
-size 124726
+oid sha256:dfe564a0c48702922c1c449447b28155dd27ec904105a82aad787fd2caa5999e
+size 132102