Checkpoint at step 40000

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a174ef7ce0c4c91f03dfaa1da446eb56194034d6ebd570bbda05f47837ed2bf0
+oid sha256:e852e31fadf2566fdba7ebc8d456578820b724fcc2d3481b296277321c7a528a
 size 828512040

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7427a6a2c676e9c0179d12cf99b17bc116b5ee3931ef4785f3721828477fb9c
+oid sha256:3d795647eb1b830bb555594115e0d0e2dd648b2292077362acfc38684c8d42be
 size 1657183499

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c03aaf8d6a80448551a2ff6a31325875df2fc4b5630b7ab23b4efffd4ba5bfe
+oid sha256:24283bdedf9664a219b45889b2cb297750d51485079a99ec4cc794379d885fef
 size 14709

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69350fe377f4fbb41ad9123978ca0927051a1fcac35b55eafcfef32a09708d4a
+oid sha256:40e39c62dc7958d0b4ec9e70f0a92b9bbef106b669c87a83bd7746958ec30824
 size 1383

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a6b200d12a55a703157e4869cf5f39ac2652159ef002f661f44b9e4322421a1
+oid sha256:4550786e39369e9c3ca0437120bbc321596c5a6d7f9b1cef8d6993ef9b199588
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.46320123520329387,
+  "epoch": 0.6176016469377251,
   "eval_steps": 500,
-  "global_step": 30000,
+  "global_step": 40000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -218,6 +218,76 @@
       "learning_rate": 1.53681658869486e-05,
       "loss": 0.01891119956970215,
       "step": 30000
+    },
+    {
+      "epoch": 0.478641276376737,
+      "grad_norm": 0.2585284411907196,
+      "learning_rate": 1.521376626985965e-05,
+      "loss": 0.019102977752685546,
+      "step": 31000
+    },
+    {
+      "epoch": 0.49408131755018014,
+      "grad_norm": 0.2984430491924286,
+      "learning_rate": 1.5059366652770702e-05,
+      "loss": 0.0187547550201416,
+      "step": 32000
+    },
+    {
+      "epoch": 0.5095213587236233,
+      "grad_norm": 0.1507445126771927,
+      "learning_rate": 1.4904967035681753e-05,
+      "loss": 0.021003768920898436,
+      "step": 33000
+    },
+    {
+      "epoch": 0.5249613998970664,
+      "grad_norm": 0.24011844396591187,
+      "learning_rate": 1.4750567418592802e-05,
+      "loss": 0.01817093276977539,
+      "step": 34000
+    },
+    {
+      "epoch": 0.5404014410705095,
+      "grad_norm": 0.2567104697227478,
+      "learning_rate": 1.4596167801503853e-05,
+      "loss": 0.017122507095336914,
+      "step": 35000
+    },
+    {
+      "epoch": 0.5558414822439527,
+      "grad_norm": 0.23033738136291504,
+      "learning_rate": 1.4441768184414904e-05,
+      "loss": 0.01751700210571289,
+      "step": 36000
+    },
+    {
+      "epoch": 0.5712815234173958,
+      "grad_norm": 0.1389719396829605,
+      "learning_rate": 1.4287368567325955e-05,
+      "loss": 0.017340713500976563,
+      "step": 37000
+    },
+    {
+      "epoch": 0.5867215645908389,
+      "grad_norm": 0.020788291469216347,
+      "learning_rate": 1.4132968950237004e-05,
+      "loss": 0.016769933700561523,
+      "step": 38000
+    },
+    {
+      "epoch": 0.602161605764282,
+      "grad_norm": 0.014325047843158245,
+      "learning_rate": 1.3978569333148055e-05,
+      "loss": 0.01715911293029785,
+      "step": 39000
+    },
+    {
+      "epoch": 0.6176016469377251,
+      "grad_norm": 0.3109320104122162,
+      "learning_rate": 1.3824169716059106e-05,
+      "loss": 0.017378036499023437,
+      "step": 40000
     }
   ],
   "logging_steps": 1000,
@@ -237,7 +307,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.638677453096141e+17,
+  "total_flos": 6.183935693796557e+17,
   "train_batch_size": 24,
   "trial_name": null,
   "trial_params": null

run/progress.json

@@ -1,35 +1,35 @@
 {
   "state": "training",
-  "global_step": 38400,
+  "global_step": 40000,
   "max_steps": 129534,
-  "epoch": 0.5928975810602162,
-  "elapsed_seconds": 14220.950442084999,
-  "steps_per_second": 2.7002414611023706,
-  "optimizer_updates_this_process": 38400,
-  "peak_cuda_GiB": 16.14784526824951,
+  "epoch": 0.6176016469377251,
+  "elapsed_seconds": 3863.9128347660007,
+  "steps_per_second": 2.5880501004121648,
+  "optimizer_updates_this_process": 10000,
+  "peak_cuda_GiB": 16.14780044555664,
   "world_size": 1,
   "effective_batch_size": 72,
   "recent_metrics": [
     {
-      "loss": 0.01756742477416992,
-      "grad_norm": 0.28143829107284546,
-      "learning_rate": 1.4441768184414904e-05,
-      "epoch": 0.5558414822439527,
-      "step": 36000
-    },
-    {
-      "loss": 0.01727944755554199,
-      "grad_norm": 0.07350185513496399,
+      "loss": 0.017340713500976563,
+      "grad_norm": 0.1389719396829605,
       "learning_rate": 1.4287368567325955e-05,
       "epoch": 0.5712815234173958,
       "step": 37000
     },
     {
-      "loss": 0.01676115608215332,
-      "grad_norm": 0.022185837849974632,
+      "loss": 0.016769933700561523,
+      "grad_norm": 0.020788291469216347,
       "learning_rate": 1.4132968950237004e-05,
       "epoch": 0.5867215645908389,
       "step": 38000
+    },
+    {
+      "loss": 0.01715911293029785,
+      "grad_norm": 0.014325047843158245,
+      "learning_rate": 1.3978569333148055e-05,
+      "epoch": 0.602161605764282,
+      "step": 39000
     }
   ]
 }

run/resume-history.jsonl

@@ -0,0 +1 @@
+{"provenance": {"scope": "universal", "ordinary_version": null, "task": "statement", "initialization": "/home/henryh/pylingual-runs/hub/Salesforce--codet5-base", "initial_weights_sha256": "bf6d33ae2678bf6c2ecbbd19049a304b4a1ad2c39f91c396020dca53b8b2a4cd", "data_manifest": {"scope": "universal", "bindings": {}, "split": "train", "task": "statement", "journal_sha256": "98cf9ae8f79afb03367e926aeae16f69d41f8dc39b4d893744ec159c7f925c96", "files": 50000, "smoke_only": false, "encoding_implementation_sha256": "127eebcd5d7b399b63667d5bfac738be6d89ae8379af37e37c2b78be4c21f9a7", "tokenizer_json_sha256": "576881c7af5310402326eff7bf5584ad6e2f88dcfd4c998d4e8b6a26f243153f", "registry_sha256": "522c85e983dcca62d12b648ca24f4fe4a5c554720eaf69aa130db6dffab495e4", "versions": {"3.6": 5556, "3.7": 5556, "3.8": 5556, "3.9": 5556, "3.10": 5556, "3.11": 5555, "3.12": 5555, "3.13": 5555, "3.14": 5555}, "corpora": {"pypi": 45000, "csn": 5000}, "raw_examples": 4663180, "all_converter_rows": true, "limits": {"statement_input": 256, "statement_target": 256, "segmentation_window": 512, "stride": 128}, "complete": true, "encoded_rows": 4663180, "dataset_fingerprint": "6c2dc84fd509e91b", "seconds": 2375.584766071057}, "tokenizer_registry_fingerprint": "c66d4184e9f396c0f2c18d45930d65f589303f7f66dd0b4f3a6b9c171af5b03b", "training_arguments": {"output_dir": "/home/henryh/pylingual-runs/universal/statement/training", "per_device_train_batch_size": 24, "num_train_epochs": 2, "max_steps": -1, "learning_rate": 2e-05, "lr_scheduler_type": "linear", "lr_scheduler_kwargs": null, "warmup_steps": 0, "optim": "adamw_torch_fused", "optim_args": null, "weight_decay": 0.01, "adam_beta1": 0.9, "adam_beta2": 0.999, "adam_epsilon": 1e-08, "optim_target_modules": null, "gradient_accumulation_steps": 3, "average_tokens_across_devices": true, "max_grad_norm": 1.0, "label_smoothing_factor": 0.0, "bf16": false, "fp16": true, "bf16_full_eval": false, "fp16_full_eval": false, "tf32": null, "gradient_checkpointing": false, "gradient_checkpointing_kwargs": null, "torch_compile": false, "torch_compile_backend": null, "torch_compile_mode": null, "use_liger_kernel": false, "liger_kernel_config": null, "use_cache": false, "neftune_noise_alpha": null, "torch_empty_cache_steps": null, "auto_find_batch_size": false, "logging_strategy": "steps", "logging_steps": 1000, "logging_first_step": false, "log_on_each_node": true, "logging_nan_inf_filter": false, "include_num_input_tokens_seen": "no", "log_level": "passive", "log_level_replica": "warning", "disable_tqdm": true, "report_to": [], "run_name": null, "project": "huggingface", "trackio_space_id": null, "trackio_bucket_id": null, "trackio_static_space_id": null, "eval_strategy": "no", "eval_steps": null, "eval_delay": 0, "per_device_eval_batch_size": 8, "prediction_loss_only": false, "eval_on_start": false, "eval_do_concat_batches": true, "eval_use_gather_object": false, "eval_accumulation_steps": null, "include_for_metrics": [], "batch_eval_metrics": false, "save_only_model": false, "save_strategy": "steps", "save_steps": 10000, "save_on_each_node": false, "save_total_limit": 2, "enable_jit_checkpoint": false, "push_to_hub": false, "hub_token": "<HUB_TOKEN>", "hub_private_repo": null, "hub_model_id": null, "hub_strategy": "every_save", "hub_always_push": false, "hub_revision": null, "load_best_model_at_end": false, "metric_for_best_model": null, "greater_is_better": null, "ignore_data_skip": false, "restore_callback_states_from_checkpoint": false, "full_determinism": false, "seed": 20261020, "data_seed": null, "use_cpu": false, "accelerator_config": {"split_batches": false, "dispatch_batches": null, "even_batches": true, "use_seedable_sampler": true, "non_blocking": false, "gradient_accumulation_kwargs": null}, "parallelism_config": null, "dataloader_drop_last": false, "dataloader_num_workers": 2, "dataloader_pin_memory": true, "dataloader_persistent_workers": false, "dataloader_prefetch_factor": null, "dataloader_multiprocessing_context": null, "dataloader_in_order": true, "remove_unused_columns": false, "label_names": null, "train_sampling_strategy": "random", "length_column_name": "length", "ddp_find_unused_parameters": false, "ddp_bucket_cap_mb": null, "ddp_broadcast_buffers": null, "ddp_static_graph": null, "ddp_backend": null, "ddp_timeout": 1800, "fsdp": null, "fsdp_config": null, "deepspeed": null, "debug": [], "skip_memory_metrics": true, "do_train": false, "do_eval": false, "do_predict": false, "resume_from_checkpoint": null, "local_rank": -1, "sortish_sampler": false, "predict_with_generate": false, "generation_max_length": null, "generation_num_beams": null, "generation_config": null}, "original_vocab": 32100, "vocab": 11577, "torch": "2.14.0+cu130", "transformers": "5.17.0", "python": "3.12.16", "hostname": "triplesec1-gpu", "smoke_only": false, "script_sha256": "0647e466755f5c2995b095ac4e1119b0a3b6380d7c4271d952d76e3d1ffeed9f", "world_size": 1, "loss_normalization": "global-supervised-token-mean-v1", "loss_normalization_sha256": "3d8e6217caddd6b6e66f35df704c9fa7b7fb0b29849e84fa2cb91d597383c274"}, "checkpoint": "/home/henryh/pylingual-runs/universal/statement/training/checkpoint-30000", "layout": {"original_world_size": 1, "world_size": 1, "original_per_device_batch": 24, "per_device_batch": 24, "effective_batch_size": 72, "changed_from_original_layout": false}, "checkpoint_world_size": 1, "observed_at": 1793306820.47653}

run/resume-verification.json

@@ -1,16 +1,16 @@
 {
-  "global_step": 0,
+  "global_step": 30000,
   "world_size": 1,
   "effective_batch_size": 72,
-  "optimizer_state_entries": 0,
-  "optimizer_step_min": null,
-  "optimizer_step_max": null,
-  "scheduler_last_epoch": 0,
+  "optimizer_state_entries": 257,
+  "optimizer_step_min": 29991.0,
+  "optimizer_step_max": 29991.0,
+  "scheduler_last_epoch": 30000,
   "scaler": {
-    "scale": 65536.0,
+    "scale": 131072.0,
     "growth_factor": 2.0,
     "backoff_factor": 0.5,
     "growth_interval": 2000,
-    "_growth_tracker": 0
+    "_growth_tracker": 509
   }
 }