initial commit of Pharia4608 embedding control model

config.yml

@@ -0,0 +1,227 @@
+{
+    "version": ".unknown.",
+    "runner": {
+        "runner_type": "pdsh",
+        "hostsfile": null,
+        "hosts": null,
+        "master_port": 29500,
+        "master_addr": null,
+        "script": "src/scaling/transformer/train.py",
+        "default_gpu_count": 8,
+        "docker_config": {
+            "docker_container": null,
+            "docker_sudo": false,
+            "docker_mounts": [
+                [
+                    "/mnt/",
+                    "/mnt/"
+                ]
+            ]
+        },
+        "use_determined": true
+    },
+    "logger": {
+        "log_level": "info",
+        "log_dir": null,
+        "metrics_ranks": null,
+        "use_wandb": true,
+        "wandb_ranks": null,
+        "wandb_host": "https://api.wandb.ai",
+        "wandb_team": "aleph-alpha",
+        "wandb_project": "ng-semantic-embedding",
+        "wandb_group": "7b_medi_from_pretrained-2024-09-17-08-51-16",
+        "wandb_api_key": "d80060c84c496915fcad15283809dbde65087d73",
+        "use_tensorboard": false,
+        "tensorboard_ranks": null,
+        "determined_metrics_ranks": null
+    },
+    "topology": {
+        "global_rank": 0,
+        "world_size": 256,
+        "local_slot": 0,
+        "model_parallel_size": 1,
+        "pipe_parallel_size": 1,
+        "data_parallel_size": 256,
+        "global_batch_size": 2048,
+        "micro_batch_size": 8,
+        "gradient_accumulation_steps": 1,
+        "pipe_partition_method": "balanced",
+        "pipe_partition_overwrite": null,
+        "activation_checkpointing_type": "every_layer",
+        "sequence_parallel": false
+    },
+    "optimizer": {
+        "method": "adamw",
+        "beta1": 0.9,
+        "beta2": 0.95,
+        "eps": 1e-15,
+        "gradient_clipping": 0.0,
+        "allreduce_bucket_size": 500000000,
+        "loss_scaler": {
+            "enable": false,
+            "initial_scale": 4294967296.0,
+            "window": 1000,
+            "hysteresis": 2.0,
+            "consecutive_hysteresis": false,
+            "min_scale": 1.0,
+            "factor": 2.0
+        },
+        "zero": true,
+        "zero_save_static": false,
+        "debug_log": false
+    },
+    "learning_rate_scheduler": {
+        "learning_rate": 2e-05,
+        "learning_rate_minimum": 0.0,
+        "learning_rate_decay_style": "linear",
+        "learning_rate_decay_iters": 938,
+        "learning_rate_warmup_steps": 50
+    },
+    "embedding_learning_rate_scheduler": {
+        "learning_rate": 0.0,
+        "learning_rate_minimum": 0.0,
+        "learning_rate_decay_style": "cosine",
+        "learning_rate_decay_iters": 0,
+        "learning_rate_warmup_steps": 0
+    },
+    "training": {
+        "weight_decay": 0.0001,
+        "finetune": true,
+        "finetunable_parameters": [
+            "embed"
+        ],
+        "parameters_exclude": [],
+        "use_separate_lr_on_embeddings": false,
+        "use_deterministic_torch_algorithms": false,
+        "loss_function_config": {
+            "loss_type": "contrastive_loss",
+            "number_of_hard_negatives": 1,
+            "use_instructions": true,
+            "query_side_only": false,
+            "scale": 50,
+            "log_verbose_metrics": true
+        }
+    },
+    "trainer": {
+        "save_dir": "checkpoints",
+        "save_interval": 938,
+        "load_dir": "/scratch/samuel/Pharia-1-LLM-7B-control",
+        "train_iterations": 938,
+        "assert_checkpoint_loaded": true,
+        "load_optimizer_states": false,
+        "delete_past_optimizer_states": true,
+        "load_context": false,
+        "allowed_missing_keys_in_checkpoint": [
+            "embed"
+        ],
+        "allowed_unexpected_keys_in_checkpoint": null,
+        "ignore_keys_in_checkpoint": null,
+        "merge_lora_after_loading_checkpoint": false,
+        "seed": 42,
+        "dataloader_num_workers": 0,
+        "dataloader_pin_memory": true,
+        "dataloader_prefetch_factor": null,
+        "eval_iterations": 1,
+        "eval_interval": null,
+        "separate_file_for_parameters": [
+            "adapter_embed"
+        ]
+    },
+    "profiler": {
+        "profile_steps": 0,
+        "profile_start_at_step": 0,
+        "profiler_output": null
+    },
+    "transformer_architecture": {
+        "vocab_size": 128000,
+        "vocab_file": "/scratch/samuel/Pharia-1-LLM-7B-control/vocab.json",
+        "hidden_size": 4608,
+        "num_layers": 27,
+        "num_attention_heads": 36,
+        "num_local_attention_heads": 0,
+        "local_attention_window_size": null,
+        "rotary_embedding_base": 1000000,
+        "rotary_percentage": 1.0,
+        "sequence_length": 2048,
+        "norm_type": "layernorm",
+        "relative_position_embedding_type": "rotary_complex",
+        "mlp_type": "default",
+        "mlp_factor": 4.0,
+        "attention_bias": true,
+        "attention_qkv_in_one": false,
+        "attention_num_kv_heads": 4,
+        "attention_use_matmul": false,
+        "mlp_bias": true,
+        "key_query_norm": false,
+        "weight_tying": false,
+        "masked_softmax": {
+            "kernel": "flash_attention",
+            "softmax_in_fp32": true,
+            "scale": 1.0,
+            "deterministic_flash_attn_bwd": false
+        },
+        "layernorm": {
+            "optimization_type": "torch",
+            "layernorm_epsilon": 1e-05
+        },
+        "precision": "bfloat16",
+        "dropout_embedding": 0.0,
+        "dropout_attention_probs": 0.0,
+        "dropout_after_attention": 0.0,
+        "dropout_after_mlp": 0.0,
+        "bitfit_bias_config": null,
+        "finetunable_token_ids": [],
+        "image_encoder": false,
+        "dropout_image_encoder": 0.0,
+        "softprompt_config": null,
+        "adapter_config": {
+            "name": "embed",
+            "attention_downsampling_factor": 0.25,
+            "mlp_downsampling_factor": 0.25,
+            "init_std": 1e-05,
+            "version": ".unknown."
+        },
+        "lora_config": null,
+        "embedding_head_config": {
+            "name": "pooling_only",
+            "proj_layers": null,
+            "pooling": "weighted_mean"
+        },
+        "lm_head": false,
+        "causal": true
+    },
+    "data": {
+        "legacy_dataset": false,
+        "load_mmap_index_to_memory": false,
+        "use_mmap": false,
+        "load_data_item_mmap_index_to_memory": false,
+        "finetuning_dataset": false,
+        "finetuning_chat_dataset": false,
+        "finetuning_dataset_memory_map": false,
+        "embedding_dataset": true,
+        "embedding_dataset_memory_map": false,
+        "data_prefixes": [
+            "/scratch/samuel/all_source_data_templated/allnli_eng.jsonl",
+            "/scratch/samuel/all_source_data_templated/hotpot_with_bm25_negatives.jsonl",
+            "/scratch/samuel/all_source_data_templated/medi_1.jsonl",
+            "/scratch/samuel/all_source_data_templated/msmarco.jsonl"
+        ],
+        "validation_data_prefixes": null,
+        "blended_dataset": {
+            "weight_by_num_documents": true,
+            "weighted_sampler_alpha": 1.0,
+            "weights": null,
+            "weight_examples_proportional": false,
+            "ep_maximum": null,
+            "ep_temperature": 1.0,
+            "minimum_dataset_size": 0,
+            "cache_directory": "/scratch/samuel/blended_dataset_cache",
+            "shuffle_dataset_indices": false,
+            "load_dataset_indices_to_memory": false
+        },
+        "only_full_sequences": false,
+        "allow_incomplete_sequences_every_n": 0
+    },
+    "determined_experiment_id": 2765,
+    "determined_trial_id": 81283
+}

model_state_layer_0_EmbeddingInput.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63f7803c3b4d22df9ef5c826e0df060f140138e5f25b9be47431ad2663802ff0
+size 1179649443

model_state_layer_10_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef63496e306c9f3e0112742300dd47b51c31fd2226fd3d7aa5c2adc1a5ffb265
+size 434219902

model_state_layer_10_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:676a207e49d6c3a2ec44a9e46f2556b14aff9338fba4e264ed2e74a5f2cd0e6e
+size 42469872

model_state_layer_11_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38ae21f20ea4e1492bdf2369f7ef91dda55015f81ab4eab9604a9362086d12cb
+size 434219902

model_state_layer_11_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cecb2254f42e26b348186f40932e45e15619b614726a2b0260c7c9ccc153020
+size 42469872

model_state_layer_12_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd902b46a5a353687a2f31e754164e3db18fe72c769a69ed13284bbff224f455
+size 434219902

model_state_layer_12_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9792a7ebe8ce63557a6d899cdf41de63ada82de5bde9c1af8eb2949704a83094
+size 42469872

model_state_layer_13_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:664c1b719da7b9218debdc2e9794798ca9d0f22568b138996e182229d79c3444
+size 434219902

model_state_layer_13_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8a6450411a45420b87e825d62d1733c5a5ecfbcd7a0a0a19842cfed1eb98172
+size 42469872

model_state_layer_14_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef77d20a502e6c8d8908cbd7cb222d04ccea69002e78042a57ac11a8a0b4362
+size 434219902

model_state_layer_14_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c084fd0149e3b81d783bc54a83026b0b9bf34b69db846af5981a57f37f3e738e
+size 42469872

model_state_layer_15_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7de81299e97471d03af358c730a40238867eb6032c93c4cebcba714f2db5d47
+size 434219902

model_state_layer_15_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:685c7a8933ff4764e0fe8df4cbd42569c956faa08c00da4530da692c065a5159
+size 42469872

model_state_layer_16_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e866357f6d06e90c4a4785e7b0cc2056b50361904168d54fb6fc314ef9f4639b
+size 434219902

model_state_layer_16_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9de4b8a1852dfbbfb30e80a42ec141062ee1f9cef355074ef2f6b5305c30a715
+size 42469872

model_state_layer_17_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03b4c380ed92cd055a067919127a7365bcd66f18fa8dfa8d38653018ac0f597c
+size 434219902

model_state_layer_17_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44c33bda7e7ced7993a85681a940aff2a79678c37c7ee02721dca5f7a1388b6a
+size 42469872

model_state_layer_18_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c34494e568a417b9adae97acb4d7c76012a9903a3e3d596e68d0020f1af6d2b0
+size 434219902

model_state_layer_18_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a208197337f96311529638ce0f8ecda2728e034e13d5a61fd1337292b6bac799
+size 42469872

model_state_layer_19_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2807c758304a165e6140b8942fa9eab390814f81c26fd327628822ce004ebfe7
+size 434219902

model_state_layer_19_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3effc3fdeb41663fd7d38516cbd2b025edc6210e8eb945523ead26a81c8618cf
+size 42469872

model_state_layer_1_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c2679a5ca251090f3a75c87b7add5f42e975a213ec8aaee90bde27da688a3d1
+size 434219882

model_state_layer_1_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d6d706fec5794b603bd590ba493d4c23fe1e6858917211027112f725d650bf4
+size 42469864

model_state_layer_20_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:197bec6a16feca3ac18653521b975673a56a071bf59ae3a9bfce737648bda775
+size 434219902

model_state_layer_20_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:834b9eda7c39ce0afee590f0e7c1a034d47bb0a0c8b827eac414f1f8fe12496b
+size 42469872

model_state_layer_21_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31eb0f91b2b326f7b869a2dc5dae07be2a6831efb72d3193d1d6d457a2d6fcff
+size 434219902

model_state_layer_21_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b0d557feef0473fa638bf2a02808d6887cc1b3fba759addcf4eecd07ca52956
+size 42469872

model_state_layer_22_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8deac3ee07d0285db53ce915d9315491287708edec05a9f16e17e3b0c2395ceb
+size 434219902

model_state_layer_22_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01c5dd99e10c4235d9caca5b13b649bb912de5bfaf4783ce07513b54b5c3b0b8
+size 42469872

model_state_layer_23_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28a4de46c226702e34dbf4d5cd8ead1d366f7cd94b5c5a0f9481dc0f5d502214
+size 434219902

model_state_layer_23_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1c041f2f5985fcf81de47980dca2a231269254757771c3fd703b6b8479d0623
+size 42469872

model_state_layer_24_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:290bde2e41ce42a45bef1ae6977cbc4f8add6ca43f1493fedeac5d17b7497248
+size 434219902

model_state_layer_24_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d0f786469b37c31f5e321fe6b20032147c4fe4860a50844e096dabc2f66c5bb
+size 42469872

model_state_layer_25_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45968daef6d99223322950b1251b22b8462b8d95fd355edb67421feaef60fc49
+size 434219902

model_state_layer_25_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:631d05d00de00be24e252a7d448c6d028d9dab8ffbd4de365708feac348b4ac0
+size 42469872

model_state_layer_26_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47274e293886f267b6ef434f55b4087c3400132b40e33ea732762fdcf544648d
+size 434219902

model_state_layer_26_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2e7c0b257e634dc8e2140981bd688dd059a43926d68426f43c80d3b3a05f92d
+size 42469872

model_state_layer_27_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f25827475fe28d36d12002c77f4e337f01af1d38524d74163c804f387ebcad4
+size 434219902

model_state_layer_27_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06e95fb59125bcf66ac9d1c257889094fa27ac8f64802ab94020b031cd05b83c
+size 42469872

model_state_layer_28_LayerNormWrapper.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cd1372d2ae21d66d644fb98394d0868cc9aeff84ccc20619a9bbf95eeebcd62
+size 20172

model_state_layer_2_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15decebd9fe0f57b1d69766411ab106d37f354b7dc1b6b280d48baca79626756
+size 434219882

model_state_layer_2_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2ad19f1d9e94cf44d1015372f7b060bc1f96bc3aee2ef8c82933051e8cd48c1
+size 42469864

model_state_layer_3_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c9465e9dcb05d29f2c9e320c53e486825e05341a6c5097fcefa86ca1384f495
+size 434219882

model_state_layer_3_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d046231c7dd30aa6f3cf077183f9c806f26f4d00683c08e4021681db492541e
+size 42469864

model_state_layer_4_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f8449ff2999f09a5ae6a621104c5d8929bb9f6969315cf4210bfe327dd0bc72
+size 434219882

model_state_layer_4_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd805b365e2e0394fac691cde5f39a6dfee441f9e3e0c167a3194bfc8bfd775a
+size 42469864

model_state_layer_5_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a4b77dab79cedd475c6335c22188eefd71bcb07cf39a11d8f80deb7b678f2ac
+size 434219882

model_state_layer_5_TransformerLayer_adapter_embed.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07ae55c82c22d0e1c80796c921607a40770741c522c34adf533d1790599ad718
+size 42469864

model_state_layer_6_TransformerLayer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca257c7e6088e46d529b4b8be5c185d1767f3f5916db5e0e799942af25c9c1b0
+size 434219882